atris 3.82.10 → 3.82.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/atris.js +14 -0
- package/commands/memory-live.js +373 -0
- package/commands/memory.js +205 -0
- package/commands/security-review.js +23 -0
- package/commands/sentinel-fix.js +8 -0
- package/commands/sentinel-powers.js +25 -4
- package/commands/sentinel-record.js +139 -0
- package/lib/known-commands.js +1 -1
- package/lib/memory-files.js +226 -0
- package/lib/memory-folder.js +602 -0
- package/lib/memory-live.js +356 -0
- package/lib/sentinel-cloud.js +1 -0
- package/lib/sentinel-record.js +877 -0
- package/lib/workspace-safety.js +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,602 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// A person's Atris memory folder on this laptop: copy it down, see what changed
|
|
4
|
+
// here, send changes back. The server is the source of truth. Everything here
|
|
5
|
+
// works from a small hidden state file that remembers, for each file, the
|
|
6
|
+
// fingerprint it had on the server at the last pull.
|
|
7
|
+
|
|
8
|
+
const fs = require('fs');
|
|
9
|
+
const path = require('path');
|
|
10
|
+
const crypto = require('crypto');
|
|
11
|
+
const transfer = require('./memory-files');
|
|
12
|
+
|
|
13
|
+
const STATE_FILE = '.atris-memory.json';
|
|
14
|
+
const WRITABLE_PREFIXES = ['wiki/', 'files/'];
|
|
15
|
+
const MAX_WRITE_BYTES = 1000000;
|
|
16
|
+
const READ_PAGE = 400;
|
|
17
|
+
const FETCH_PARALLEL = 6;
|
|
18
|
+
const MAX_FOLDERS = 2000;
|
|
19
|
+
const MAX_LOCAL_FILES = 20000;
|
|
20
|
+
const MAX_DEPTH = 24;
|
|
21
|
+
|
|
22
|
+
const utf8 = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true });
|
|
23
|
+
|
|
24
|
+
function sha256(data) {
|
|
25
|
+
return crypto.createHash('sha256').update(data).digest('hex');
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// A path inside the folder, or null. Rejects '..', absolute paths, drive
|
|
29
|
+
// letters, backslashes, empty parts and hidden parts (the state file is hidden,
|
|
30
|
+
// so hidden names are never copied in either direction).
|
|
31
|
+
function cleanRel(rel) {
|
|
32
|
+
if (typeof rel !== 'string' || !rel || rel.length > 400) return null;
|
|
33
|
+
if (rel.startsWith('/') || rel.includes('\\') || rel.includes('\0') || /^[A-Za-z]:/.test(rel)) return null;
|
|
34
|
+
const parts = rel.split('/');
|
|
35
|
+
if (parts.some((part) => part === '' || part === '.' || part === '..' || part.startsWith('.'))) return null;
|
|
36
|
+
return rel;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function isWritable(rel) {
|
|
40
|
+
return WRITABLE_PREFIXES.some((prefix) => rel.startsWith(prefix));
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// Pages and files that push --prune may remove from the server. wiki/learning.md
|
|
44
|
+
// is a note the server keeps in step with remember, so it is never removed.
|
|
45
|
+
const SERVER_NOTES = new Set(['wiki/learning.md']);
|
|
46
|
+
|
|
47
|
+
function isRemovable(rel) {
|
|
48
|
+
return isWritable(rel) && !SERVER_NOTES.has(rel);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function insideRoot(rootReal, target) {
|
|
52
|
+
const rel = path.relative(rootReal, target);
|
|
53
|
+
return rel !== '' && !rel.startsWith('..') && !path.isAbsolute(rel);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// Where a file lives on disk. Builds its folders one step at a time and refuses
|
|
57
|
+
// any step that is a link or a file, so nothing can lead out of the target
|
|
58
|
+
// folder. The file itself must not be a link either.
|
|
59
|
+
function localTarget(rootReal, rel) {
|
|
60
|
+
const clean = cleanRel(rel);
|
|
61
|
+
if (!clean) throw new Error(`unsafe path: ${String(rel).slice(0, 80)}`);
|
|
62
|
+
const parts = clean.split('/');
|
|
63
|
+
let current = rootReal;
|
|
64
|
+
for (const part of parts.slice(0, -1)) {
|
|
65
|
+
current = path.join(current, part);
|
|
66
|
+
let info = null;
|
|
67
|
+
try { info = fs.lstatSync(current); } catch { /* not there yet */ }
|
|
68
|
+
if (!info) fs.mkdirSync(current);
|
|
69
|
+
else if (!info.isDirectory()) throw new Error(`not a plain folder: ${path.relative(rootReal, current)}`);
|
|
70
|
+
}
|
|
71
|
+
const abs = path.join(current, parts[parts.length - 1]);
|
|
72
|
+
if (!insideRoot(rootReal, abs)) throw new Error(`unsafe path: ${clean}`);
|
|
73
|
+
let info = null;
|
|
74
|
+
try { info = fs.lstatSync(abs); } catch { /* new file */ }
|
|
75
|
+
if (info && !info.isFile()) throw new Error(`not a plain file: ${clean}`);
|
|
76
|
+
return abs;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function writeFileAtomic(abs, text) {
|
|
80
|
+
const temp = path.join(path.dirname(abs), `.${path.basename(abs)}.atris-tmp-${process.pid}`);
|
|
81
|
+
try {
|
|
82
|
+
fs.writeFileSync(temp, text, 'utf8');
|
|
83
|
+
fs.renameSync(temp, abs);
|
|
84
|
+
} catch (error) {
|
|
85
|
+
try { fs.unlinkSync(temp); } catch { /* nothing to clean */ }
|
|
86
|
+
throw error;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function loadState(rootDir) {
|
|
91
|
+
let raw;
|
|
92
|
+
try {
|
|
93
|
+
raw = fs.readFileSync(path.join(rootDir, STATE_FILE), 'utf8');
|
|
94
|
+
} catch (error) {
|
|
95
|
+
if (error.code === 'ENOENT') return null;
|
|
96
|
+
throw error;
|
|
97
|
+
}
|
|
98
|
+
let state;
|
|
99
|
+
try { state = JSON.parse(raw); } catch { state = null; }
|
|
100
|
+
if (!state || typeof state !== 'object' || !state.files || typeof state.files !== 'object') {
|
|
101
|
+
throw new Error(`${STATE_FILE} is unreadable. Delete it and pull again; your files are kept.`);
|
|
102
|
+
}
|
|
103
|
+
return state;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function saveState(rootDir, state) {
|
|
107
|
+
writeFileAtomic(path.join(rootDir, STATE_FILE), `${JSON.stringify(state, null, 2)}\n`);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// Every plain file under the folder, hidden names and links skipped.
|
|
111
|
+
function scanLocal(rootDir) {
|
|
112
|
+
const files = new Map();
|
|
113
|
+
const skipped = [];
|
|
114
|
+
const walk = (dir, prefix, depth) => {
|
|
115
|
+
if (depth > MAX_DEPTH) return;
|
|
116
|
+
let entries;
|
|
117
|
+
try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
|
|
118
|
+
entries.sort((a, b) => (a.name < b.name ? -1 : 1));
|
|
119
|
+
for (const entry of entries) {
|
|
120
|
+
if (entry.name.startsWith('.')) continue;
|
|
121
|
+
const rel = prefix + entry.name;
|
|
122
|
+
if (entry.isSymbolicLink()) { skipped.push({ path: rel, reason: 'is a link' }); continue; }
|
|
123
|
+
if (entry.isDirectory()) { walk(path.join(dir, entry.name), `${rel}/`, depth + 1); continue; }
|
|
124
|
+
if (!entry.isFile()) continue;
|
|
125
|
+
if (!cleanRel(rel)) { skipped.push({ path: rel, reason: 'has an unsafe name' }); continue; }
|
|
126
|
+
if (files.size >= MAX_LOCAL_FILES) { skipped.push({ path: rel, reason: 'too many files' }); continue; }
|
|
127
|
+
files.set(rel, path.join(dir, entry.name));
|
|
128
|
+
}
|
|
129
|
+
};
|
|
130
|
+
walk(rootDir, '', 0);
|
|
131
|
+
return { files, skipped };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// What differs from the last pull. Files are compared by fingerprint.
|
|
135
|
+
function diffLocal(rootDir, state) {
|
|
136
|
+
const { files, skipped } = scanLocal(rootDir);
|
|
137
|
+
const modified = [];
|
|
138
|
+
const added = [];
|
|
139
|
+
let unchanged = 0;
|
|
140
|
+
for (const [rel, abs] of files) {
|
|
141
|
+
const known = state.files[rel];
|
|
142
|
+
let hash;
|
|
143
|
+
try { hash = transfer.hashFileSync(abs).sha256; } catch { skipped.push({ path: rel, reason: 'could not be read' }); continue; }
|
|
144
|
+
if (!known) added.push(rel);
|
|
145
|
+
else if (known.sha256 !== hash) modified.push(rel);
|
|
146
|
+
else unchanged += 1;
|
|
147
|
+
}
|
|
148
|
+
const deleted = Object.keys(state.files).filter((rel) => !files.has(rel));
|
|
149
|
+
return { modified, added, deleted, unchanged, skipped };
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function decodeText(buffer) {
|
|
153
|
+
try { return utf8.decode(buffer); } catch { return null; }
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
async function mapLimit(items, limit, fn) {
|
|
157
|
+
const results = new Array(items.length);
|
|
158
|
+
let next = 0;
|
|
159
|
+
const worker = async () => {
|
|
160
|
+
while (next < items.length) {
|
|
161
|
+
const index = next;
|
|
162
|
+
next += 1;
|
|
163
|
+
results[index] = await fn(items[index], index);
|
|
164
|
+
}
|
|
165
|
+
};
|
|
166
|
+
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
|
167
|
+
return results;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function apiError(result, what) {
|
|
171
|
+
if (result && result.status === 401) {
|
|
172
|
+
const error = new Error('your Atris login was not accepted. Run: atris login');
|
|
173
|
+
error.status = 401;
|
|
174
|
+
return error;
|
|
175
|
+
}
|
|
176
|
+
const detail = result && result.error;
|
|
177
|
+
const message = detail && typeof detail === 'object' ? JSON.stringify(detail) : String(detail || 'request failed');
|
|
178
|
+
const error = new Error(`${what}: ${message.replace(/\s+/g, ' ').slice(0, 200)}`);
|
|
179
|
+
error.status = result ? result.status : 0;
|
|
180
|
+
return error;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// The few server calls this needs. `request` is the CLI's API helper.
|
|
184
|
+
function memoryClient(request, token) {
|
|
185
|
+
const call = (pathname, options = {}) => request(pathname, { token, timeoutMs: 30000, ...options });
|
|
186
|
+
const query = (value) => encodeURIComponent(value);
|
|
187
|
+
const client = {
|
|
188
|
+
usedLineFallback: false,
|
|
189
|
+
rawFileMissing: false,
|
|
190
|
+
filesSupported: null, // false once the server shows it has no file links yet
|
|
191
|
+
sizes: new Map(), // bytes of each file, from the listing
|
|
192
|
+
|
|
193
|
+
async list() {
|
|
194
|
+
const found = [];
|
|
195
|
+
let level = [''];
|
|
196
|
+
let folders = 0;
|
|
197
|
+
while (level.length) {
|
|
198
|
+
folders += level.length;
|
|
199
|
+
if (folders > MAX_FOLDERS) throw new Error('the folder has too many sub-folders to copy');
|
|
200
|
+
const pages = await mapLimit(level, 4, async (folder) => {
|
|
201
|
+
const result = await call(`/memory/ls?folder=${query(folder)}`);
|
|
202
|
+
if (!result.ok) throw apiError(result, 'could not list the folder');
|
|
203
|
+
return { folder, items: (result.data && result.data.items) || [] };
|
|
204
|
+
});
|
|
205
|
+
level = [];
|
|
206
|
+
for (const { folder, items } of pages) {
|
|
207
|
+
for (const item of items) {
|
|
208
|
+
const name = String(item.name || '');
|
|
209
|
+
const full = folder ? `${folder}/${name}` : name;
|
|
210
|
+
if (!name.endsWith('/')) { found.push(full); client.sizes.set(full, Number(item.size)); }
|
|
211
|
+
else if (cleanRel(full.slice(0, -1))) level.push(full.slice(0, -1));
|
|
212
|
+
else found.push(full); // never descended; pull counts it as an unsafe name
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
return found.sort();
|
|
217
|
+
},
|
|
218
|
+
|
|
219
|
+
// { exists: false } when the server has no such file; throws on any other trouble.
|
|
220
|
+
async read(rel) {
|
|
221
|
+
if (!client.rawFileMissing) {
|
|
222
|
+
const result = await call(`/memory/file?path=${query(rel)}`);
|
|
223
|
+
if (result.ok && result.data && typeof result.data.text === 'string') {
|
|
224
|
+
return { exists: true, text: result.data.text };
|
|
225
|
+
}
|
|
226
|
+
if (result.status === 404) client.rawFileMissing = true;
|
|
227
|
+
else if (/^No file at/.test(String(result.error || ''))) return { exists: false };
|
|
228
|
+
else throw apiError(result, `could not read ${rel}`);
|
|
229
|
+
}
|
|
230
|
+
// The server is older than the exact-file call: page through numbered lines.
|
|
231
|
+
client.usedLineFallback = true;
|
|
232
|
+
const lines = [];
|
|
233
|
+
for (let start = 1; ; start += READ_PAGE) {
|
|
234
|
+
const result = await call(`/memory/read?path=${query(rel)}&start=${start}&count=${READ_PAGE}`);
|
|
235
|
+
if (!result.ok) {
|
|
236
|
+
if (start === 1 && /^No file at/.test(String(result.error || ''))) return { exists: false };
|
|
237
|
+
throw apiError(result, `could not read ${rel}`);
|
|
238
|
+
}
|
|
239
|
+
const data = result.data || {};
|
|
240
|
+
for (const line of data.lines || []) lines.push(String(line.text));
|
|
241
|
+
if (!data.more || start > 2000000) break;
|
|
242
|
+
}
|
|
243
|
+
return { exists: true, text: lines.length ? `${lines.join('\n')}\n` : '' };
|
|
244
|
+
},
|
|
245
|
+
|
|
246
|
+
async write(rel, text) {
|
|
247
|
+
const result = await call('/memory/write', { method: 'PUT', body: { path: rel, text } });
|
|
248
|
+
if (!result.ok) throw apiError(result, `could not save ${rel}`);
|
|
249
|
+
},
|
|
250
|
+
|
|
251
|
+
// true when removed, false when the server had no such file.
|
|
252
|
+
async remove(rel) {
|
|
253
|
+
const result = await call(`/memory/file?path=${query(rel)}`, { method: 'DELETE' });
|
|
254
|
+
if (result.ok) return true;
|
|
255
|
+
if (result.status === 404 && /^No file at/.test(String(result.error || ''))) return false;
|
|
256
|
+
if (result.status === 404 || result.status === 405) {
|
|
257
|
+
const error = new Error('the Atris server cannot delete pages yet');
|
|
258
|
+
error.status = result.status;
|
|
259
|
+
throw error;
|
|
260
|
+
}
|
|
261
|
+
throw apiError(result, `could not remove ${rel}`);
|
|
262
|
+
},
|
|
263
|
+
|
|
264
|
+
// A short-lived download link plus size and fingerprint, or { exists: false }.
|
|
265
|
+
async link(rel) {
|
|
266
|
+
const result = await call(`/memory/link?path=${query(rel)}`);
|
|
267
|
+
if (result.ok && result.data && result.data.url) {
|
|
268
|
+
client.filesSupported = true;
|
|
269
|
+
return { exists: true, ...result.data };
|
|
270
|
+
}
|
|
271
|
+
if (result.status === 404 && /^No file at/.test(String(result.error || ''))) return { exists: false };
|
|
272
|
+
if (result.status === 404 || result.status === 405) throw noFiles(client);
|
|
273
|
+
throw apiError(result, `could not look up ${rel}`);
|
|
274
|
+
},
|
|
275
|
+
|
|
276
|
+
// The server checks the upload like a write and signs a form for exactly this file.
|
|
277
|
+
async startUpload(rel, bytes) {
|
|
278
|
+
const result = await call('/memory/upload', { method: 'POST', body: { path: rel, size: bytes } });
|
|
279
|
+
if (result.ok && result.data && result.data.url && result.data.token) return result.data;
|
|
280
|
+
if ((result.status === 404 && result.error === 'Not Found') || result.status === 405) throw noFiles(client);
|
|
281
|
+
throw apiError(result, `could not send ${rel}`);
|
|
282
|
+
},
|
|
283
|
+
|
|
284
|
+
async finishUpload(token) {
|
|
285
|
+
const result = await call('/memory/upload/done', { method: 'POST', body: { token }, timeoutMs: 120000 });
|
|
286
|
+
if (result.ok && result.data) return result.data;
|
|
287
|
+
throw apiError(result, 'the server could not confirm it arrived');
|
|
288
|
+
},
|
|
289
|
+
};
|
|
290
|
+
return client;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
function noFiles(client) {
|
|
294
|
+
client.filesSupported = false;
|
|
295
|
+
const error = new Error('the Atris server cannot take files yet');
|
|
296
|
+
error.noFiles = true;
|
|
297
|
+
return error;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
// Whether the server copy of rel may be replaced or removed. "same": it already
|
|
301
|
+
// holds these bytes. "clear": it holds what was pulled last, or nothing.
|
|
302
|
+
// "changed": it changed since the last pull, so it is never overwritten.
|
|
303
|
+
// Files fetched as files are compared by S3 fingerprint, pages by their text.
|
|
304
|
+
async function serverCheck(client, rel, base, local, { asFile = false } = {}) {
|
|
305
|
+
if ((asFile || (base && base.etag)) && client.filesSupported !== false) {
|
|
306
|
+
const info = await client.link(rel);
|
|
307
|
+
if (!info.exists) return { verdict: 'clear', exists: false };
|
|
308
|
+
if (local && info.md5 && info.md5 === local.md5 && info.bytes === local.bytes) {
|
|
309
|
+
return { verdict: 'same', exists: true, etag: info.etag };
|
|
310
|
+
}
|
|
311
|
+
if (base && base.etag && base.etag === info.etag) return { verdict: 'clear', exists: true };
|
|
312
|
+
if (base && base.md5 && info.md5 && base.md5 === info.md5) return { verdict: 'clear', exists: true };
|
|
313
|
+
if (!base || base.etag) return { verdict: 'changed', exists: true };
|
|
314
|
+
// A page pulled as text: compare its text below.
|
|
315
|
+
}
|
|
316
|
+
const current = await client.read(rel);
|
|
317
|
+
if (!current.exists) return { verdict: 'clear', exists: false };
|
|
318
|
+
const serverSha = sha256(Buffer.from(current.text, 'utf8'));
|
|
319
|
+
if (local && serverSha === local.sha256) return { verdict: 'same', exists: true };
|
|
320
|
+
return { verdict: base && serverSha === base.sha256 ? 'clear' : 'changed', exists: true };
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
// Copy one file down as a file: streamed to a hidden temp name beside it,
|
|
324
|
+
// checked against S3's size and fingerprint, then moved into place. Same rules
|
|
325
|
+
// as a page: an edit here or a change on both sides is never overwritten.
|
|
326
|
+
// false when the server has no file links yet, so the caller copies it as text.
|
|
327
|
+
async function pullFile({ rootReal, rel, client, state, out, onProgress }) {
|
|
328
|
+
let temp = null;
|
|
329
|
+
try {
|
|
330
|
+
const link = await client.link(rel);
|
|
331
|
+
if (!link.exists) return true;
|
|
332
|
+
const abs = localTarget(rootReal, rel);
|
|
333
|
+
const base = state.files[rel] || null;
|
|
334
|
+
const local = fs.existsSync(abs) ? transfer.hashFileSync(abs) : null;
|
|
335
|
+
const record = (print) => ({ sha256: print.sha256, bytes: print.bytes, md5: print.md5, etag: link.etag });
|
|
336
|
+
if (base && base.etag && base.etag === link.etag && local) {
|
|
337
|
+
// The server copy is the one pulled last time: nothing to fetch.
|
|
338
|
+
if (local.sha256 === base.sha256) out.same += 1; else out.keptLocal.push(rel);
|
|
339
|
+
return true;
|
|
340
|
+
}
|
|
341
|
+
if (local && link.md5 && local.md5 === link.md5 && local.bytes === link.bytes) {
|
|
342
|
+
out.same += 1;
|
|
343
|
+
state.files[rel] = record(local);
|
|
344
|
+
return true;
|
|
345
|
+
}
|
|
346
|
+
temp = transfer.tempBeside(abs);
|
|
347
|
+
const progress = onProgress && ((done, total) => onProgress({ rel, verb: 'Fetching', done, total }));
|
|
348
|
+
let got;
|
|
349
|
+
try {
|
|
350
|
+
got = await transfer.fetchFile({ url: link.url, temp, bytes: link.bytes, onProgress: progress });
|
|
351
|
+
} finally {
|
|
352
|
+
if (onProgress) onProgress({ rel, verb: 'Fetching', done: link.bytes, total: link.bytes, finished: true });
|
|
353
|
+
}
|
|
354
|
+
if (got.bytes !== link.bytes || (link.md5 && got.md5 !== link.md5)) {
|
|
355
|
+
throw new Error('it arrived damaged (its fingerprint did not match), so it was not kept. Pull again');
|
|
356
|
+
}
|
|
357
|
+
if (local && local.sha256 === got.sha256) {
|
|
358
|
+
out.same += 1;
|
|
359
|
+
} else if (!local || (base && local.sha256 === base.sha256)) {
|
|
360
|
+
fs.renameSync(temp, abs);
|
|
361
|
+
temp = null;
|
|
362
|
+
out[local ? 'updated' : 'added'].push(rel);
|
|
363
|
+
} else {
|
|
364
|
+
out[base && got.sha256 === base.sha256 ? 'keptLocal' : 'conflicts'].push(rel);
|
|
365
|
+
return true;
|
|
366
|
+
}
|
|
367
|
+
state.files[rel] = record(got);
|
|
368
|
+
return true;
|
|
369
|
+
} catch (error) {
|
|
370
|
+
if (error.noFiles) return false;
|
|
371
|
+
out.failed.push({ path: rel, error: error.message });
|
|
372
|
+
return true;
|
|
373
|
+
} finally {
|
|
374
|
+
if (temp) { try { fs.unlinkSync(temp); } catch { /* never made */ } }
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
// Send one big or binary file under files/: check the server copy, ask for a
|
|
379
|
+
// signed form, stream the file to storage, then confirm. A failed send is
|
|
380
|
+
// still confirmed, so the room the server held for it is given back.
|
|
381
|
+
async function pushFile({ rootReal, rel, small, client, state, out, onProgress }) {
|
|
382
|
+
const abs = path.join(rootReal, ...rel.split('/'));
|
|
383
|
+
const base = state.files[rel] || null;
|
|
384
|
+
try {
|
|
385
|
+
const local = transfer.hashFileSync(abs);
|
|
386
|
+
const check = await serverCheck(client, rel, base, local, { asFile: true });
|
|
387
|
+
if (check.verdict === 'same') {
|
|
388
|
+
state.files[rel] = { sha256: local.sha256, bytes: local.bytes, md5: local.md5, etag: check.etag };
|
|
389
|
+
out.alreadyThere.push(rel);
|
|
390
|
+
return;
|
|
391
|
+
}
|
|
392
|
+
if (check.verdict === 'changed') { out.conflicts.push(rel); return; }
|
|
393
|
+
const form = await client.startUpload(rel, local.bytes);
|
|
394
|
+
const progress = onProgress && ((done, total) => onProgress({ rel, verb: 'Sending', done, total }));
|
|
395
|
+
let sent;
|
|
396
|
+
try {
|
|
397
|
+
sent = await transfer.sendFile({ url: form.url, fields: form.fields, abs, bytes: local.bytes, onProgress: progress });
|
|
398
|
+
} catch (error) {
|
|
399
|
+
await client.finishUpload(form.token).catch(() => {});
|
|
400
|
+
throw error;
|
|
401
|
+
} finally {
|
|
402
|
+
if (onProgress) onProgress({ rel, verb: 'Sending', done: local.bytes, total: local.bytes, finished: true });
|
|
403
|
+
}
|
|
404
|
+
const done = await client.finishUpload(form.token);
|
|
405
|
+
if (Number(done.bytes) !== sent.bytes || (done.md5 && done.md5 !== sent.md5)) {
|
|
406
|
+
throw new Error('it arrived damaged (its fingerprint did not match). Push again');
|
|
407
|
+
}
|
|
408
|
+
state.files[rel] = { sha256: sent.sha256, bytes: sent.bytes, md5: sent.md5, etag: done.etag };
|
|
409
|
+
out.pushed.push(rel);
|
|
410
|
+
} catch (error) {
|
|
411
|
+
if (error.noFiles) out[small ? 'notText' : 'tooBig'].push(rel); // an older server: as before
|
|
412
|
+
else out.failed.push({ path: rel, error: error.message });
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
// Small files in parallel, large ones one at a time so each gets its own progress line.
|
|
417
|
+
async function moveFiles(list, sizeOf, fn) {
|
|
418
|
+
await mapLimit(list.filter((item) => sizeOf(item) < transfer.LARGE_BYTES), 4, fn);
|
|
419
|
+
for (const item of list.filter((each) => sizeOf(each) >= transfer.LARGE_BYTES)) await fn(item);
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
function checkAccount(state, account) {
|
|
423
|
+
if (state.account && account && state.account !== account) {
|
|
424
|
+
throw new Error('this folder was pulled from a different Atris account. Use another folder, or run: atris whoami');
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
async function pullFolder({ dir, client, account = null, now = () => new Date(), onProgress = null }) {
|
|
429
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
430
|
+
const rootReal = fs.realpathSync(dir);
|
|
431
|
+
const state = loadState(rootReal) || { version: 1, files: {} };
|
|
432
|
+
checkAccount(state, account);
|
|
433
|
+
if (account) state.account = account;
|
|
434
|
+
|
|
435
|
+
const remote = await client.list();
|
|
436
|
+
const out = { total: remote.length, added: [], updated: [], same: 0, keptLocal: [], conflicts: [], failed: [], unsafe: [], gone: 0 };
|
|
437
|
+
const movedAsFile = (rel) => Boolean(state.files[rel] && state.files[rel].etag);
|
|
438
|
+
const asFiles = remote.filter((rel) => cleanRel(rel) && (transfer.goesAsFile(rel, client.sizes.get(rel)) || movedAsFile(rel)));
|
|
439
|
+
|
|
440
|
+
const pullText = async (rel) => {
|
|
441
|
+
if (!cleanRel(rel)) { out.unsafe.push(rel); return; }
|
|
442
|
+
let file;
|
|
443
|
+
try { file = await client.read(rel); } catch (error) {
|
|
444
|
+
// A .csv or .txt that is not UTF-8, sent from another laptop as a file: fetch it as one.
|
|
445
|
+
if (rel.startsWith('files/') && client.filesSupported !== false
|
|
446
|
+
&& await pullFile({ rootReal, rel, client, state, out, onProgress })) return;
|
|
447
|
+
out.failed.push({ path: rel, error: error.message });
|
|
448
|
+
return;
|
|
449
|
+
}
|
|
450
|
+
if (!file.exists) return;
|
|
451
|
+
const remoteSha = sha256(Buffer.from(file.text, 'utf8'));
|
|
452
|
+
const base = state.files[rel] ? state.files[rel].sha256 : null;
|
|
453
|
+
let abs;
|
|
454
|
+
try {
|
|
455
|
+
abs = localTarget(rootReal, rel);
|
|
456
|
+
let localSha = null;
|
|
457
|
+
try { localSha = transfer.hashFileSync(abs).sha256; } catch { /* no local copy */ }
|
|
458
|
+
const { md5 } = transfer.printOf(Buffer.from(file.text, 'utf8'));
|
|
459
|
+
const record = { sha256: remoteSha, bytes: Buffer.byteLength(file.text, 'utf8'), md5 };
|
|
460
|
+
if (localSha === remoteSha) {
|
|
461
|
+
out.same += 1;
|
|
462
|
+
state.files[rel] = record;
|
|
463
|
+
} else if (localSha === null) {
|
|
464
|
+
writeFileAtomic(abs, file.text);
|
|
465
|
+
state.files[rel] = record;
|
|
466
|
+
out.added.push(rel);
|
|
467
|
+
} else if (base && localSha === base) {
|
|
468
|
+
writeFileAtomic(abs, file.text);
|
|
469
|
+
state.files[rel] = record;
|
|
470
|
+
out.updated.push(rel);
|
|
471
|
+
} else if (base && remoteSha === base) {
|
|
472
|
+
out.keptLocal.push(rel);
|
|
473
|
+
} else {
|
|
474
|
+
out.conflicts.push(rel);
|
|
475
|
+
}
|
|
476
|
+
} catch (error) {
|
|
477
|
+
out.failed.push({ path: rel, error: error.message });
|
|
478
|
+
}
|
|
479
|
+
};
|
|
480
|
+
const fileSet = new Set(asFiles);
|
|
481
|
+
await mapLimit(remote.filter((rel) => !fileSet.has(rel)), FETCH_PARALLEL, pullText);
|
|
482
|
+
await moveFiles(asFiles, (rel) => client.sizes.get(rel) || 0, async (rel) => {
|
|
483
|
+
const done = client.filesSupported !== false && await pullFile({ rootReal, rel, client, state, out, onProgress });
|
|
484
|
+
if (!done) await pullText(rel); // an older server: copy it as text, as before
|
|
485
|
+
});
|
|
486
|
+
|
|
487
|
+
const onServer = new Set(remote);
|
|
488
|
+
for (const rel of Object.keys(state.files)) {
|
|
489
|
+
if (!onServer.has(rel)) { delete state.files[rel]; out.gone += 1; }
|
|
490
|
+
}
|
|
491
|
+
state.pulled_at = now().toISOString();
|
|
492
|
+
saveState(rootReal, state);
|
|
493
|
+
for (const key of ['added', 'updated', 'keptLocal', 'conflicts', 'unsafe']) out[key].sort();
|
|
494
|
+
out.usedLineFallback = client.usedLineFallback;
|
|
495
|
+
return out;
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
// Remove from the server the pages deleted here since the last pull. Only
|
|
499
|
+
// wiki/ and files/, and only while the server copy is still the one pulled, so
|
|
500
|
+
// a page someone changed elsewhere is never lost. onPrune hears the list first.
|
|
501
|
+
async function pruneDeleted({ state, client, deleted, out, onPrune }) {
|
|
502
|
+
const doomed = deleted.filter(isRemovable).sort();
|
|
503
|
+
Object.assign(out, {
|
|
504
|
+
pruned: true, removed: [], alreadyGone: [], changedOnServer: [],
|
|
505
|
+
notRemovable: deleted.filter((rel) => !isRemovable(rel)).sort(),
|
|
506
|
+
});
|
|
507
|
+
if (onPrune && doomed.length) onPrune(doomed);
|
|
508
|
+
await mapLimit(doomed, 4, async (rel) => {
|
|
509
|
+
try {
|
|
510
|
+
const current = await serverCheck(client, rel, state.files[rel], null);
|
|
511
|
+
if (current.verdict === 'changed') {
|
|
512
|
+
out.changedOnServer.push(rel);
|
|
513
|
+
return;
|
|
514
|
+
}
|
|
515
|
+
const removed = current.exists && await client.remove(rel);
|
|
516
|
+
out[removed ? 'removed' : 'alreadyGone'].push(rel);
|
|
517
|
+
delete state.files[rel];
|
|
518
|
+
} catch (error) {
|
|
519
|
+
out.failed.push({ path: rel, error: error.message, removing: true });
|
|
520
|
+
}
|
|
521
|
+
});
|
|
522
|
+
for (const key of ['removed', 'alreadyGone', 'changedOnServer']) out[key].sort();
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
async function pushFolder({ dir, client, account = null, prune = false, onPrune = null, onProgress = null }) {
|
|
526
|
+
let rootReal;
|
|
527
|
+
try { rootReal = fs.realpathSync(dir); } catch { rootReal = null; }
|
|
528
|
+
const state = rootReal ? loadState(rootReal) : null;
|
|
529
|
+
if (!state) throw new Error(`${dir} has not been pulled yet. Run: atris memory pull`);
|
|
530
|
+
checkAccount(state, account);
|
|
531
|
+
|
|
532
|
+
const change = diffLocal(rootReal, state);
|
|
533
|
+
const changed = [...change.modified, ...change.added].sort();
|
|
534
|
+
const out = {
|
|
535
|
+
pushed: [], alreadyThere: [], conflicts: [], failed: [], tooBig: [], notText: [],
|
|
536
|
+
notPushable: changed.filter((rel) => !isWritable(rel)),
|
|
537
|
+
deletedHere: change.deleted, skipped: change.skipped,
|
|
538
|
+
};
|
|
539
|
+
|
|
540
|
+
const asFiles = [];
|
|
541
|
+
await mapLimit(changed.filter(isWritable), 4, async (rel) => {
|
|
542
|
+
const abs = path.join(rootReal, ...rel.split('/'));
|
|
543
|
+
let bytes;
|
|
544
|
+
try { bytes = fs.statSync(abs).size; } catch (error) { out.failed.push({ path: rel, error: error.message }); return; }
|
|
545
|
+
// Only a file small enough to be a page is read whole; anything bigger is streamed.
|
|
546
|
+
const buffer = bytes <= MAX_WRITE_BYTES ? fs.readFileSync(abs) : null;
|
|
547
|
+
const text = buffer && decodeText(buffer);
|
|
548
|
+
if (text === null || text === undefined) {
|
|
549
|
+
if (rel.startsWith('files/')) asFiles.push({ rel, bytes, small: Boolean(buffer) });
|
|
550
|
+
else out[buffer ? 'notText' : 'tooBig'].push(rel); // pages under wiki/ stay text up to 1 MB
|
|
551
|
+
return;
|
|
552
|
+
}
|
|
553
|
+
const local = transfer.printOf(buffer);
|
|
554
|
+
try {
|
|
555
|
+
const current = await serverCheck(client, rel, state.files[rel], local);
|
|
556
|
+
if (current.verdict === 'same') {
|
|
557
|
+
state.files[rel] = { sha256: local.sha256, bytes: buffer.length, md5: local.md5 };
|
|
558
|
+
out.alreadyThere.push(rel);
|
|
559
|
+
return;
|
|
560
|
+
}
|
|
561
|
+
if (current.verdict === 'changed') { out.conflicts.push(rel); return; }
|
|
562
|
+
await client.write(rel, text);
|
|
563
|
+
state.files[rel] = { sha256: local.sha256, bytes: buffer.length, md5: local.md5 };
|
|
564
|
+
out.pushed.push(rel);
|
|
565
|
+
} catch (error) {
|
|
566
|
+
out.failed.push({ path: rel, error: error.message });
|
|
567
|
+
}
|
|
568
|
+
});
|
|
569
|
+
await moveFiles(asFiles, (item) => item.bytes, (item) => pushFile({ rootReal, ...item, client, state, out, onProgress }));
|
|
570
|
+
if (prune) await pruneDeleted({ state, client, deleted: change.deleted, out, onPrune });
|
|
571
|
+
|
|
572
|
+
saveState(rootReal, state);
|
|
573
|
+
for (const key of ['pushed', 'alreadyThere', 'conflicts', 'tooBig', 'notText']) out[key].sort();
|
|
574
|
+
return out;
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
function statusFolder({ dir }) {
|
|
578
|
+
let rootReal;
|
|
579
|
+
try { rootReal = fs.realpathSync(dir); } catch { rootReal = null; }
|
|
580
|
+
const state = rootReal ? loadState(rootReal) : null;
|
|
581
|
+
if (!state) throw new Error(`${dir} has not been pulled yet. Run: atris memory pull`);
|
|
582
|
+
const change = diffLocal(rootReal, state);
|
|
583
|
+
return {
|
|
584
|
+
modified: change.modified.sort(),
|
|
585
|
+
added: change.added.sort(),
|
|
586
|
+
deleted: change.deleted.sort(),
|
|
587
|
+
unchanged: change.unchanged,
|
|
588
|
+
skipped: change.skipped,
|
|
589
|
+
pulledAt: state.pulled_at || null,
|
|
590
|
+
};
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
module.exports = {
|
|
594
|
+
STATE_FILE,
|
|
595
|
+
cleanRel,
|
|
596
|
+
isRemovable,
|
|
597
|
+
isWritable,
|
|
598
|
+
memoryClient,
|
|
599
|
+
pullFolder,
|
|
600
|
+
pushFolder,
|
|
601
|
+
statusFolder,
|
|
602
|
+
};
|