@solidactions/cli 3.24.0 → 3.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/command-manifest.json +115 -1
- package/dist/commands/connection-connect.js +77 -0
- package/dist/commands/connection-list.js +26 -16
- package/dist/commands/connection-wait.js +234 -0
- package/dist/commands/device-login.js +24 -16
- package/dist/commands/doc-pull.js +674 -172
- package/dist/index.js +18 -0
- package/dist/utils/api.js +17 -3
- package/dist/utils/connect-link.js +63 -0
- package/dist/utils/doc-pull-writes.js +342 -0
- package/dist/utils/docs-manifest.js +22 -3
- package/dist/utils/mutating-commands.js +3 -0
- package/dist/utils/output.js +22 -0
- package/package.json +1 -1
|
@@ -24,6 +24,9 @@ exports.sha256Hex = exports.DOCS_MANIFEST = void 0;
|
|
|
24
24
|
exports.sanitizeSegment = sanitizeSegment;
|
|
25
25
|
exports.sanitizeTitle = sanitizeTitle;
|
|
26
26
|
exports.docPullWithConfig = docPullWithConfig;
|
|
27
|
+
exports.decideOutcomes = decideOutcomes;
|
|
28
|
+
exports.buildManifest = buildManifest;
|
|
29
|
+
exports.manifestProblem = manifestProblem;
|
|
27
30
|
exports.docPull = docPull;
|
|
28
31
|
const fs_1 = __importDefault(require("fs"));
|
|
29
32
|
const path_1 = __importDefault(require("path"));
|
|
@@ -36,14 +39,15 @@ const source_provenance_1 = require("../utils/source-provenance");
|
|
|
36
39
|
const docs_manifest_1 = require("../utils/docs-manifest");
|
|
37
40
|
Object.defineProperty(exports, "DOCS_MANIFEST", { enumerable: true, get: function () { return docs_manifest_1.DOCS_MANIFEST; } });
|
|
38
41
|
Object.defineProperty(exports, "sha256Hex", { enumerable: true, get: function () { return docs_manifest_1.sha256Hex; } });
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
'image/
|
|
43
|
-
'image/
|
|
44
|
-
'image/
|
|
45
|
-
'
|
|
46
|
-
|
|
42
|
+
const doc_pull_writes_1 = require("../utils/doc-pull-writes");
|
|
43
|
+
/** Extension appended to a media file's sanitized title when the title itself has none. A Map: a server MIME is never an inherited name (PM ruling 14). */
|
|
44
|
+
const MIME_EXTENSIONS = new Map([
|
|
45
|
+
['image/png', '.png'],
|
|
46
|
+
['image/jpeg', '.jpg'],
|
|
47
|
+
['image/gif', '.gif'],
|
|
48
|
+
['image/webp', '.webp'],
|
|
49
|
+
['application/pdf', '.pdf'],
|
|
50
|
+
]);
|
|
47
51
|
const CHUNK_SIZE = 50;
|
|
48
52
|
/** Characters that are unsafe as filesystem path segments, plus control chars. */
|
|
49
53
|
// eslint-disable-next-line no-control-regex
|
|
@@ -133,7 +137,7 @@ async function resolveMedia(config, doc) {
|
|
|
133
137
|
* file is missing locally is never reported as modified — graceful
|
|
134
138
|
* degradation to "no detection" rather than a false refusal.
|
|
135
139
|
*/
|
|
136
|
-
function detectLocalModifications(destination, manifest) {
|
|
140
|
+
function detectLocalModifications(destination, manifest, seen) {
|
|
137
141
|
const modified = [];
|
|
138
142
|
for (const [relPath, entry] of Object.entries(manifest.docs)) {
|
|
139
143
|
if (entry.body_sha256 == null)
|
|
@@ -145,18 +149,25 @@ function detectLocalModifications(destination, manifest) {
|
|
|
145
149
|
try {
|
|
146
150
|
stat = fs_1.default.statSync(absPath);
|
|
147
151
|
}
|
|
148
|
-
catch {
|
|
152
|
+
catch (error) {
|
|
153
|
+
if (error.code === 'ENOENT')
|
|
154
|
+
observe(seen, relPath, { kind: 'absent' });
|
|
149
155
|
continue; // missing locally — graceful degradation, never a false refusal
|
|
150
156
|
}
|
|
151
157
|
if (!stat.isFile())
|
|
152
158
|
continue;
|
|
153
159
|
const currentHash = (0, docs_manifest_1.sha256Hex)(fs_1.default.readFileSync(absPath));
|
|
160
|
+
observe(seen, relPath, { kind: 'sha256', sha256: currentHash });
|
|
154
161
|
if (currentHash !== entry.body_sha256) {
|
|
155
162
|
modified.push(relPath);
|
|
156
163
|
}
|
|
157
164
|
}
|
|
158
165
|
return modified;
|
|
159
166
|
}
|
|
167
|
+
function observe(seen, relPath, state) {
|
|
168
|
+
if (!seen.has(relPath))
|
|
169
|
+
seen.set(relPath, state);
|
|
170
|
+
}
|
|
160
171
|
/** The last non-empty path segment of a '/'-separated folder path. */
|
|
161
172
|
function lastSegment(folderPath) {
|
|
162
173
|
const segments = folderPath.split('/').filter(Boolean);
|
|
@@ -171,6 +182,23 @@ async function listTree(config, folderPath) {
|
|
|
171
182
|
const rows = [];
|
|
172
183
|
const queue = [{ folder_path: folderPath, relative: '' }];
|
|
173
184
|
let first = true;
|
|
185
|
+
// Per parent folder: the segment each sanitised name was given, and the keys taken, so folders that differ
|
|
186
|
+
// only by case or normalisation (or take a reserved name) get distinct segments (cli#168).
|
|
187
|
+
const segmentsByParent = new Map();
|
|
188
|
+
const segmentFor = (parent, sanitized) => {
|
|
189
|
+
const taken = segmentsByParent.get(parent) ?? { given: new Map(), keys: new Set() };
|
|
190
|
+
segmentsByParent.set(parent, taken);
|
|
191
|
+
const known = taken.given.get(sanitized);
|
|
192
|
+
if (known !== undefined)
|
|
193
|
+
return known;
|
|
194
|
+
const start = (0, doc_pull_writes_1.isReservedName)(sanitized, docs_manifest_1.DOCS_MANIFEST) ? `_${sanitized}` : sanitized;
|
|
195
|
+
let segment = start;
|
|
196
|
+
for (let suffix = 2; taken.keys.has((0, doc_pull_writes_1.nameKey)(segment)); suffix++)
|
|
197
|
+
segment = `${start}-${suffix}`;
|
|
198
|
+
taken.given.set(sanitized, segment);
|
|
199
|
+
taken.keys.add((0, doc_pull_writes_1.nameKey)(segment));
|
|
200
|
+
return segment;
|
|
201
|
+
};
|
|
174
202
|
while (queue.length > 0) {
|
|
175
203
|
const { folder_path, relative } = queue.shift();
|
|
176
204
|
const result = await (0, mcp_1.callDocsTool)(config, { action: 'list', folder_path });
|
|
@@ -182,7 +210,7 @@ async function listTree(config, folderPath) {
|
|
|
182
210
|
}
|
|
183
211
|
first = false;
|
|
184
212
|
for (const folder of result.data?.folders ?? []) {
|
|
185
|
-
const safeName = sanitizeSegment(folder.name);
|
|
213
|
+
const safeName = segmentFor(relative, sanitizeSegment(folder.name));
|
|
186
214
|
const childRelative = relative ? `${relative}/${safeName}` : safeName;
|
|
187
215
|
queue.push({ folder_path: folder.folder_path, relative: childRelative });
|
|
188
216
|
}
|
|
@@ -202,17 +230,20 @@ async function listTree(config, folderPath) {
|
|
|
202
230
|
const BULK_READ_OK_STATUS = 'found';
|
|
203
231
|
/**
|
|
204
232
|
* Fetch bodies + revisions for every collected row via bulk_read, chunked at
|
|
205
|
-
* CHUNK_SIZE ids. A row whose status isn't `BULK_READ_OK_STATUS` (
|
|
206
|
-
*
|
|
207
|
-
*
|
|
208
|
-
*
|
|
233
|
+
* CHUNK_SIZE ids. A row whose status isn't `BULK_READ_OK_STATUS` (`error`,
|
|
234
|
+
* `not_found`, an unknown or a missing status), a `found` row without a
|
|
235
|
+
* string body (no `body` key, or `null`; `""` is content), or a requested id
|
|
236
|
+
* absent from the response entirely, is a doc the server still lists but whose
|
|
237
|
+
* content could not be fetched: it is returned in `failed`, never treated as gone
|
|
238
|
+
* (cli#209). Its file and tracking are left as they were, and the pull names
|
|
239
|
+
* it on one line and exits 1.
|
|
209
240
|
*/
|
|
210
241
|
async function fetchBodies(config, rows) {
|
|
211
242
|
const byId = new Map();
|
|
212
243
|
for (const row of rows)
|
|
213
244
|
byId.set(row.id, row);
|
|
214
245
|
const fetched = [];
|
|
215
|
-
const
|
|
246
|
+
const failed = [];
|
|
216
247
|
for (let i = 0; i < rows.length; i += CHUNK_SIZE) {
|
|
217
248
|
const chunk = rows.slice(i, i + CHUNK_SIZE);
|
|
218
249
|
const result = await (0, mcp_1.callDocsTool)(config, {
|
|
@@ -226,30 +257,25 @@ async function fetchBodies(config, rows) {
|
|
|
226
257
|
process.exit(1);
|
|
227
258
|
}
|
|
228
259
|
const resultRows = result.data?.results ?? [];
|
|
229
|
-
const
|
|
260
|
+
const fetchedIds = new Set();
|
|
230
261
|
for (const row of resultRows) {
|
|
231
262
|
const original = byId.get(row.id);
|
|
232
|
-
if (!original)
|
|
233
|
-
continue;
|
|
234
|
-
seenIds.add(row.id);
|
|
235
|
-
if (row.status !== BULK_READ_OK_STATUS) {
|
|
236
|
-
warnings.push(`warn: skipping doc ${row.id} (${shown(original.title)}): bulk_read returned status "${shown(row.status ?? 'unknown')}"`);
|
|
263
|
+
if (!original || row.status !== BULK_READ_OK_STATUS || typeof row.body !== 'string')
|
|
237
264
|
continue;
|
|
238
|
-
|
|
265
|
+
fetchedIds.add(row.id);
|
|
239
266
|
fetched.push({
|
|
240
267
|
...original,
|
|
241
|
-
body: row.body
|
|
268
|
+
body: row.body,
|
|
242
269
|
current_revision_id: row.current_revision_id ?? null,
|
|
243
270
|
properties: row.properties ?? {},
|
|
244
271
|
});
|
|
245
272
|
}
|
|
246
273
|
for (const requested of chunk) {
|
|
247
|
-
if (!
|
|
248
|
-
|
|
249
|
-
}
|
|
274
|
+
if (!fetchedIds.has(requested.id))
|
|
275
|
+
failed.push(requested);
|
|
250
276
|
}
|
|
251
277
|
}
|
|
252
|
-
return { fetched,
|
|
278
|
+
return { fetched, failed };
|
|
253
279
|
}
|
|
254
280
|
/**
|
|
255
281
|
* Fill in the doc type for fetched docs whose list row did not carry a
|
|
@@ -284,16 +310,20 @@ async function backfillDocTypes(config, fetched) {
|
|
|
284
310
|
}
|
|
285
311
|
return warnings;
|
|
286
312
|
}
|
|
313
|
+
/** A name the pull keeps for itself (the manifest, its lock, the temp-file prefix) takes a leading `_`; the usual `-N` suffix cannot leave a reserved prefix. */
|
|
314
|
+
const withoutReservedName = (base, ext) => ((0, doc_pull_writes_1.isReservedName)(`${base}${ext}`, docs_manifest_1.DOCS_MANIFEST) ? `_${base}` : base);
|
|
287
315
|
/**
|
|
288
316
|
* Allocate the first free `<base>[ -N]<ext>` name not present in `used`,
|
|
289
|
-
* recording the winner in `used`.
|
|
317
|
+
* recording the winner in `used`. Names are compared by `nameKey` (NFC, case-folded), so two
|
|
318
|
+
* docs that differ only by case or by Unicode normalisation never share a file (cli#168).
|
|
290
319
|
*/
|
|
291
320
|
function allocateName(used, base, ext) {
|
|
292
|
-
|
|
321
|
+
const first = withoutReservedName(base, ext);
|
|
322
|
+
let candidate = first;
|
|
293
323
|
let suffix = 2;
|
|
294
|
-
const
|
|
295
|
-
while (
|
|
296
|
-
candidate = `${
|
|
324
|
+
const usedKeys = new Set([...used].map(doc_pull_writes_1.nameKey));
|
|
325
|
+
while (usedKeys.has((0, doc_pull_writes_1.nameKey)(`${candidate}${ext}`))) {
|
|
326
|
+
candidate = `${first}-${suffix}`;
|
|
297
327
|
suffix++;
|
|
298
328
|
}
|
|
299
329
|
const fileName = `${candidate}${ext}`;
|
|
@@ -329,8 +359,10 @@ function isCollisionVariant(file, base, ext) {
|
|
|
329
359
|
* reallocates. Folder pulls reserve only paths whose title, directory and
|
|
330
360
|
* extension stay unchanged, keeping their exact spelling regardless of list
|
|
331
361
|
* order without reserving the old names of docs that move away.
|
|
362
|
+
* `heldPaths` are the tracked paths of listed docs whose content could not be
|
|
363
|
+
* fetched (cli#209): they keep their file and entry, so no doc is given one.
|
|
332
364
|
*/
|
|
333
|
-
async function planDocs(docs, config, reserved, previousManifest) {
|
|
365
|
+
async function planDocs(docs, config, reserved, previousManifest, heldPaths = []) {
|
|
334
366
|
const planned = [];
|
|
335
367
|
const warnings = [];
|
|
336
368
|
const usedNamesByDir = new Map();
|
|
@@ -339,6 +371,13 @@ async function planDocs(docs, config, reserved, previousManifest) {
|
|
|
339
371
|
usedNamesByDir.set(dir, new Set(names));
|
|
340
372
|
}
|
|
341
373
|
}
|
|
374
|
+
for (const rel of heldPaths) {
|
|
375
|
+
const slash = rel.lastIndexOf('/');
|
|
376
|
+
const dir = slash === -1 ? '' : rel.slice(0, slash);
|
|
377
|
+
const used = usedNamesByDir.get(dir) ?? new Set();
|
|
378
|
+
used.add(rel.slice(slash + 1));
|
|
379
|
+
usedNamesByDir.set(dir, used);
|
|
380
|
+
}
|
|
342
381
|
const pathById = reserved?.pathById ?? new Map();
|
|
343
382
|
const titleById = reserved?.titleById ?? new Map();
|
|
344
383
|
if (!reserved && previousManifest) {
|
|
@@ -365,7 +404,7 @@ async function planDocs(docs, config, reserved, previousManifest) {
|
|
|
365
404
|
}
|
|
366
405
|
else {
|
|
367
406
|
base = sanitized;
|
|
368
|
-
ext = (media.mime && MIME_EXTENSIONS
|
|
407
|
+
ext = (media.mime && MIME_EXTENSIONS.get(media.mime)) ?? '';
|
|
369
408
|
}
|
|
370
409
|
}
|
|
371
410
|
else if (doc.docType === 'visual') {
|
|
@@ -383,7 +422,7 @@ async function planDocs(docs, config, reserved, previousManifest) {
|
|
|
383
422
|
}
|
|
384
423
|
else {
|
|
385
424
|
// Download failure: warning already recorded above; the doc is still
|
|
386
|
-
// tracked (manifest entry
|
|
425
|
+
// tracked (manifest entry recorded after the writes) but there is nothing
|
|
387
426
|
// to write to disk.
|
|
388
427
|
bodySha256 = null;
|
|
389
428
|
}
|
|
@@ -400,7 +439,7 @@ async function planDocs(docs, config, reserved, previousManifest) {
|
|
|
400
439
|
const slash = ownRelPath.lastIndexOf('/');
|
|
401
440
|
const ownDir = slash === -1 ? '' : ownRelPath.slice(0, slash);
|
|
402
441
|
const ownFile = slash === -1 ? ownRelPath : ownRelPath.slice(slash + 1);
|
|
403
|
-
return ownDir === doc.relative && isCollisionVariant(ownFile, base, ext) ? ownRelPath : undefined;
|
|
442
|
+
return ownDir === doc.relative && !(0, doc_pull_writes_1.isReservedName)(ownFile, docs_manifest_1.DOCS_MANIFEST) && isCollisionVariant(ownFile, base, ext) ? ownRelPath : undefined;
|
|
404
443
|
};
|
|
405
444
|
if (!reserved) {
|
|
406
445
|
for (const { doc, base, ext } of prepared) {
|
|
@@ -430,42 +469,16 @@ async function planDocs(docs, config, reserved, previousManifest) {
|
|
|
430
469
|
}
|
|
431
470
|
return { planned, warnings };
|
|
432
471
|
}
|
|
433
|
-
/**
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
const files = [];
|
|
444
|
-
for (const p of planned) {
|
|
445
|
-
const dirAbs = p.dirRel ? path_1.default.join(destination, ...p.dirRel.split('/')) : destination;
|
|
446
|
-
fs_1.default.mkdirSync(dirAbs, { recursive: true });
|
|
447
|
-
const targetAbs = path_1.default.join(dirAbs, p.fileName);
|
|
448
|
-
if (p.isMedia) {
|
|
449
|
-
if (p.mediaBytes) {
|
|
450
|
-
fs_1.default.writeFileSync(targetAbs, p.mediaBytes);
|
|
451
|
-
files.push({ path: p.relPath, action: 'written' });
|
|
452
|
-
}
|
|
453
|
-
// else: download failure, warning already recorded by planDocs; skip the
|
|
454
|
-
// write but still record the manifest entry below so the doc isn't lost.
|
|
455
|
-
}
|
|
456
|
-
else {
|
|
457
|
-
fs_1.default.writeFileSync(targetAbs, p.doc.body, 'utf8');
|
|
458
|
-
files.push({ path: p.relPath, action: 'written' });
|
|
459
|
-
}
|
|
460
|
-
manifestDocs[p.relPath] = {
|
|
461
|
-
id: p.doc.id,
|
|
462
|
-
title: p.doc.title,
|
|
463
|
-
current_revision_id: p.doc.current_revision_id,
|
|
464
|
-
media: p.isMedia,
|
|
465
|
-
body_sha256: p.bodySha256,
|
|
466
|
-
};
|
|
467
|
-
}
|
|
468
|
-
return { manifestDocs, files };
|
|
472
|
+
/** The manifest entry for a planned doc, in the shape the manifest stores. Writes nothing. */
|
|
473
|
+
function manifestEntryFor(p) {
|
|
474
|
+
// A failed media download writes nothing, but its entry (no hash) is still how the doc is tracked.
|
|
475
|
+
return {
|
|
476
|
+
id: p.doc.id,
|
|
477
|
+
title: p.doc.title,
|
|
478
|
+
current_revision_id: p.doc.current_revision_id,
|
|
479
|
+
media: p.isMedia,
|
|
480
|
+
body_sha256: p.bodySha256,
|
|
481
|
+
};
|
|
469
482
|
}
|
|
470
483
|
/**
|
|
471
484
|
* Manifest-clobber protection, against the RESOLVED folder: the argument itself for a folder
|
|
@@ -482,13 +495,76 @@ function refuseManifestClobber(previousManifest, resolvedFolder, argument, desti
|
|
|
482
495
|
process.stderr.write(chalk_1.default.red('Pull into a different directory, or pass --overwrite to replace the manifest.\n'));
|
|
483
496
|
process.exit(1);
|
|
484
497
|
}
|
|
498
|
+
/** True when `abs` is an existing directory; any stat failure is left to the destination checks below. */
|
|
499
|
+
function isExistingDirectory(abs) {
|
|
500
|
+
try {
|
|
501
|
+
return fs_1.default.statSync(abs).isDirectory();
|
|
502
|
+
}
|
|
503
|
+
catch {
|
|
504
|
+
return false;
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
/**
|
|
508
|
+
* Take the destination's lock (spec §1.4): one O_EXCL file, `<destination>/.solidactions-docs.json.lock`, holding this
|
|
509
|
+
* pid. An existing one refuses with nothing else changed; it is never removed or liveness-tested except by the pull that
|
|
510
|
+
* created it, so a killed pull's lock stays until the user deletes it.
|
|
511
|
+
*/
|
|
512
|
+
function takeLock(destination, lock) {
|
|
513
|
+
try {
|
|
514
|
+
(0, doc_pull_writes_1.acquireLock)(destination, docs_manifest_1.DOCS_MANIFEST);
|
|
515
|
+
}
|
|
516
|
+
catch (error) {
|
|
517
|
+
if (error instanceof doc_pull_writes_1.LockHeldError) {
|
|
518
|
+
process.stderr.write(chalk_1.default.red(`error: ${shown(path_1.default.join(destination, error.lockName))} exists: another doc pull may be writing to ${shown(destination)}. If none is running, delete that file and pull again.\n`));
|
|
519
|
+
process.exit(1);
|
|
520
|
+
}
|
|
521
|
+
try {
|
|
522
|
+
fs_1.default.readdirSync(destination);
|
|
523
|
+
}
|
|
524
|
+
catch (readError) {
|
|
525
|
+
process.stderr.write(chalk_1.default.red(`error: cannot read ${shown(destination)}: ${shown(readError.message)}\n`));
|
|
526
|
+
process.exit(1);
|
|
527
|
+
}
|
|
528
|
+
process.stderr.write(chalk_1.default.red(`error: cannot write ${shown((0, doc_pull_writes_1.lockNameFor)(docs_manifest_1.DOCS_MANIFEST))}: ${shown(error.message)} — nothing was changed.\n`));
|
|
529
|
+
process.exit(1);
|
|
530
|
+
}
|
|
531
|
+
lock.destination = destination;
|
|
532
|
+
}
|
|
485
533
|
/**
|
|
486
534
|
* Core implementation — accepts an injected config so tests can point at a
|
|
487
535
|
* stub server without touching the filesystem config.
|
|
488
536
|
*/
|
|
489
537
|
async function docPullWithConfig(folderPath, dest, options, config) {
|
|
538
|
+
// The lock taken below is removed on every way out: a process.exit anywhere in the pull (refusal, error, a "no"
|
|
539
|
+
// at the prompt) runs the exit listener, and an interrupt exits through it too. Only a SIGKILL leaves it (spec §1.6).
|
|
540
|
+
const lock = { destination: null };
|
|
541
|
+
const releaseHeld = () => {
|
|
542
|
+
if (lock.destination !== null)
|
|
543
|
+
(0, doc_pull_writes_1.releaseLock)(lock.destination, docs_manifest_1.DOCS_MANIFEST);
|
|
544
|
+
lock.destination = null;
|
|
545
|
+
};
|
|
546
|
+
const onInterrupt = () => process.exit(130);
|
|
547
|
+
const onTerminate = () => process.exit(143);
|
|
548
|
+
process.on('exit', releaseHeld);
|
|
549
|
+
process.on('SIGINT', onInterrupt);
|
|
550
|
+
process.on('SIGTERM', onTerminate);
|
|
551
|
+
try {
|
|
552
|
+
await pullInto(folderPath, dest, options, config, lock);
|
|
553
|
+
}
|
|
554
|
+
finally {
|
|
555
|
+
process.removeListener('exit', releaseHeld);
|
|
556
|
+
process.removeListener('SIGINT', onInterrupt);
|
|
557
|
+
process.removeListener('SIGTERM', onTerminate);
|
|
558
|
+
releaseHeld();
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
async function pullInto(folderPath, dest, options, config, lock) {
|
|
490
562
|
const destInput = dest ?? `./${lastSegment(folderPath)}`;
|
|
491
563
|
const destination = path_1.default.resolve(destInput);
|
|
564
|
+
// cli#168: take the lock before anything reads the destination, so the plan below is built on a manifest
|
|
565
|
+
// no other pull can change while this one runs (spec §1.3 step 1, §1.4).
|
|
566
|
+
if (isExistingDirectory(destination))
|
|
567
|
+
takeLock(destination, lock);
|
|
492
568
|
// Captured once, before the walk, so deletion propagation (below) can diff
|
|
493
569
|
// against the folder tree as it stood before this pull. Already read below
|
|
494
570
|
// for overwrite protection — hoisted here rather than read twice.
|
|
@@ -499,12 +575,25 @@ async function docPullWithConfig(folderPath, dest, options, config) {
|
|
|
499
575
|
// in report() — only once the server walk is known can it tell a real conflict (the
|
|
500
576
|
// doc still exists remotely) from an edited orphan (kept + warned, no flag needed).
|
|
501
577
|
if (fs_1.default.existsSync(destination)) {
|
|
502
|
-
|
|
503
|
-
|
|
578
|
+
let entries;
|
|
579
|
+
try {
|
|
580
|
+
if (!fs_1.default.statSync(destination).isDirectory()) {
|
|
581
|
+
process.stderr.write(chalk_1.default.red(`error: destination "${shown(destination)}" exists and is not a directory.\n`));
|
|
582
|
+
process.exit(1);
|
|
583
|
+
}
|
|
584
|
+
entries = fs_1.default.readdirSync(destination).filter((name) => name !== (0, doc_pull_writes_1.lockNameFor)(docs_manifest_1.DOCS_MANIFEST));
|
|
585
|
+
}
|
|
586
|
+
catch (error) {
|
|
587
|
+
// cli#191: one line naming the destination, never a raw scandir stack.
|
|
588
|
+
process.stderr.write(chalk_1.default.red(`error: cannot read ${shown(destination)}: ${shown(error.message)}\n`));
|
|
504
589
|
process.exit(1);
|
|
505
590
|
}
|
|
506
|
-
const entries = fs_1.default.readdirSync(destination);
|
|
507
591
|
if (entries.length > 0 && !options.yes && !options.overwrite) {
|
|
592
|
+
if (process.stdin.isTTY !== true) {
|
|
593
|
+
// cli#176: nobody can answer the prompt; a script must not read "Cancelled" as success.
|
|
594
|
+
process.stderr.write(chalk_1.default.red(`error: ${shown(destination)} is not empty and there is no terminal to confirm the pull; pass -y to pull into it.\n`));
|
|
595
|
+
process.exit(1);
|
|
596
|
+
}
|
|
508
597
|
console.log(chalk_1.default.yellow(`Destination "${shown(destination)}" is not empty (${entries.length} items).`));
|
|
509
598
|
console.log(chalk_1.default.yellow("Pulling overwrites tracked files; local files the folder doesn't track are refused unless --overwrite."));
|
|
510
599
|
const response = await (0, prompts_1.default)({
|
|
@@ -544,21 +633,17 @@ async function docPullWithConfig(folderPath, dest, options, config) {
|
|
|
544
633
|
process.exit(1);
|
|
545
634
|
}
|
|
546
635
|
const data = readResult.data;
|
|
547
|
-
const
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
body: data.body ?? '',
|
|
554
|
-
current_revision_id: data.current_revision_id ?? null,
|
|
555
|
-
properties: data.properties ?? {},
|
|
556
|
-
}];
|
|
636
|
+
const row = { id: data.id, title: data.title, relative: '', docType: data.doc_type?.slug ?? null, docTypeKnown: true };
|
|
637
|
+
// An answer without a string body (no `body` key, or `null`) is a failed fetch, as in a folder pull (cli#209).
|
|
638
|
+
const hasBody = typeof data.body === 'string';
|
|
639
|
+
const fetched = hasBody
|
|
640
|
+
? [{ ...row, body: data.body, current_revision_id: data.current_revision_id ?? null, properties: data.properties ?? {} }]
|
|
641
|
+
: [];
|
|
557
642
|
// The doc's real folder, not the argument (cli#153): a later `doc push` creates
|
|
558
643
|
// untracked files under the manifest's folder_path.
|
|
559
644
|
const docFolder = typeof data.folder_path === 'string' ? data.folder_path : (dir === '.' ? '' : dir);
|
|
560
645
|
refuseManifestClobber(previousManifest, docFolder, folderPath, destination, options);
|
|
561
|
-
await report(destination, docFolder, fetched, options, config, [], previousManifest, usedSingleDocFallback);
|
|
646
|
+
await report(destination, docFolder, fetched, options, config, [], previousManifest, usedSingleDocFallback, new Set([data.id]), lock, hasBody ? [] : [row]);
|
|
562
647
|
return;
|
|
563
648
|
}
|
|
564
649
|
else {
|
|
@@ -566,9 +651,9 @@ async function docPullWithConfig(folderPath, dest, options, config) {
|
|
|
566
651
|
process.exit(1);
|
|
567
652
|
return;
|
|
568
653
|
}
|
|
569
|
-
const { fetched,
|
|
654
|
+
const { fetched, failed } = await fetchBodies(config, rows);
|
|
570
655
|
const typeWarnings = await backfillDocTypes(config, fetched);
|
|
571
|
-
await report(destination, folderPath, fetched, options, config,
|
|
656
|
+
await report(destination, folderPath, fetched, options, config, typeWarnings, previousManifest, usedSingleDocFallback, new Set(rows.map((row) => row.id)), lock, failed);
|
|
572
657
|
}
|
|
573
658
|
/**
|
|
574
659
|
* Safety guards shared by deletion propagation and rename cleanup (cli#157):
|
|
@@ -685,7 +770,7 @@ function resolveOrExplain(destination, rel, target) {
|
|
|
685
770
|
/**
|
|
686
771
|
* Refuse, before any write, a pull that would write a planned doc through a
|
|
687
772
|
* symbolic link or to a place that resolves outside the destination. Covers
|
|
688
|
-
* every planned doc, including a failed media download:
|
|
773
|
+
* every planned doc, including a failed media download: the write still
|
|
689
774
|
* creates its directory. Also refuse, unless `--overwrite`, a planned write
|
|
690
775
|
* over a regular file the pull does not own (cli#167): one the previous
|
|
691
776
|
* manifest does not track with a hash and whose bytes differ from the
|
|
@@ -693,7 +778,7 @@ function resolveOrExplain(destination, rel, target) {
|
|
|
693
778
|
* rename's own source file (a case-only retitle on a case-insensitive
|
|
694
779
|
* filesystem) is never "untracked".
|
|
695
780
|
*/
|
|
696
|
-
function checkPlannedWrites(destination, planned, previousManifest, options, renameMoves) {
|
|
781
|
+
function checkPlannedWrites(destination, planned, previousManifest, options, renameMoves, seen) {
|
|
697
782
|
// First, so a destination that is itself a looping link gets one line, not a raw ELOOP from the sidecar check below.
|
|
698
783
|
let realDest;
|
|
699
784
|
try {
|
|
@@ -731,7 +816,7 @@ function checkPlannedWrites(destination, planned, previousManifest, options, ren
|
|
|
731
816
|
for (const p of planned) {
|
|
732
817
|
if (p.isMedia && p.mediaBytes === null)
|
|
733
818
|
continue; // a failed download writes nothing
|
|
734
|
-
const tracked = previousManifest
|
|
819
|
+
const tracked = (0, docs_manifest_1.entryAt)(previousManifest, p.relPath);
|
|
735
820
|
if (tracked !== undefined && tracked.body_sha256 != null)
|
|
736
821
|
continue; // checked as unpushed local changes
|
|
737
822
|
const targetAbs = path_1.default.join(destination, ...p.relPath.split('/'));
|
|
@@ -745,11 +830,14 @@ function checkPlannedWrites(destination, planned, previousManifest, options, ren
|
|
|
745
830
|
}
|
|
746
831
|
catch (error) {
|
|
747
832
|
const code = error.code ?? 'ERROR';
|
|
748
|
-
if (code === 'ENOENT')
|
|
833
|
+
if (code === 'ENOENT') {
|
|
834
|
+
observe(seen, p.relPath, { kind: 'absent' });
|
|
749
835
|
continue; // absent: nothing to overwrite here
|
|
836
|
+
}
|
|
750
837
|
untracked.push(`${p.relPath} (cannot be read: ${code})`); // present but unverifiable: not owned
|
|
751
838
|
continue;
|
|
752
839
|
}
|
|
840
|
+
observe(seen, p.relPath, { kind: 'sha256', sha256: (0, docs_manifest_1.sha256Hex)(current) });
|
|
753
841
|
if ((0, docs_manifest_1.sha256Hex)(current) === p.bodySha256)
|
|
754
842
|
continue;
|
|
755
843
|
const ownSource = renameMoves.find((m) => m.id === p.doc.id)?.sourceIdentity;
|
|
@@ -766,8 +854,365 @@ function checkPlannedWrites(destination, planned, previousManifest, options, ren
|
|
|
766
854
|
process.exit(1);
|
|
767
855
|
}
|
|
768
856
|
}
|
|
857
|
+
/** True when anything at all (a link included) is at `abs`. */
|
|
858
|
+
function entryExists(abs) {
|
|
859
|
+
try {
|
|
860
|
+
fs_1.default.lstatSync(abs);
|
|
861
|
+
return true;
|
|
862
|
+
}
|
|
863
|
+
catch {
|
|
864
|
+
return false;
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
/** The error code and Node's description of a filesystem failure, without the path Node appends: `EACCES: permission denied`. */
|
|
868
|
+
function describeError(error) {
|
|
869
|
+
const match = /^[A-Z0-9]+: ([^,]+)/.exec(error instanceof Error ? error.message : '');
|
|
870
|
+
return match === null ? errnoOf(error) : `${errnoOf(error)}: ${match[1]}`;
|
|
871
|
+
}
|
|
872
|
+
/** What `lstat` finds at `rel`: nothing (only ENOENT), the entry, or why it cannot be checked. */
|
|
873
|
+
function inspect(destination, rel) {
|
|
874
|
+
try {
|
|
875
|
+
return { kind: 'entry', stat: fs_1.default.lstatSync(path_1.default.join(destination, ...rel.split('/'))) };
|
|
876
|
+
}
|
|
877
|
+
catch (error) {
|
|
878
|
+
return errnoOf(error) === 'ENOENT' ? { kind: 'absent' } : { kind: 'error', reason: `cannot check it (${describeError(error)})` };
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
/**
|
|
882
|
+
* What is at a path a doc's outcome depends on (cli#168, issuecomment-6025529901): `absent` only on ENOENT; a regular file
|
|
883
|
+
* (no link) and the hash of its bytes; otherwise a refusal naming why the pull cannot vouch for it.
|
|
884
|
+
*/
|
|
885
|
+
function localState(destination, rel) {
|
|
886
|
+
const found = inspect(destination, rel);
|
|
887
|
+
if (found.kind === 'absent')
|
|
888
|
+
return found;
|
|
889
|
+
if (found.kind === 'error')
|
|
890
|
+
return { kind: 'refused', reason: found.reason };
|
|
891
|
+
if (found.stat.isDirectory())
|
|
892
|
+
return { kind: 'refused', reason: 'is a folder, not a file' };
|
|
893
|
+
if (found.stat.isSymbolicLink())
|
|
894
|
+
return { kind: 'refused', reason: 'is a symbolic link, not a file' };
|
|
895
|
+
if (!found.stat.isFile())
|
|
896
|
+
return { kind: 'refused', reason: 'is not a regular file' };
|
|
897
|
+
try {
|
|
898
|
+
return { kind: 'file', hash: (0, docs_manifest_1.sha256Hex)(fs_1.default.readFileSync(path_1.default.join(destination, ...rel.split('/')))) };
|
|
899
|
+
}
|
|
900
|
+
catch (error) {
|
|
901
|
+
return { kind: 'refused', reason: `cannot read it (${describeError(error)})` };
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
/**
|
|
905
|
+
* One outcome per planned doc, after the writes: the files this pull placed, and for every other doc what its
|
|
906
|
+
* manifest entry should be (cli#183, cli#190, cli#167). A failed download keeps the tracking it had while the file at its
|
|
907
|
+
* path is the one the entry names; a doc the pull never got to keeps its earlier entry. An earlier entry with a hash is
|
|
908
|
+
* kept only while a file is at its path: a missing file drops the doc's tracking with a warning (manager ruling on cli#168,
|
|
909
|
+
* issuecomment-6013479498, I1(b)), so the gate never meets a kept entry with a hash for a missing file.
|
|
910
|
+
* "Missing" is only ENOENT: every path an outcome depends on is classified by `localState`, and an inspection error or a
|
|
911
|
+
* non-regular entry there makes the outcome carry a `refusal`, which the gate refuses the manifest with, so no tracking is
|
|
912
|
+
* dropped or relabelled (cli#168, issuecomment-6025529901).
|
|
913
|
+
* `pendingWarnings` receives the `! …` lines printed after a successful pull.
|
|
914
|
+
*/
|
|
915
|
+
function decideOutcomes(destination, planned, placedPaths, stopped, previousManifest, folderPath, renameMoves, listedIds, pendingWarnings) {
|
|
916
|
+
const outcomes = [];
|
|
917
|
+
const earlierById = new Map();
|
|
918
|
+
if (previousManifest !== null && previousManifest.folder_path === folderPath) {
|
|
919
|
+
for (const [relPath, entry] of Object.entries(previousManifest.docs)) {
|
|
920
|
+
const earlier = earlierById.get(entry.id) ?? new Map();
|
|
921
|
+
earlier.set(relPath, entry);
|
|
922
|
+
earlierById.set(entry.id, earlier);
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
const renamedIds = new Set(renameMoves.map((m) => m.id));
|
|
926
|
+
const plannedIds = new Set(planned.map((q) => q.doc.id));
|
|
927
|
+
const failedDownload = (p) => p.isMedia && p.mediaBytes === null;
|
|
928
|
+
for (const p of planned) {
|
|
929
|
+
const entry = manifestEntryFor(p);
|
|
930
|
+
if (!failedDownload(p)) {
|
|
931
|
+
if (placedPaths.has(p.relPath)) {
|
|
932
|
+
// A stopped pull never removes a renamed doc's old twin, so that file keeps its tracking too (listed first, so
|
|
933
|
+
// the next pull sees the rename and finishes it). A twin that is gone keeps no entry (I1(b)); one the pull
|
|
934
|
+
// cannot vouch for keeps its entry and refuses the manifest. A complete pull retires the old entry (cli#157),
|
|
935
|
+
// but an old name it cannot even check could hold edits nobody verified, so that refuses too; a folder or a
|
|
936
|
+
// link there is still nothing to keep or remove.
|
|
937
|
+
const move = renameMoves.find((m) => m.id === p.doc.id);
|
|
938
|
+
const oldEntry = move === undefined ? undefined : (0, docs_manifest_1.entryAt)(previousManifest, move.oldRel);
|
|
939
|
+
const outcome = { id: p.doc.id, kind: 'placed', placed: new Map([[p.relPath, entry]]), kept: new Map() };
|
|
940
|
+
if (move?.sourceError !== undefined) {
|
|
941
|
+
if (oldEntry !== undefined)
|
|
942
|
+
outcome.kept.set(move.oldRel, oldEntry);
|
|
943
|
+
outcome.refusal = { rel: move.oldRel, reason: move.sourceError };
|
|
944
|
+
}
|
|
945
|
+
else if (move !== undefined && oldEntry !== undefined && stopped) {
|
|
946
|
+
const old = localState(destination, move.oldRel);
|
|
947
|
+
if (old.kind !== 'absent')
|
|
948
|
+
outcome.kept.set(move.oldRel, oldEntry);
|
|
949
|
+
if (old.kind === 'refused')
|
|
950
|
+
outcome.refusal = { rel: move.oldRel, reason: old.reason };
|
|
951
|
+
}
|
|
952
|
+
else if (move !== undefined && oldEntry !== undefined) {
|
|
953
|
+
const old = inspect(destination, move.oldRel);
|
|
954
|
+
if (old.kind === 'error')
|
|
955
|
+
outcome.refusal = { rel: move.oldRel, reason: old.reason };
|
|
956
|
+
}
|
|
957
|
+
outcomes.push(outcome);
|
|
958
|
+
}
|
|
959
|
+
else {
|
|
960
|
+
outcomes.push({ id: p.doc.id, kind: 'refused', placed: new Map(), kept: new Map(earlierById.get(p.doc.id)) });
|
|
961
|
+
}
|
|
962
|
+
continue;
|
|
963
|
+
}
|
|
964
|
+
if (renamedIds.has(p.doc.id))
|
|
965
|
+
continue; // decided with its rename below
|
|
966
|
+
// cli#183: a failed download at a path the previous manifest tracked for the same doc keeps
|
|
967
|
+
// that entry unchanged (revision and hash), as a failed rename does: nothing was written,
|
|
968
|
+
// so the manifest must not claim a newer revision or drop the hash of the file still there.
|
|
969
|
+
// The entry is kept only while the file is the one it names: exactly its hash (INV-C), or no file for an entry
|
|
970
|
+
// with no hash. A missing file drops the tracking (I1(b)).
|
|
971
|
+
//
|
|
972
|
+
// cli#167: a media doc whose download failed must not be tracked over a
|
|
973
|
+
// local file the previous manifest does not track, with a hash, for that
|
|
974
|
+
// same doc: nothing is written, so the manifest would claim bytes the
|
|
975
|
+
// pull never gave it. A rename keeps its restore-old-entry handling below.
|
|
976
|
+
//
|
|
977
|
+
// cli#190: if that local file is another doc's (tracked with a hash at this path), that
|
|
978
|
+
// doc's tracking is kept when it is still on the server and not moved by this pull.
|
|
979
|
+
//
|
|
980
|
+
// An inspection error or a non-regular entry at the path is never "absent": the doc keeps its earlier entries
|
|
981
|
+
// and the gate refuses the manifest. The previous owner's entry is never replaced by this doc's id with no hash.
|
|
982
|
+
const tracked = (0, docs_manifest_1.entryAt)(previousManifest, p.relPath);
|
|
983
|
+
const local = localState(destination, p.relPath);
|
|
984
|
+
if (local.kind === 'refused') {
|
|
985
|
+
outcomes.push({ id: p.doc.id, kind: 'refused', placed: new Map(), kept: new Map(earlierById.get(p.doc.id)), refusal: { rel: p.relPath, reason: local.reason } });
|
|
986
|
+
continue;
|
|
987
|
+
}
|
|
988
|
+
const ownEntry = tracked !== undefined && tracked.id === p.doc.id ? tracked : undefined;
|
|
989
|
+
if (ownEntry !== undefined && (ownEntry.body_sha256 == null ? local.kind === 'absent' : local.kind === 'file' && local.hash === ownEntry.body_sha256)) {
|
|
990
|
+
outcomes.push({ id: p.doc.id, kind: 'kept-previous', placed: new Map(), kept: new Map([[p.relPath, ownEntry]]) });
|
|
991
|
+
continue;
|
|
992
|
+
}
|
|
993
|
+
if (ownEntry !== undefined && ownEntry.body_sha256 != null && local.kind === 'absent') {
|
|
994
|
+
outcomes.push({ id: p.doc.id, kind: 'dropped', placed: new Map(), kept: new Map() });
|
|
995
|
+
pendingWarnings.push(`! doc ${p.doc.id} ("${shown(p.doc.title)}") failed to download and ${shown(p.relPath)} is not present locally; not tracking it — pull again later.`);
|
|
996
|
+
continue;
|
|
997
|
+
}
|
|
998
|
+
const otherOwner = tracked !== undefined && tracked.id !== p.doc.id ? tracked : undefined;
|
|
999
|
+
if (local.kind === 'absent' && otherOwner === undefined) {
|
|
1000
|
+
outcomes.push({ id: p.doc.id, kind: 'placed', placed: new Map([[p.relPath, entry]]), kept: new Map() });
|
|
1001
|
+
continue;
|
|
1002
|
+
}
|
|
1003
|
+
const otherDoc = otherOwner !== undefined && otherOwner.body_sha256 != null ? otherOwner : undefined;
|
|
1004
|
+
outcomes.push({ id: p.doc.id, kind: 'dropped', placed: new Map(), kept: new Map() });
|
|
1005
|
+
if (otherDoc !== undefined && local.kind === 'file' && local.hash === otherDoc.body_sha256 && listedIds.has(otherDoc.id) && !plannedIds.has(otherDoc.id)) {
|
|
1006
|
+
outcomes.push({ id: otherDoc.id, kind: 'kept-previous', placed: new Map(), kept: new Map([[p.relPath, otherDoc]]) });
|
|
1007
|
+
pendingWarnings.push(`! doc ${p.doc.id} ("${shown(p.doc.title)}") failed to download and ${shown(p.relPath)} holds doc ${otherDoc.id}'s file ("${shown(otherDoc.title)}"); still tracking it as doc ${otherDoc.id} — pull again later`);
|
|
1008
|
+
continue;
|
|
1009
|
+
}
|
|
1010
|
+
// A pull that stopped keeps every earlier entry of a doc it has no outcome for, so it takes no tracking from that doc.
|
|
1011
|
+
const lost = otherOwner === undefined || stopped ? '' : ` Doc ${otherOwner.id} ("${shown(otherOwner.title)}") was tracked at ${shown(p.relPath)} before and is no longer tracked there.`;
|
|
1012
|
+
if (local.kind === 'absent') {
|
|
1013
|
+
pendingWarnings.push(`! doc ${p.doc.id} ("${shown(p.doc.title)}") failed to download and ${shown(p.relPath)} is not present locally; not tracking it — pull again later.${lost}`);
|
|
1014
|
+
}
|
|
1015
|
+
else {
|
|
1016
|
+
pendingWarnings.push(`! doc ${p.doc.id} ("${shown(p.doc.title)}") failed to download and ${shown(p.relPath)} holds a local file; not tracking it — pull again later.${lost}`);
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
// A rename whose replacement is NOT written (a failed media download plans a new path but
|
|
1020
|
+
// writes nothing) keeps its old file and its old manifest entry: deleting it would destroy
|
|
1021
|
+
// the only good copy. An old entry with a hash whose file is gone is not kept (I1(b)): the doc is then not
|
|
1022
|
+
// tracked at all, neither at the old path nor (with no hash) at a new path that may hold an untracked file. The
|
|
1023
|
+
// refusals in `report` guarantee no other doc in this pull claims that old path.
|
|
1024
|
+
for (const m of renameMoves) {
|
|
1025
|
+
if (m.replacementWritten)
|
|
1026
|
+
continue;
|
|
1027
|
+
const oldEntry = (0, docs_manifest_1.entryAt)(previousManifest, m.oldRel);
|
|
1028
|
+
const local = m.sourceError === undefined ? localState(destination, m.oldRel) : { kind: 'refused', reason: m.sourceError };
|
|
1029
|
+
if (local.kind === 'refused') {
|
|
1030
|
+
outcomes.push({ id: m.id, kind: 'refused', placed: new Map(), kept: new Map(earlierById.get(m.id)), refusal: { rel: m.oldRel, reason: local.reason } });
|
|
1031
|
+
continue;
|
|
1032
|
+
}
|
|
1033
|
+
if (oldEntry !== undefined && oldEntry.body_sha256 != null && local.kind === 'absent') {
|
|
1034
|
+
outcomes.push({ id: m.id, kind: 'dropped', placed: new Map(), kept: new Map() });
|
|
1035
|
+
pendingWarnings.push(`! doc ${m.id} ("${shown(m.title)}") failed to download and ${shown(m.oldRel)} is not present locally; not tracking it — pull again later.`);
|
|
1036
|
+
continue;
|
|
1037
|
+
}
|
|
1038
|
+
outcomes.push({ id: m.id, kind: 'kept-previous', placed: new Map(), kept: oldEntry === undefined ? new Map() : new Map([[m.oldRel, oldEntry]]) });
|
|
1039
|
+
if (local.kind === 'file') {
|
|
1040
|
+
pendingWarnings.push(`! kept ${shown(m.oldRel)} — doc ${m.id} download failed; still tracked as ${shown(m.oldRel)}`);
|
|
1041
|
+
}
|
|
1042
|
+
else {
|
|
1043
|
+
pendingWarnings.push(`! doc ${m.id} download failed; still tracked as ${shown(m.oldRel)}, which is not present locally`);
|
|
1044
|
+
}
|
|
1045
|
+
}
|
|
1046
|
+
return outcomes;
|
|
1047
|
+
}
|
|
1048
|
+
/**
|
|
1049
|
+
* The manifest as a pure function of the previous manifest, the outcomes and whether this pull speaks for one doc
|
|
1050
|
+
* (spec §1.3, rule 3). A single-doc pull, or one that stopped before it could propagate deletions, keeps every earlier
|
|
1051
|
+
* entry whose doc has no outcome (cli#153); an outcome's own doc never gets its earlier entries back. Bytes this run
|
|
1052
|
+
* placed always win: an earlier entry at a name a placed entry holds, or one equal to it under `nameKey`, is dropped, and
|
|
1053
|
+
* the doc that lost it gets a "no longer tracked there" line in `pendingWarnings` (a doc's own old twin loses silently).
|
|
1054
|
+
* The overlap is settled here, before anything is collapsed by name, so no retained entry can stand for placed bytes.
|
|
1055
|
+
* Entries are collected in a Map and published as a null-prototype dictionary, so every path is an own key (PM ruling 14).
|
|
1056
|
+
*/
|
|
1057
|
+
function buildManifest(folderPath, previousManifest, outcomes, keepUnlisted, pendingWarnings) {
|
|
1058
|
+
const placedByKey = new Map();
|
|
1059
|
+
for (const outcome of outcomes) {
|
|
1060
|
+
for (const [relPath, entry] of outcome.placed)
|
|
1061
|
+
placedByKey.set((0, doc_pull_writes_1.nameKey)(relPath), { relPath, entry });
|
|
1062
|
+
}
|
|
1063
|
+
const docs = new Map();
|
|
1064
|
+
const warned = new Set();
|
|
1065
|
+
const keep = (relPath, entry) => {
|
|
1066
|
+
const winner = placedByKey.get((0, doc_pull_writes_1.nameKey)(relPath));
|
|
1067
|
+
if (winner === undefined) {
|
|
1068
|
+
docs.set(relPath, entry);
|
|
1069
|
+
return;
|
|
1070
|
+
}
|
|
1071
|
+
if (winner.entry.id !== entry.id && !warned.has(relPath)) {
|
|
1072
|
+
warned.add(relPath);
|
|
1073
|
+
pendingWarnings.push(`! ${shown(winner.relPath)} now holds the file this pull wrote for doc ${winner.entry.id} ("${shown(winner.entry.title)}"); doc ${entry.id} ("${shown(entry.title)}") was tracked at ${shown(relPath)} before and is no longer tracked there`);
|
|
1074
|
+
}
|
|
1075
|
+
};
|
|
1076
|
+
if (keepUnlisted && previousManifest !== null && previousManifest.folder_path === folderPath) {
|
|
1077
|
+
const outcomeIds = new Set(outcomes.map((outcome) => outcome.id));
|
|
1078
|
+
for (const [relPath, entry] of Object.entries(previousManifest.docs)) {
|
|
1079
|
+
if (!outcomeIds.has(entry.id))
|
|
1080
|
+
keep(relPath, entry);
|
|
1081
|
+
}
|
|
1082
|
+
}
|
|
1083
|
+
for (const outcome of outcomes) {
|
|
1084
|
+
for (const [relPath, entry] of outcome.kept)
|
|
1085
|
+
keep(relPath, entry);
|
|
1086
|
+
for (const [relPath, entry] of outcome.placed)
|
|
1087
|
+
docs.set(relPath, entry);
|
|
1088
|
+
}
|
|
1089
|
+
return { folder_path: folderPath, docs: (0, docs_manifest_1.docsFrom)(docs) };
|
|
1090
|
+
}
|
|
1091
|
+
/** The error code of a filesystem failure, for the gate's wording. */
|
|
1092
|
+
const errnoOf = (error) => error.code ?? 'ERROR';
|
|
1093
|
+
/**
|
|
1094
|
+
* INV-C as a runtime gate (spec §1.2, rule 4), checked just before the manifest is written, on the FINAL manifest: no two
|
|
1095
|
+
* of its paths may be one entry on some filesystem, every file this run placed must be in it with the hash this run wrote
|
|
1096
|
+
* (PM ruling 14: a placed file the manifest would not record refuses), and every entry with a hash must be one this run
|
|
1097
|
+
* can vouch for:
|
|
1098
|
+
* - A file this run placed (`placedFiles`): it must be a regular file with the `dev:ino` this run renamed in, and the
|
|
1099
|
+
* entry must be the hash of the bytes it wrote. The bytes are read back; only a permission failure (EACCES, or EPERM,
|
|
1100
|
+
* the same failure on some platforms) on that same file (a write-only mode it kept from the file it replaced) falls
|
|
1101
|
+
* back to the hash this run wrote.
|
|
1102
|
+
* - An entry kept for a doc this run acted on (a kept-previous outcome, or the old twin of a placed rename): lstat must
|
|
1103
|
+
* show a regular file (no link, no folder) whose bytes hash to the entry. A missing file refuses like any other lstat
|
|
1104
|
+
* or read failure: `decideOutcomes` keeps no such entry.
|
|
1105
|
+
* - An entry carried unchanged for a doc this run did not write (a refused doc's, or one it never listed) was not
|
|
1106
|
+
* touched by this run, and is not read: a folder or an edited file at that name is the refusal's own state.
|
|
1107
|
+
* The three exceptions to PM ruling 13 (a refused doc's carried entry is not read; a kept entry's missing file is NOT
|
|
1108
|
+
* one; EPERM like EACCES for a placed file) are the manager ruling on cli#168, issuecomment-6013479498.
|
|
1109
|
+
* First of all, an outcome's own `refusal` (a path it depends on that `decideOutcomes` could not vouch for: an inspection
|
|
1110
|
+
* error or a non-regular entry, cli#168 issuecomment-6025529901) is the problem.
|
|
1111
|
+
* Returns the first problem, or null.
|
|
1112
|
+
*/
|
|
1113
|
+
function manifestProblem(destination, manifest, outcomes, placedFiles) {
|
|
1114
|
+
for (const outcome of outcomes) {
|
|
1115
|
+
if (outcome.refusal !== undefined)
|
|
1116
|
+
return outcome.refusal;
|
|
1117
|
+
}
|
|
1118
|
+
const byKey = new Map();
|
|
1119
|
+
for (const rel of Object.keys(manifest.docs)) {
|
|
1120
|
+
const key = (0, doc_pull_writes_1.nameKey)(rel);
|
|
1121
|
+
const other = byKey.get(key);
|
|
1122
|
+
if (other !== undefined)
|
|
1123
|
+
return { rel, reason: `the same file as ${other} on a case-insensitive or Unicode-normalising filesystem` };
|
|
1124
|
+
byKey.set(key, rel);
|
|
1125
|
+
}
|
|
1126
|
+
for (const [rel, placed] of placedFiles) {
|
|
1127
|
+
const entry = (0, docs_manifest_1.entryAt)(manifest, rel);
|
|
1128
|
+
if (entry === undefined)
|
|
1129
|
+
return { rel, reason: 'this pull wrote it but the manifest would not record it' };
|
|
1130
|
+
if (entry.body_sha256 !== placed.hash)
|
|
1131
|
+
return { rel, reason: 'its bytes are not the ones the manifest would record' };
|
|
1132
|
+
}
|
|
1133
|
+
const verified = new Set();
|
|
1134
|
+
for (const outcome of outcomes) {
|
|
1135
|
+
if (outcome.kind === 'refused')
|
|
1136
|
+
continue;
|
|
1137
|
+
for (const rel of outcome.kept.keys())
|
|
1138
|
+
verified.add(rel);
|
|
1139
|
+
}
|
|
1140
|
+
for (const [rel, entry] of Object.entries(manifest.docs)) {
|
|
1141
|
+
if (entry.body_sha256 == null)
|
|
1142
|
+
continue;
|
|
1143
|
+
const placed = placedFiles.get(rel);
|
|
1144
|
+
if (placed === undefined && !verified.has(rel))
|
|
1145
|
+
continue;
|
|
1146
|
+
const abs = path_1.default.join(destination, ...rel.split('/'));
|
|
1147
|
+
let stat;
|
|
1148
|
+
try {
|
|
1149
|
+
stat = fs_1.default.lstatSync(abs);
|
|
1150
|
+
}
|
|
1151
|
+
catch (error) {
|
|
1152
|
+
if (errnoOf(error) !== 'ENOENT')
|
|
1153
|
+
return { rel, reason: `it cannot be checked (${errnoOf(error)})` };
|
|
1154
|
+
return { rel, reason: placed !== undefined ? 'the file this pull wrote is not there' : 'the file it tracks is not there' };
|
|
1155
|
+
}
|
|
1156
|
+
if (!stat.isFile())
|
|
1157
|
+
return { rel, reason: 'it is not a regular file now' };
|
|
1158
|
+
if (placed !== undefined && (placed.hash !== entry.body_sha256 || placed.identity !== `${stat.dev}:${stat.ino}`))
|
|
1159
|
+
return { rel, reason: 'its bytes are not the ones the manifest would record' };
|
|
1160
|
+
let actual;
|
|
1161
|
+
try {
|
|
1162
|
+
actual = (0, docs_manifest_1.sha256Hex)(fs_1.default.readFileSync(abs));
|
|
1163
|
+
}
|
|
1164
|
+
catch (error) {
|
|
1165
|
+
if (placed === undefined || (errnoOf(error) !== 'EACCES' && errnoOf(error) !== 'EPERM'))
|
|
1166
|
+
return { rel, reason: `it cannot be read back (${errnoOf(error)})` };
|
|
1167
|
+
actual = placed.hash;
|
|
1168
|
+
}
|
|
1169
|
+
if (actual !== entry.body_sha256)
|
|
1170
|
+
return { rel, reason: 'its bytes are not the ones the manifest would record' };
|
|
1171
|
+
}
|
|
1172
|
+
return null;
|
|
1173
|
+
}
|
|
1174
|
+
/**
|
|
1175
|
+
* The one line naming every listed doc whose content could not be fetched (cli#209), printed once the manifest step is
|
|
1176
|
+
* done; the caller exits 1. Nothing when every doc was fetched.
|
|
1177
|
+
*/
|
|
1178
|
+
function reportFetchFailed(fetchFailed) {
|
|
1179
|
+
if (fetchFailed.length === 0)
|
|
1180
|
+
return;
|
|
1181
|
+
const names = fetchFailed.map((row) => `${row.id} ("${shown(row.title)}")`).join(', ');
|
|
1182
|
+
process.stderr.write(chalk_1.default.red(`error: could not fetch ${fetchFailed.length} doc(s) from the server: ${names} — their local files and tracking were left as they were; pull again.\n`));
|
|
1183
|
+
}
|
|
1184
|
+
/**
|
|
1185
|
+
* The line for the error or refusal the writes stopped at, then exit 1 (spec §1.5). `tracked` says the manifest
|
|
1186
|
+
* recorded the files written before the stop; when it did not, the manifest's own line already said so.
|
|
1187
|
+
*/
|
|
1188
|
+
function reportStop(stop, planned, placed, writes, tracked) {
|
|
1189
|
+
const { relPath } = stop.write;
|
|
1190
|
+
const tail = tracked ? ` — ${placed.length} of ${writes.length} files were updated and are tracked` : '';
|
|
1191
|
+
if (stop.error instanceof doc_pull_writes_1.LinkOnTheWayError) {
|
|
1192
|
+
const linked = planned.find((q) => q.relPath === relPath);
|
|
1193
|
+
if (linked !== undefined)
|
|
1194
|
+
refuseLink(relPath, stop.error.component, linked.doc);
|
|
1195
|
+
}
|
|
1196
|
+
if (stop.error instanceof doc_pull_writes_1.PublicationRefusedError) {
|
|
1197
|
+
process.stderr.write(chalk_1.default.red(`error: ${shown(relPath)} changed after doc pull checked it${tail}${tracked ? '. Pull again, or pass --overwrite to replace it.' : ''}\n`));
|
|
1198
|
+
}
|
|
1199
|
+
else if (stop.error instanceof doc_pull_writes_1.UnsupportedTargetError) {
|
|
1200
|
+
// Ruling 9: a folder at a target is left alone, never moved or deleted.
|
|
1201
|
+
process.stderr.write(chalk_1.default.red(`error: ${shown(relPath)} is a folder (doc pull writes a file there)${tail}${tracked ? '. Move it aside and pull again.' : ''}\n`));
|
|
1202
|
+
}
|
|
1203
|
+
else {
|
|
1204
|
+
process.stderr.write(chalk_1.default.red(`error: cannot write ${shown(relPath)}: ${shown(stop.error.message)}${tail}${tracked ? '; pull again.' : ''}\n`));
|
|
1205
|
+
}
|
|
1206
|
+
process.exit(1);
|
|
1207
|
+
}
|
|
769
1208
|
/** Write the docs + manifest to disk and print the result (chalk lines or --json). */
|
|
770
|
-
async function report(destination, folderPath, fetched, options, config, extraWarnings, previousManifest, usedSingleDocFallback) {
|
|
1209
|
+
async function report(destination, folderPath, fetched, options, config, extraWarnings, previousManifest, usedSingleDocFallback, listedIds, lock, fetchFailed) {
|
|
1210
|
+
// A listed doc whose content could not be fetched (cli#209) is refused: its earlier entries are carried unchanged
|
|
1211
|
+
// and its file is not touched, so no other doc is planned at those paths. A doc with no earlier entry is not tracked.
|
|
1212
|
+
const fetchFailedIds = new Set(fetchFailed.map((row) => row.id));
|
|
1213
|
+
const heldEntries = new Map(previousManifest !== null && previousManifest.folder_path === folderPath
|
|
1214
|
+
? Object.entries(previousManifest.docs).filter(([, entry]) => fetchFailedIds.has(entry.id))
|
|
1215
|
+
: []);
|
|
771
1216
|
// Single-doc fallback merging into a manifest that tracks the same folder
|
|
772
1217
|
// must not steal a filename another tracked doc owns (cli#153 C1): the
|
|
773
1218
|
// pulled doc reuses its own tracked path, and the allocator avoids every
|
|
@@ -792,7 +1237,23 @@ async function report(destination, folderPath, fetched, options, config, extraWa
|
|
|
792
1237
|
}
|
|
793
1238
|
reserved = { usedNames, pathById, titleById };
|
|
794
1239
|
}
|
|
795
|
-
const { planned, warnings } = await planDocs(fetched, config, reserved, previousManifest);
|
|
1240
|
+
const { planned, warnings } = await planDocs(fetched, config, reserved, previousManifest, [...heldEntries.keys()]);
|
|
1241
|
+
const seen = new Map();
|
|
1242
|
+
// Two docs whose names are one entry on some filesystem (case, Unicode normalisation) can never both keep
|
|
1243
|
+
// their own bytes: refuse before anything is written (cli#168).
|
|
1244
|
+
const plannedByKey = new Map();
|
|
1245
|
+
for (const p of planned) {
|
|
1246
|
+
if (p.isMedia && p.mediaBytes === null)
|
|
1247
|
+
continue; // a failed download writes nothing
|
|
1248
|
+
const key = (0, doc_pull_writes_1.nameKey)(p.relPath);
|
|
1249
|
+
const other = plannedByKey.get(key);
|
|
1250
|
+
if (other !== undefined) {
|
|
1251
|
+
process.stderr.write(chalk_1.default.red(`error: ${shown(other.relPath)} (doc ${other.doc.id}) and ${shown(p.relPath)} (doc ${p.doc.id}) would be the same file on a case-insensitive or Unicode-normalising filesystem; this pull would write different docs through those names.\n`));
|
|
1252
|
+
process.stderr.write(chalk_1.default.red('Nothing was written. Rename one of the docs on the server and pull again.\n'));
|
|
1253
|
+
process.exit(1);
|
|
1254
|
+
}
|
|
1255
|
+
plannedByKey.set(key, p);
|
|
1256
|
+
}
|
|
796
1257
|
// Same doc id under a different path is a rename (PM ruling 2, cli#157):
|
|
797
1258
|
// the previous manifest tracks a planned doc under a DIFFERENT path `old`,
|
|
798
1259
|
// whatever is on disk at `old` (cli#157 FR4-C1). What is at `old` decides
|
|
@@ -818,26 +1279,30 @@ async function report(destination, folderPath, fetched, options, config, extraWa
|
|
|
818
1279
|
continue;
|
|
819
1280
|
const absOld = path_1.default.join(destination, ...old.split('/'));
|
|
820
1281
|
let sourceStat = null;
|
|
1282
|
+
let sourceError;
|
|
821
1283
|
try {
|
|
822
1284
|
const stat = fs_1.default.statSync(absOld);
|
|
823
1285
|
if (stat.isFile())
|
|
824
1286
|
sourceStat = stat;
|
|
825
1287
|
}
|
|
826
|
-
catch {
|
|
1288
|
+
catch (error) {
|
|
827
1289
|
sourceStat = null;
|
|
1290
|
+
// Only ENOENT is "no source"; any other failure is a source nobody could check for edits.
|
|
1291
|
+
if (errnoOf(error) !== 'ENOENT')
|
|
1292
|
+
sourceError = `cannot check it (${describeError(error)})`;
|
|
828
1293
|
}
|
|
829
1294
|
let modified = false;
|
|
830
1295
|
let targetIsSource = false;
|
|
1296
|
+
let currentHash = null;
|
|
831
1297
|
if (sourceStat !== null) {
|
|
832
|
-
const entry = previousManifest
|
|
833
|
-
let currentHash = null;
|
|
1298
|
+
const entry = (0, docs_manifest_1.entryAt)(previousManifest, old);
|
|
834
1299
|
try {
|
|
835
1300
|
currentHash = (0, docs_manifest_1.sha256Hex)(fs_1.default.readFileSync(absOld));
|
|
836
1301
|
}
|
|
837
1302
|
catch {
|
|
838
1303
|
currentHash = null;
|
|
839
1304
|
}
|
|
840
|
-
modified = currentHash === null || currentHash !== entry
|
|
1305
|
+
modified = currentHash === null || currentHash !== entry?.body_sha256;
|
|
841
1306
|
try {
|
|
842
1307
|
const targetStat = fs_1.default.statSync(path_1.default.join(destination, ...p.relPath.split('/')));
|
|
843
1308
|
targetIsSource = targetStat.dev === sourceStat.dev && targetStat.ino === sourceStat.ino;
|
|
@@ -853,10 +1318,12 @@ async function report(destination, folderPath, fetched, options, config, extraWa
|
|
|
853
1318
|
title: p.doc.title,
|
|
854
1319
|
sourcePresent: sourceStat !== null,
|
|
855
1320
|
sourceIdentity: sourceStat === null ? null : `${sourceStat.dev}:${sourceStat.ino}`,
|
|
1321
|
+
sourceHash: currentHash,
|
|
856
1322
|
modified,
|
|
857
1323
|
// A failed media download plans a new path but writes nothing there.
|
|
858
1324
|
replacementWritten: !p.isMedia || p.mediaBytes !== null,
|
|
859
1325
|
targetIsSource,
|
|
1326
|
+
...(sourceError === undefined ? {} : { sourceError }),
|
|
860
1327
|
});
|
|
861
1328
|
}
|
|
862
1329
|
const plannedPaths = new Map(planned.map((p) => [p.relPath, p]));
|
|
@@ -889,6 +1356,17 @@ async function report(destination, folderPath, fetched, options, config, extraWa
|
|
|
889
1356
|
?? (m.sourceIdentity === null
|
|
890
1357
|
? undefined
|
|
891
1358
|
: plannedByIdentity.get(m.sourceIdentity)?.find((p) => p.doc.id !== m.id));
|
|
1359
|
+
// Rule 5 (spec §1.1) at every rename-source decision: a source already holding the bytes this pull writes for its
|
|
1360
|
+
// doc, or for the doc whose write lands on that file, holds no edit (an interrupted pull placed them, and the old
|
|
1361
|
+
// name aliases the placed file or another doc's write took it). Real edits are still refused below.
|
|
1362
|
+
const plannedByDocId = new Map(planned.map((p) => [p.doc.id, p]));
|
|
1363
|
+
for (const m of renameMoves) {
|
|
1364
|
+
if (!m.modified || m.sourceHash === null)
|
|
1365
|
+
continue;
|
|
1366
|
+
const pulledHere = [plannedByDocId.get(m.id), plannedTargetForSource(m)];
|
|
1367
|
+
if (pulledHere.some((q) => q !== undefined && q.bodySha256 !== null && q.bodySha256 === m.sourceHash))
|
|
1368
|
+
m.modified = false;
|
|
1369
|
+
}
|
|
892
1370
|
// Edited sources cannot be discarded by --overwrite during a rename.
|
|
893
1371
|
// Report a cross-doc collision first, including an alias's actual path.
|
|
894
1372
|
for (const m of renameMoves) {
|
|
@@ -897,7 +1375,7 @@ async function report(destination, folderPath, fetched, options, config, extraWa
|
|
|
897
1375
|
const taker = plannedTargetForSource(m);
|
|
898
1376
|
if (!taker)
|
|
899
1377
|
continue;
|
|
900
|
-
const oldTitle = previousManifest
|
|
1378
|
+
const oldTitle = (0, docs_manifest_1.entryAt)(previousManifest, m.oldRel)?.title ?? `#${m.id}`;
|
|
901
1379
|
process.stderr.write(chalk_1.default.red(`error: ${shown(m.oldRel)} holds unpublished edits for doc ${m.id} ("${shown(oldTitle)}", now written as ${shown(m.newRel)}), but this pull would write doc ${taker.doc.id} ("${shown(taker.doc.title)}") to ${shown(taker.relPath)}, the same file.\n`));
|
|
902
1380
|
process.stderr.write(chalk_1.default.red(`Move or rename the edited ${shown(m.oldRel)} (or push it first) and pull again.\n`));
|
|
903
1381
|
process.exit(1);
|
|
@@ -983,7 +1461,7 @@ async function report(destination, folderPath, fetched, options, config, extraWa
|
|
|
983
1461
|
for (const m of renameMoves) {
|
|
984
1462
|
if (!m.replacementWritten || m.targetIsSource)
|
|
985
1463
|
continue;
|
|
986
|
-
const targetEntry = previousManifest
|
|
1464
|
+
const targetEntry = (0, docs_manifest_1.entryAt)(previousManifest, m.newRel);
|
|
987
1465
|
if (targetEntry !== undefined && targetEntry.body_sha256 != null)
|
|
988
1466
|
continue;
|
|
989
1467
|
const absNew = path_1.default.join(destination, ...m.newRel.split('/'));
|
|
@@ -996,10 +1474,14 @@ async function report(destination, folderPath, fetched, options, config, extraWa
|
|
|
996
1474
|
}
|
|
997
1475
|
catch (error) {
|
|
998
1476
|
const code = error.code ?? 'ERROR';
|
|
999
|
-
if (code === 'ENOENT')
|
|
1477
|
+
if (code === 'ENOENT') {
|
|
1478
|
+
observe(seen, m.newRel, { kind: 'absent' });
|
|
1000
1479
|
continue;
|
|
1480
|
+
}
|
|
1001
1481
|
unreadable = ` (cannot be read: ${code})`; // present but unverifiable: not owned
|
|
1002
1482
|
}
|
|
1483
|
+
if (existing !== null)
|
|
1484
|
+
observe(seen, m.newRel, { kind: 'sha256', sha256: (0, docs_manifest_1.sha256Hex)(existing) });
|
|
1003
1485
|
if (existing !== null && (0, docs_manifest_1.sha256Hex)(existing) === plannedPaths.get(m.newRel)?.bodySha256)
|
|
1004
1486
|
continue;
|
|
1005
1487
|
const state = (targetEntry === undefined ? 'exists locally but is not tracked' : `holds local bytes with no recorded hash (tracked for doc ${targetEntry.id})`) + unreadable;
|
|
@@ -1012,18 +1494,26 @@ async function report(destination, folderPath, fetched, options, config, extraWa
|
|
|
1012
1494
|
const handledOldPaths = new Set(renameMoves.map((m) => m.oldRel));
|
|
1013
1495
|
// After the rename block (which keeps its own messages for rename cases) and before anything
|
|
1014
1496
|
// else that reads or writes the plan: no planned write may go through a link or outside.
|
|
1015
|
-
checkPlannedWrites(destination, planned, previousManifest, options, renameMoves);
|
|
1497
|
+
checkPlannedWrites(destination, planned, previousManifest, options, renameMoves, seen);
|
|
1016
1498
|
// Unpushed-local-changes protection: now that the server walk is known, refuse only
|
|
1017
1499
|
// for a file whose doc still exists remotely — i.e. its relative path is part of this
|
|
1018
|
-
// pull's plan and would be overwritten by
|
|
1500
|
+
// pull's plan and would be overwritten by the writes below. A modified file whose
|
|
1019
1501
|
// relPath is NOT in the plan is an orphan, not a conflict: it is left alone here and
|
|
1020
1502
|
// the deletion-propagation block further down keeps it and warns (no --overwrite
|
|
1021
|
-
// needed). This check must run before
|
|
1503
|
+
// needed). This check must run before any write — nothing may be written to disk
|
|
1022
1504
|
// before a real conflict has had the chance to refuse.
|
|
1023
1505
|
if (previousManifest && !options.overwrite) {
|
|
1024
|
-
const modified = detectLocalModifications(destination, previousManifest);
|
|
1025
|
-
const
|
|
1026
|
-
|
|
1506
|
+
const modified = detectLocalModifications(destination, previousManifest, seen);
|
|
1507
|
+
const plannedByPath = new Map(planned.map((p) => [p.relPath, p]));
|
|
1508
|
+
// A file already holding the bytes this pull would write is never a conflict (spec §1.3): the next pull
|
|
1509
|
+
// after an interrupted or failed one adopts what that one placed.
|
|
1510
|
+
const conflicts = modified.filter((relPath) => {
|
|
1511
|
+
const p = plannedByPath.get(relPath);
|
|
1512
|
+
if (p === undefined)
|
|
1513
|
+
return false;
|
|
1514
|
+
const state = seen.get(relPath);
|
|
1515
|
+
return !(p.bodySha256 !== null && state?.kind === 'sha256' && state.sha256 === p.bodySha256);
|
|
1516
|
+
});
|
|
1027
1517
|
if (conflicts.length > 0) {
|
|
1028
1518
|
const noun = conflicts.length === 1 ? 'file has' : 'files have';
|
|
1029
1519
|
process.stderr.write(chalk_1.default.red(`${conflicts.length} ${noun} unpushed local changes:\n`));
|
|
@@ -1034,9 +1524,8 @@ async function report(destination, folderPath, fetched, options, config, extraWa
|
|
|
1034
1524
|
process.exit(1);
|
|
1035
1525
|
}
|
|
1036
1526
|
}
|
|
1037
|
-
// A destination path that already exists as a directory would make
|
|
1038
|
-
//
|
|
1039
|
-
// is saved — a half-pulled tree with a stale sidecar. Refuse before any write.
|
|
1527
|
+
// A destination path that already exists as a directory would make the write fail mid-walk, after earlier
|
|
1528
|
+
// docs were written. Refuse before any write.
|
|
1040
1529
|
for (const p of planned) {
|
|
1041
1530
|
const targetAbs = path_1.default.join(destination, ...p.relPath.split('/'));
|
|
1042
1531
|
if (fs_1.default.existsSync(targetAbs) && !fs_1.default.statSync(targetAbs).isFile()) {
|
|
@@ -1044,36 +1533,73 @@ async function report(destination, folderPath, fetched, options, config, extraWa
|
|
|
1044
1533
|
process.exit(1);
|
|
1045
1534
|
}
|
|
1046
1535
|
}
|
|
1047
|
-
//
|
|
1048
|
-
//
|
|
1049
|
-
//
|
|
1536
|
+
// The state each target was authorized in is the state the checks above saw (spec §1.3 step 2); the write
|
|
1537
|
+
// refuses a target that is no longer in it (INV-B). Nothing is read again here. A target no check saw
|
|
1538
|
+
// is authorized as absent, so anything found there is refused.
|
|
1539
|
+
const faults = (0, doc_pull_writes_1.faultsFromEnv)();
|
|
1540
|
+
faults.afterChecks(destination);
|
|
1541
|
+
const writes = [];
|
|
1542
|
+
for (const p of planned) {
|
|
1543
|
+
const data = p.isMedia ? p.mediaBytes : p.doc.body;
|
|
1544
|
+
if (data === null)
|
|
1545
|
+
continue; // a failed media download writes nothing
|
|
1546
|
+
const authorized = options.overwrite ? { kind: 'any' } : (seen.get(p.relPath) ?? { kind: 'absent' });
|
|
1547
|
+
writes.push({ relPath: p.relPath, dirRel: p.dirRel, data, authorized });
|
|
1548
|
+
}
|
|
1549
|
+
// Every throw below can carry a server-derived path (a title-named file): the command
|
|
1550
|
+
// boundary in `docPull` prints it sanitised.
|
|
1050
1551
|
fs_1.default.mkdirSync(destination, { recursive: true });
|
|
1552
|
+
if (lock.destination === null) {
|
|
1553
|
+
// A destination made just now was planned as new, with no manifest: take its lock, then check that no other
|
|
1554
|
+
// pull wrote one while this one was running (spec §1.3 step 1). The plan is built on the manifest read under
|
|
1555
|
+
// the lock, or the pull refuses before any write.
|
|
1556
|
+
takeLock(destination, lock);
|
|
1557
|
+
if (entryExists(path_1.default.join(destination, docs_manifest_1.DOCS_MANIFEST))) {
|
|
1558
|
+
process.stderr.write(chalk_1.default.red(`error: another doc pull wrote to ${shown(destination)} while this one was running — nothing was changed. Pull again.\n`));
|
|
1559
|
+
process.exit(1);
|
|
1560
|
+
}
|
|
1561
|
+
}
|
|
1051
1562
|
const destPrefix = path_1.default.resolve(destination) + path_1.default.sep;
|
|
1052
1563
|
const realDest = fs_1.default.realpathSync(destination);
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
const
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1564
|
+
// Write in plan order, stopping at the first error or refusal; then record exactly what was written, once (spec §1.3).
|
|
1565
|
+
const { placed, stop } = (0, doc_pull_writes_1.writeAll)(destination, writes, faults);
|
|
1566
|
+
faults.afterWrites(destination);
|
|
1567
|
+
const pendingWarnings = [];
|
|
1568
|
+
const outcomes = decideOutcomes(destination, planned, new Set(placed.map((write) => write.relPath)), stop !== null, previousManifest, folderPath, renameMoves, listedIds, pendingWarnings);
|
|
1569
|
+
for (const row of fetchFailed) {
|
|
1570
|
+
outcomes.push({ id: row.id, kind: 'refused', placed: new Map(), kept: new Map([...heldEntries].filter(([, entry]) => entry.id === row.id)) });
|
|
1571
|
+
}
|
|
1572
|
+
const manifest = buildManifest(folderPath, previousManifest, outcomes, usedSingleDocFallback || stop !== null, pendingWarnings);
|
|
1573
|
+
const updated = `${placed.length} of ${writes.length} files were updated`;
|
|
1574
|
+
let manifestRecorded = false;
|
|
1575
|
+
const problem = manifestProblem(destination, manifest, outcomes, new Map(placed.map((write) => [write.relPath, { hash: (0, docs_manifest_1.sha256Hex)(write.data), identity: write.identity }])));
|
|
1576
|
+
if (problem !== null) {
|
|
1577
|
+
process.stderr.write(chalk_1.default.red(`error: doc pull stopped before recording a manifest that would not match the files (${shown(problem.rel)}: ${shown(problem.reason)}) — ${updated}; the manifest was not changed.\n`));
|
|
1578
|
+
}
|
|
1579
|
+
else {
|
|
1066
1580
|
try {
|
|
1067
|
-
|
|
1581
|
+
faults.beforeManifestTemp();
|
|
1582
|
+
(0, docs_manifest_1.writeManifest)(destination, manifest, faults.beforeManifestRename);
|
|
1583
|
+
manifestRecorded = true;
|
|
1068
1584
|
}
|
|
1069
|
-
catch {
|
|
1070
|
-
|
|
1585
|
+
catch (error) {
|
|
1586
|
+
process.stderr.write(chalk_1.default.red(`error: cannot write ${docs_manifest_1.DOCS_MANIFEST}: ${shown(error.message)} — ${updated}; the manifest was not changed.\n`));
|
|
1071
1587
|
}
|
|
1072
|
-
if (!holdsLocalFile)
|
|
1073
|
-
continue;
|
|
1074
|
-
delete manifestDocs[p.relPath];
|
|
1075
|
-
process.stderr.write(chalk_1.default.yellow(`! doc ${p.doc.id} ("${shown(p.doc.title)}") failed to download and ${shown(p.relPath)} holds a local file; not tracking it — pull again later\n`));
|
|
1076
1588
|
}
|
|
1589
|
+
// The tracking decisions describe the manifest just recorded, so they come before the error of a pull that stopped
|
|
1590
|
+
// (spec §1.5); a manifest that was not recorded changed no tracking, so says nothing.
|
|
1591
|
+
if (manifestRecorded) {
|
|
1592
|
+
for (const line of pendingWarnings)
|
|
1593
|
+
process.stderr.write(chalk_1.default.yellow(`${line}\n`));
|
|
1594
|
+
}
|
|
1595
|
+
if (stop !== null || !manifestRecorded)
|
|
1596
|
+
reportFetchFailed(fetchFailed);
|
|
1597
|
+
if (stop !== null)
|
|
1598
|
+
reportStop(stop, planned, placed, writes, manifestRecorded);
|
|
1599
|
+
if (!manifestRecorded)
|
|
1600
|
+
process.exit(1);
|
|
1601
|
+
const files = placed
|
|
1602
|
+
.map((write) => ({ path: write.relPath, action: 'written' }));
|
|
1077
1603
|
// Identity of every file this pull actually wrote, keyed by dev:ino. Used by the
|
|
1078
1604
|
// deletion-propagation loop below to recognize an "orphan" that is really just the
|
|
1079
1605
|
// same file this pull wrote under a different manifest key (e.g. a case-only title
|
|
@@ -1094,32 +1620,13 @@ async function report(destination, folderPath, fetched, options, config, extraWa
|
|
|
1094
1620
|
continue;
|
|
1095
1621
|
}
|
|
1096
1622
|
}
|
|
1097
|
-
// Rename cleanup (PM ruling 2, cli#157): each unmodified old twin is
|
|
1098
|
-
// now that the new file is written — but never one this pull just wrote
|
|
1099
|
-
//
|
|
1100
|
-
//
|
|
1101
|
-
//
|
|
1102
|
-
// a new path but writes nothing) keeps its old file and its old manifest
|
|
1103
|
-
// entry — deleting it would destroy the only good copy. Without an old
|
|
1104
|
-
// file the old entry is still kept, so the doc is not tracked (with no
|
|
1105
|
-
// hash) at a new path that may hold an untracked file. The refusals above
|
|
1106
|
-
// guarantee no other doc in this pull claims that old path.
|
|
1623
|
+
// Rename cleanup (PM ruling 2, cli#157), after publication only: each unmodified old twin is
|
|
1624
|
+
// gone now that the new file is written — but never one this pull just wrote for another
|
|
1625
|
+
// doc (e.g. a new markdown doc that now takes `page.md`). Edited sources were all refused
|
|
1626
|
+
// before any write, even with --overwrite. A twin whose replacement was not written was
|
|
1627
|
+
// kept above.
|
|
1107
1628
|
for (const m of renameMoves) {
|
|
1108
|
-
if (!m.replacementWritten)
|
|
1109
|
-
delete manifestDocs[m.newRel];
|
|
1110
|
-
const oldEntry = previousManifest?.docs[m.oldRel];
|
|
1111
|
-
if (oldEntry !== undefined) {
|
|
1112
|
-
manifestDocs[m.oldRel] = oldEntry;
|
|
1113
|
-
}
|
|
1114
|
-
if (m.sourcePresent) {
|
|
1115
|
-
process.stderr.write(chalk_1.default.yellow(`! kept ${shown(m.oldRel)} — doc ${m.id} download failed; still tracked as ${shown(m.oldRel)}\n`));
|
|
1116
|
-
}
|
|
1117
|
-
else {
|
|
1118
|
-
process.stderr.write(chalk_1.default.yellow(`! doc ${m.id} download failed; still tracked as ${shown(m.oldRel)}, which is not present locally\n`));
|
|
1119
|
-
}
|
|
1120
|
-
continue;
|
|
1121
|
-
}
|
|
1122
|
-
if (!m.sourcePresent)
|
|
1629
|
+
if (!m.replacementWritten || !m.sourcePresent)
|
|
1123
1630
|
continue;
|
|
1124
1631
|
const absOld = path_1.default.resolve(destination, ...m.oldRel.split('/'));
|
|
1125
1632
|
// cli#167 M2: the old path is the same file as one this pull just
|
|
@@ -1150,16 +1657,6 @@ async function report(destination, folderPath, fetched, options, config, extraWa
|
|
|
1150
1657
|
continue;
|
|
1151
1658
|
}
|
|
1152
1659
|
}
|
|
1153
|
-
let docs = manifestDocs;
|
|
1154
|
-
if (usedSingleDocFallback && previousManifest !== null && previousManifest.folder_path === folderPath) {
|
|
1155
|
-
// A single-doc pull speaks for one doc only: keep every other tracked entry (cli#153),
|
|
1156
|
-
// dropping any stale key that pointed at this same doc id.
|
|
1157
|
-
const pulledIds = new Set(Object.values(manifestDocs).map((entry) => entry.id));
|
|
1158
|
-
docs = Object.fromEntries(Object.entries(previousManifest.docs).filter(([, entry]) => !pulledIds.has(entry.id)));
|
|
1159
|
-
Object.assign(docs, manifestDocs);
|
|
1160
|
-
}
|
|
1161
|
-
const manifest = { folder_path: folderPath, docs };
|
|
1162
|
-
(0, docs_manifest_1.writeManifest)(destination, manifest);
|
|
1163
1660
|
for (const warning of [...extraWarnings, ...warnings]) {
|
|
1164
1661
|
process.stderr.write(chalk_1.default.yellow(`${warning}\n`));
|
|
1165
1662
|
}
|
|
@@ -1183,12 +1680,15 @@ async function report(destination, folderPath, fetched, options, config, extraWa
|
|
|
1183
1680
|
const keptModifiedMedia = new Set();
|
|
1184
1681
|
if (canPropagateDeletions) {
|
|
1185
1682
|
for (const [relPath, entry] of Object.entries(previousManifest.docs)) {
|
|
1186
|
-
if (
|
|
1683
|
+
if ((0, docs_manifest_1.entryAt)(manifest, relPath) !== undefined)
|
|
1187
1684
|
continue;
|
|
1188
1685
|
// Rename cleanup (cli#157) already handled this path: never a
|
|
1189
1686
|
// "deleted remotely" orphan warning for it.
|
|
1190
1687
|
if (handledOldPaths.has(relPath))
|
|
1191
1688
|
continue;
|
|
1689
|
+
// A doc the server listed in this pull was not deleted remotely, whatever its outcome (cli#209).
|
|
1690
|
+
if (listedIds.has(entry.id))
|
|
1691
|
+
continue;
|
|
1192
1692
|
const absPath = path_1.default.resolve(destination, ...relPath.split('/'));
|
|
1193
1693
|
// The whole entry is guarded: the new manifest is already on disk by now, so an
|
|
1194
1694
|
// uncaught throw would abort the report, silently skip every later orphan, and
|
|
@@ -1216,7 +1716,8 @@ async function report(destination, folderPath, fetched, options, config, extraWa
|
|
|
1216
1716
|
}
|
|
1217
1717
|
if (options.json) {
|
|
1218
1718
|
console.log((0, source_provenance_1.escapeJsonDisplayText)(JSON.stringify({ manifest, files, removed, kept_modified: keptModified })));
|
|
1219
|
-
|
|
1719
|
+
reportFetchFailed(fetchFailed);
|
|
1720
|
+
process.exit(fetchFailed.length > 0 ? 1 : 0);
|
|
1220
1721
|
}
|
|
1221
1722
|
console.log(chalk_1.default.green(`pulled ${files.length} doc${files.length === 1 ? '' : 's'} → ${shown(destination)}`));
|
|
1222
1723
|
for (const file of files) {
|
|
@@ -1234,7 +1735,8 @@ async function report(destination, folderPath, fetched, options, config, extraWa
|
|
|
1234
1735
|
process.stderr.write(chalk_1.default.yellow(' (it is now untracked; `doc push` will re-create it)\n'));
|
|
1235
1736
|
}
|
|
1236
1737
|
}
|
|
1237
|
-
|
|
1738
|
+
reportFetchFailed(fetchFailed);
|
|
1739
|
+
process.exit(fetchFailed.length > 0 ? 1 : 0);
|
|
1238
1740
|
}
|
|
1239
1741
|
/**
|
|
1240
1742
|
* Entry point called from index.ts.
|