@indigoai-us/hq-cloud 6.16.56 → 6.16.58-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +14 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +88 -8
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +204 -1
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +12 -3
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +65 -3
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +172 -1
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +51 -2
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +409 -165
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +19 -2
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +40 -5
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.test.js +47 -0
- package/dist/bin/sync-runner-watch-routes.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +65 -5
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +48 -4
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +1553 -192
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js +300 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +1 -0
- package/dist/cli/onepass-pull-planner.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner.test.js +266 -0
- package/dist/cli/onepass-pull-planner.test.js.map +1 -0
- package/dist/cli/share.d.ts +19 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +55 -0
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +87 -1
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +15 -0
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +1 -0
- package/dist/cli/streaming-pull-planner.bench.test.js +273 -0
- package/dist/cli/streaming-pull-planner.bench.test.js.map +1 -0
- package/dist/cli/sync-scope.test.js +8 -0
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +12 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +297 -125
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +255 -2
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/journal-row-store.d.ts +40 -5
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +240 -21
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-row-store.test.js +54 -0
- package/dist/journal-row-store.test.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +11 -6
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal-store.test.js +324 -7
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +51 -16
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +303 -79
- package/dist/journal.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +15 -0
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +15 -0
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +10 -3
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +35 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +9 -2
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +59 -2
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +1 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +20 -2
- package/dist/s3.js.map +1 -1
- package/dist/s3.list-cost.test.d.ts +2 -0
- package/dist/s3.list-cost.test.d.ts.map +1 -0
- package/dist/s3.list-cost.test.js +51 -0
- package/dist/s3.list-cost.test.js.map +1 -0
- package/dist/sync/area-ledger.d.ts +27 -3
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +149 -39
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/area-ledger.test.js +192 -5
- package/dist/sync/area-ledger.test.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +29 -0
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +14 -6
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +136 -16
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync/remote-file-spool.d.ts +16 -0
- package/dist/sync/remote-file-spool.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.js +91 -0
- package/dist/sync/remote-file-spool.js.map +1 -0
- package/dist/sync/remote-file-spool.test.d.ts +2 -0
- package/dist/sync/remote-file-spool.test.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.test.js +77 -0
- package/dist/sync/remote-file-spool.test.js.map +1 -0
- package/dist/sync/state-store.d.ts +6 -0
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +14 -2
- package/dist/sync/state-store.js.map +1 -1
- package/dist/watcher.d.ts +12 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +131 -0
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +63 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +2 -1
package/dist/cli/sync.js
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import * as fs from "fs";
|
|
8
8
|
import * as path from "path";
|
|
9
|
+
import { spoolRemoteFiles, collectRemoteAncestors } from "../sync/remote-file-spool.js";
|
|
9
10
|
import { VaultAuthError, VaultClient } from "../vault-client.js";
|
|
10
11
|
import { readlinkOrNull } from "../lib/readlink-safe.js";
|
|
11
12
|
import { describeError } from "../lib/describe-error.js";
|
|
@@ -16,7 +17,7 @@ import { ensureCompanyVaultTransport } from "../company-vault-transport.js";
|
|
|
16
17
|
import { createSyncProgressRecorder } from "../sync-progress.js";
|
|
17
18
|
import { canonicalVaultKeySpelling, localPathForVaultKey, vaultKeyForLocalPath, } from "../local-path-codec.js";
|
|
18
19
|
import { SKILLS_KEY_PREFIX } from "./doctor.js";
|
|
19
|
-
import { downloadFile,
|
|
20
|
+
import { downloadFile, streamRemoteFiles, headRemoteFile, primeObjectTransport, toPosixKey, classifyVaultKey, DanglingSymlinkParentError, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
|
|
20
21
|
import { readJournalForPass, writeJournal, getStateDir, openJournalStoreSession, buildJournalBaseline, journalDeltaFromBaseline, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, updateEntry, removeEntry, getEntry, normalizeEtag, migrateToV2, gcTombstones, isTombstone, tombstoneEntry, lastPullRecord, appendPullRecord, generatePullId, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, } from "../journal.js";
|
|
21
22
|
import { isGeneratedCoreMirrorKey, PERSONAL_VAULT_MANIFEST_KEY, } from "../personal-vault.js";
|
|
22
23
|
import { isPersonalVaultExcluded } from "../personal-vault-exclusions.js";
|
|
@@ -185,6 +186,11 @@ export function targetedJournalSessionCensusSizes() {
|
|
|
185
186
|
* dirty entries through a run-scoped journal writer.
|
|
186
187
|
*/
|
|
187
188
|
const DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE = 10_000;
|
|
189
|
+
/**
|
|
190
|
+
* A full-pull hand-off unit. Every member has finished its transfer and the
|
|
191
|
+
* resulting journal delta is durable before realtime can acquire the writer.
|
|
192
|
+
*/
|
|
193
|
+
export const FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE = 500;
|
|
188
194
|
/**
|
|
189
195
|
* Server cap on files per `/v1/notify/file-added` report. The endpoint rejects
|
|
190
196
|
* an oversized batch wholesale, so the client MUST split a large report into
|
|
@@ -318,36 +324,53 @@ export async function sync(options) {
|
|
|
318
324
|
return withOperationLock(options.hqRoot, "sync", () => syncWithOperationLockHeld(options));
|
|
319
325
|
}
|
|
320
326
|
async function syncWithOperationLockHeld(options) {
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
const
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
327
|
+
let carried;
|
|
328
|
+
for (;;) {
|
|
329
|
+
const run = await buildPullContext(options);
|
|
330
|
+
if (carried)
|
|
331
|
+
run.carriedProgress = carried;
|
|
332
|
+
try {
|
|
333
|
+
const plan = await planPull(run);
|
|
334
|
+
emitPullPlan(run.emit, plan);
|
|
335
|
+
reportInvalidRemoteKeys(run, plan);
|
|
336
|
+
const scopeRun = run.keyedJournal
|
|
337
|
+
? emptyTargetedScopeRun(run)
|
|
338
|
+
: executeScopeShrink(run, planScopeShrink(run));
|
|
339
|
+
const counters = createPullCounters();
|
|
340
|
+
// A streaming all-mode planner retains only executor-relevant items. Its
|
|
341
|
+
// ordinary skips have already been counted while the page was live.
|
|
342
|
+
counters.filesSkipped = plan.preplannedFilesSkipped;
|
|
343
|
+
counters.filesOutOfScope = plan.preplannedFilesOutOfScope;
|
|
344
|
+
seedCarriedPullProgress(counters, carried);
|
|
345
|
+
const transferConcurrency = resolveTransferConcurrency(run.hostLoadAtStart);
|
|
346
|
+
const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
|
|
347
|
+
if (conflictRun.abortResult) {
|
|
348
|
+
return conflictRun.abortResult;
|
|
349
|
+
}
|
|
350
|
+
await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
|
|
351
|
+
// The following added-by and tombstone phases may still make authenticated
|
|
352
|
+
// vault calls. Make a completed sub-threshold download tail durable before
|
|
353
|
+
// they begin, so their failure cannot turn a successful bulk transfer into
|
|
354
|
+
// a full replay on the next pass.
|
|
355
|
+
flushDownloadedJournalTail(run);
|
|
356
|
+
await emitAndReportNewFiles(run, plan);
|
|
357
|
+
await verifyPlannedJournalTombstones(run, plan);
|
|
358
|
+
executeJournalTombstoneDeletes(run, plan, counters);
|
|
359
|
+
emitScopeMaterializationGap(run, plan);
|
|
360
|
+
return finalizePullRun(run, plan, scopeRun, counters);
|
|
361
|
+
}
|
|
362
|
+
catch (error) {
|
|
363
|
+
if (error instanceof CooperativePullHandoff) {
|
|
364
|
+
carried = error.completed;
|
|
365
|
+
continue;
|
|
366
|
+
}
|
|
367
|
+
throw error;
|
|
368
|
+
}
|
|
369
|
+
finally {
|
|
370
|
+
// Release a pass-local lease even on conflict, transfer, or auth failure.
|
|
371
|
+
// The one bounded hot lease remains available only for its same vault.
|
|
372
|
+
run.releaseKeyedJournal?.();
|
|
373
|
+
}
|
|
351
374
|
}
|
|
352
375
|
}
|
|
353
376
|
async function buildPullContext(options) {
|
|
@@ -436,8 +459,9 @@ async function buildPullContext(options) {
|
|
|
436
459
|
? targetedJournalSession(journalSlug)
|
|
437
460
|
: undefined;
|
|
438
461
|
const keyedJournal = keyedJournalLease?.store;
|
|
462
|
+
let remoteFiles;
|
|
439
463
|
try {
|
|
440
|
-
|
|
464
|
+
remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
|
|
441
465
|
const fileTombstones = options.fileTombstones ??
|
|
442
466
|
await fetchFileTombstones(vaultConfig, options.personalMode === true
|
|
443
467
|
? { personal: true }
|
|
@@ -485,43 +509,27 @@ async function buildPullContext(options) {
|
|
|
485
509
|
catch (error) {
|
|
486
510
|
// `buildPullContext` can fail during LIST/freshness. Do not leave the
|
|
487
511
|
// leased projection marked in use when no PullRunContext is returned.
|
|
512
|
+
if (remoteFiles && isStreamingRemoteFileSource(remoteFiles))
|
|
513
|
+
remoteFiles.dispose();
|
|
488
514
|
keyedJournalLease?.release();
|
|
489
515
|
throw error;
|
|
490
516
|
}
|
|
491
517
|
}
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
* thousands of RemoteFile objects on large vaults) only for `computePullPlan`
|
|
499
|
-
* to discard most of it as `skip-out-of-scope`. Scoped pulls now list ONLY the
|
|
500
|
-
* in-scope prefixes via `listRemoteForScope` (per-prefix ListObjectsV2 calls,
|
|
501
|
-
* bounded parallel, deduped), which bounds the listing's memory to the scope's
|
|
502
|
-
* actual footprint.
|
|
503
|
-
*
|
|
504
|
-
* `all`-mode (an unnarrowed full-vault pull, `listedPrefixSet` `[""]`) keeps
|
|
505
|
-
* the single broad listing: fanning `all` out by top-level prefix would need a
|
|
506
|
-
* delimiter LIST to discover the top-level names first and still union the
|
|
507
|
-
* same total object count in one plan, so it bounds nothing today. (A
|
|
508
|
-
* streaming planner is the real fix for `all` and is out of scope here.)
|
|
509
|
-
*
|
|
510
|
-
* An empty `listedPrefixSet` in a scoped mode means "nothing in scope" —
|
|
511
|
-
* issue zero LIST calls. Consumers already treat an empty listing correctly:
|
|
512
|
-
* the deletion sweep in `computePullPlan` only considers journal keys covered
|
|
513
|
-
* by the LISTED scope, so keys absent because they were never listed can never
|
|
514
|
-
* be mistaken for remote deletes.
|
|
515
|
-
*/
|
|
518
|
+
function isStreamingRemoteFileSource(source) {
|
|
519
|
+
return !Array.isArray(source);
|
|
520
|
+
}
|
|
521
|
+
function allModeRemoteFileStream(ctx) {
|
|
522
|
+
return spoolRemoteFiles(streamRemoteFiles(ctx));
|
|
523
|
+
}
|
|
516
524
|
async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargetedPull) {
|
|
517
525
|
const entries = toScopePrefixEntries(listedPrefixSet);
|
|
518
526
|
if (entries.some((e) => e.prefix === ""))
|
|
519
|
-
return
|
|
527
|
+
return allModeRemoteFileStream(ctx);
|
|
520
528
|
if (entries.length === 0) {
|
|
521
529
|
// `all` mode always carries `[""]`, so an empty set only occurs for a
|
|
522
530
|
// scoped mode (empty grants, or a targeted intersection that excluded
|
|
523
531
|
// everything). Defensive `all` branch: never silently narrow a full pull.
|
|
524
|
-
return syncMode === "all" ?
|
|
532
|
+
return syncMode === "all" ? allModeRemoteFileStream(ctx) : [];
|
|
525
533
|
}
|
|
526
534
|
return listRemoteForScope({
|
|
527
535
|
ctx,
|
|
@@ -542,8 +550,14 @@ async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargete
|
|
|
542
550
|
},
|
|
543
551
|
});
|
|
544
552
|
}
|
|
545
|
-
function planPull(run) {
|
|
546
|
-
|
|
553
|
+
async function planPull(run) {
|
|
554
|
+
try {
|
|
555
|
+
return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount);
|
|
556
|
+
}
|
|
557
|
+
finally {
|
|
558
|
+
if (isStreamingRemoteFileSource(run.remoteFiles))
|
|
559
|
+
run.remoteFiles.dispose();
|
|
560
|
+
}
|
|
547
561
|
}
|
|
548
562
|
/** Build only the rows a targeted remote LIST can observe. */
|
|
549
563
|
function journalForTargetedPull(store, prefixes) {
|
|
@@ -656,9 +670,55 @@ function createPullCounters() {
|
|
|
656
670
|
conflictPaths: [],
|
|
657
671
|
changedPaths: [],
|
|
658
672
|
appliedUpserts: [],
|
|
673
|
+
newFiles: [],
|
|
659
674
|
hasRetryableOutcome: false,
|
|
660
675
|
};
|
|
661
676
|
}
|
|
677
|
+
/** Fold a previous attempt's completed work into a restarted run's counters. */
|
|
678
|
+
function seedCarriedPullProgress(counters, carried) {
|
|
679
|
+
if (!carried)
|
|
680
|
+
return;
|
|
681
|
+
counters.filesDownloaded += carried.filesDownloaded;
|
|
682
|
+
counters.bytesDownloaded += carried.bytesDownloaded;
|
|
683
|
+
counters.conflicts += carried.conflicts;
|
|
684
|
+
counters.filesTombstoned += carried.filesTombstoned;
|
|
685
|
+
counters.conflictPaths.push(...carried.conflictPaths);
|
|
686
|
+
counters.changedPaths.push(...carried.changedPaths);
|
|
687
|
+
counters.appliedUpserts.push(...carried.appliedUpserts);
|
|
688
|
+
if (carried.hasRetryableOutcome)
|
|
689
|
+
counters.hasRetryableOutcome = true;
|
|
690
|
+
}
|
|
691
|
+
/**
|
|
692
|
+
* Snapshot what survives the hand-off. The numeric and path axes already
|
|
693
|
+
* include everything carried into this attempt; `newFiles` is accumulated
|
|
694
|
+
* separately so it stays a record of pre-restart downloads only, and cannot
|
|
695
|
+
* double up with the current plan's own new files.
|
|
696
|
+
*/
|
|
697
|
+
function carryPullProgress(previous, counters) {
|
|
698
|
+
return {
|
|
699
|
+
filesDownloaded: counters.filesDownloaded,
|
|
700
|
+
bytesDownloaded: counters.bytesDownloaded,
|
|
701
|
+
conflicts: counters.conflicts,
|
|
702
|
+
conflictPaths: [...counters.conflictPaths],
|
|
703
|
+
filesTombstoned: counters.filesTombstoned,
|
|
704
|
+
changedPaths: [...counters.changedPaths],
|
|
705
|
+
appliedUpserts: [...counters.appliedUpserts],
|
|
706
|
+
newFiles: [...(previous?.newFiles ?? []), ...counters.newFiles],
|
|
707
|
+
hasRetryableOutcome: counters.hasRetryableOutcome,
|
|
708
|
+
};
|
|
709
|
+
}
|
|
710
|
+
/**
|
|
711
|
+
* Every new file this pull is responsible for reporting: the ones a previous
|
|
712
|
+
* attempt downloaded before handing off, then this plan's own. A carried path
|
|
713
|
+
* that the fresh plan also lists is reported once.
|
|
714
|
+
*/
|
|
715
|
+
function reportableNewFiles(run, plan) {
|
|
716
|
+
const carried = run.carriedProgress?.newFiles;
|
|
717
|
+
if (carried === undefined || carried.length === 0)
|
|
718
|
+
return plan.newFiles;
|
|
719
|
+
const planned = new Set(plan.newFiles.map((newFile) => newFile.path));
|
|
720
|
+
return [...carried.filter((newFile) => !planned.has(newFile.path)), ...plan.newFiles];
|
|
721
|
+
}
|
|
662
722
|
function planScopeShrink(run) {
|
|
663
723
|
const lastRecord = lastPullRecord(run.journal, run.ctx.uid);
|
|
664
724
|
const lastPrefixSet = lastRecord && lastRecord.prefixSet.length > 0
|
|
@@ -1119,6 +1179,7 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
1119
1179
|
if (resolution === "abort") {
|
|
1120
1180
|
run.emit({ type: "new-files", files: [] });
|
|
1121
1181
|
persistPullJournal(run);
|
|
1182
|
+
const abortedNewFiles = reportableNewFiles(run, plan);
|
|
1122
1183
|
return {
|
|
1123
1184
|
filesDownloaded: counters.filesDownloaded,
|
|
1124
1185
|
bytesDownloaded: counters.bytesDownloaded,
|
|
@@ -1126,8 +1187,8 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
1126
1187
|
conflicts: counters.conflicts,
|
|
1127
1188
|
conflictPaths: counters.conflictPaths,
|
|
1128
1189
|
aborted: true,
|
|
1129
|
-
newFiles:
|
|
1130
|
-
newFilesCount:
|
|
1190
|
+
newFiles: abortedNewFiles,
|
|
1191
|
+
newFilesCount: abortedNewFiles.length,
|
|
1131
1192
|
filesExcludedByPolicy: plan.filesExcludedByPolicy,
|
|
1132
1193
|
filesTombstoned: 0,
|
|
1133
1194
|
filesOutOfScope: counters.filesOutOfScope,
|
|
@@ -1253,10 +1314,21 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
1253
1314
|
discardProbe();
|
|
1254
1315
|
}
|
|
1255
1316
|
}
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1317
|
+
/**
|
|
1318
|
+
* The old plan may have tombstone evidence that predates a realtime hand-off.
|
|
1319
|
+
* Rebuilding from disk after a hand-off makes the continuation observe the
|
|
1320
|
+
* realtime journal and a fresh LIST before it can perform another mutation.
|
|
1321
|
+
*/
|
|
1322
|
+
class CooperativePullHandoff extends Error {
|
|
1323
|
+
/** Completed work the restarted pull must keep reporting. */
|
|
1324
|
+
completed;
|
|
1325
|
+
constructor(completed) {
|
|
1326
|
+
super("full pull handed off to realtime work");
|
|
1327
|
+
this.name = "CooperativePullHandoff";
|
|
1328
|
+
this.completed = completed;
|
|
1329
|
+
}
|
|
1330
|
+
}
|
|
1331
|
+
async function executeDownloadQueue(run, downloadItems, transferConcurrency, counters, deferJournalCheckpoint) {
|
|
1260
1332
|
const queue = [...downloadItems];
|
|
1261
1333
|
const inFlight = new Set();
|
|
1262
1334
|
const workerErrors = [];
|
|
@@ -1271,7 +1343,7 @@ async function executeDownloadExecutor(run, downloadItems, transferConcurrency,
|
|
|
1271
1343
|
inFlight.size < transferConcurrency &&
|
|
1272
1344
|
queue.length > 0) {
|
|
1273
1345
|
const downloadItem = queue.shift();
|
|
1274
|
-
const download = () => downloadOne(run, downloadItem, counters);
|
|
1346
|
+
const download = () => downloadOne(run, downloadItem, counters, deferJournalCheckpoint);
|
|
1275
1347
|
const p = (run.options.transferSemaphore
|
|
1276
1348
|
? run.options.transferSemaphore.withPermit(download)
|
|
1277
1349
|
: download())
|
|
@@ -1295,16 +1367,44 @@ async function executeDownloadExecutor(run, downloadItems, transferConcurrency,
|
|
|
1295
1367
|
break;
|
|
1296
1368
|
}
|
|
1297
1369
|
}
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1370
|
+
return workerErrors;
|
|
1371
|
+
}
|
|
1372
|
+
function throwDownloadWorkerErrors(run, workerErrors) {
|
|
1373
|
+
persistPullJournal(run);
|
|
1374
|
+
const first = workerErrors[0];
|
|
1375
|
+
if (workerErrors.length > 1) {
|
|
1376
|
+
first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
|
|
1377
|
+
}
|
|
1378
|
+
throw first;
|
|
1379
|
+
}
|
|
1380
|
+
async function executeDownloadExecutor(run, downloadItems, transferConcurrency, counters) {
|
|
1381
|
+
if (downloadItems.length === 0)
|
|
1382
|
+
return;
|
|
1383
|
+
await primeObjectTransport(run.ctx, "get", downloadItems.map((d) => d.remoteFile.key));
|
|
1384
|
+
// Ordinary and targeted pulls retain the existing 10k crash-recovery cadence.
|
|
1385
|
+
// Only a watch chore that supplied the cooperative seam pays for drained
|
|
1386
|
+
// units and can return its root lock to queued realtime work.
|
|
1387
|
+
if (!run.options.cooperativeCheckpoint) {
|
|
1388
|
+
const workerErrors = await executeDownloadQueue(run, downloadItems, transferConcurrency, counters, false);
|
|
1389
|
+
if (workerErrors.length > 0)
|
|
1390
|
+
throwDownloadWorkerErrors(run, workerErrors);
|
|
1391
|
+
return;
|
|
1392
|
+
}
|
|
1393
|
+
for (let start = 0; start < downloadItems.length; start += FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE) {
|
|
1394
|
+
const chunk = downloadItems.slice(start, start + FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE);
|
|
1395
|
+
const workerErrors = await executeDownloadQueue(run, chunk, transferConcurrency, counters, true);
|
|
1396
|
+
// `executeDownloadQueue` does not return until every member of this chunk
|
|
1397
|
+
// has settled. Persist before the callback so the suspended chore owns no
|
|
1398
|
+
// mutable pass state while realtime uses the root lock.
|
|
1399
|
+
flushDownloadedJournalTail(run);
|
|
1400
|
+
if (workerErrors.length > 0)
|
|
1401
|
+
throwDownloadWorkerErrors(run, workerErrors);
|
|
1402
|
+
if (await run.options.cooperativeCheckpoint() === true) {
|
|
1403
|
+
throw new CooperativePullHandoff(carryPullProgress(run.carriedProgress, counters));
|
|
1303
1404
|
}
|
|
1304
|
-
throw first;
|
|
1305
1405
|
}
|
|
1306
1406
|
}
|
|
1307
|
-
async function downloadOne(run, downloadItem, counters) {
|
|
1407
|
+
async function downloadOne(run, downloadItem, counters, deferJournalCheckpoint = false) {
|
|
1308
1408
|
const { remoteFile, localPath } = downloadItem;
|
|
1309
1409
|
await refreshRunContextIfExpiring(run);
|
|
1310
1410
|
if (!isDownloadWritePathStillContained(run.companyRoot, remoteFile.key, localPath)) {
|
|
@@ -1373,6 +1473,9 @@ async function downloadOne(run, downloadItem, counters) {
|
|
|
1373
1473
|
counters.filesDownloaded++;
|
|
1374
1474
|
counters.bytesDownloaded += size;
|
|
1375
1475
|
counters.changedPaths.push(remoteFile.key);
|
|
1476
|
+
if (downloadItem.isNew) {
|
|
1477
|
+
counters.newFiles.push({ path: remoteFile.key, bytes: remoteFile.size });
|
|
1478
|
+
}
|
|
1376
1479
|
// The event-push emitter hashes regular file bodies. Symlink journal hashes
|
|
1377
1480
|
// describe link targets instead, and the startup baseline excludes them for
|
|
1378
1481
|
// the same reason, so only hand regular-file downloads to that ledger.
|
|
@@ -1461,7 +1564,8 @@ async function downloadOne(run, downloadItem, counters) {
|
|
|
1461
1564
|
return;
|
|
1462
1565
|
}
|
|
1463
1566
|
run.downloadsSinceJournalCheckpoint++;
|
|
1464
|
-
if (
|
|
1567
|
+
if (!deferJournalCheckpoint &&
|
|
1568
|
+
run.downloadsSinceJournalCheckpoint >= DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE) {
|
|
1465
1569
|
flushDownloadedJournalTail(run);
|
|
1466
1570
|
}
|
|
1467
1571
|
}
|
|
@@ -1494,15 +1598,19 @@ function persistPullJournal(run) {
|
|
|
1494
1598
|
}
|
|
1495
1599
|
async function emitAndReportNewFiles(run, plan) {
|
|
1496
1600
|
const enrichedNewFiles = [];
|
|
1601
|
+
// New files a cooperative hand-off already downloaded are reported here too:
|
|
1602
|
+
// the restarted plan cannot see them as new, so this is their only
|
|
1603
|
+
// notification.
|
|
1604
|
+
const newFiles = reportableNewFiles(run, plan);
|
|
1497
1605
|
// Batch-mint the GET presigns once (chunked, breaker-aware) so the per-file
|
|
1498
1606
|
// created-by HEADs below reuse the cache instead of each minting its own
|
|
1499
1607
|
// presign. Without this, a big catch-up pull (hundreds of new files) bursts
|
|
1500
1608
|
// the presign endpoint, trips the circuit breaker, and every enrichment HEAD
|
|
1501
1609
|
// then fails. Mirrors the tombstone HEAD-verify pre-prime.
|
|
1502
|
-
await primeObjectTransport(run.ctx, "get",
|
|
1610
|
+
await primeObjectTransport(run.ctx, "get", newFiles.map((nf) => nf.path));
|
|
1503
1611
|
const HEAD_CONCURRENCY = 5;
|
|
1504
|
-
for (let i = 0; i <
|
|
1505
|
-
const batch =
|
|
1612
|
+
for (let i = 0; i < newFiles.length; i += HEAD_CONCURRENCY) {
|
|
1613
|
+
const batch = newFiles.slice(i, i + HEAD_CONCURRENCY);
|
|
1506
1614
|
const results = await Promise.all(batch.map(async (nf) => {
|
|
1507
1615
|
let addedBy = null;
|
|
1508
1616
|
try {
|
|
@@ -1663,6 +1771,7 @@ function finalizePullRun(run, plan, scopeRun, counters) {
|
|
|
1663
1771
|
// best-effort: a post-sync refresh failure never fails the sync
|
|
1664
1772
|
}
|
|
1665
1773
|
}
|
|
1774
|
+
const newFiles = reportableNewFiles(run, plan);
|
|
1666
1775
|
const result = {
|
|
1667
1776
|
filesDownloaded: counters.filesDownloaded,
|
|
1668
1777
|
bytesDownloaded: counters.bytesDownloaded,
|
|
@@ -1670,8 +1779,8 @@ function finalizePullRun(run, plan, scopeRun, counters) {
|
|
|
1670
1779
|
conflicts: counters.conflicts,
|
|
1671
1780
|
conflictPaths: counters.conflictPaths,
|
|
1672
1781
|
aborted: false,
|
|
1673
|
-
newFiles
|
|
1674
|
-
newFilesCount:
|
|
1782
|
+
newFiles,
|
|
1783
|
+
newFilesCount: newFiles.length,
|
|
1675
1784
|
filesExcludedByPolicy: plan.filesExcludedByPolicy,
|
|
1676
1785
|
filesTombstoned: counters.filesTombstoned,
|
|
1677
1786
|
filesOutOfScope: counters.filesOutOfScope,
|
|
@@ -1900,7 +2009,8 @@ class LocalSnapshotChangedError extends Error {
|
|
|
1900
2009
|
* the real outcome of the upcoming Stage-2 execution rather than an
|
|
1901
2010
|
* upper-bound guess.
|
|
1902
2011
|
*
|
|
1903
|
-
*
|
|
2012
|
+
* It performs no writes, journal mutations, or event emission. An all-mode
|
|
2013
|
+
* source performs its paged classification LIST while it is consumed; the
|
|
1904
2014
|
* caller (`sync()`) is responsible for emitting the resulting plan event
|
|
1905
2015
|
* before iterating `items`.
|
|
1906
2016
|
*/
|
|
@@ -1968,6 +2078,36 @@ excludePrefixes = [],
|
|
|
1968
2078
|
// callers preserve the pre-policy, watcher-intent-only behavior.
|
|
1969
2079
|
propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows(journal).size) {
|
|
1970
2080
|
const items = [];
|
|
2081
|
+
const streaming = isStreamingRemoteFileSource(remoteFiles);
|
|
2082
|
+
const listedFiles = isStreamingRemoteFileSource(remoteFiles) ? remoteFiles.files : remoteFiles;
|
|
2083
|
+
let preplannedFilesSkipped = 0;
|
|
2084
|
+
let preplannedFilesOutOfScope = 0;
|
|
2085
|
+
let preplannedFilesExcludedByPolicy = 0;
|
|
2086
|
+
if (streaming) {
|
|
2087
|
+
const retain = items.push.bind(items);
|
|
2088
|
+
items.push = (...next) => {
|
|
2089
|
+
for (const item of next) {
|
|
2090
|
+
// These actions still need the executor for an event or a state change.
|
|
2091
|
+
if (item.action === "download" ||
|
|
2092
|
+
item.action === "conflict" ||
|
|
2093
|
+
item.action === "tombstone-delete" ||
|
|
2094
|
+
item.action === "skip-junk-key-spelling" ||
|
|
2095
|
+
item.action === "skip-unreadable-link") {
|
|
2096
|
+
retain(item);
|
|
2097
|
+
}
|
|
2098
|
+
else if (item.action === "skip-out-of-scope") {
|
|
2099
|
+
preplannedFilesOutOfScope++;
|
|
2100
|
+
}
|
|
2101
|
+
else if (item.action === "skip-excluded-policy") {
|
|
2102
|
+
preplannedFilesExcludedByPolicy++;
|
|
2103
|
+
}
|
|
2104
|
+
else {
|
|
2105
|
+
preplannedFilesSkipped++;
|
|
2106
|
+
}
|
|
2107
|
+
}
|
|
2108
|
+
return items.length;
|
|
2109
|
+
};
|
|
2110
|
+
}
|
|
1971
2111
|
// Remote keys the server will never presign — collected so the caller can
|
|
1972
2112
|
// warn + emit telemetry without allowing one poison object to fail a pull.
|
|
1973
2113
|
const invalidRemoteKeys = [];
|
|
@@ -1983,8 +2123,35 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
1983
2123
|
// never gated by this map (see the guard below), so pre-existing families
|
|
1984
2124
|
// keep syncing untouched until the vault doctor (US-003) collapses them.
|
|
1985
2125
|
const journaledKeyByCanonical = new Map();
|
|
2126
|
+
// The delete sweep starts with journaled keys and clears them as the streamed
|
|
2127
|
+
// LIST proves each one present. This retains existing journal-key references
|
|
2128
|
+
// instead of a second, whole-vault population of long remote key strings.
|
|
2129
|
+
const unseenJournalKeys = new Set();
|
|
2130
|
+
// Most journal keys have already been normalized to POSIX and can be cleared
|
|
2131
|
+
// directly. Keep the legacy Windows spelling defense-in-depth without
|
|
2132
|
+
// imposing a second map entry on every current journal row.
|
|
2133
|
+
const legacyJournalKeysByPosix = new Map();
|
|
2134
|
+
// The junk-spelling delete guard needs all journal twins, while
|
|
2135
|
+
// `journaledKeyByCanonical` above deliberately retains only the preferred
|
|
2136
|
+
// representative for classification.
|
|
2137
|
+
const journaledSkillKeysByCanonical = new Map();
|
|
1986
2138
|
for (const journaledKey of journalRows(journal).keys()) {
|
|
1987
2139
|
onHeartbeat?.();
|
|
2140
|
+
const posixJournaledKey = toPosixKey(journaledKey);
|
|
2141
|
+
// A scoped LIST cannot prove anything about keys outside its prefixes.
|
|
2142
|
+
// Do not copy the entire vault's journal into its presence bookkeeping.
|
|
2143
|
+
const inListedScope = isCoveredByAny(posixJournaledKey, prefixSet);
|
|
2144
|
+
if (inListedScope)
|
|
2145
|
+
unseenJournalKeys.add(journaledKey);
|
|
2146
|
+
if (inListedScope && posixJournaledKey !== journaledKey) {
|
|
2147
|
+
const legacyKeys = legacyJournalKeysByPosix.get(posixJournaledKey);
|
|
2148
|
+
if (legacyKeys === undefined) {
|
|
2149
|
+
legacyJournalKeysByPosix.set(posixJournaledKey, [journaledKey]);
|
|
2150
|
+
}
|
|
2151
|
+
else {
|
|
2152
|
+
legacyKeys.push(journaledKey);
|
|
2153
|
+
}
|
|
2154
|
+
}
|
|
1988
2155
|
if (!journaledKey.startsWith(SKILLS_KEY_PREFIX))
|
|
1989
2156
|
continue;
|
|
1990
2157
|
const canonical = canonicalVaultKeySpelling(journaledKey);
|
|
@@ -1992,6 +2159,15 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
1992
2159
|
if (existing === undefined || journaledKey === canonical) {
|
|
1993
2160
|
journaledKeyByCanonical.set(canonical, journaledKey);
|
|
1994
2161
|
}
|
|
2162
|
+
if (!inListedScope)
|
|
2163
|
+
continue;
|
|
2164
|
+
const skillKeys = journaledSkillKeysByCanonical.get(canonical);
|
|
2165
|
+
if (skillKeys === undefined) {
|
|
2166
|
+
journaledSkillKeysByCanonical.set(canonical, [journaledKey]);
|
|
2167
|
+
}
|
|
2168
|
+
else {
|
|
2169
|
+
skillKeys.push(journaledKey);
|
|
2170
|
+
}
|
|
1995
2171
|
}
|
|
1996
2172
|
// Remote keys that ALSO appear as an ancestor of another remote key — i.e.
|
|
1997
2173
|
// keys that carry child objects under `${key}/…` in this same LIST. The
|
|
@@ -2001,22 +2177,40 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2001
2177
|
// objects beneath it. A REAL directory, by contrast, only ever syncs as
|
|
2002
2178
|
// those child objects — never as a single object AT its own key. So a single
|
|
2003
2179
|
// remote object sitting at a key that also has children is unambiguously a
|
|
2004
|
-
// directory-overlay marker, not a regular file.
|
|
2005
|
-
//
|
|
2006
|
-
//
|
|
2007
|
-
//
|
|
2008
|
-
const remoteKeysWithChildren =
|
|
2009
|
-
|
|
2180
|
+
// directory-overlay marker, not a regular file. Evidence is complete before
|
|
2181
|
+
// classification for both the spooled all-mode LIST and scoped arrays.
|
|
2182
|
+
// Build scoped evidence incrementally: flatMap would transiently retain an
|
|
2183
|
+
// ancestor string for every row/depth before Set deduplication.
|
|
2184
|
+
const remoteKeysWithChildren = isStreamingRemoteFileSource(remoteFiles)
|
|
2185
|
+
? remoteFiles.remoteKeysWithChildren
|
|
2186
|
+
: collectRemoteAncestors(remoteFiles);
|
|
2187
|
+
// Both sources replay synchronously. for-await on a scoped array introduces
|
|
2188
|
+
// promise/microtask work per object even though no I/O is being awaited.
|
|
2189
|
+
for (const remoteFile of listedFiles) {
|
|
2010
2190
|
onHeartbeat?.();
|
|
2011
|
-
const
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2191
|
+
const posixRemoteKey = toPosixKey(remoteFile.key);
|
|
2192
|
+
// S3 object keys are POSIX. Match the old delete-sweep membership check:
|
|
2193
|
+
// a legacy backslash journal entry counts as present only when its POSIX
|
|
2194
|
+
// form appears in the LIST, while a malformed backslash remote key does
|
|
2195
|
+
// not accidentally change that behavior.
|
|
2196
|
+
if (remoteFile.key === posixRemoteKey) {
|
|
2197
|
+
unseenJournalKeys.delete(remoteFile.key);
|
|
2198
|
+
for (const legacyKey of legacyJournalKeysByPosix.get(remoteFile.key) ?? []) {
|
|
2199
|
+
unseenJournalKeys.delete(legacyKey);
|
|
2200
|
+
}
|
|
2201
|
+
}
|
|
2202
|
+
if (remoteFile.key.startsWith(SKILLS_KEY_PREFIX)) {
|
|
2203
|
+
// A remote junk spelling means its canonical journal twin is still
|
|
2204
|
+
// remotely present. Keep that twin out of the delete sweep unless an
|
|
2205
|
+
// exact FILE_TOMBSTONE is authoritative, precisely matching the old
|
|
2206
|
+
// remoteCanonicalSkillKeys protection without retaining remote keys.
|
|
2207
|
+
const canonical = canonicalVaultKeySpelling(remoteFile.key);
|
|
2208
|
+
for (const journaledKey of journaledSkillKeysByCanonical.get(canonical) ?? []) {
|
|
2209
|
+
if (fileTombstones.get(toPosixKey(journaledKey)) === undefined) {
|
|
2210
|
+
unseenJournalKeys.delete(journaledKey);
|
|
2211
|
+
}
|
|
2212
|
+
}
|
|
2016
2213
|
}
|
|
2017
|
-
}
|
|
2018
|
-
for (const remoteFile of remoteFiles) {
|
|
2019
|
-
onHeartbeat?.();
|
|
2020
2214
|
const invalidKey = classifyVaultKey(remoteFile.key, personalMode ? "personal" : "company");
|
|
2021
2215
|
if (invalidKey !== null) {
|
|
2022
2216
|
invalidRemoteKeys.push({
|
|
@@ -2669,6 +2863,9 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2669
2863
|
filesToSkip++;
|
|
2670
2864
|
}
|
|
2671
2865
|
}
|
|
2866
|
+
filesToSkip += preplannedFilesSkipped;
|
|
2867
|
+
filesExcludedByPolicy += preplannedFilesExcludedByPolicy;
|
|
2868
|
+
filesOutOfScope += preplannedFilesOutOfScope;
|
|
2672
2869
|
// Bug #9 — cross-machine delete propagation. The peer's push leg
|
|
2673
2870
|
// already removed the file from S3 (verified in the deep-test
|
|
2674
2871
|
// addendum: `aws s3 ls` showed the keys absent post-push). Pre-fix
|
|
@@ -2677,38 +2874,21 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2677
2874
|
// that-was-there-before" — so the file stayed forever on every
|
|
2678
2875
|
// receiver and `filesTombstoned: 0` showed up on every pull.
|
|
2679
2876
|
//
|
|
2680
|
-
// Closing the loop: walk the
|
|
2681
|
-
//
|
|
2877
|
+
// Closing the loop: walk journal keys the streamed LIST did NOT clear, AND
|
|
2878
|
+
// whose path passes the current
|
|
2682
2879
|
// ignore filter (paths newly excluded by .hqignore must not trigger
|
|
2683
2880
|
// mass-delete), AND — in personalMode — survives the same companies/*
|
|
2684
2881
|
// gating the download branch applies. The executor will apply each
|
|
2685
2882
|
// as a local delete + journal removal. Symmetric to the push side's
|
|
2686
2883
|
// `propagateDeletes` plan in share.ts.
|
|
2687
|
-
const remoteKeySet = new Set();
|
|
2688
|
-
// Skill-farm twin protection (companion to the junk-spelling pull guard):
|
|
2689
|
-
// when the LIST holds only a junk spelling of a journaled canonical key,
|
|
2690
|
-
// the logical object is still remotely present — a raw has() would class
|
|
2691
|
-
// the canonical journal entry as remote-deleted, unlink the good local
|
|
2692
|
-
// wrapper, and drop the journal entry (after which the next pull would
|
|
2693
|
-
// materialize the junk spelling the guard just refused). Track canonical
|
|
2694
|
-
// forms of listed skill keys and consult them below.
|
|
2695
|
-
const remoteCanonicalSkillKeys = new Set();
|
|
2696
|
-
for (const rf of remoteFiles) {
|
|
2697
|
-
remoteKeySet.add(rf.key);
|
|
2698
|
-
if (rf.key.startsWith(SKILLS_KEY_PREFIX)) {
|
|
2699
|
-
remoteCanonicalSkillKeys.add(canonicalVaultKeySpelling(rf.key));
|
|
2700
|
-
}
|
|
2701
|
-
}
|
|
2702
2884
|
const tombstones = [];
|
|
2703
|
-
for (const key of
|
|
2885
|
+
for (const key of unseenJournalKeys) {
|
|
2704
2886
|
// Compare membership in POSIX space. A pre-5.47.2 Windows journal key can
|
|
2705
2887
|
// carry backslash separators; the remote LIST is always forward-slash, so a
|
|
2706
2888
|
// raw `has(key)` would miss the match and class a live local file as
|
|
2707
2889
|
// remote-deleted. normalizeJournalKeys() defuses this on load, but the
|
|
2708
2890
|
// POSIX compare is defense-in-depth (ridge data-loss, feedback_b8d09d0f).
|
|
2709
2891
|
const posixKey = toPosixKey(key);
|
|
2710
|
-
if (remoteKeySet.has(posixKey))
|
|
2711
|
-
continue;
|
|
2712
2892
|
// Listing-scope guard (targeted sync / scoped listing). The remote LIST
|
|
2713
2893
|
// now covers ONLY `prefixSet` (scoped modes list per-prefix; a targeted
|
|
2714
2894
|
// pull narrows further), so "absent from the LIST" is evidence of a remote
|
|
@@ -2720,16 +2900,6 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2720
2900
|
// `all` mode passes `[""]`, which covers every key — no behavior change.
|
|
2721
2901
|
if (!isCoveredByAny(posixKey, prefixSet))
|
|
2722
2902
|
continue;
|
|
2723
|
-
if (posixKey.startsWith(SKILLS_KEY_PREFIX) &&
|
|
2724
|
-
remoteCanonicalSkillKeys.has(canonicalVaultKeySpelling(posixKey)) &&
|
|
2725
|
-
// An EXACT FILE_TOMBSTONE for the journaled key is an authoritative
|
|
2726
|
-
// delete and wins over the twin protection — a lingering junk
|
|
2727
|
-
// spelling in the LIST must not keep a deliberately-deleted wrapper
|
|
2728
|
-
// alive forever. (The key is absent from the LIST here, so there is
|
|
2729
|
-
// no newer re-create to defer to.)
|
|
2730
|
-
fileTombstones.get(posixKey) === undefined) {
|
|
2731
|
-
continue;
|
|
2732
|
-
}
|
|
2733
2903
|
// Fast-path ignore, BEFORE containment canonicalization. The tombstone
|
|
2734
2904
|
// sweep runs for every journaled key, and a stale journal can carry
|
|
2735
2905
|
// hundreds of thousands of entries under an ignored runtime tree (e.g.
|
|
@@ -2838,6 +3008,8 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2838
3008
|
}
|
|
2839
3009
|
return {
|
|
2840
3010
|
items,
|
|
3011
|
+
preplannedFilesSkipped,
|
|
3012
|
+
preplannedFilesOutOfScope,
|
|
2841
3013
|
filesToDownload,
|
|
2842
3014
|
bytesToDownload,
|
|
2843
3015
|
filesToSkip,
|