@indigoai-us/hq-cloud 6.16.57 → 6.16.58-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. package/dist/bin/sync-runner-company.d.ts +14 -0
  2. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  3. package/dist/bin/sync-runner-company.js +42 -8
  4. package/dist/bin/sync-runner-company.js.map +1 -1
  5. package/dist/bin/sync-runner-company.test.js +55 -0
  6. package/dist/bin/sync-runner-company.test.js.map +1 -1
  7. package/dist/bin/sync-runner-planning.d.ts +12 -3
  8. package/dist/bin/sync-runner-planning.d.ts.map +1 -1
  9. package/dist/bin/sync-runner-planning.js +65 -3
  10. package/dist/bin/sync-runner-planning.js.map +1 -1
  11. package/dist/bin/sync-runner-planning.test.js +172 -1
  12. package/dist/bin/sync-runner-planning.test.js.map +1 -1
  13. package/dist/bin/sync-runner-watch-loop.d.ts +51 -2
  14. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  15. package/dist/bin/sync-runner-watch-loop.js +409 -165
  16. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  17. package/dist/bin/sync-runner-watch-routes.d.ts +19 -2
  18. package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
  19. package/dist/bin/sync-runner-watch-routes.js +40 -5
  20. package/dist/bin/sync-runner-watch-routes.js.map +1 -1
  21. package/dist/bin/sync-runner-watch-routes.test.js +47 -0
  22. package/dist/bin/sync-runner-watch-routes.test.js.map +1 -1
  23. package/dist/bin/sync-runner.d.ts +59 -5
  24. package/dist/bin/sync-runner.d.ts.map +1 -1
  25. package/dist/bin/sync-runner.js +48 -4
  26. package/dist/bin/sync-runner.js.map +1 -1
  27. package/dist/bin/sync-runner.test.js +1553 -192
  28. package/dist/bin/sync-runner.test.js.map +1 -1
  29. package/dist/cli/onepass-pull-planner-regressions.test.d.ts +2 -0
  30. package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +1 -0
  31. package/dist/cli/onepass-pull-planner-regressions.test.js +300 -0
  32. package/dist/cli/onepass-pull-planner-regressions.test.js.map +1 -0
  33. package/dist/cli/onepass-pull-planner.test.d.ts +2 -0
  34. package/dist/cli/onepass-pull-planner.test.d.ts.map +1 -0
  35. package/dist/cli/onepass-pull-planner.test.js +266 -0
  36. package/dist/cli/onepass-pull-planner.test.js.map +1 -0
  37. package/dist/cli/share.d.ts +19 -0
  38. package/dist/cli/share.d.ts.map +1 -1
  39. package/dist/cli/share.js +55 -0
  40. package/dist/cli/share.js.map +1 -1
  41. package/dist/cli/share.test.js +87 -1
  42. package/dist/cli/share.test.js.map +1 -1
  43. package/dist/cli/streaming-pull-planner.bench.test.d.ts +15 -0
  44. package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +1 -0
  45. package/dist/cli/streaming-pull-planner.bench.test.js +273 -0
  46. package/dist/cli/streaming-pull-planner.bench.test.js.map +1 -0
  47. package/dist/cli/sync-scope.test.js +8 -0
  48. package/dist/cli/sync-scope.test.js.map +1 -1
  49. package/dist/cli/sync.d.ts +12 -0
  50. package/dist/cli/sync.d.ts.map +1 -1
  51. package/dist/cli/sync.js +297 -125
  52. package/dist/cli/sync.js.map +1 -1
  53. package/dist/cli/sync.test.js +255 -2
  54. package/dist/cli/sync.test.js.map +1 -1
  55. package/dist/journal-row-store.d.ts +40 -5
  56. package/dist/journal-row-store.d.ts.map +1 -1
  57. package/dist/journal-row-store.js +240 -21
  58. package/dist/journal-row-store.js.map +1 -1
  59. package/dist/journal-row-store.test.js +54 -0
  60. package/dist/journal-row-store.test.js.map +1 -1
  61. package/dist/journal-snapshot-v4.d.ts.map +1 -1
  62. package/dist/journal-snapshot-v4.js +11 -6
  63. package/dist/journal-snapshot-v4.js.map +1 -1
  64. package/dist/journal-store.test.js +324 -7
  65. package/dist/journal-store.test.js.map +1 -1
  66. package/dist/journal.d.ts +51 -16
  67. package/dist/journal.d.ts.map +1 -1
  68. package/dist/journal.js +303 -79
  69. package/dist/journal.js.map +1 -1
  70. package/dist/lib/exit-codes.d.ts +15 -0
  71. package/dist/lib/exit-codes.d.ts.map +1 -1
  72. package/dist/lib/exit-codes.js +15 -0
  73. package/dist/lib/exit-codes.js.map +1 -1
  74. package/dist/personal-vault.d.ts.map +1 -1
  75. package/dist/personal-vault.js +10 -3
  76. package/dist/personal-vault.js.map +1 -1
  77. package/dist/personal-vault.test.js +35 -0
  78. package/dist/personal-vault.test.js.map +1 -1
  79. package/dist/remote-pull.d.ts.map +1 -1
  80. package/dist/remote-pull.js +9 -2
  81. package/dist/remote-pull.js.map +1 -1
  82. package/dist/remote-pull.test.js +59 -2
  83. package/dist/remote-pull.test.js.map +1 -1
  84. package/dist/s3.d.ts +1 -0
  85. package/dist/s3.d.ts.map +1 -1
  86. package/dist/s3.js +20 -2
  87. package/dist/s3.js.map +1 -1
  88. package/dist/s3.list-cost.test.d.ts +2 -0
  89. package/dist/s3.list-cost.test.d.ts.map +1 -0
  90. package/dist/s3.list-cost.test.js +51 -0
  91. package/dist/s3.list-cost.test.js.map +1 -0
  92. package/dist/sync/area-ledger.d.ts +27 -3
  93. package/dist/sync/area-ledger.d.ts.map +1 -1
  94. package/dist/sync/area-ledger.js +149 -39
  95. package/dist/sync/area-ledger.js.map +1 -1
  96. package/dist/sync/area-ledger.test.js +192 -5
  97. package/dist/sync/area-ledger.test.js.map +1 -1
  98. package/dist/sync/parallel-sync-coordinator.test.js +29 -0
  99. package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
  100. package/dist/sync/realtime-telemetry.d.ts +1 -1
  101. package/dist/sync/realtime-telemetry.d.ts.map +1 -1
  102. package/dist/sync/realtime-telemetry.js +14 -6
  103. package/dist/sync/realtime-telemetry.js.map +1 -1
  104. package/dist/sync/realtime-telemetry.test.js +136 -16
  105. package/dist/sync/realtime-telemetry.test.js.map +1 -1
  106. package/dist/sync/remote-file-spool.d.ts +16 -0
  107. package/dist/sync/remote-file-spool.d.ts.map +1 -0
  108. package/dist/sync/remote-file-spool.js +91 -0
  109. package/dist/sync/remote-file-spool.js.map +1 -0
  110. package/dist/sync/remote-file-spool.test.d.ts +2 -0
  111. package/dist/sync/remote-file-spool.test.d.ts.map +1 -0
  112. package/dist/sync/remote-file-spool.test.js +77 -0
  113. package/dist/sync/remote-file-spool.test.js.map +1 -0
  114. package/dist/sync/state-store.d.ts +6 -0
  115. package/dist/sync/state-store.d.ts.map +1 -1
  116. package/dist/sync/state-store.js +14 -2
  117. package/dist/sync/state-store.js.map +1 -1
  118. package/dist/watcher.d.ts +12 -0
  119. package/dist/watcher.d.ts.map +1 -1
  120. package/dist/watcher.js +131 -0
  121. package/dist/watcher.js.map +1 -1
  122. package/dist/watcher.test.js +63 -0
  123. package/dist/watcher.test.js.map +1 -1
  124. package/package.json +2 -1
package/dist/cli/sync.js CHANGED
@@ -6,6 +6,7 @@
6
6
  */
7
7
  import * as fs from "fs";
8
8
  import * as path from "path";
9
+ import { spoolRemoteFiles, collectRemoteAncestors } from "../sync/remote-file-spool.js";
9
10
  import { VaultAuthError, VaultClient } from "../vault-client.js";
10
11
  import { readlinkOrNull } from "../lib/readlink-safe.js";
11
12
  import { describeError } from "../lib/describe-error.js";
@@ -16,7 +17,7 @@ import { ensureCompanyVaultTransport } from "../company-vault-transport.js";
16
17
  import { createSyncProgressRecorder } from "../sync-progress.js";
17
18
  import { canonicalVaultKeySpelling, localPathForVaultKey, vaultKeyForLocalPath, } from "../local-path-codec.js";
18
19
  import { SKILLS_KEY_PREFIX } from "./doctor.js";
19
- import { downloadFile, listRemoteFiles, headRemoteFile, primeObjectTransport, toPosixKey, classifyVaultKey, DanglingSymlinkParentError, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
20
+ import { downloadFile, streamRemoteFiles, headRemoteFile, primeObjectTransport, toPosixKey, classifyVaultKey, DanglingSymlinkParentError, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
20
21
  import { readJournalForPass, writeJournal, getStateDir, openJournalStoreSession, buildJournalBaseline, journalDeltaFromBaseline, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, updateEntry, removeEntry, getEntry, normalizeEtag, migrateToV2, gcTombstones, isTombstone, tombstoneEntry, lastPullRecord, appendPullRecord, generatePullId, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, } from "../journal.js";
21
22
  import { isGeneratedCoreMirrorKey, PERSONAL_VAULT_MANIFEST_KEY, } from "../personal-vault.js";
22
23
  import { isPersonalVaultExcluded } from "../personal-vault-exclusions.js";
@@ -185,6 +186,11 @@ export function targetedJournalSessionCensusSizes() {
185
186
  * dirty entries through a run-scoped journal writer.
186
187
  */
187
188
  const DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE = 10_000;
189
+ /**
190
+ * A full-pull hand-off unit. Every member has finished its transfer and the
191
+ * resulting journal delta is durable before realtime can acquire the writer.
192
+ */
193
+ export const FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE = 500;
188
194
  /**
189
195
  * Server cap on files per `/v1/notify/file-added` report. The endpoint rejects
190
196
  * an oversized batch wholesale, so the client MUST split a large report into
@@ -318,36 +324,53 @@ export async function sync(options) {
318
324
  return withOperationLock(options.hqRoot, "sync", () => syncWithOperationLockHeld(options));
319
325
  }
320
326
  async function syncWithOperationLockHeld(options) {
321
- const run = await buildPullContext(options);
322
- try {
323
- const plan = planPull(run);
324
- emitPullPlan(run.emit, plan);
325
- reportInvalidRemoteKeys(run, plan);
326
- const scopeRun = run.keyedJournal
327
- ? emptyTargetedScopeRun(run)
328
- : executeScopeShrink(run, planScopeShrink(run));
329
- const counters = createPullCounters();
330
- const transferConcurrency = resolveTransferConcurrency(run.hostLoadAtStart);
331
- const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
332
- if (conflictRun.abortResult) {
333
- return conflictRun.abortResult;
334
- }
335
- await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
336
- // The following added-by and tombstone phases may still make authenticated
337
- // vault calls. Make a completed sub-threshold download tail durable before
338
- // they begin, so their failure cannot turn a successful bulk transfer into
339
- // a full replay on the next pass.
340
- flushDownloadedJournalTail(run);
341
- await emitAndReportNewFiles(run, plan);
342
- await verifyPlannedJournalTombstones(run, plan);
343
- executeJournalTombstoneDeletes(run, plan, counters);
344
- emitScopeMaterializationGap(run, plan);
345
- return finalizePullRun(run, plan, scopeRun, counters);
346
- }
347
- finally {
348
- // Release a pass-local lease even on conflict, transfer, or auth failure.
349
- // The one bounded hot lease remains available only for its same vault.
350
- run.releaseKeyedJournal?.();
327
+ let carried;
328
+ for (;;) {
329
+ const run = await buildPullContext(options);
330
+ if (carried)
331
+ run.carriedProgress = carried;
332
+ try {
333
+ const plan = await planPull(run);
334
+ emitPullPlan(run.emit, plan);
335
+ reportInvalidRemoteKeys(run, plan);
336
+ const scopeRun = run.keyedJournal
337
+ ? emptyTargetedScopeRun(run)
338
+ : executeScopeShrink(run, planScopeShrink(run));
339
+ const counters = createPullCounters();
340
+ // A streaming all-mode planner retains only executor-relevant items. Its
341
+ // ordinary skips have already been counted while the page was live.
342
+ counters.filesSkipped = plan.preplannedFilesSkipped;
343
+ counters.filesOutOfScope = plan.preplannedFilesOutOfScope;
344
+ seedCarriedPullProgress(counters, carried);
345
+ const transferConcurrency = resolveTransferConcurrency(run.hostLoadAtStart);
346
+ const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
347
+ if (conflictRun.abortResult) {
348
+ return conflictRun.abortResult;
349
+ }
350
+ await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
351
+ // The following added-by and tombstone phases may still make authenticated
352
+ // vault calls. Make a completed sub-threshold download tail durable before
353
+ // they begin, so their failure cannot turn a successful bulk transfer into
354
+ // a full replay on the next pass.
355
+ flushDownloadedJournalTail(run);
356
+ await emitAndReportNewFiles(run, plan);
357
+ await verifyPlannedJournalTombstones(run, plan);
358
+ executeJournalTombstoneDeletes(run, plan, counters);
359
+ emitScopeMaterializationGap(run, plan);
360
+ return finalizePullRun(run, plan, scopeRun, counters);
361
+ }
362
+ catch (error) {
363
+ if (error instanceof CooperativePullHandoff) {
364
+ carried = error.completed;
365
+ continue;
366
+ }
367
+ throw error;
368
+ }
369
+ finally {
370
+ // Release a pass-local lease even on conflict, transfer, or auth failure.
371
+ // The one bounded hot lease remains available only for its same vault.
372
+ run.releaseKeyedJournal?.();
373
+ }
351
374
  }
352
375
  }
353
376
  async function buildPullContext(options) {
@@ -436,8 +459,9 @@ async function buildPullContext(options) {
436
459
  ? targetedJournalSession(journalSlug)
437
460
  : undefined;
438
461
  const keyedJournal = keyedJournalLease?.store;
462
+ let remoteFiles;
439
463
  try {
440
- const remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
464
+ remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
441
465
  const fileTombstones = options.fileTombstones ??
442
466
  await fetchFileTombstones(vaultConfig, options.personalMode === true
443
467
  ? { personal: true }
@@ -485,43 +509,27 @@ async function buildPullContext(options) {
485
509
  catch (error) {
486
510
  // `buildPullContext` can fail during LIST/freshness. Do not leave the
487
511
  // leased projection marked in use when no PullRunContext is returned.
512
+ if (remoteFiles && isStreamingRemoteFileSource(remoteFiles))
513
+ remoteFiles.dispose();
488
514
  keyedJournalLease?.release();
489
515
  throw error;
490
516
  }
491
517
  }
492
- /**
493
- * List the remote objects the current pull can actually materialize.
494
- *
495
- * Historically this was ALWAYS a full-bucket `listRemoteFiles(ctx)` — even for
496
- * a `shared`/`custom` membership whose effective scope is a handful of
497
- * prefixes — so every pull materialized the entire remote listing (tens of
498
- * thousands of RemoteFile objects on large vaults) only for `computePullPlan`
499
- * to discard most of it as `skip-out-of-scope`. Scoped pulls now list ONLY the
500
- * in-scope prefixes via `listRemoteForScope` (per-prefix ListObjectsV2 calls,
501
- * bounded parallel, deduped), which bounds the listing's memory to the scope's
502
- * actual footprint.
503
- *
504
- * `all`-mode (an unnarrowed full-vault pull, `listedPrefixSet` `[""]`) keeps
505
- * the single broad listing: fanning `all` out by top-level prefix would need a
506
- * delimiter LIST to discover the top-level names first and still union the
507
- * same total object count in one plan, so it bounds nothing today. (A
508
- * streaming planner is the real fix for `all` and is out of scope here.)
509
- *
510
- * An empty `listedPrefixSet` in a scoped mode means "nothing in scope" —
511
- * issue zero LIST calls. Consumers already treat an empty listing correctly:
512
- * the deletion sweep in `computePullPlan` only considers journal keys covered
513
- * by the LISTED scope, so keys absent because they were never listed can never
514
- * be mistaken for remote deletes.
515
- */
518
+ function isStreamingRemoteFileSource(source) {
519
+ return !Array.isArray(source);
520
+ }
521
+ function allModeRemoteFileStream(ctx) {
522
+ return spoolRemoteFiles(streamRemoteFiles(ctx));
523
+ }
516
524
  async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargetedPull) {
517
525
  const entries = toScopePrefixEntries(listedPrefixSet);
518
526
  if (entries.some((e) => e.prefix === ""))
519
- return listRemoteFiles(ctx);
527
+ return allModeRemoteFileStream(ctx);
520
528
  if (entries.length === 0) {
521
529
  // `all` mode always carries `[""]`, so an empty set only occurs for a
522
530
  // scoped mode (empty grants, or a targeted intersection that excluded
523
531
  // everything). Defensive `all` branch: never silently narrow a full pull.
524
- return syncMode === "all" ? listRemoteFiles(ctx) : [];
532
+ return syncMode === "all" ? allModeRemoteFileStream(ctx) : [];
525
533
  }
526
534
  return listRemoteForScope({
527
535
  ctx,
@@ -542,8 +550,14 @@ async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargete
542
550
  },
543
551
  });
544
552
  }
545
- function planPull(run) {
546
- return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount);
553
+ async function planPull(run) {
554
+ try {
555
+ return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount);
556
+ }
557
+ finally {
558
+ if (isStreamingRemoteFileSource(run.remoteFiles))
559
+ run.remoteFiles.dispose();
560
+ }
547
561
  }
548
562
  /** Build only the rows a targeted remote LIST can observe. */
549
563
  function journalForTargetedPull(store, prefixes) {
@@ -656,9 +670,55 @@ function createPullCounters() {
656
670
  conflictPaths: [],
657
671
  changedPaths: [],
658
672
  appliedUpserts: [],
673
+ newFiles: [],
659
674
  hasRetryableOutcome: false,
660
675
  };
661
676
  }
677
+ /** Fold a previous attempt's completed work into a restarted run's counters. */
678
+ function seedCarriedPullProgress(counters, carried) {
679
+ if (!carried)
680
+ return;
681
+ counters.filesDownloaded += carried.filesDownloaded;
682
+ counters.bytesDownloaded += carried.bytesDownloaded;
683
+ counters.conflicts += carried.conflicts;
684
+ counters.filesTombstoned += carried.filesTombstoned;
685
+ counters.conflictPaths.push(...carried.conflictPaths);
686
+ counters.changedPaths.push(...carried.changedPaths);
687
+ counters.appliedUpserts.push(...carried.appliedUpserts);
688
+ if (carried.hasRetryableOutcome)
689
+ counters.hasRetryableOutcome = true;
690
+ }
691
+ /**
692
+ * Snapshot what survives the hand-off. The numeric and path axes already
693
+ * include everything carried into this attempt; `newFiles` is accumulated
694
+ * separately so it stays a record of pre-restart downloads only, and cannot
695
+ * double up with the current plan's own new files.
696
+ */
697
+ function carryPullProgress(previous, counters) {
698
+ return {
699
+ filesDownloaded: counters.filesDownloaded,
700
+ bytesDownloaded: counters.bytesDownloaded,
701
+ conflicts: counters.conflicts,
702
+ conflictPaths: [...counters.conflictPaths],
703
+ filesTombstoned: counters.filesTombstoned,
704
+ changedPaths: [...counters.changedPaths],
705
+ appliedUpserts: [...counters.appliedUpserts],
706
+ newFiles: [...(previous?.newFiles ?? []), ...counters.newFiles],
707
+ hasRetryableOutcome: counters.hasRetryableOutcome,
708
+ };
709
+ }
710
+ /**
711
+ * Every new file this pull is responsible for reporting: the ones a previous
712
+ * attempt downloaded before handing off, then this plan's own. A carried path
713
+ * that the fresh plan also lists is reported once.
714
+ */
715
+ function reportableNewFiles(run, plan) {
716
+ const carried = run.carriedProgress?.newFiles;
717
+ if (carried === undefined || carried.length === 0)
718
+ return plan.newFiles;
719
+ const planned = new Set(plan.newFiles.map((newFile) => newFile.path));
720
+ return [...carried.filter((newFile) => !planned.has(newFile.path)), ...plan.newFiles];
721
+ }
662
722
  function planScopeShrink(run) {
663
723
  const lastRecord = lastPullRecord(run.journal, run.ctx.uid);
664
724
  const lastPrefixSet = lastRecord && lastRecord.prefixSet.length > 0
@@ -1119,6 +1179,7 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
1119
1179
  if (resolution === "abort") {
1120
1180
  run.emit({ type: "new-files", files: [] });
1121
1181
  persistPullJournal(run);
1182
+ const abortedNewFiles = reportableNewFiles(run, plan);
1122
1183
  return {
1123
1184
  filesDownloaded: counters.filesDownloaded,
1124
1185
  bytesDownloaded: counters.bytesDownloaded,
@@ -1126,8 +1187,8 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
1126
1187
  conflicts: counters.conflicts,
1127
1188
  conflictPaths: counters.conflictPaths,
1128
1189
  aborted: true,
1129
- newFiles: plan.newFiles,
1130
- newFilesCount: plan.newFilesCount,
1190
+ newFiles: abortedNewFiles,
1191
+ newFilesCount: abortedNewFiles.length,
1131
1192
  filesExcludedByPolicy: plan.filesExcludedByPolicy,
1132
1193
  filesTombstoned: 0,
1133
1194
  filesOutOfScope: counters.filesOutOfScope,
@@ -1253,10 +1314,21 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
1253
1314
  discardProbe();
1254
1315
  }
1255
1316
  }
1256
- async function executeDownloadExecutor(run, downloadItems, transferConcurrency, counters) {
1257
- if (downloadItems.length === 0)
1258
- return;
1259
- await primeObjectTransport(run.ctx, "get", downloadItems.map((d) => d.remoteFile.key));
1317
+ /**
1318
+ * The old plan may have tombstone evidence that predates a realtime hand-off.
1319
+ * Rebuilding from disk after a hand-off makes the continuation observe the
1320
+ * realtime journal and a fresh LIST before it can perform another mutation.
1321
+ */
1322
+ class CooperativePullHandoff extends Error {
1323
+ /** Completed work the restarted pull must keep reporting. */
1324
+ completed;
1325
+ constructor(completed) {
1326
+ super("full pull handed off to realtime work");
1327
+ this.name = "CooperativePullHandoff";
1328
+ this.completed = completed;
1329
+ }
1330
+ }
1331
+ async function executeDownloadQueue(run, downloadItems, transferConcurrency, counters, deferJournalCheckpoint) {
1260
1332
  const queue = [...downloadItems];
1261
1333
  const inFlight = new Set();
1262
1334
  const workerErrors = [];
@@ -1271,7 +1343,7 @@ async function executeDownloadExecutor(run, downloadItems, transferConcurrency,
1271
1343
  inFlight.size < transferConcurrency &&
1272
1344
  queue.length > 0) {
1273
1345
  const downloadItem = queue.shift();
1274
- const download = () => downloadOne(run, downloadItem, counters);
1346
+ const download = () => downloadOne(run, downloadItem, counters, deferJournalCheckpoint);
1275
1347
  const p = (run.options.transferSemaphore
1276
1348
  ? run.options.transferSemaphore.withPermit(download)
1277
1349
  : download())
@@ -1295,16 +1367,44 @@ async function executeDownloadExecutor(run, downloadItems, transferConcurrency,
1295
1367
  break;
1296
1368
  }
1297
1369
  }
1298
- if (workerErrors.length > 0) {
1299
- persistPullJournal(run);
1300
- const first = workerErrors[0];
1301
- if (workerErrors.length > 1) {
1302
- first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
1370
+ return workerErrors;
1371
+ }
1372
+ function throwDownloadWorkerErrors(run, workerErrors) {
1373
+ persistPullJournal(run);
1374
+ const first = workerErrors[0];
1375
+ if (workerErrors.length > 1) {
1376
+ first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
1377
+ }
1378
+ throw first;
1379
+ }
1380
+ async function executeDownloadExecutor(run, downloadItems, transferConcurrency, counters) {
1381
+ if (downloadItems.length === 0)
1382
+ return;
1383
+ await primeObjectTransport(run.ctx, "get", downloadItems.map((d) => d.remoteFile.key));
1384
+ // Ordinary and targeted pulls retain the existing 10k crash-recovery cadence.
1385
+ // Only a watch chore that supplied the cooperative seam pays for drained
1386
+ // units and can return its root lock to queued realtime work.
1387
+ if (!run.options.cooperativeCheckpoint) {
1388
+ const workerErrors = await executeDownloadQueue(run, downloadItems, transferConcurrency, counters, false);
1389
+ if (workerErrors.length > 0)
1390
+ throwDownloadWorkerErrors(run, workerErrors);
1391
+ return;
1392
+ }
1393
+ for (let start = 0; start < downloadItems.length; start += FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE) {
1394
+ const chunk = downloadItems.slice(start, start + FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE);
1395
+ const workerErrors = await executeDownloadQueue(run, chunk, transferConcurrency, counters, true);
1396
+ // `executeDownloadQueue` does not return until every member of this chunk
1397
+ // has settled. Persist before the callback so the suspended chore owns no
1398
+ // mutable pass state while realtime uses the root lock.
1399
+ flushDownloadedJournalTail(run);
1400
+ if (workerErrors.length > 0)
1401
+ throwDownloadWorkerErrors(run, workerErrors);
1402
+ if (await run.options.cooperativeCheckpoint() === true) {
1403
+ throw new CooperativePullHandoff(carryPullProgress(run.carriedProgress, counters));
1303
1404
  }
1304
- throw first;
1305
1405
  }
1306
1406
  }
1307
- async function downloadOne(run, downloadItem, counters) {
1407
+ async function downloadOne(run, downloadItem, counters, deferJournalCheckpoint = false) {
1308
1408
  const { remoteFile, localPath } = downloadItem;
1309
1409
  await refreshRunContextIfExpiring(run);
1310
1410
  if (!isDownloadWritePathStillContained(run.companyRoot, remoteFile.key, localPath)) {
@@ -1373,6 +1473,9 @@ async function downloadOne(run, downloadItem, counters) {
1373
1473
  counters.filesDownloaded++;
1374
1474
  counters.bytesDownloaded += size;
1375
1475
  counters.changedPaths.push(remoteFile.key);
1476
+ if (downloadItem.isNew) {
1477
+ counters.newFiles.push({ path: remoteFile.key, bytes: remoteFile.size });
1478
+ }
1376
1479
  // The event-push emitter hashes regular file bodies. Symlink journal hashes
1377
1480
  // describe link targets instead, and the startup baseline excludes them for
1378
1481
  // the same reason, so only hand regular-file downloads to that ledger.
@@ -1461,7 +1564,8 @@ async function downloadOne(run, downloadItem, counters) {
1461
1564
  return;
1462
1565
  }
1463
1566
  run.downloadsSinceJournalCheckpoint++;
1464
- if (run.downloadsSinceJournalCheckpoint >= DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE) {
1567
+ if (!deferJournalCheckpoint &&
1568
+ run.downloadsSinceJournalCheckpoint >= DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE) {
1465
1569
  flushDownloadedJournalTail(run);
1466
1570
  }
1467
1571
  }
@@ -1494,15 +1598,19 @@ function persistPullJournal(run) {
1494
1598
  }
1495
1599
  async function emitAndReportNewFiles(run, plan) {
1496
1600
  const enrichedNewFiles = [];
1601
+ // New files a cooperative hand-off already downloaded are reported here too:
1602
+ // the restarted plan cannot see them as new, so this is their only
1603
+ // notification.
1604
+ const newFiles = reportableNewFiles(run, plan);
1497
1605
  // Batch-mint the GET presigns once (chunked, breaker-aware) so the per-file
1498
1606
  // created-by HEADs below reuse the cache instead of each minting its own
1499
1607
  // presign. Without this, a big catch-up pull (hundreds of new files) bursts
1500
1608
  // the presign endpoint, trips the circuit breaker, and every enrichment HEAD
1501
1609
  // then fails. Mirrors the tombstone HEAD-verify pre-prime.
1502
- await primeObjectTransport(run.ctx, "get", plan.newFiles.map((nf) => nf.path));
1610
+ await primeObjectTransport(run.ctx, "get", newFiles.map((nf) => nf.path));
1503
1611
  const HEAD_CONCURRENCY = 5;
1504
- for (let i = 0; i < plan.newFiles.length; i += HEAD_CONCURRENCY) {
1505
- const batch = plan.newFiles.slice(i, i + HEAD_CONCURRENCY);
1612
+ for (let i = 0; i < newFiles.length; i += HEAD_CONCURRENCY) {
1613
+ const batch = newFiles.slice(i, i + HEAD_CONCURRENCY);
1506
1614
  const results = await Promise.all(batch.map(async (nf) => {
1507
1615
  let addedBy = null;
1508
1616
  try {
@@ -1663,6 +1771,7 @@ function finalizePullRun(run, plan, scopeRun, counters) {
1663
1771
  // best-effort: a post-sync refresh failure never fails the sync
1664
1772
  }
1665
1773
  }
1774
+ const newFiles = reportableNewFiles(run, plan);
1666
1775
  const result = {
1667
1776
  filesDownloaded: counters.filesDownloaded,
1668
1777
  bytesDownloaded: counters.bytesDownloaded,
@@ -1670,8 +1779,8 @@ function finalizePullRun(run, plan, scopeRun, counters) {
1670
1779
  conflicts: counters.conflicts,
1671
1780
  conflictPaths: counters.conflictPaths,
1672
1781
  aborted: false,
1673
- newFiles: plan.newFiles,
1674
- newFilesCount: plan.newFilesCount,
1782
+ newFiles,
1783
+ newFilesCount: newFiles.length,
1675
1784
  filesExcludedByPolicy: plan.filesExcludedByPolicy,
1676
1785
  filesTombstoned: counters.filesTombstoned,
1677
1786
  filesOutOfScope: counters.filesOutOfScope,
@@ -1900,7 +2009,8 @@ class LocalSnapshotChangedError extends Error {
1900
2009
  * the real outcome of the upcoming Stage-2 execution rather than an
1901
2010
  * upper-bound guess.
1902
2011
  *
1903
- * Pure function: no S3 calls, no journal writes, no event emission. The
2012
+ * It performs no writes, journal mutations, or event emission. An all-mode
2013
+ * source performs its paged classification LIST while it is consumed; the
1904
2014
  * caller (`sync()`) is responsible for emitting the resulting plan event
1905
2015
  * before iterating `items`.
1906
2016
  */
@@ -1968,6 +2078,36 @@ excludePrefixes = [],
1968
2078
  // callers preserve the pre-policy, watcher-intent-only behavior.
1969
2079
  propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows(journal).size) {
1970
2080
  const items = [];
2081
+ const streaming = isStreamingRemoteFileSource(remoteFiles);
2082
+ const listedFiles = isStreamingRemoteFileSource(remoteFiles) ? remoteFiles.files : remoteFiles;
2083
+ let preplannedFilesSkipped = 0;
2084
+ let preplannedFilesOutOfScope = 0;
2085
+ let preplannedFilesExcludedByPolicy = 0;
2086
+ if (streaming) {
2087
+ const retain = items.push.bind(items);
2088
+ items.push = (...next) => {
2089
+ for (const item of next) {
2090
+ // These actions still need the executor for an event or a state change.
2091
+ if (item.action === "download" ||
2092
+ item.action === "conflict" ||
2093
+ item.action === "tombstone-delete" ||
2094
+ item.action === "skip-junk-key-spelling" ||
2095
+ item.action === "skip-unreadable-link") {
2096
+ retain(item);
2097
+ }
2098
+ else if (item.action === "skip-out-of-scope") {
2099
+ preplannedFilesOutOfScope++;
2100
+ }
2101
+ else if (item.action === "skip-excluded-policy") {
2102
+ preplannedFilesExcludedByPolicy++;
2103
+ }
2104
+ else {
2105
+ preplannedFilesSkipped++;
2106
+ }
2107
+ }
2108
+ return items.length;
2109
+ };
2110
+ }
1971
2111
  // Remote keys the server will never presign — collected so the caller can
1972
2112
  // warn + emit telemetry without allowing one poison object to fail a pull.
1973
2113
  const invalidRemoteKeys = [];
@@ -1983,8 +2123,35 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
1983
2123
  // never gated by this map (see the guard below), so pre-existing families
1984
2124
  // keep syncing untouched until the vault doctor (US-003) collapses them.
1985
2125
  const journaledKeyByCanonical = new Map();
2126
+ // The delete sweep starts with journaled keys and clears them as the streamed
2127
+ // LIST proves each one present. This retains existing journal-key references
2128
+ // instead of a second, whole-vault population of long remote key strings.
2129
+ const unseenJournalKeys = new Set();
2130
+ // Most journal keys have already been normalized to POSIX and can be cleared
2131
+ // directly. Keep the legacy Windows spelling defense-in-depth without
2132
+ // imposing a second map entry on every current journal row.
2133
+ const legacyJournalKeysByPosix = new Map();
2134
+ // The junk-spelling delete guard needs all journal twins, while
2135
+ // `journaledKeyByCanonical` above deliberately retains only the preferred
2136
+ // representative for classification.
2137
+ const journaledSkillKeysByCanonical = new Map();
1986
2138
  for (const journaledKey of journalRows(journal).keys()) {
1987
2139
  onHeartbeat?.();
2140
+ const posixJournaledKey = toPosixKey(journaledKey);
2141
+ // A scoped LIST cannot prove anything about keys outside its prefixes.
2142
+ // Do not copy the entire vault's journal into its presence bookkeeping.
2143
+ const inListedScope = isCoveredByAny(posixJournaledKey, prefixSet);
2144
+ if (inListedScope)
2145
+ unseenJournalKeys.add(journaledKey);
2146
+ if (inListedScope && posixJournaledKey !== journaledKey) {
2147
+ const legacyKeys = legacyJournalKeysByPosix.get(posixJournaledKey);
2148
+ if (legacyKeys === undefined) {
2149
+ legacyJournalKeysByPosix.set(posixJournaledKey, [journaledKey]);
2150
+ }
2151
+ else {
2152
+ legacyKeys.push(journaledKey);
2153
+ }
2154
+ }
1988
2155
  if (!journaledKey.startsWith(SKILLS_KEY_PREFIX))
1989
2156
  continue;
1990
2157
  const canonical = canonicalVaultKeySpelling(journaledKey);
@@ -1992,6 +2159,15 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
1992
2159
  if (existing === undefined || journaledKey === canonical) {
1993
2160
  journaledKeyByCanonical.set(canonical, journaledKey);
1994
2161
  }
2162
+ if (!inListedScope)
2163
+ continue;
2164
+ const skillKeys = journaledSkillKeysByCanonical.get(canonical);
2165
+ if (skillKeys === undefined) {
2166
+ journaledSkillKeysByCanonical.set(canonical, [journaledKey]);
2167
+ }
2168
+ else {
2169
+ skillKeys.push(journaledKey);
2170
+ }
1995
2171
  }
1996
2172
  // Remote keys that ALSO appear as an ancestor of another remote key — i.e.
1997
2173
  // keys that carry child objects under `${key}/…` in this same LIST. The
@@ -2001,22 +2177,40 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
2001
2177
  // objects beneath it. A REAL directory, by contrast, only ever syncs as
2002
2178
  // those child objects — never as a single object AT its own key. So a single
2003
2179
  // remote object sitting at a key that also has children is unambiguously a
2004
- // directory-overlay marker, not a regular file. Precomputed once (pure path
2005
- // derivation, no I/O — the planner is synchronous) so the dir-vs-object
2006
- // collision branch can tell an inert overlay marker apart from a genuine
2007
- // file-vs-directory structural collision. feedback_d2082110.
2008
- const remoteKeysWithChildren = new Set();
2009
- for (const rf of remoteFiles) {
2180
+ // directory-overlay marker, not a regular file. Evidence is complete before
2181
+ // classification for both the spooled all-mode LIST and scoped arrays.
2182
+ // Build scoped evidence incrementally: flatMap would transiently retain an
2183
+ // ancestor string for every row/depth before Set deduplication.
2184
+ const remoteKeysWithChildren = isStreamingRemoteFileSource(remoteFiles)
2185
+ ? remoteFiles.remoteKeysWithChildren
2186
+ : collectRemoteAncestors(remoteFiles);
2187
+ // Both sources replay synchronously. for-await on a scoped array introduces
2188
+ // promise/microtask work per object even though no I/O is being awaited.
2189
+ for (const remoteFile of listedFiles) {
2010
2190
  onHeartbeat?.();
2011
- const key = toPosixKey(rf.key);
2012
- let slash = key.indexOf("/");
2013
- while (slash !== -1) {
2014
- remoteKeysWithChildren.add(key.slice(0, slash));
2015
- slash = key.indexOf("/", slash + 1);
2191
+ const posixRemoteKey = toPosixKey(remoteFile.key);
2192
+ // S3 object keys are POSIX. Match the old delete-sweep membership check:
2193
+ // a legacy backslash journal entry counts as present only when its POSIX
2194
+ // form appears in the LIST, while a malformed backslash remote key does
2195
+ // not accidentally change that behavior.
2196
+ if (remoteFile.key === posixRemoteKey) {
2197
+ unseenJournalKeys.delete(remoteFile.key);
2198
+ for (const legacyKey of legacyJournalKeysByPosix.get(remoteFile.key) ?? []) {
2199
+ unseenJournalKeys.delete(legacyKey);
2200
+ }
2201
+ }
2202
+ if (remoteFile.key.startsWith(SKILLS_KEY_PREFIX)) {
2203
+ // A remote junk spelling means its canonical journal twin is still
2204
+ // remotely present. Keep that twin out of the delete sweep unless an
2205
+ // exact FILE_TOMBSTONE is authoritative, precisely matching the old
2206
+ // remoteCanonicalSkillKeys protection without retaining remote keys.
2207
+ const canonical = canonicalVaultKeySpelling(remoteFile.key);
2208
+ for (const journaledKey of journaledSkillKeysByCanonical.get(canonical) ?? []) {
2209
+ if (fileTombstones.get(toPosixKey(journaledKey)) === undefined) {
2210
+ unseenJournalKeys.delete(journaledKey);
2211
+ }
2212
+ }
2016
2213
  }
2017
- }
2018
- for (const remoteFile of remoteFiles) {
2019
- onHeartbeat?.();
2020
2214
  const invalidKey = classifyVaultKey(remoteFile.key, personalMode ? "personal" : "company");
2021
2215
  if (invalidKey !== null) {
2022
2216
  invalidRemoteKeys.push({
@@ -2669,6 +2863,9 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
2669
2863
  filesToSkip++;
2670
2864
  }
2671
2865
  }
2866
+ filesToSkip += preplannedFilesSkipped;
2867
+ filesExcludedByPolicy += preplannedFilesExcludedByPolicy;
2868
+ filesOutOfScope += preplannedFilesOutOfScope;
2672
2869
  // Bug #9 — cross-machine delete propagation. The peer's push leg
2673
2870
  // already removed the file from S3 (verified in the deep-test
2674
2871
  // addendum: `aws s3 ls` showed the keys absent post-push). Pre-fix
@@ -2677,38 +2874,21 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
2677
2874
  // that-was-there-before" — so the file stayed forever on every
2678
2875
  // receiver and `filesTombstoned: 0` showed up on every pull.
2679
2876
  //
2680
- // Closing the loop: walk the journal, find every entry whose key is
2681
- // NOT in the remote LIST set, AND whose path passes the current
2877
+ // Closing the loop: walk journal keys the streamed LIST did NOT clear, AND
2878
+ // whose path passes the current
2682
2879
  // ignore filter (paths newly excluded by .hqignore must not trigger
2683
2880
  // mass-delete), AND — in personalMode — survives the same companies/*
2684
2881
  // gating the download branch applies. The executor will apply each
2685
2882
  // as a local delete + journal removal. Symmetric to the push side's
2686
2883
  // `propagateDeletes` plan in share.ts.
2687
- const remoteKeySet = new Set();
2688
- // Skill-farm twin protection (companion to the junk-spelling pull guard):
2689
- // when the LIST holds only a junk spelling of a journaled canonical key,
2690
- // the logical object is still remotely present — a raw has() would class
2691
- // the canonical journal entry as remote-deleted, unlink the good local
2692
- // wrapper, and drop the journal entry (after which the next pull would
2693
- // materialize the junk spelling the guard just refused). Track canonical
2694
- // forms of listed skill keys and consult them below.
2695
- const remoteCanonicalSkillKeys = new Set();
2696
- for (const rf of remoteFiles) {
2697
- remoteKeySet.add(rf.key);
2698
- if (rf.key.startsWith(SKILLS_KEY_PREFIX)) {
2699
- remoteCanonicalSkillKeys.add(canonicalVaultKeySpelling(rf.key));
2700
- }
2701
- }
2702
2884
  const tombstones = [];
2703
- for (const key of journalRows(journal).keys()) {
2885
+ for (const key of unseenJournalKeys) {
2704
2886
  // Compare membership in POSIX space. A pre-5.47.2 Windows journal key can
2705
2887
  // carry backslash separators; the remote LIST is always forward-slash, so a
2706
2888
  // raw `has(key)` would miss the match and class a live local file as
2707
2889
  // remote-deleted. normalizeJournalKeys() defuses this on load, but the
2708
2890
  // POSIX compare is defense-in-depth (ridge data-loss, feedback_b8d09d0f).
2709
2891
  const posixKey = toPosixKey(key);
2710
- if (remoteKeySet.has(posixKey))
2711
- continue;
2712
2892
  // Listing-scope guard (targeted sync / scoped listing). The remote LIST
2713
2893
  // now covers ONLY `prefixSet` (scoped modes list per-prefix; a targeted
2714
2894
  // pull narrows further), so "absent from the LIST" is evidence of a remote
@@ -2720,16 +2900,6 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
2720
2900
  // `all` mode passes `[""]`, which covers every key — no behavior change.
2721
2901
  if (!isCoveredByAny(posixKey, prefixSet))
2722
2902
  continue;
2723
- if (posixKey.startsWith(SKILLS_KEY_PREFIX) &&
2724
- remoteCanonicalSkillKeys.has(canonicalVaultKeySpelling(posixKey)) &&
2725
- // An EXACT FILE_TOMBSTONE for the journaled key is an authoritative
2726
- // delete and wins over the twin protection — a lingering junk
2727
- // spelling in the LIST must not keep a deliberately-deleted wrapper
2728
- // alive forever. (The key is absent from the LIST here, so there is
2729
- // no newer re-create to defer to.)
2730
- fileTombstones.get(posixKey) === undefined) {
2731
- continue;
2732
- }
2733
2903
  // Fast-path ignore, BEFORE containment canonicalization. The tombstone
2734
2904
  // sweep runs for every journaled key, and a stale journal can carry
2735
2905
  // hundreds of thousands of entries under an ignored runtime tree (e.g.
@@ -2838,6 +3008,8 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
2838
3008
  }
2839
3009
  return {
2840
3010
  items,
3011
+ preplannedFilesSkipped,
3012
+ preplannedFilesOutOfScope,
2841
3013
  filesToDownload,
2842
3014
  bytesToDownload,
2843
3015
  filesToSkip,