@indigoai-us/hq-cloud 6.16.53 → 6.16.54-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. package/dist/backup-prune.d.ts +1 -23
  2. package/dist/backup-prune.d.ts.map +1 -1
  3. package/dist/backup-prune.js +8 -92
  4. package/dist/backup-prune.js.map +1 -1
  5. package/dist/backup-prune.test.js +0 -36
  6. package/dist/backup-prune.test.js.map +1 -1
  7. package/dist/bin/rescue-runner.js +1 -2
  8. package/dist/bin/rescue-runner.js.map +1 -1
  9. package/dist/bin/sync-runner-company.d.ts +6 -0
  10. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  11. package/dist/bin/sync-runner-company.js +30 -1
  12. package/dist/bin/sync-runner-company.js.map +1 -1
  13. package/dist/bin/sync-runner-company.test.js +57 -126
  14. package/dist/bin/sync-runner-company.test.js.map +1 -1
  15. package/dist/bin/sync-runner-planning.d.ts +12 -3
  16. package/dist/bin/sync-runner-planning.d.ts.map +1 -1
  17. package/dist/bin/sync-runner-planning.js +65 -3
  18. package/dist/bin/sync-runner-planning.js.map +1 -1
  19. package/dist/bin/sync-runner-planning.test.js +108 -1
  20. package/dist/bin/sync-runner-planning.test.js.map +1 -1
  21. package/dist/bin/sync-runner-watch-loop.d.ts +48 -2
  22. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  23. package/dist/bin/sync-runner-watch-loop.js +318 -120
  24. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  25. package/dist/bin/sync-runner.d.ts +33 -5
  26. package/dist/bin/sync-runner.d.ts.map +1 -1
  27. package/dist/bin/sync-runner.js +34 -4
  28. package/dist/bin/sync-runner.js.map +1 -1
  29. package/dist/bin/sync-runner.test.js +1079 -79
  30. package/dist/bin/sync-runner.test.js.map +1 -1
  31. package/dist/cli/onepass-pull-planner-regressions.test.d.ts +2 -0
  32. package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +1 -0
  33. package/dist/cli/onepass-pull-planner-regressions.test.js +300 -0
  34. package/dist/cli/onepass-pull-planner-regressions.test.js.map +1 -0
  35. package/dist/cli/onepass-pull-planner.test.d.ts +2 -0
  36. package/dist/cli/onepass-pull-planner.test.d.ts.map +1 -0
  37. package/dist/cli/onepass-pull-planner.test.js +266 -0
  38. package/dist/cli/onepass-pull-planner.test.js.map +1 -0
  39. package/dist/cli/rescue-core.d.ts +0 -40
  40. package/dist/cli/rescue-core.d.ts.map +1 -1
  41. package/dist/cli/rescue-core.js +162 -1454
  42. package/dist/cli/rescue-core.js.map +1 -1
  43. package/dist/cli/rescue-rsync-preflight.test.js +4 -256
  44. package/dist/cli/rescue-rsync-preflight.test.js.map +1 -1
  45. package/dist/cli/rescue.d.ts +0 -4
  46. package/dist/cli/rescue.d.ts.map +1 -1
  47. package/dist/cli/rescue.js +1 -20
  48. package/dist/cli/rescue.js.map +1 -1
  49. package/dist/cli/rescue.test.js +1 -16
  50. package/dist/cli/rescue.test.js.map +1 -1
  51. package/dist/cli/share-presign-transport.test.js +0 -2
  52. package/dist/cli/share-presign-transport.test.js.map +1 -1
  53. package/dist/cli/share.d.ts +19 -0
  54. package/dist/cli/share.d.ts.map +1 -1
  55. package/dist/cli/share.js +150 -467
  56. package/dist/cli/share.js.map +1 -1
  57. package/dist/cli/share.test.js +95 -608
  58. package/dist/cli/share.test.js.map +1 -1
  59. package/dist/cli/streaming-pull-planner.bench.test.d.ts +15 -0
  60. package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +1 -0
  61. package/dist/cli/streaming-pull-planner.bench.test.js +273 -0
  62. package/dist/cli/streaming-pull-planner.bench.test.js.map +1 -0
  63. package/dist/cli/sync-scope.test.js +8 -0
  64. package/dist/cli/sync-scope.test.js.map +1 -1
  65. package/dist/cli/sync.d.ts +13 -15
  66. package/dist/cli/sync.d.ts.map +1 -1
  67. package/dist/cli/sync.js +226 -118
  68. package/dist/cli/sync.js.map +1 -1
  69. package/dist/cli/sync.test.js +215 -2
  70. package/dist/cli/sync.test.js.map +1 -1
  71. package/dist/cli/watch-event-push-conflict.test.js +0 -2
  72. package/dist/cli/watch-event-push-conflict.test.js.map +1 -1
  73. package/dist/cognito-auth.d.ts +0 -30
  74. package/dist/cognito-auth.d.ts.map +1 -1
  75. package/dist/cognito-auth.js +0 -175
  76. package/dist/cognito-auth.js.map +1 -1
  77. package/dist/index.d.ts +2 -2
  78. package/dist/index.d.ts.map +1 -1
  79. package/dist/index.js +1 -1
  80. package/dist/index.js.map +1 -1
  81. package/dist/journal-row-store.d.ts +40 -5
  82. package/dist/journal-row-store.d.ts.map +1 -1
  83. package/dist/journal-row-store.js +240 -21
  84. package/dist/journal-row-store.js.map +1 -1
  85. package/dist/journal-row-store.test.js +54 -0
  86. package/dist/journal-row-store.test.js.map +1 -1
  87. package/dist/journal-snapshot-v4.d.ts.map +1 -1
  88. package/dist/journal-snapshot-v4.js +11 -6
  89. package/dist/journal-snapshot-v4.js.map +1 -1
  90. package/dist/journal-store.test.js +259 -7
  91. package/dist/journal-store.test.js.map +1 -1
  92. package/dist/journal.d.ts +43 -15
  93. package/dist/journal.d.ts.map +1 -1
  94. package/dist/journal.js +297 -79
  95. package/dist/journal.js.map +1 -1
  96. package/dist/lib/describe-error.d.ts.map +1 -1
  97. package/dist/lib/describe-error.js +0 -7
  98. package/dist/lib/describe-error.js.map +1 -1
  99. package/dist/lib/describe-error.test.js +0 -11
  100. package/dist/lib/describe-error.test.js.map +1 -1
  101. package/dist/lib/exit-codes.d.ts +15 -0
  102. package/dist/lib/exit-codes.d.ts.map +1 -1
  103. package/dist/lib/exit-codes.js +15 -0
  104. package/dist/lib/exit-codes.js.map +1 -1
  105. package/dist/machine-auth.test.js +0 -295
  106. package/dist/machine-auth.test.js.map +1 -1
  107. package/dist/object-io.d.ts +2 -2
  108. package/dist/object-io.d.ts.map +1 -1
  109. package/dist/object-io.js +3 -18
  110. package/dist/object-io.js.map +1 -1
  111. package/dist/personal-vault.d.ts.map +1 -1
  112. package/dist/personal-vault.js +10 -3
  113. package/dist/personal-vault.js.map +1 -1
  114. package/dist/personal-vault.test.js +35 -0
  115. package/dist/personal-vault.test.js.map +1 -1
  116. package/dist/remote-pull.d.ts.map +1 -1
  117. package/dist/remote-pull.js +9 -2
  118. package/dist/remote-pull.js.map +1 -1
  119. package/dist/remote-pull.test.js +59 -2
  120. package/dist/remote-pull.test.js.map +1 -1
  121. package/dist/s3.d.ts +1 -18
  122. package/dist/s3.d.ts.map +1 -1
  123. package/dist/s3.js +24 -47
  124. package/dist/s3.js.map +1 -1
  125. package/dist/s3.list-cost.test.d.ts +2 -0
  126. package/dist/s3.list-cost.test.d.ts.map +1 -0
  127. package/dist/s3.list-cost.test.js +51 -0
  128. package/dist/s3.list-cost.test.js.map +1 -0
  129. package/dist/s3.test.js +1 -27
  130. package/dist/s3.test.js.map +1 -1
  131. package/dist/sync/area-ledger.d.ts +2 -2
  132. package/dist/sync/area-ledger.d.ts.map +1 -1
  133. package/dist/sync/area-ledger.js +66 -34
  134. package/dist/sync/area-ledger.js.map +1 -1
  135. package/dist/sync/area-ledger.test.js +90 -5
  136. package/dist/sync/area-ledger.test.js.map +1 -1
  137. package/dist/sync/parallel-sync-coordinator.test.js +29 -0
  138. package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
  139. package/dist/sync/realtime-telemetry.d.ts +1 -1
  140. package/dist/sync/realtime-telemetry.d.ts.map +1 -1
  141. package/dist/sync/realtime-telemetry.js +14 -6
  142. package/dist/sync/realtime-telemetry.js.map +1 -1
  143. package/dist/sync/realtime-telemetry.test.js +136 -16
  144. package/dist/sync/realtime-telemetry.test.js.map +1 -1
  145. package/dist/sync/remote-file-spool.d.ts +16 -0
  146. package/dist/sync/remote-file-spool.d.ts.map +1 -0
  147. package/dist/sync/remote-file-spool.js +91 -0
  148. package/dist/sync/remote-file-spool.js.map +1 -0
  149. package/dist/sync/remote-file-spool.test.d.ts +2 -0
  150. package/dist/sync/remote-file-spool.test.d.ts.map +1 -0
  151. package/dist/sync/remote-file-spool.test.js +77 -0
  152. package/dist/sync/remote-file-spool.test.js.map +1 -0
  153. package/dist/sync/state-store.d.ts +6 -0
  154. package/dist/sync/state-store.d.ts.map +1 -1
  155. package/dist/sync/state-store.js +14 -2
  156. package/dist/sync/state-store.js.map +1 -1
  157. package/dist/sync-core.d.ts +4 -6
  158. package/dist/sync-core.d.ts.map +1 -1
  159. package/dist/sync-core.js +4 -10
  160. package/dist/sync-core.js.map +1 -1
  161. package/dist/sync-core.test.js +1 -25
  162. package/dist/sync-core.test.js.map +1 -1
  163. package/dist/vault-client.d.ts +5 -26
  164. package/dist/vault-client.d.ts.map +1 -1
  165. package/dist/vault-client.js +20 -43
  166. package/dist/vault-client.js.map +1 -1
  167. package/dist/vault-client.test.js +0 -32
  168. package/dist/vault-client.test.js.map +1 -1
  169. package/package.json +2 -1
  170. package/dist/cli/rescue-breaker.test.d.ts +0 -2
  171. package/dist/cli/rescue-breaker.test.d.ts.map +0 -1
  172. package/dist/cli/rescue-breaker.test.js +0 -334
  173. package/dist/cli/rescue-breaker.test.js.map +0 -1
  174. package/dist/cli/rescue-realpath.test.d.ts +0 -2
  175. package/dist/cli/rescue-realpath.test.d.ts.map +0 -1
  176. package/dist/cli/rescue-realpath.test.js +0 -109
  177. package/dist/cli/rescue-realpath.test.js.map +0 -1
  178. package/dist/cli/rescue-snapshot-copy.test.d.ts +0 -2
  179. package/dist/cli/rescue-snapshot-copy.test.d.ts.map +0 -1
  180. package/dist/cli/rescue-snapshot-copy.test.js +0 -627
  181. package/dist/cli/rescue-snapshot-copy.test.js.map +0 -1
  182. package/dist/cli/rescue-snapshot-recovery.test.d.ts +0 -2
  183. package/dist/cli/rescue-snapshot-recovery.test.d.ts.map +0 -1
  184. package/dist/cli/rescue-snapshot-recovery.test.js +0 -260
  185. package/dist/cli/rescue-snapshot-recovery.test.js.map +0 -1
  186. package/dist/cli/rescue-source-switch-floor.test.d.ts +0 -2
  187. package/dist/cli/rescue-source-switch-floor.test.d.ts.map +0 -1
  188. package/dist/cli/rescue-source-switch-floor.test.js +0 -210
  189. package/dist/cli/rescue-source-switch-floor.test.js.map +0 -1
package/dist/cli/sync.js CHANGED
@@ -6,6 +6,7 @@
6
6
  */
7
7
  import * as fs from "fs";
8
8
  import * as path from "path";
9
+ import { spoolRemoteFiles, collectRemoteAncestors } from "../sync/remote-file-spool.js";
9
10
  import { VaultAuthError, VaultClient } from "../vault-client.js";
10
11
  import { readlinkOrNull } from "../lib/readlink-safe.js";
11
12
  import { describeError } from "../lib/describe-error.js";
@@ -16,7 +17,7 @@ import { ensureCompanyVaultTransport } from "../company-vault-transport.js";
16
17
  import { createSyncProgressRecorder } from "../sync-progress.js";
17
18
  import { canonicalVaultKeySpelling, localPathForVaultKey, vaultKeyForLocalPath, } from "../local-path-codec.js";
18
19
  import { SKILLS_KEY_PREFIX } from "./doctor.js";
19
- import { downloadFile, listRemoteFiles, headRemoteFile, primeObjectTransport, toPosixKey, classifyVaultKey, DanglingSymlinkParentError, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
20
+ import { downloadFile, streamRemoteFiles, headRemoteFile, primeObjectTransport, toPosixKey, classifyVaultKey, DanglingSymlinkParentError, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
20
21
  import { readJournalForPass, writeJournal, getStateDir, openJournalStoreSession, buildJournalBaseline, journalDeltaFromBaseline, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, updateEntry, removeEntry, getEntry, normalizeEtag, migrateToV2, gcTombstones, isTombstone, tombstoneEntry, lastPullRecord, appendPullRecord, generatePullId, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, } from "../journal.js";
21
22
  import { isGeneratedCoreMirrorKey, PERSONAL_VAULT_MANIFEST_KEY, } from "../personal-vault.js";
22
23
  import { isPersonalVaultExcluded } from "../personal-vault-exclusions.js";
@@ -185,6 +186,11 @@ export function targetedJournalSessionCensusSizes() {
185
186
  * dirty entries through a run-scoped journal writer.
186
187
  */
187
188
  const DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE = 10_000;
189
+ /**
190
+ * A full-pull hand-off unit. Every member has finished its transfer and the
191
+ * resulting journal delta is durable before realtime can acquire the writer.
192
+ */
193
+ export const FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE = 500;
188
194
  /**
189
195
  * Server cap on files per `/v1/notify/file-added` report. The endpoint rejects
190
196
  * an oversized batch wholesale, so the client MUST split a large report into
@@ -318,36 +324,47 @@ export async function sync(options) {
318
324
  return withOperationLock(options.hqRoot, "sync", () => syncWithOperationLockHeld(options));
319
325
  }
320
326
  async function syncWithOperationLockHeld(options) {
321
- const run = await buildPullContext(options);
322
- try {
323
- const plan = planPull(run);
324
- emitPullPlan(run.emit, plan);
325
- reportInvalidRemoteKeys(run, plan);
326
- const scopeRun = run.keyedJournal
327
- ? emptyTargetedScopeRun(run)
328
- : executeScopeShrink(run, planScopeShrink(run));
329
- const counters = createPullCounters();
330
- const transferConcurrency = resolveTransferConcurrency(run.hostLoadAtStart);
331
- const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
332
- if (conflictRun.abortResult) {
333
- return conflictRun.abortResult;
334
- }
335
- await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
336
- // The following added-by and tombstone phases may still make authenticated
337
- // vault calls. Make a completed sub-threshold download tail durable before
338
- // they begin, so their failure cannot turn a successful bulk transfer into
339
- // a full replay on the next pass.
340
- flushDownloadedJournalTail(run);
341
- await emitAndReportNewFiles(run, plan);
342
- await verifyPlannedJournalTombstones(run, plan);
343
- executeJournalTombstoneDeletes(run, plan, counters);
344
- emitScopeMaterializationGap(run, plan);
345
- return finalizePullRun(run, plan, scopeRun, counters);
346
- }
347
- finally {
348
- // Release a pass-local lease even on conflict, transfer, or auth failure.
349
- // The one bounded hot lease remains available only for its same vault.
350
- run.releaseKeyedJournal?.();
327
+ for (;;) {
328
+ const run = await buildPullContext(options);
329
+ try {
330
+ const plan = await planPull(run);
331
+ emitPullPlan(run.emit, plan);
332
+ reportInvalidRemoteKeys(run, plan);
333
+ const scopeRun = run.keyedJournal
334
+ ? emptyTargetedScopeRun(run)
335
+ : executeScopeShrink(run, planScopeShrink(run));
336
+ const counters = createPullCounters();
337
+ // A streaming all-mode planner retains only executor-relevant items. Its
338
+ // ordinary skips have already been counted while the page was live.
339
+ counters.filesSkipped = plan.preplannedFilesSkipped;
340
+ counters.filesOutOfScope = plan.preplannedFilesOutOfScope;
341
+ const transferConcurrency = resolveTransferConcurrency(run.hostLoadAtStart);
342
+ const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
343
+ if (conflictRun.abortResult) {
344
+ return conflictRun.abortResult;
345
+ }
346
+ await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
347
+ // The following added-by and tombstone phases may still make authenticated
348
+ // vault calls. Make a completed sub-threshold download tail durable before
349
+ // they begin, so their failure cannot turn a successful bulk transfer into
350
+ // a full replay on the next pass.
351
+ flushDownloadedJournalTail(run);
352
+ await emitAndReportNewFiles(run, plan);
353
+ await verifyPlannedJournalTombstones(run, plan);
354
+ executeJournalTombstoneDeletes(run, plan, counters);
355
+ emitScopeMaterializationGap(run, plan);
356
+ return finalizePullRun(run, plan, scopeRun, counters);
357
+ }
358
+ catch (error) {
359
+ if (error instanceof CooperativePullHandoff)
360
+ continue;
361
+ throw error;
362
+ }
363
+ finally {
364
+ // Release a pass-local lease even on conflict, transfer, or auth failure.
365
+ // The one bounded hot lease remains available only for its same vault.
366
+ run.releaseKeyedJournal?.();
367
+ }
351
368
  }
352
369
  }
353
370
  async function buildPullContext(options) {
@@ -436,8 +453,9 @@ async function buildPullContext(options) {
436
453
  ? targetedJournalSession(journalSlug)
437
454
  : undefined;
438
455
  const keyedJournal = keyedJournalLease?.store;
456
+ let remoteFiles;
439
457
  try {
440
- const remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
458
+ remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
441
459
  const fileTombstones = options.fileTombstones ??
442
460
  await fetchFileTombstones(vaultConfig, options.personalMode === true
443
461
  ? { personal: true }
@@ -485,43 +503,27 @@ async function buildPullContext(options) {
485
503
  catch (error) {
486
504
  // `buildPullContext` can fail during LIST/freshness. Do not leave the
487
505
  // leased projection marked in use when no PullRunContext is returned.
506
+ if (remoteFiles && isStreamingRemoteFileSource(remoteFiles))
507
+ remoteFiles.dispose();
488
508
  keyedJournalLease?.release();
489
509
  throw error;
490
510
  }
491
511
  }
492
- /**
493
- * List the remote objects the current pull can actually materialize.
494
- *
495
- * Historically this was ALWAYS a full-bucket `listRemoteFiles(ctx)` — even for
496
- * a `shared`/`custom` membership whose effective scope is a handful of
497
- * prefixes — so every pull materialized the entire remote listing (tens of
498
- * thousands of RemoteFile objects on large vaults) only for `computePullPlan`
499
- * to discard most of it as `skip-out-of-scope`. Scoped pulls now list ONLY the
500
- * in-scope prefixes via `listRemoteForScope` (per-prefix ListObjectsV2 calls,
501
- * bounded parallel, deduped), which bounds the listing's memory to the scope's
502
- * actual footprint.
503
- *
504
- * `all`-mode (an unnarrowed full-vault pull, `listedPrefixSet` `[""]`) keeps
505
- * the single broad listing: fanning `all` out by top-level prefix would need a
506
- * delimiter LIST to discover the top-level names first and still union the
507
- * same total object count in one plan, so it bounds nothing today. (A
508
- * streaming planner is the real fix for `all` and is out of scope here.)
509
- *
510
- * An empty `listedPrefixSet` in a scoped mode means "nothing in scope" —
511
- * issue zero LIST calls. Consumers already treat an empty listing correctly:
512
- * the deletion sweep in `computePullPlan` only considers journal keys covered
513
- * by the LISTED scope, so keys absent because they were never listed can never
514
- * be mistaken for remote deletes.
515
- */
512
+ function isStreamingRemoteFileSource(source) {
513
+ return !Array.isArray(source);
514
+ }
515
+ function allModeRemoteFileStream(ctx) {
516
+ return spoolRemoteFiles(streamRemoteFiles(ctx));
517
+ }
516
518
  async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargetedPull) {
517
519
  const entries = toScopePrefixEntries(listedPrefixSet);
518
520
  if (entries.some((e) => e.prefix === ""))
519
- return listRemoteFiles(ctx);
521
+ return allModeRemoteFileStream(ctx);
520
522
  if (entries.length === 0) {
521
523
  // `all` mode always carries `[""]`, so an empty set only occurs for a
522
524
  // scoped mode (empty grants, or a targeted intersection that excluded
523
525
  // everything). Defensive `all` branch: never silently narrow a full pull.
524
- return syncMode === "all" ? listRemoteFiles(ctx) : [];
526
+ return syncMode === "all" ? allModeRemoteFileStream(ctx) : [];
525
527
  }
526
528
  return listRemoteForScope({
527
529
  ctx,
@@ -542,8 +544,14 @@ async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargete
542
544
  },
543
545
  });
544
546
  }
545
- function planPull(run) {
546
- return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount);
547
+ async function planPull(run) {
548
+ try {
549
+ return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount);
550
+ }
551
+ finally {
552
+ if (isStreamingRemoteFileSource(run.remoteFiles))
553
+ run.remoteFiles.dispose();
554
+ }
547
555
  }
548
556
  /** Build only the rows a targeted remote LIST can observe. */
549
557
  function journalForTargetedPull(store, prefixes) {
@@ -1253,10 +1261,18 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
1253
1261
  discardProbe();
1254
1262
  }
1255
1263
  }
1256
- async function executeDownloadExecutor(run, downloadItems, transferConcurrency, counters) {
1257
- if (downloadItems.length === 0)
1258
- return;
1259
- await primeObjectTransport(run.ctx, "get", downloadItems.map((d) => d.remoteFile.key));
1264
+ /**
1265
+ * The old plan may have tombstone evidence that predates a realtime hand-off.
1266
+ * Rebuilding from disk after a hand-off makes the continuation observe the
1267
+ * realtime journal and a fresh LIST before it can perform another mutation.
1268
+ */
1269
+ class CooperativePullHandoff extends Error {
1270
+ constructor() {
1271
+ super("full pull handed off to realtime work");
1272
+ this.name = "CooperativePullHandoff";
1273
+ }
1274
+ }
1275
+ async function executeDownloadQueue(run, downloadItems, transferConcurrency, counters, deferJournalCheckpoint) {
1260
1276
  const queue = [...downloadItems];
1261
1277
  const inFlight = new Set();
1262
1278
  const workerErrors = [];
@@ -1271,7 +1287,7 @@ async function executeDownloadExecutor(run, downloadItems, transferConcurrency,
1271
1287
  inFlight.size < transferConcurrency &&
1272
1288
  queue.length > 0) {
1273
1289
  const downloadItem = queue.shift();
1274
- const download = () => downloadOne(run, downloadItem, counters);
1290
+ const download = () => downloadOne(run, downloadItem, counters, deferJournalCheckpoint);
1275
1291
  const p = (run.options.transferSemaphore
1276
1292
  ? run.options.transferSemaphore.withPermit(download)
1277
1293
  : download())
@@ -1295,16 +1311,44 @@ async function executeDownloadExecutor(run, downloadItems, transferConcurrency,
1295
1311
  break;
1296
1312
  }
1297
1313
  }
1298
- if (workerErrors.length > 0) {
1299
- persistPullJournal(run);
1300
- const first = workerErrors[0];
1301
- if (workerErrors.length > 1) {
1302
- first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
1314
+ return workerErrors;
1315
+ }
1316
+ function throwDownloadWorkerErrors(run, workerErrors) {
1317
+ persistPullJournal(run);
1318
+ const first = workerErrors[0];
1319
+ if (workerErrors.length > 1) {
1320
+ first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
1321
+ }
1322
+ throw first;
1323
+ }
1324
+ async function executeDownloadExecutor(run, downloadItems, transferConcurrency, counters) {
1325
+ if (downloadItems.length === 0)
1326
+ return;
1327
+ await primeObjectTransport(run.ctx, "get", downloadItems.map((d) => d.remoteFile.key));
1328
+ // Ordinary and targeted pulls retain the existing 10k crash-recovery cadence.
1329
+ // Only a watch chore that supplied the cooperative seam pays for drained
1330
+ // units and can return its root lock to queued realtime work.
1331
+ if (!run.options.cooperativeCheckpoint) {
1332
+ const workerErrors = await executeDownloadQueue(run, downloadItems, transferConcurrency, counters, false);
1333
+ if (workerErrors.length > 0)
1334
+ throwDownloadWorkerErrors(run, workerErrors);
1335
+ return;
1336
+ }
1337
+ for (let start = 0; start < downloadItems.length; start += FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE) {
1338
+ const chunk = downloadItems.slice(start, start + FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE);
1339
+ const workerErrors = await executeDownloadQueue(run, chunk, transferConcurrency, counters, true);
1340
+ // `executeDownloadQueue` does not return until every member of this chunk
1341
+ // has settled. Persist before the callback so the suspended chore owns no
1342
+ // mutable pass state while realtime uses the root lock.
1343
+ flushDownloadedJournalTail(run);
1344
+ if (workerErrors.length > 0)
1345
+ throwDownloadWorkerErrors(run, workerErrors);
1346
+ if (await run.options.cooperativeCheckpoint() === true) {
1347
+ throw new CooperativePullHandoff();
1303
1348
  }
1304
- throw first;
1305
1349
  }
1306
1350
  }
1307
- async function downloadOne(run, downloadItem, counters) {
1351
+ async function downloadOne(run, downloadItem, counters, deferJournalCheckpoint = false) {
1308
1352
  const { remoteFile, localPath } = downloadItem;
1309
1353
  await refreshRunContextIfExpiring(run);
1310
1354
  if (!isDownloadWritePathStillContained(run.companyRoot, remoteFile.key, localPath)) {
@@ -1461,7 +1505,8 @@ async function downloadOne(run, downloadItem, counters) {
1461
1505
  return;
1462
1506
  }
1463
1507
  run.downloadsSinceJournalCheckpoint++;
1464
- if (run.downloadsSinceJournalCheckpoint >= DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE) {
1508
+ if (!deferJournalCheckpoint &&
1509
+ run.downloadsSinceJournalCheckpoint >= DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE) {
1465
1510
  flushDownloadedJournalTail(run);
1466
1511
  }
1467
1512
  }
@@ -1900,7 +1945,8 @@ class LocalSnapshotChangedError extends Error {
1900
1945
  * the real outcome of the upcoming Stage-2 execution rather than an
1901
1946
  * upper-bound guess.
1902
1947
  *
1903
- * Pure function: no S3 calls, no journal writes, no event emission. The
1948
+ * It performs no writes, journal mutations, or event emission. An all-mode
1949
+ * source performs its paged classification LIST while it is consumed; the
1904
1950
  * caller (`sync()`) is responsible for emitting the resulting plan event
1905
1951
  * before iterating `items`.
1906
1952
  */
@@ -1968,6 +2014,36 @@ excludePrefixes = [],
1968
2014
  // callers preserve the pre-policy, watcher-intent-only behavior.
1969
2015
  propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows(journal).size) {
1970
2016
  const items = [];
2017
+ const streaming = isStreamingRemoteFileSource(remoteFiles);
2018
+ const listedFiles = isStreamingRemoteFileSource(remoteFiles) ? remoteFiles.files : remoteFiles;
2019
+ let preplannedFilesSkipped = 0;
2020
+ let preplannedFilesOutOfScope = 0;
2021
+ let preplannedFilesExcludedByPolicy = 0;
2022
+ if (streaming) {
2023
+ const retain = items.push.bind(items);
2024
+ items.push = (...next) => {
2025
+ for (const item of next) {
2026
+ // These actions still need the executor for an event or a state change.
2027
+ if (item.action === "download" ||
2028
+ item.action === "conflict" ||
2029
+ item.action === "tombstone-delete" ||
2030
+ item.action === "skip-junk-key-spelling" ||
2031
+ item.action === "skip-unreadable-link") {
2032
+ retain(item);
2033
+ }
2034
+ else if (item.action === "skip-out-of-scope") {
2035
+ preplannedFilesOutOfScope++;
2036
+ }
2037
+ else if (item.action === "skip-excluded-policy") {
2038
+ preplannedFilesExcludedByPolicy++;
2039
+ }
2040
+ else {
2041
+ preplannedFilesSkipped++;
2042
+ }
2043
+ }
2044
+ return items.length;
2045
+ };
2046
+ }
1971
2047
  // Remote keys the server will never presign — collected so the caller can
1972
2048
  // warn + emit telemetry without allowing one poison object to fail a pull.
1973
2049
  const invalidRemoteKeys = [];
@@ -1983,8 +2059,35 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
1983
2059
  // never gated by this map (see the guard below), so pre-existing families
1984
2060
  // keep syncing untouched until the vault doctor (US-003) collapses them.
1985
2061
  const journaledKeyByCanonical = new Map();
2062
+ // The delete sweep starts with journaled keys and clears them as the streamed
2063
+ // LIST proves each one present. This retains existing journal-key references
2064
+ // instead of a second, whole-vault population of long remote key strings.
2065
+ const unseenJournalKeys = new Set();
2066
+ // Most journal keys have already been normalized to POSIX and can be cleared
2067
+ // directly. Keep the legacy Windows spelling defense-in-depth without
2068
+ // imposing a second map entry on every current journal row.
2069
+ const legacyJournalKeysByPosix = new Map();
2070
+ // The junk-spelling delete guard needs all journal twins, while
2071
+ // `journaledKeyByCanonical` above deliberately retains only the preferred
2072
+ // representative for classification.
2073
+ const journaledSkillKeysByCanonical = new Map();
1986
2074
  for (const journaledKey of journalRows(journal).keys()) {
1987
2075
  onHeartbeat?.();
2076
+ const posixJournaledKey = toPosixKey(journaledKey);
2077
+ // A scoped LIST cannot prove anything about keys outside its prefixes.
2078
+ // Do not copy the entire vault's journal into its presence bookkeeping.
2079
+ const inListedScope = isCoveredByAny(posixJournaledKey, prefixSet);
2080
+ if (inListedScope)
2081
+ unseenJournalKeys.add(journaledKey);
2082
+ if (inListedScope && posixJournaledKey !== journaledKey) {
2083
+ const legacyKeys = legacyJournalKeysByPosix.get(posixJournaledKey);
2084
+ if (legacyKeys === undefined) {
2085
+ legacyJournalKeysByPosix.set(posixJournaledKey, [journaledKey]);
2086
+ }
2087
+ else {
2088
+ legacyKeys.push(journaledKey);
2089
+ }
2090
+ }
1988
2091
  if (!journaledKey.startsWith(SKILLS_KEY_PREFIX))
1989
2092
  continue;
1990
2093
  const canonical = canonicalVaultKeySpelling(journaledKey);
@@ -1992,6 +2095,15 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
1992
2095
  if (existing === undefined || journaledKey === canonical) {
1993
2096
  journaledKeyByCanonical.set(canonical, journaledKey);
1994
2097
  }
2098
+ if (!inListedScope)
2099
+ continue;
2100
+ const skillKeys = journaledSkillKeysByCanonical.get(canonical);
2101
+ if (skillKeys === undefined) {
2102
+ journaledSkillKeysByCanonical.set(canonical, [journaledKey]);
2103
+ }
2104
+ else {
2105
+ skillKeys.push(journaledKey);
2106
+ }
1995
2107
  }
1996
2108
  // Remote keys that ALSO appear as an ancestor of another remote key — i.e.
1997
2109
  // keys that carry child objects under `${key}/…` in this same LIST. The
@@ -2001,22 +2113,40 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
2001
2113
  // objects beneath it. A REAL directory, by contrast, only ever syncs as
2002
2114
  // those child objects — never as a single object AT its own key. So a single
2003
2115
  // remote object sitting at a key that also has children is unambiguously a
2004
- // directory-overlay marker, not a regular file. Precomputed once (pure path
2005
- // derivation, no I/O — the planner is synchronous) so the dir-vs-object
2006
- // collision branch can tell an inert overlay marker apart from a genuine
2007
- // file-vs-directory structural collision. feedback_d2082110.
2008
- const remoteKeysWithChildren = new Set();
2009
- for (const rf of remoteFiles) {
2116
+ // directory-overlay marker, not a regular file. Evidence is complete before
2117
+ // classification for both the spooled all-mode LIST and scoped arrays.
2118
+ // Build scoped evidence incrementally: flatMap would transiently retain an
2119
+ // ancestor string for every row/depth before Set deduplication.
2120
+ const remoteKeysWithChildren = isStreamingRemoteFileSource(remoteFiles)
2121
+ ? remoteFiles.remoteKeysWithChildren
2122
+ : collectRemoteAncestors(remoteFiles);
2123
+ // Both sources replay synchronously. for-await on a scoped array introduces
2124
+ // promise/microtask work per object even though no I/O is being awaited.
2125
+ for (const remoteFile of listedFiles) {
2010
2126
  onHeartbeat?.();
2011
- const key = toPosixKey(rf.key);
2012
- let slash = key.indexOf("/");
2013
- while (slash !== -1) {
2014
- remoteKeysWithChildren.add(key.slice(0, slash));
2015
- slash = key.indexOf("/", slash + 1);
2127
+ const posixRemoteKey = toPosixKey(remoteFile.key);
2128
+ // S3 object keys are POSIX. Match the old delete-sweep membership check:
2129
+ // a legacy backslash journal entry counts as present only when its POSIX
2130
+ // form appears in the LIST, while a malformed backslash remote key does
2131
+ // not accidentally change that behavior.
2132
+ if (remoteFile.key === posixRemoteKey) {
2133
+ unseenJournalKeys.delete(remoteFile.key);
2134
+ for (const legacyKey of legacyJournalKeysByPosix.get(remoteFile.key) ?? []) {
2135
+ unseenJournalKeys.delete(legacyKey);
2136
+ }
2137
+ }
2138
+ if (remoteFile.key.startsWith(SKILLS_KEY_PREFIX)) {
2139
+ // A remote junk spelling means its canonical journal twin is still
2140
+ // remotely present. Keep that twin out of the delete sweep unless an
2141
+ // exact FILE_TOMBSTONE is authoritative, precisely matching the old
2142
+ // remoteCanonicalSkillKeys protection without retaining remote keys.
2143
+ const canonical = canonicalVaultKeySpelling(remoteFile.key);
2144
+ for (const journaledKey of journaledSkillKeysByCanonical.get(canonical) ?? []) {
2145
+ if (fileTombstones.get(toPosixKey(journaledKey)) === undefined) {
2146
+ unseenJournalKeys.delete(journaledKey);
2147
+ }
2148
+ }
2016
2149
  }
2017
- }
2018
- for (const remoteFile of remoteFiles) {
2019
- onHeartbeat?.();
2020
2150
  const invalidKey = classifyVaultKey(remoteFile.key, personalMode ? "personal" : "company");
2021
2151
  if (invalidKey !== null) {
2022
2152
  invalidRemoteKeys.push({
@@ -2669,6 +2799,9 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
2669
2799
  filesToSkip++;
2670
2800
  }
2671
2801
  }
2802
+ filesToSkip += preplannedFilesSkipped;
2803
+ filesExcludedByPolicy += preplannedFilesExcludedByPolicy;
2804
+ filesOutOfScope += preplannedFilesOutOfScope;
2672
2805
  // Bug #9 — cross-machine delete propagation. The peer's push leg
2673
2806
  // already removed the file from S3 (verified in the deep-test
2674
2807
  // addendum: `aws s3 ls` showed the keys absent post-push). Pre-fix
@@ -2677,38 +2810,21 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
2677
2810
  // that-was-there-before" — so the file stayed forever on every
2678
2811
  // receiver and `filesTombstoned: 0` showed up on every pull.
2679
2812
  //
2680
- // Closing the loop: walk the journal, find every entry whose key is
2681
- // NOT in the remote LIST set, AND whose path passes the current
2813
+ // Closing the loop: walk journal keys the streamed LIST did NOT clear, AND
2814
+ // whose path passes the current
2682
2815
  // ignore filter (paths newly excluded by .hqignore must not trigger
2683
2816
  // mass-delete), AND — in personalMode — survives the same companies/*
2684
2817
  // gating the download branch applies. The executor will apply each
2685
2818
  // as a local delete + journal removal. Symmetric to the push side's
2686
2819
  // `propagateDeletes` plan in share.ts.
2687
- const remoteKeySet = new Set();
2688
- // Skill-farm twin protection (companion to the junk-spelling pull guard):
2689
- // when the LIST holds only a junk spelling of a journaled canonical key,
2690
- // the logical object is still remotely present — a raw has() would class
2691
- // the canonical journal entry as remote-deleted, unlink the good local
2692
- // wrapper, and drop the journal entry (after which the next pull would
2693
- // materialize the junk spelling the guard just refused). Track canonical
2694
- // forms of listed skill keys and consult them below.
2695
- const remoteCanonicalSkillKeys = new Set();
2696
- for (const rf of remoteFiles) {
2697
- remoteKeySet.add(rf.key);
2698
- if (rf.key.startsWith(SKILLS_KEY_PREFIX)) {
2699
- remoteCanonicalSkillKeys.add(canonicalVaultKeySpelling(rf.key));
2700
- }
2701
- }
2702
2820
  const tombstones = [];
2703
- for (const key of journalRows(journal).keys()) {
2821
+ for (const key of unseenJournalKeys) {
2704
2822
  // Compare membership in POSIX space. A pre-5.47.2 Windows journal key can
2705
2823
  // carry backslash separators; the remote LIST is always forward-slash, so a
2706
2824
  // raw `has(key)` would miss the match and class a live local file as
2707
2825
  // remote-deleted. normalizeJournalKeys() defuses this on load, but the
2708
2826
  // POSIX compare is defense-in-depth (ridge data-loss, feedback_b8d09d0f).
2709
2827
  const posixKey = toPosixKey(key);
2710
- if (remoteKeySet.has(posixKey))
2711
- continue;
2712
2828
  // Listing-scope guard (targeted sync / scoped listing). The remote LIST
2713
2829
  // now covers ONLY `prefixSet` (scoped modes list per-prefix; a targeted
2714
2830
  // pull narrows further), so "absent from the LIST" is evidence of a remote
@@ -2720,16 +2836,6 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
2720
2836
  // `all` mode passes `[""]`, which covers every key — no behavior change.
2721
2837
  if (!isCoveredByAny(posixKey, prefixSet))
2722
2838
  continue;
2723
- if (posixKey.startsWith(SKILLS_KEY_PREFIX) &&
2724
- remoteCanonicalSkillKeys.has(canonicalVaultKeySpelling(posixKey)) &&
2725
- // An EXACT FILE_TOMBSTONE for the journaled key is an authoritative
2726
- // delete and wins over the twin protection — a lingering junk
2727
- // spelling in the LIST must not keep a deliberately-deleted wrapper
2728
- // alive forever. (The key is absent from the LIST here, so there is
2729
- // no newer re-create to defer to.)
2730
- fileTombstones.get(posixKey) === undefined) {
2731
- continue;
2732
- }
2733
2839
  // Fast-path ignore, BEFORE containment canonicalization. The tombstone
2734
2840
  // sweep runs for every journaled key, and a stale journal can carry
2735
2841
  // hundreds of thousands of entries under an ignored runtime tree (e.g.
@@ -2838,6 +2944,8 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
2838
2944
  }
2839
2945
  return {
2840
2946
  items,
2947
+ preplannedFilesSkipped,
2948
+ preplannedFilesOutOfScope,
2841
2949
  filesToDownload,
2842
2950
  bytesToDownload,
2843
2951
  filesToSkip,