@indigoai-us/hq-cloud 6.16.51-alpha.0 → 6.16.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/dist/backup-prune.d.ts +20 -0
  2. package/dist/backup-prune.d.ts.map +1 -1
  3. package/dist/backup-prune.js +62 -3
  4. package/dist/backup-prune.js.map +1 -1
  5. package/dist/backup-prune.test.js +25 -0
  6. package/dist/backup-prune.test.js.map +1 -1
  7. package/dist/bin/rescue-runner.js +2 -1
  8. package/dist/bin/rescue-runner.js.map +1 -1
  9. package/dist/bin/sync-runner-company.d.ts +0 -6
  10. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  11. package/dist/bin/sync-runner-company.js +1 -30
  12. package/dist/bin/sync-runner-company.js.map +1 -1
  13. package/dist/bin/sync-runner-company.test.js +0 -55
  14. package/dist/bin/sync-runner-company.test.js.map +1 -1
  15. package/dist/bin/sync-runner-planning.d.ts +3 -12
  16. package/dist/bin/sync-runner-planning.d.ts.map +1 -1
  17. package/dist/bin/sync-runner-planning.js +3 -65
  18. package/dist/bin/sync-runner-planning.js.map +1 -1
  19. package/dist/bin/sync-runner-planning.test.js +1 -108
  20. package/dist/bin/sync-runner-planning.test.js.map +1 -1
  21. package/dist/bin/sync-runner-watch-loop.d.ts +2 -48
  22. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  23. package/dist/bin/sync-runner-watch-loop.js +113 -281
  24. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  25. package/dist/bin/sync-runner.d.ts +5 -33
  26. package/dist/bin/sync-runner.d.ts.map +1 -1
  27. package/dist/bin/sync-runner.js +4 -34
  28. package/dist/bin/sync-runner.js.map +1 -1
  29. package/dist/bin/sync-runner.test.js +78 -555
  30. package/dist/bin/sync-runner.test.js.map +1 -1
  31. package/dist/cli/rescue-breaker.test.d.ts +2 -0
  32. package/dist/cli/rescue-breaker.test.d.ts.map +1 -0
  33. package/dist/cli/rescue-breaker.test.js +334 -0
  34. package/dist/cli/rescue-breaker.test.js.map +1 -0
  35. package/dist/cli/rescue-core.d.ts +40 -0
  36. package/dist/cli/rescue-core.d.ts.map +1 -1
  37. package/dist/cli/rescue-core.js +1205 -109
  38. package/dist/cli/rescue-core.js.map +1 -1
  39. package/dist/cli/rescue-realpath.test.d.ts +2 -0
  40. package/dist/cli/rescue-realpath.test.d.ts.map +1 -0
  41. package/dist/cli/rescue-realpath.test.js +109 -0
  42. package/dist/cli/rescue-realpath.test.js.map +1 -0
  43. package/dist/cli/rescue-rsync-preflight.test.js +256 -4
  44. package/dist/cli/rescue-rsync-preflight.test.js.map +1 -1
  45. package/dist/cli/rescue-snapshot-copy.test.d.ts +2 -0
  46. package/dist/cli/rescue-snapshot-copy.test.d.ts.map +1 -0
  47. package/dist/cli/rescue-snapshot-copy.test.js +435 -0
  48. package/dist/cli/rescue-snapshot-copy.test.js.map +1 -0
  49. package/dist/cli/rescue-snapshot-recovery.test.d.ts +2 -0
  50. package/dist/cli/rescue-snapshot-recovery.test.d.ts.map +1 -0
  51. package/dist/cli/rescue-snapshot-recovery.test.js +260 -0
  52. package/dist/cli/rescue-snapshot-recovery.test.js.map +1 -0
  53. package/dist/cli/rescue-source-switch-floor.test.d.ts +2 -0
  54. package/dist/cli/rescue-source-switch-floor.test.d.ts.map +1 -0
  55. package/dist/cli/rescue-source-switch-floor.test.js +210 -0
  56. package/dist/cli/rescue-source-switch-floor.test.js.map +1 -0
  57. package/dist/cli/rescue.d.ts +4 -0
  58. package/dist/cli/rescue.d.ts.map +1 -1
  59. package/dist/cli/rescue.js +20 -1
  60. package/dist/cli/rescue.js.map +1 -1
  61. package/dist/cli/rescue.test.js +16 -1
  62. package/dist/cli/rescue.test.js.map +1 -1
  63. package/dist/cli/share-presign-transport.test.js +2 -0
  64. package/dist/cli/share-presign-transport.test.js.map +1 -1
  65. package/dist/cli/share.d.ts +0 -19
  66. package/dist/cli/share.d.ts.map +1 -1
  67. package/dist/cli/share.js +37 -57
  68. package/dist/cli/share.js.map +1 -1
  69. package/dist/cli/share.test.js +93 -87
  70. package/dist/cli/share.test.js.map +1 -1
  71. package/dist/cli/sync-scope.test.js +0 -8
  72. package/dist/cli/sync-scope.test.js.map +1 -1
  73. package/dist/cli/sync.d.ts +0 -12
  74. package/dist/cli/sync.d.ts.map +1 -1
  75. package/dist/cli/sync.js +118 -226
  76. package/dist/cli/sync.js.map +1 -1
  77. package/dist/cli/sync.test.js +2 -215
  78. package/dist/cli/sync.test.js.map +1 -1
  79. package/dist/cli/watch-event-push-conflict.test.js +2 -0
  80. package/dist/cli/watch-event-push-conflict.test.js.map +1 -1
  81. package/dist/cognito-auth.d.ts +30 -0
  82. package/dist/cognito-auth.d.ts.map +1 -1
  83. package/dist/cognito-auth.js +175 -0
  84. package/dist/cognito-auth.js.map +1 -1
  85. package/dist/index.d.ts +2 -2
  86. package/dist/index.d.ts.map +1 -1
  87. package/dist/index.js +1 -1
  88. package/dist/index.js.map +1 -1
  89. package/dist/journal-store.test.js +7 -130
  90. package/dist/journal-store.test.js.map +1 -1
  91. package/dist/journal.d.ts +1 -9
  92. package/dist/journal.d.ts.map +1 -1
  93. package/dist/journal.js +21 -30
  94. package/dist/journal.js.map +1 -1
  95. package/dist/lib/exit-codes.d.ts +0 -15
  96. package/dist/lib/exit-codes.d.ts.map +1 -1
  97. package/dist/lib/exit-codes.js +0 -15
  98. package/dist/lib/exit-codes.js.map +1 -1
  99. package/dist/machine-auth.test.js +295 -0
  100. package/dist/machine-auth.test.js.map +1 -1
  101. package/dist/object-io.d.ts +2 -2
  102. package/dist/object-io.d.ts.map +1 -1
  103. package/dist/object-io.js +18 -3
  104. package/dist/object-io.js.map +1 -1
  105. package/dist/personal-vault.d.ts.map +1 -1
  106. package/dist/personal-vault.js +3 -10
  107. package/dist/personal-vault.js.map +1 -1
  108. package/dist/personal-vault.test.js +0 -35
  109. package/dist/personal-vault.test.js.map +1 -1
  110. package/dist/remote-pull.d.ts.map +1 -1
  111. package/dist/remote-pull.js +2 -9
  112. package/dist/remote-pull.js.map +1 -1
  113. package/dist/remote-pull.test.js +2 -59
  114. package/dist/remote-pull.test.js.map +1 -1
  115. package/dist/s3.d.ts +18 -1
  116. package/dist/s3.d.ts.map +1 -1
  117. package/dist/s3.js +47 -24
  118. package/dist/s3.js.map +1 -1
  119. package/dist/s3.test.js +27 -1
  120. package/dist/s3.test.js.map +1 -1
  121. package/dist/sync/realtime-telemetry.d.ts +1 -1
  122. package/dist/sync/realtime-telemetry.d.ts.map +1 -1
  123. package/dist/sync/realtime-telemetry.js +6 -14
  124. package/dist/sync/realtime-telemetry.js.map +1 -1
  125. package/dist/sync/realtime-telemetry.test.js +16 -136
  126. package/dist/sync/realtime-telemetry.test.js.map +1 -1
  127. package/dist/sync-core.d.ts +6 -4
  128. package/dist/sync-core.d.ts.map +1 -1
  129. package/dist/sync-core.js +10 -4
  130. package/dist/sync-core.js.map +1 -1
  131. package/dist/sync-core.test.js +25 -1
  132. package/dist/sync-core.test.js.map +1 -1
  133. package/package.json +1 -2
  134. package/dist/cli/onepass-pull-planner-regressions.test.d.ts +0 -2
  135. package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +0 -1
  136. package/dist/cli/onepass-pull-planner-regressions.test.js +0 -300
  137. package/dist/cli/onepass-pull-planner-regressions.test.js.map +0 -1
  138. package/dist/cli/onepass-pull-planner.test.d.ts +0 -2
  139. package/dist/cli/onepass-pull-planner.test.d.ts.map +0 -1
  140. package/dist/cli/onepass-pull-planner.test.js +0 -266
  141. package/dist/cli/onepass-pull-planner.test.js.map +0 -1
  142. package/dist/cli/streaming-pull-planner.bench.test.d.ts +0 -15
  143. package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +0 -1
  144. package/dist/cli/streaming-pull-planner.bench.test.js +0 -273
  145. package/dist/cli/streaming-pull-planner.bench.test.js.map +0 -1
  146. package/dist/s3.list-cost.test.d.ts +0 -2
  147. package/dist/s3.list-cost.test.d.ts.map +0 -1
  148. package/dist/s3.list-cost.test.js +0 -51
  149. package/dist/s3.list-cost.test.js.map +0 -1
  150. package/dist/sync/remote-file-spool.d.ts +0 -16
  151. package/dist/sync/remote-file-spool.d.ts.map +0 -1
  152. package/dist/sync/remote-file-spool.js +0 -91
  153. package/dist/sync/remote-file-spool.js.map +0 -1
  154. package/dist/sync/remote-file-spool.test.d.ts +0 -2
  155. package/dist/sync/remote-file-spool.test.d.ts.map +0 -1
  156. package/dist/sync/remote-file-spool.test.js +0 -77
  157. package/dist/sync/remote-file-spool.test.js.map +0 -1
package/dist/cli/sync.js CHANGED
@@ -6,7 +6,6 @@
6
6
  */
7
7
  import * as fs from "fs";
8
8
  import * as path from "path";
9
- import { spoolRemoteFiles, collectRemoteAncestors } from "../sync/remote-file-spool.js";
10
9
  import { VaultAuthError, VaultClient } from "../vault-client.js";
11
10
  import { readlinkOrNull } from "../lib/readlink-safe.js";
12
11
  import { describeError } from "../lib/describe-error.js";
@@ -17,7 +16,7 @@ import { ensureCompanyVaultTransport } from "../company-vault-transport.js";
17
16
  import { createSyncProgressRecorder } from "../sync-progress.js";
18
17
  import { canonicalVaultKeySpelling, localPathForVaultKey, vaultKeyForLocalPath, } from "../local-path-codec.js";
19
18
  import { SKILLS_KEY_PREFIX } from "./doctor.js";
20
- import { downloadFile, streamRemoteFiles, headRemoteFile, primeObjectTransport, toPosixKey, classifyVaultKey, DanglingSymlinkParentError, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
19
+ import { downloadFile, listRemoteFiles, headRemoteFile, primeObjectTransport, toPosixKey, classifyVaultKey, DanglingSymlinkParentError, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
21
20
  import { readJournalForPass, writeJournal, getStateDir, openJournalStoreSession, buildJournalBaseline, journalDeltaFromBaseline, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, updateEntry, removeEntry, getEntry, normalizeEtag, migrateToV2, gcTombstones, isTombstone, tombstoneEntry, lastPullRecord, appendPullRecord, generatePullId, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, } from "../journal.js";
22
21
  import { isGeneratedCoreMirrorKey, PERSONAL_VAULT_MANIFEST_KEY, } from "../personal-vault.js";
23
22
  import { isPersonalVaultExcluded } from "../personal-vault-exclusions.js";
@@ -186,11 +185,6 @@ export function targetedJournalSessionCensusSizes() {
186
185
  * dirty entries through a run-scoped journal writer.
187
186
  */
188
187
  const DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE = 10_000;
189
- /**
190
- * A full-pull hand-off unit. Every member has finished its transfer and the
191
- * resulting journal delta is durable before realtime can acquire the writer.
192
- */
193
- export const FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE = 500;
194
188
  /**
195
189
  * Server cap on files per `/v1/notify/file-added` report. The endpoint rejects
196
190
  * an oversized batch wholesale, so the client MUST split a large report into
@@ -324,47 +318,36 @@ export async function sync(options) {
324
318
  return withOperationLock(options.hqRoot, "sync", () => syncWithOperationLockHeld(options));
325
319
  }
326
320
  async function syncWithOperationLockHeld(options) {
327
- for (;;) {
328
- const run = await buildPullContext(options);
329
- try {
330
- const plan = await planPull(run);
331
- emitPullPlan(run.emit, plan);
332
- reportInvalidRemoteKeys(run, plan);
333
- const scopeRun = run.keyedJournal
334
- ? emptyTargetedScopeRun(run)
335
- : executeScopeShrink(run, planScopeShrink(run));
336
- const counters = createPullCounters();
337
- // A streaming all-mode planner retains only executor-relevant items. Its
338
- // ordinary skips have already been counted while the page was live.
339
- counters.filesSkipped = plan.preplannedFilesSkipped;
340
- counters.filesOutOfScope = plan.preplannedFilesOutOfScope;
341
- const transferConcurrency = resolveTransferConcurrency(run.hostLoadAtStart);
342
- const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
343
- if (conflictRun.abortResult) {
344
- return conflictRun.abortResult;
345
- }
346
- await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
347
- // The following added-by and tombstone phases may still make authenticated
348
- // vault calls. Make a completed sub-threshold download tail durable before
349
- // they begin, so their failure cannot turn a successful bulk transfer into
350
- // a full replay on the next pass.
351
- flushDownloadedJournalTail(run);
352
- await emitAndReportNewFiles(run, plan);
353
- await verifyPlannedJournalTombstones(run, plan);
354
- executeJournalTombstoneDeletes(run, plan, counters);
355
- emitScopeMaterializationGap(run, plan);
356
- return finalizePullRun(run, plan, scopeRun, counters);
357
- }
358
- catch (error) {
359
- if (error instanceof CooperativePullHandoff)
360
- continue;
361
- throw error;
362
- }
363
- finally {
364
- // Release a pass-local lease even on conflict, transfer, or auth failure.
365
- // The one bounded hot lease remains available only for its same vault.
366
- run.releaseKeyedJournal?.();
367
- }
321
+ const run = await buildPullContext(options);
322
+ try {
323
+ const plan = planPull(run);
324
+ emitPullPlan(run.emit, plan);
325
+ reportInvalidRemoteKeys(run, plan);
326
+ const scopeRun = run.keyedJournal
327
+ ? emptyTargetedScopeRun(run)
328
+ : executeScopeShrink(run, planScopeShrink(run));
329
+ const counters = createPullCounters();
330
+ const transferConcurrency = resolveTransferConcurrency(run.hostLoadAtStart);
331
+ const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
332
+ if (conflictRun.abortResult) {
333
+ return conflictRun.abortResult;
334
+ }
335
+ await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
336
+ // The following added-by and tombstone phases may still make authenticated
337
+ // vault calls. Make a completed sub-threshold download tail durable before
338
+ // they begin, so their failure cannot turn a successful bulk transfer into
339
+ // a full replay on the next pass.
340
+ flushDownloadedJournalTail(run);
341
+ await emitAndReportNewFiles(run, plan);
342
+ await verifyPlannedJournalTombstones(run, plan);
343
+ executeJournalTombstoneDeletes(run, plan, counters);
344
+ emitScopeMaterializationGap(run, plan);
345
+ return finalizePullRun(run, plan, scopeRun, counters);
346
+ }
347
+ finally {
348
+ // Release a pass-local lease even on conflict, transfer, or auth failure.
349
+ // The one bounded hot lease remains available only for its same vault.
350
+ run.releaseKeyedJournal?.();
368
351
  }
369
352
  }
370
353
  async function buildPullContext(options) {
@@ -453,9 +436,8 @@ async function buildPullContext(options) {
453
436
  ? targetedJournalSession(journalSlug)
454
437
  : undefined;
455
438
  const keyedJournal = keyedJournalLease?.store;
456
- let remoteFiles;
457
439
  try {
458
- remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
440
+ const remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
459
441
  const fileTombstones = options.fileTombstones ??
460
442
  await fetchFileTombstones(vaultConfig, options.personalMode === true
461
443
  ? { personal: true }
@@ -503,27 +485,43 @@ async function buildPullContext(options) {
503
485
  catch (error) {
504
486
  // `buildPullContext` can fail during LIST/freshness. Do not leave the
505
487
  // leased projection marked in use when no PullRunContext is returned.
506
- if (remoteFiles && isStreamingRemoteFileSource(remoteFiles))
507
- remoteFiles.dispose();
508
488
  keyedJournalLease?.release();
509
489
  throw error;
510
490
  }
511
491
  }
512
- function isStreamingRemoteFileSource(source) {
513
- return !Array.isArray(source);
514
- }
515
- function allModeRemoteFileStream(ctx) {
516
- return spoolRemoteFiles(streamRemoteFiles(ctx));
517
- }
492
+ /**
493
+ * List the remote objects the current pull can actually materialize.
494
+ *
495
+ * Historically this was ALWAYS a full-bucket `listRemoteFiles(ctx)` — even for
496
+ * a `shared`/`custom` membership whose effective scope is a handful of
497
+ * prefixes — so every pull materialized the entire remote listing (tens of
498
+ * thousands of RemoteFile objects on large vaults) only for `computePullPlan`
499
+ * to discard most of it as `skip-out-of-scope`. Scoped pulls now list ONLY the
500
+ * in-scope prefixes via `listRemoteForScope` (per-prefix ListObjectsV2 calls,
501
+ * bounded parallel, deduped), which bounds the listing's memory to the scope's
502
+ * actual footprint.
503
+ *
504
+ * `all`-mode (an unnarrowed full-vault pull, `listedPrefixSet` `[""]`) keeps
505
+ * the single broad listing: fanning `all` out by top-level prefix would need a
506
+ * delimiter LIST to discover the top-level names first and still union the
507
+ * same total object count in one plan, so it bounds nothing today. (A
508
+ * streaming planner is the real fix for `all` and is out of scope here.)
509
+ *
510
+ * An empty `listedPrefixSet` in a scoped mode means "nothing in scope" —
511
+ * issue zero LIST calls. Consumers already treat an empty listing correctly:
512
+ * the deletion sweep in `computePullPlan` only considers journal keys covered
513
+ * by the LISTED scope, so keys absent because they were never listed can never
514
+ * be mistaken for remote deletes.
515
+ */
518
516
  async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargetedPull) {
519
517
  const entries = toScopePrefixEntries(listedPrefixSet);
520
518
  if (entries.some((e) => e.prefix === ""))
521
- return allModeRemoteFileStream(ctx);
519
+ return listRemoteFiles(ctx);
522
520
  if (entries.length === 0) {
523
521
  // `all` mode always carries `[""]`, so an empty set only occurs for a
524
522
  // scoped mode (empty grants, or a targeted intersection that excluded
525
523
  // everything). Defensive `all` branch: never silently narrow a full pull.
526
- return syncMode === "all" ? allModeRemoteFileStream(ctx) : [];
524
+ return syncMode === "all" ? listRemoteFiles(ctx) : [];
527
525
  }
528
526
  return listRemoteForScope({
529
527
  ctx,
@@ -544,14 +542,8 @@ async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargete
544
542
  },
545
543
  });
546
544
  }
547
- async function planPull(run) {
548
- try {
549
- return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount);
550
- }
551
- finally {
552
- if (isStreamingRemoteFileSource(run.remoteFiles))
553
- run.remoteFiles.dispose();
554
- }
545
+ function planPull(run) {
546
+ return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount);
555
547
  }
556
548
  /** Build only the rows a targeted remote LIST can observe. */
557
549
  function journalForTargetedPull(store, prefixes) {
@@ -1261,18 +1253,10 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
1261
1253
  discardProbe();
1262
1254
  }
1263
1255
  }
1264
- /**
1265
- * The old plan may have tombstone evidence that predates a realtime hand-off.
1266
- * Rebuilding from disk after a hand-off makes the continuation observe the
1267
- * realtime journal and a fresh LIST before it can perform another mutation.
1268
- */
1269
- class CooperativePullHandoff extends Error {
1270
- constructor() {
1271
- super("full pull handed off to realtime work");
1272
- this.name = "CooperativePullHandoff";
1273
- }
1274
- }
1275
- async function executeDownloadQueue(run, downloadItems, transferConcurrency, counters, deferJournalCheckpoint) {
1256
+ async function executeDownloadExecutor(run, downloadItems, transferConcurrency, counters) {
1257
+ if (downloadItems.length === 0)
1258
+ return;
1259
+ await primeObjectTransport(run.ctx, "get", downloadItems.map((d) => d.remoteFile.key));
1276
1260
  const queue = [...downloadItems];
1277
1261
  const inFlight = new Set();
1278
1262
  const workerErrors = [];
@@ -1287,7 +1271,7 @@ async function executeDownloadQueue(run, downloadItems, transferConcurrency, cou
1287
1271
  inFlight.size < transferConcurrency &&
1288
1272
  queue.length > 0) {
1289
1273
  const downloadItem = queue.shift();
1290
- const download = () => downloadOne(run, downloadItem, counters, deferJournalCheckpoint);
1274
+ const download = () => downloadOne(run, downloadItem, counters);
1291
1275
  const p = (run.options.transferSemaphore
1292
1276
  ? run.options.transferSemaphore.withPermit(download)
1293
1277
  : download())
@@ -1311,44 +1295,16 @@ async function executeDownloadQueue(run, downloadItems, transferConcurrency, cou
1311
1295
  break;
1312
1296
  }
1313
1297
  }
1314
- return workerErrors;
1315
- }
1316
- function throwDownloadWorkerErrors(run, workerErrors) {
1317
- persistPullJournal(run);
1318
- const first = workerErrors[0];
1319
- if (workerErrors.length > 1) {
1320
- first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
1321
- }
1322
- throw first;
1323
- }
1324
- async function executeDownloadExecutor(run, downloadItems, transferConcurrency, counters) {
1325
- if (downloadItems.length === 0)
1326
- return;
1327
- await primeObjectTransport(run.ctx, "get", downloadItems.map((d) => d.remoteFile.key));
1328
- // Ordinary and targeted pulls retain the existing 10k crash-recovery cadence.
1329
- // Only a watch chore that supplied the cooperative seam pays for drained
1330
- // units and can return its root lock to queued realtime work.
1331
- if (!run.options.cooperativeCheckpoint) {
1332
- const workerErrors = await executeDownloadQueue(run, downloadItems, transferConcurrency, counters, false);
1333
- if (workerErrors.length > 0)
1334
- throwDownloadWorkerErrors(run, workerErrors);
1335
- return;
1336
- }
1337
- for (let start = 0; start < downloadItems.length; start += FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE) {
1338
- const chunk = downloadItems.slice(start, start + FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE);
1339
- const workerErrors = await executeDownloadQueue(run, chunk, transferConcurrency, counters, true);
1340
- // `executeDownloadQueue` does not return until every member of this chunk
1341
- // has settled. Persist before the callback so the suspended chore owns no
1342
- // mutable pass state while realtime uses the root lock.
1343
- flushDownloadedJournalTail(run);
1344
- if (workerErrors.length > 0)
1345
- throwDownloadWorkerErrors(run, workerErrors);
1346
- if (await run.options.cooperativeCheckpoint() === true) {
1347
- throw new CooperativePullHandoff();
1298
+ if (workerErrors.length > 0) {
1299
+ persistPullJournal(run);
1300
+ const first = workerErrors[0];
1301
+ if (workerErrors.length > 1) {
1302
+ first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
1348
1303
  }
1304
+ throw first;
1349
1305
  }
1350
1306
  }
1351
- async function downloadOne(run, downloadItem, counters, deferJournalCheckpoint = false) {
1307
+ async function downloadOne(run, downloadItem, counters) {
1352
1308
  const { remoteFile, localPath } = downloadItem;
1353
1309
  await refreshRunContextIfExpiring(run);
1354
1310
  if (!isDownloadWritePathStillContained(run.companyRoot, remoteFile.key, localPath)) {
@@ -1505,8 +1461,7 @@ async function downloadOne(run, downloadItem, counters, deferJournalCheckpoint =
1505
1461
  return;
1506
1462
  }
1507
1463
  run.downloadsSinceJournalCheckpoint++;
1508
- if (!deferJournalCheckpoint &&
1509
- run.downloadsSinceJournalCheckpoint >= DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE) {
1464
+ if (run.downloadsSinceJournalCheckpoint >= DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE) {
1510
1465
  flushDownloadedJournalTail(run);
1511
1466
  }
1512
1467
  }
@@ -1945,8 +1900,7 @@ class LocalSnapshotChangedError extends Error {
1945
1900
  * the real outcome of the upcoming Stage-2 execution rather than an
1946
1901
  * upper-bound guess.
1947
1902
  *
1948
- * It performs no writes, journal mutations, or event emission. An all-mode
1949
- * source performs its paged classification LIST while it is consumed; the
1903
+ * Pure function: no S3 calls, no journal writes, no event emission. The
1950
1904
  * caller (`sync()`) is responsible for emitting the resulting plan event
1951
1905
  * before iterating `items`.
1952
1906
  */
@@ -2014,36 +1968,6 @@ excludePrefixes = [],
2014
1968
  // callers preserve the pre-policy, watcher-intent-only behavior.
2015
1969
  propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows(journal).size) {
2016
1970
  const items = [];
2017
- const streaming = isStreamingRemoteFileSource(remoteFiles);
2018
- const listedFiles = isStreamingRemoteFileSource(remoteFiles) ? remoteFiles.files : remoteFiles;
2019
- let preplannedFilesSkipped = 0;
2020
- let preplannedFilesOutOfScope = 0;
2021
- let preplannedFilesExcludedByPolicy = 0;
2022
- if (streaming) {
2023
- const retain = items.push.bind(items);
2024
- items.push = (...next) => {
2025
- for (const item of next) {
2026
- // These actions still need the executor for an event or a state change.
2027
- if (item.action === "download" ||
2028
- item.action === "conflict" ||
2029
- item.action === "tombstone-delete" ||
2030
- item.action === "skip-junk-key-spelling" ||
2031
- item.action === "skip-unreadable-link") {
2032
- retain(item);
2033
- }
2034
- else if (item.action === "skip-out-of-scope") {
2035
- preplannedFilesOutOfScope++;
2036
- }
2037
- else if (item.action === "skip-excluded-policy") {
2038
- preplannedFilesExcludedByPolicy++;
2039
- }
2040
- else {
2041
- preplannedFilesSkipped++;
2042
- }
2043
- }
2044
- return items.length;
2045
- };
2046
- }
2047
1971
  // Remote keys the server will never presign — collected so the caller can
2048
1972
  // warn + emit telemetry without allowing one poison object to fail a pull.
2049
1973
  const invalidRemoteKeys = [];
@@ -2059,35 +1983,8 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
2059
1983
  // never gated by this map (see the guard below), so pre-existing families
2060
1984
  // keep syncing untouched until the vault doctor (US-003) collapses them.
2061
1985
  const journaledKeyByCanonical = new Map();
2062
- // The delete sweep starts with journaled keys and clears them as the streamed
2063
- // LIST proves each one present. This retains existing journal-key references
2064
- // instead of a second, whole-vault population of long remote key strings.
2065
- const unseenJournalKeys = new Set();
2066
- // Most journal keys have already been normalized to POSIX and can be cleared
2067
- // directly. Keep the legacy Windows spelling defense-in-depth without
2068
- // imposing a second map entry on every current journal row.
2069
- const legacyJournalKeysByPosix = new Map();
2070
- // The junk-spelling delete guard needs all journal twins, while
2071
- // `journaledKeyByCanonical` above deliberately retains only the preferred
2072
- // representative for classification.
2073
- const journaledSkillKeysByCanonical = new Map();
2074
1986
  for (const journaledKey of journalRows(journal).keys()) {
2075
1987
  onHeartbeat?.();
2076
- const posixJournaledKey = toPosixKey(journaledKey);
2077
- // A scoped LIST cannot prove anything about keys outside its prefixes.
2078
- // Do not copy the entire vault's journal into its presence bookkeeping.
2079
- const inListedScope = isCoveredByAny(posixJournaledKey, prefixSet);
2080
- if (inListedScope)
2081
- unseenJournalKeys.add(journaledKey);
2082
- if (inListedScope && posixJournaledKey !== journaledKey) {
2083
- const legacyKeys = legacyJournalKeysByPosix.get(posixJournaledKey);
2084
- if (legacyKeys === undefined) {
2085
- legacyJournalKeysByPosix.set(posixJournaledKey, [journaledKey]);
2086
- }
2087
- else {
2088
- legacyKeys.push(journaledKey);
2089
- }
2090
- }
2091
1988
  if (!journaledKey.startsWith(SKILLS_KEY_PREFIX))
2092
1989
  continue;
2093
1990
  const canonical = canonicalVaultKeySpelling(journaledKey);
@@ -2095,15 +1992,6 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
2095
1992
  if (existing === undefined || journaledKey === canonical) {
2096
1993
  journaledKeyByCanonical.set(canonical, journaledKey);
2097
1994
  }
2098
- if (!inListedScope)
2099
- continue;
2100
- const skillKeys = journaledSkillKeysByCanonical.get(canonical);
2101
- if (skillKeys === undefined) {
2102
- journaledSkillKeysByCanonical.set(canonical, [journaledKey]);
2103
- }
2104
- else {
2105
- skillKeys.push(journaledKey);
2106
- }
2107
1995
  }
2108
1996
  // Remote keys that ALSO appear as an ancestor of another remote key — i.e.
2109
1997
  // keys that carry child objects under `${key}/…` in this same LIST. The
@@ -2113,40 +2001,22 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
2113
2001
  // objects beneath it. A REAL directory, by contrast, only ever syncs as
2114
2002
  // those child objects — never as a single object AT its own key. So a single
2115
2003
  // remote object sitting at a key that also has children is unambiguously a
2116
- // directory-overlay marker, not a regular file. Evidence is complete before
2117
- // classification for both the spooled all-mode LIST and scoped arrays.
2118
- // Build scoped evidence incrementally: flatMap would transiently retain an
2119
- // ancestor string for every row/depth before Set deduplication.
2120
- const remoteKeysWithChildren = isStreamingRemoteFileSource(remoteFiles)
2121
- ? remoteFiles.remoteKeysWithChildren
2122
- : collectRemoteAncestors(remoteFiles);
2123
- // Both sources replay synchronously. for-await on a scoped array introduces
2124
- // promise/microtask work per object even though no I/O is being awaited.
2125
- for (const remoteFile of listedFiles) {
2004
+ // directory-overlay marker, not a regular file. Precomputed once (pure path
2005
+ // derivation, no I/O — the planner is synchronous) so the dir-vs-object
2006
+ // collision branch can tell an inert overlay marker apart from a genuine
2007
+ // file-vs-directory structural collision. feedback_d2082110.
2008
+ const remoteKeysWithChildren = new Set();
2009
+ for (const rf of remoteFiles) {
2126
2010
  onHeartbeat?.();
2127
- const posixRemoteKey = toPosixKey(remoteFile.key);
2128
- // S3 object keys are POSIX. Match the old delete-sweep membership check:
2129
- // a legacy backslash journal entry counts as present only when its POSIX
2130
- // form appears in the LIST, while a malformed backslash remote key does
2131
- // not accidentally change that behavior.
2132
- if (remoteFile.key === posixRemoteKey) {
2133
- unseenJournalKeys.delete(remoteFile.key);
2134
- for (const legacyKey of legacyJournalKeysByPosix.get(remoteFile.key) ?? []) {
2135
- unseenJournalKeys.delete(legacyKey);
2136
- }
2137
- }
2138
- if (remoteFile.key.startsWith(SKILLS_KEY_PREFIX)) {
2139
- // A remote junk spelling means its canonical journal twin is still
2140
- // remotely present. Keep that twin out of the delete sweep unless an
2141
- // exact FILE_TOMBSTONE is authoritative, precisely matching the old
2142
- // remoteCanonicalSkillKeys protection without retaining remote keys.
2143
- const canonical = canonicalVaultKeySpelling(remoteFile.key);
2144
- for (const journaledKey of journaledSkillKeysByCanonical.get(canonical) ?? []) {
2145
- if (fileTombstones.get(toPosixKey(journaledKey)) === undefined) {
2146
- unseenJournalKeys.delete(journaledKey);
2147
- }
2148
- }
2011
+ const key = toPosixKey(rf.key);
2012
+ let slash = key.indexOf("/");
2013
+ while (slash !== -1) {
2014
+ remoteKeysWithChildren.add(key.slice(0, slash));
2015
+ slash = key.indexOf("/", slash + 1);
2149
2016
  }
2017
+ }
2018
+ for (const remoteFile of remoteFiles) {
2019
+ onHeartbeat?.();
2150
2020
  const invalidKey = classifyVaultKey(remoteFile.key, personalMode ? "personal" : "company");
2151
2021
  if (invalidKey !== null) {
2152
2022
  invalidRemoteKeys.push({
@@ -2799,9 +2669,6 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
2799
2669
  filesToSkip++;
2800
2670
  }
2801
2671
  }
2802
- filesToSkip += preplannedFilesSkipped;
2803
- filesExcludedByPolicy += preplannedFilesExcludedByPolicy;
2804
- filesOutOfScope += preplannedFilesOutOfScope;
2805
2672
  // Bug #9 — cross-machine delete propagation. The peer's push leg
2806
2673
  // already removed the file from S3 (verified in the deep-test
2807
2674
  // addendum: `aws s3 ls` showed the keys absent post-push). Pre-fix
@@ -2810,21 +2677,38 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
2810
2677
  // that-was-there-before" — so the file stayed forever on every
2811
2678
  // receiver and `filesTombstoned: 0` showed up on every pull.
2812
2679
  //
2813
- // Closing the loop: walk journal keys the streamed LIST did NOT clear, AND
2814
- // whose path passes the current
2680
+ // Closing the loop: walk the journal, find every entry whose key is
2681
+ // NOT in the remote LIST set, AND whose path passes the current
2815
2682
  // ignore filter (paths newly excluded by .hqignore must not trigger
2816
2683
  // mass-delete), AND — in personalMode — survives the same companies/*
2817
2684
  // gating the download branch applies. The executor will apply each
2818
2685
  // as a local delete + journal removal. Symmetric to the push side's
2819
2686
  // `propagateDeletes` plan in share.ts.
2687
+ const remoteKeySet = new Set();
2688
+ // Skill-farm twin protection (companion to the junk-spelling pull guard):
2689
+ // when the LIST holds only a junk spelling of a journaled canonical key,
2690
+ // the logical object is still remotely present — a raw has() would class
2691
+ // the canonical journal entry as remote-deleted, unlink the good local
2692
+ // wrapper, and drop the journal entry (after which the next pull would
2693
+ // materialize the junk spelling the guard just refused). Track canonical
2694
+ // forms of listed skill keys and consult them below.
2695
+ const remoteCanonicalSkillKeys = new Set();
2696
+ for (const rf of remoteFiles) {
2697
+ remoteKeySet.add(rf.key);
2698
+ if (rf.key.startsWith(SKILLS_KEY_PREFIX)) {
2699
+ remoteCanonicalSkillKeys.add(canonicalVaultKeySpelling(rf.key));
2700
+ }
2701
+ }
2820
2702
  const tombstones = [];
2821
- for (const key of unseenJournalKeys) {
2703
+ for (const key of journalRows(journal).keys()) {
2822
2704
  // Compare membership in POSIX space. A pre-5.47.2 Windows journal key can
2823
2705
  // carry backslash separators; the remote LIST is always forward-slash, so a
2824
2706
  // raw `has(key)` would miss the match and class a live local file as
2825
2707
  // remote-deleted. normalizeJournalKeys() defuses this on load, but the
2826
2708
  // POSIX compare is defense-in-depth (ridge data-loss, feedback_b8d09d0f).
2827
2709
  const posixKey = toPosixKey(key);
2710
+ if (remoteKeySet.has(posixKey))
2711
+ continue;
2828
2712
  // Listing-scope guard (targeted sync / scoped listing). The remote LIST
2829
2713
  // now covers ONLY `prefixSet` (scoped modes list per-prefix; a targeted
2830
2714
  // pull narrows further), so "absent from the LIST" is evidence of a remote
@@ -2836,6 +2720,16 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
2836
2720
  // `all` mode passes `[""]`, which covers every key — no behavior change.
2837
2721
  if (!isCoveredByAny(posixKey, prefixSet))
2838
2722
  continue;
2723
+ if (posixKey.startsWith(SKILLS_KEY_PREFIX) &&
2724
+ remoteCanonicalSkillKeys.has(canonicalVaultKeySpelling(posixKey)) &&
2725
+ // An EXACT FILE_TOMBSTONE for the journaled key is an authoritative
2726
+ // delete and wins over the twin protection — a lingering junk
2727
+ // spelling in the LIST must not keep a deliberately-deleted wrapper
2728
+ // alive forever. (The key is absent from the LIST here, so there is
2729
+ // no newer re-create to defer to.)
2730
+ fileTombstones.get(posixKey) === undefined) {
2731
+ continue;
2732
+ }
2839
2733
  // Fast-path ignore, BEFORE containment canonicalization. The tombstone
2840
2734
  // sweep runs for every journaled key, and a stale journal can carry
2841
2735
  // hundreds of thousands of entries under an ignored runtime tree (e.g.
@@ -2944,8 +2838,6 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
2944
2838
  }
2945
2839
  return {
2946
2840
  items,
2947
- preplannedFilesSkipped,
2948
- preplannedFilesOutOfScope,
2949
2841
  filesToDownload,
2950
2842
  bytesToDownload,
2951
2843
  filesToSkip,