@indigoai-us/hq-cloud 6.16.49 → 6.16.51-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/dist/bin/rescue-runner.js +1 -1
  2. package/dist/bin/sync-runner-company.d.ts +6 -0
  3. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  4. package/dist/bin/sync-runner-company.js +30 -1
  5. package/dist/bin/sync-runner-company.js.map +1 -1
  6. package/dist/bin/sync-runner-company.test.js +55 -0
  7. package/dist/bin/sync-runner-company.test.js.map +1 -1
  8. package/dist/bin/sync-runner-planning.d.ts +12 -3
  9. package/dist/bin/sync-runner-planning.d.ts.map +1 -1
  10. package/dist/bin/sync-runner-planning.js +65 -3
  11. package/dist/bin/sync-runner-planning.js.map +1 -1
  12. package/dist/bin/sync-runner-planning.test.js +108 -1
  13. package/dist/bin/sync-runner-planning.test.js.map +1 -1
  14. package/dist/bin/sync-runner-watch-loop.d.ts +48 -2
  15. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  16. package/dist/bin/sync-runner-watch-loop.js +281 -113
  17. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  18. package/dist/bin/sync-runner.d.ts +33 -5
  19. package/dist/bin/sync-runner.d.ts.map +1 -1
  20. package/dist/bin/sync-runner.js +34 -4
  21. package/dist/bin/sync-runner.js.map +1 -1
  22. package/dist/bin/sync-runner.test.js +555 -78
  23. package/dist/bin/sync-runner.test.js.map +1 -1
  24. package/dist/cli/onepass-pull-planner-regressions.test.d.ts +2 -0
  25. package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +1 -0
  26. package/dist/cli/onepass-pull-planner-regressions.test.js +300 -0
  27. package/dist/cli/onepass-pull-planner-regressions.test.js.map +1 -0
  28. package/dist/cli/onepass-pull-planner.test.d.ts +2 -0
  29. package/dist/cli/onepass-pull-planner.test.d.ts.map +1 -0
  30. package/dist/cli/onepass-pull-planner.test.js +266 -0
  31. package/dist/cli/onepass-pull-planner.test.js.map +1 -0
  32. package/dist/cli/rescue-core.d.ts +0 -29
  33. package/dist/cli/rescue-core.d.ts.map +1 -1
  34. package/dist/cli/rescue-core.js +39 -236
  35. package/dist/cli/rescue-core.js.map +1 -1
  36. package/dist/cli/rescue-rsync-preflight.test.js +4 -256
  37. package/dist/cli/rescue-rsync-preflight.test.js.map +1 -1
  38. package/dist/cli/rescue.d.ts +0 -6
  39. package/dist/cli/rescue.d.ts.map +1 -1
  40. package/dist/cli/rescue.js +0 -12
  41. package/dist/cli/rescue.js.map +1 -1
  42. package/dist/cli/rescue.test.js +0 -6
  43. package/dist/cli/rescue.test.js.map +1 -1
  44. package/dist/cli/share-presign-transport.test.js +0 -2
  45. package/dist/cli/share-presign-transport.test.js.map +1 -1
  46. package/dist/cli/share.d.ts +19 -0
  47. package/dist/cli/share.d.ts.map +1 -1
  48. package/dist/cli/share.js +57 -37
  49. package/dist/cli/share.js.map +1 -1
  50. package/dist/cli/share.test.js +87 -93
  51. package/dist/cli/share.test.js.map +1 -1
  52. package/dist/cli/streaming-pull-planner.bench.test.d.ts +15 -0
  53. package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +1 -0
  54. package/dist/cli/streaming-pull-planner.bench.test.js +273 -0
  55. package/dist/cli/streaming-pull-planner.bench.test.js.map +1 -0
  56. package/dist/cli/sync-scope.test.js +8 -0
  57. package/dist/cli/sync-scope.test.js.map +1 -1
  58. package/dist/cli/sync.d.ts +12 -0
  59. package/dist/cli/sync.d.ts.map +1 -1
  60. package/dist/cli/sync.js +226 -118
  61. package/dist/cli/sync.js.map +1 -1
  62. package/dist/cli/sync.test.js +215 -2
  63. package/dist/cli/sync.test.js.map +1 -1
  64. package/dist/cli/watch-event-push-conflict.test.js +0 -2
  65. package/dist/cli/watch-event-push-conflict.test.js.map +1 -1
  66. package/dist/cognito-auth.d.ts +0 -30
  67. package/dist/cognito-auth.d.ts.map +1 -1
  68. package/dist/cognito-auth.js +0 -175
  69. package/dist/cognito-auth.js.map +1 -1
  70. package/dist/index.d.ts +2 -2
  71. package/dist/index.d.ts.map +1 -1
  72. package/dist/index.js +1 -1
  73. package/dist/index.js.map +1 -1
  74. package/dist/journal-store.test.js +130 -7
  75. package/dist/journal-store.test.js.map +1 -1
  76. package/dist/journal.d.ts +9 -1
  77. package/dist/journal.d.ts.map +1 -1
  78. package/dist/journal.js +30 -21
  79. package/dist/journal.js.map +1 -1
  80. package/dist/lib/exit-codes.d.ts +15 -0
  81. package/dist/lib/exit-codes.d.ts.map +1 -1
  82. package/dist/lib/exit-codes.js +15 -0
  83. package/dist/lib/exit-codes.js.map +1 -1
  84. package/dist/machine-auth.test.js +0 -295
  85. package/dist/machine-auth.test.js.map +1 -1
  86. package/dist/object-io.d.ts +2 -2
  87. package/dist/object-io.d.ts.map +1 -1
  88. package/dist/object-io.js +3 -18
  89. package/dist/object-io.js.map +1 -1
  90. package/dist/personal-vault.d.ts.map +1 -1
  91. package/dist/personal-vault.js +10 -3
  92. package/dist/personal-vault.js.map +1 -1
  93. package/dist/personal-vault.test.js +35 -0
  94. package/dist/personal-vault.test.js.map +1 -1
  95. package/dist/remote-pull.d.ts.map +1 -1
  96. package/dist/remote-pull.js +9 -2
  97. package/dist/remote-pull.js.map +1 -1
  98. package/dist/remote-pull.test.js +59 -2
  99. package/dist/remote-pull.test.js.map +1 -1
  100. package/dist/s3.d.ts +1 -18
  101. package/dist/s3.d.ts.map +1 -1
  102. package/dist/s3.js +24 -47
  103. package/dist/s3.js.map +1 -1
  104. package/dist/s3.list-cost.test.d.ts +2 -0
  105. package/dist/s3.list-cost.test.d.ts.map +1 -0
  106. package/dist/s3.list-cost.test.js +51 -0
  107. package/dist/s3.list-cost.test.js.map +1 -0
  108. package/dist/s3.test.js +1 -27
  109. package/dist/s3.test.js.map +1 -1
  110. package/dist/sync/realtime-telemetry.d.ts +1 -1
  111. package/dist/sync/realtime-telemetry.d.ts.map +1 -1
  112. package/dist/sync/realtime-telemetry.js +14 -6
  113. package/dist/sync/realtime-telemetry.js.map +1 -1
  114. package/dist/sync/realtime-telemetry.test.js +136 -16
  115. package/dist/sync/realtime-telemetry.test.js.map +1 -1
  116. package/dist/sync/remote-file-spool.d.ts +16 -0
  117. package/dist/sync/remote-file-spool.d.ts.map +1 -0
  118. package/dist/sync/remote-file-spool.js +91 -0
  119. package/dist/sync/remote-file-spool.js.map +1 -0
  120. package/dist/sync/remote-file-spool.test.d.ts +2 -0
  121. package/dist/sync/remote-file-spool.test.d.ts.map +1 -0
  122. package/dist/sync/remote-file-spool.test.js +77 -0
  123. package/dist/sync/remote-file-spool.test.js.map +1 -0
  124. package/dist/sync-core.d.ts +4 -6
  125. package/dist/sync-core.d.ts.map +1 -1
  126. package/dist/sync-core.js +4 -10
  127. package/dist/sync-core.js.map +1 -1
  128. package/dist/sync-core.test.js +1 -25
  129. package/dist/sync-core.test.js.map +1 -1
  130. package/package.json +2 -1
  131. package/dist/cli/rescue-breaker.test.d.ts +0 -2
  132. package/dist/cli/rescue-breaker.test.d.ts.map +0 -1
  133. package/dist/cli/rescue-breaker.test.js +0 -314
  134. package/dist/cli/rescue-breaker.test.js.map +0 -1
  135. package/dist/cli/rescue-realpath.test.d.ts +0 -2
  136. package/dist/cli/rescue-realpath.test.d.ts.map +0 -1
  137. package/dist/cli/rescue-realpath.test.js +0 -109
  138. package/dist/cli/rescue-realpath.test.js.map +0 -1
  139. package/dist/cli/rescue-source-switch-floor.test.d.ts +0 -2
  140. package/dist/cli/rescue-source-switch-floor.test.d.ts.map +0 -1
  141. package/dist/cli/rescue-source-switch-floor.test.js +0 -210
  142. package/dist/cli/rescue-source-switch-floor.test.js.map +0 -1
package/dist/cli/share.js CHANGED
@@ -11,7 +11,7 @@ import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../c
11
11
  import { ensureCompanyVaultTransport } from "../company-vault-transport.js";
12
12
  import { createSyncProgressRecorder } from "../sync-progress.js";
13
13
  import { localPathForVaultKey, vaultKeyForLocalPath } from "../local-path-codec.js";
14
- import { uploadFile, uploadSymlink, toPosixKey, headRemoteFile, readVerifiedRemoteSymlinkTarget, deleteRemoteFile, downloadFile, primeObjectTransport, primeUploads, publishedContentHash, setUploadContentHash, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
14
+ import { uploadFile, uploadSymlink, toPosixKey, headRemoteFile, readVerifiedRemoteSymlinkTarget, deleteRemoteFile, downloadFile, primeObjectTransport, primeUploads, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
15
15
  import * as crypto from "crypto";
16
16
  import { isUnderAnyRoot, readJournalForPass, readJournalScoped, writeJournal, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, timestampsTooCoarseToTrust, updateEntry, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, migrateToV2, } from "../journal.js";
17
17
  import { createIgnoreFilter, hasControlCharacters, isExpectedIgnore, isWithinSizeLimit, } from "../ignore.js";
@@ -628,10 +628,33 @@ export function wrapFilterWithIgnoreVisibility(underlying, hqRoot, onIgnoreExclu
628
628
  return false;
629
629
  };
630
630
  }
631
+ class CooperativePushHandoff extends Error {
632
+ constructor() {
633
+ super("Realtime work ran at a settled full-push boundary");
634
+ this.name = "CooperativePushHandoff";
635
+ }
636
+ }
631
637
  /**
632
638
  * Share local file(s) to the entity vault.
639
+ *
640
+ * A cooperative hand-off discards the old plan instead of continuing it after
641
+ * another writer has run. Journal delta computation treats missing keys as
642
+ * deletes, so reusing a pre-handoff plan would be unsafe even though the
643
+ * hand-off itself only happens at a durable unit boundary.
633
644
  */
634
645
  export async function share(options) {
646
+ for (;;) {
647
+ try {
648
+ return await shareOnce(options);
649
+ }
650
+ catch (error) {
651
+ if (error instanceof CooperativePushHandoff)
652
+ continue;
653
+ throw error;
654
+ }
655
+ }
656
+ }
657
+ async function shareOnce(options) {
635
658
  const run = await createPushRunContext(options);
636
659
  const counters = createShareCounters();
637
660
  const filesRefusedStalePaths = [];
@@ -1333,7 +1356,6 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
1333
1356
  key: it.relativePath,
1334
1357
  localPath: it.absolutePath,
1335
1358
  isSymlink: it.kind === "symlink",
1336
- ...(it.kind === "file" ? { contentHash: it.localHash } : {}),
1337
1359
  author: run.options.author,
1338
1360
  })));
1339
1361
  // Warm the GET presigns the per-item conflict HEAD (remoteMeta) reuses, so a
@@ -1505,39 +1527,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
1505
1527
  }
1506
1528
  if (remoteMeta) {
1507
1529
  const localChanged = !!journalEntry && journalEntry.hash !== localHash;
1508
- const remoteContentHash = item.kind === "file" ? publishedContentHash(remoteMeta.metadata) : undefined;
1509
- const remoteChanged = !!journalEntry &&
1510
- hasRemoteChanged({ ...remoteMeta, contentHash: remoteContentHash }, journalEntry);
1511
- // A matching published digest is a content no-op even when SSE-KMS gave
1512
- // this object a fresh ETag. Reconcile the journal to that version without
1513
- // a needless PUT or a phantom two-sided conflict.
1514
- if (remoteContentHash === localHash) {
1515
- const lstat = fs.lstatSync(absolutePath);
1516
- updateEntry(run.journal, relativePath, localHash, lstat.size, journalEntry?.direction ?? "up", absolutePath, {
1517
- remoteEtag: remoteMeta.etag,
1518
- mtimeMs: lstat.mtimeMs,
1519
- ctimeMs: ctimeToStamp(item.kind === "file" ? item.statAtHash : undefined, lstat),
1520
- kind: "file",
1521
- });
1522
- checkpointShareJournal(run);
1523
- run.emit({ type: "reconciled", path: relativePath, direction: "push" });
1524
- counters.filesSkipped++;
1525
- return;
1526
- }
1527
- // This machine's journal baseline is a superseded content generation.
1528
- // Replaying that unchanged baseline is a stale revert, not an edit. A
1529
- // user who deliberately restores some other historic content has a hash
1530
- // different from the journal baseline and still reaches normal conflict
1531
- // resolution below.
1532
- if (journalEntry !== undefined &&
1533
- remoteContentHash !== undefined &&
1534
- journalEntry.hash === localHash &&
1535
- remoteContentHash !== localHash) {
1536
- conflictPaths.push(relativePath);
1537
- run.emit({ type: "conflict", path: relativePath, direction: "push", resolution: "skip" });
1538
- counters.filesSkipped++;
1539
- return;
1540
- }
1530
+ const remoteChanged = !!journalEntry && hasRemoteChanged(remoteMeta, journalEntry);
1541
1531
  // Every path below that needs the remote BODY — the no-row equality
1542
1532
  // probe, and a `keep` that adopts the remote — is impossible against a
1543
1533
  // DEEP_ARCHIVE session transcript: each GET fails with InvalidObjectState.
@@ -1770,8 +1760,6 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
1770
1760
  const skillContentAtPut = !isSymlinkUpload && CANONICAL_COMPANY_SKILL_RE.test(relativePath)
1771
1761
  ? fs.readFileSync(absolutePath, "utf8")
1772
1762
  : undefined;
1773
- if (!isSymlinkUpload)
1774
- setUploadContentHash(absolutePath, localHash);
1775
1763
  const { etag } = isSymlinkUpload
1776
1764
  ? await uploadSymlink(run.ctx, item.target, relativePath, run.options.author, pc)
1777
1765
  : await uploadFile(run.ctx, absolutePath, relativePath, run.options.author, pc);
@@ -2015,9 +2003,16 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
2015
2003
  // this flag only halts admission, so the refusal still surfaces via
2016
2004
  // throwUploadWorkerErrors below. In-flight legs still drain.
2017
2005
  let sessionAuthAborted = false;
2006
+ // Cooperative work is admitted in bounded units, then fully drained before
2007
+ // the root lock can change owners. Skipped rows are not durable mutations
2008
+ // and therefore do not count toward the unit.
2009
+ let admittedSinceCooperativeCheckpoint = 0;
2018
2010
  while (queue.length > 0 || inFlight.size > 0) {
2019
2011
  while (!aborted &&
2020
2012
  !sessionAuthAborted &&
2013
+ (!run.options.cooperativeCheckpoint ||
2014
+ admittedSinceCooperativeCheckpoint <
2015
+ FULL_PUSH_COOPERATIVE_CHECKPOINT_BATCH_SIZE) &&
2021
2016
  inFlight.size < TRANSFER_CONCURRENCY &&
2022
2017
  queue.length > 0) {
2023
2018
  const previewItem = queue[0];
@@ -2044,6 +2039,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
2044
2039
  if (inFlight.size > 0 && inFlightBytes + nextBytes > byteBudget)
2045
2040
  break;
2046
2041
  queue.shift();
2042
+ admittedSinceCooperativeCheckpoint++;
2047
2043
  inFlightBytes += nextBytes;
2048
2044
  const upload = () => processUploadItem(item);
2049
2045
  const p = (run.options.transferSemaphore
@@ -2071,6 +2067,24 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
2071
2067
  if (inFlight.size > 0) {
2072
2068
  await Promise.race(Array.from(inFlight));
2073
2069
  }
2070
+ else if (run.options.cooperativeCheckpoint &&
2071
+ !aborted &&
2072
+ !sessionAuthAborted &&
2073
+ !hadUploadFailure &&
2074
+ admittedSinceCooperativeCheckpoint > 0 &&
2075
+ (admittedSinceCooperativeCheckpoint >=
2076
+ FULL_PUSH_COOPERATIVE_CHECKPOINT_BATCH_SIZE ||
2077
+ queue.length === 0)) {
2078
+ // Every admitted upload has settled, so `processUploadItem` has
2079
+ // finished its mutable journal updates. Persist this durable unit
2080
+ // before yielding. A real hand-off restarts share() above and thereby
2081
+ // discards its now-stale upload/delete plan.
2082
+ flushShareJournal(run);
2083
+ admittedSinceCooperativeCheckpoint = 0;
2084
+ if ((await run.options.cooperativeCheckpoint()) === true) {
2085
+ throw new CooperativePushHandoff();
2086
+ }
2087
+ }
2074
2088
  else {
2075
2089
  break;
2076
2090
  }
@@ -2585,6 +2599,12 @@ async function executeDeletes(run, deletePlan, decommissionPlan, counters, files
2585
2599
  */
2586
2600
  const CHECKPOINT_MIN_APPLIED = 250;
2587
2601
  const CHECKPOINT_MIN_INTERVAL_MS = 5_000;
2602
+ /**
2603
+ * Maximum completed upload mutations between cooperative full-push hand-offs.
2604
+ * The executor drains a unit before yielding, so realtime never overlaps an
2605
+ * upload body or the push's mutable journal state.
2606
+ */
2607
+ export const FULL_PUSH_COOPERATIVE_CHECKPOINT_BATCH_SIZE = 500;
2588
2608
  /**
2589
2609
  * Fraction of the journal's current size that must be re-applied before another
2590
2610
  * checkpoint is worth its write. `writeJournal` publishes a whole compact