@indigoai-us/hq-cloud 6.16.49 → 6.16.51-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/dist/bin/rescue-runner.js +1 -1
  2. package/dist/bin/sync-runner-company.d.ts +6 -0
  3. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  4. package/dist/bin/sync-runner-company.js +30 -1
  5. package/dist/bin/sync-runner-company.js.map +1 -1
  6. package/dist/bin/sync-runner-company.test.js +55 -0
  7. package/dist/bin/sync-runner-company.test.js.map +1 -1
  8. package/dist/bin/sync-runner-planning.d.ts +12 -3
  9. package/dist/bin/sync-runner-planning.d.ts.map +1 -1
  10. package/dist/bin/sync-runner-planning.js +65 -3
  11. package/dist/bin/sync-runner-planning.js.map +1 -1
  12. package/dist/bin/sync-runner-planning.test.js +108 -1
  13. package/dist/bin/sync-runner-planning.test.js.map +1 -1
  14. package/dist/bin/sync-runner-watch-loop.d.ts +48 -2
  15. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  16. package/dist/bin/sync-runner-watch-loop.js +281 -113
  17. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  18. package/dist/bin/sync-runner.d.ts +33 -5
  19. package/dist/bin/sync-runner.d.ts.map +1 -1
  20. package/dist/bin/sync-runner.js +34 -4
  21. package/dist/bin/sync-runner.js.map +1 -1
  22. package/dist/bin/sync-runner.test.js +555 -78
  23. package/dist/bin/sync-runner.test.js.map +1 -1
  24. package/dist/cli/onepass-pull-planner-regressions.test.d.ts +2 -0
  25. package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +1 -0
  26. package/dist/cli/onepass-pull-planner-regressions.test.js +300 -0
  27. package/dist/cli/onepass-pull-planner-regressions.test.js.map +1 -0
  28. package/dist/cli/onepass-pull-planner.test.d.ts +2 -0
  29. package/dist/cli/onepass-pull-planner.test.d.ts.map +1 -0
  30. package/dist/cli/onepass-pull-planner.test.js +266 -0
  31. package/dist/cli/onepass-pull-planner.test.js.map +1 -0
  32. package/dist/cli/rescue-core.d.ts +0 -29
  33. package/dist/cli/rescue-core.d.ts.map +1 -1
  34. package/dist/cli/rescue-core.js +39 -236
  35. package/dist/cli/rescue-core.js.map +1 -1
  36. package/dist/cli/rescue-rsync-preflight.test.js +4 -256
  37. package/dist/cli/rescue-rsync-preflight.test.js.map +1 -1
  38. package/dist/cli/rescue.d.ts +0 -6
  39. package/dist/cli/rescue.d.ts.map +1 -1
  40. package/dist/cli/rescue.js +0 -12
  41. package/dist/cli/rescue.js.map +1 -1
  42. package/dist/cli/rescue.test.js +0 -6
  43. package/dist/cli/rescue.test.js.map +1 -1
  44. package/dist/cli/share-presign-transport.test.js +0 -2
  45. package/dist/cli/share-presign-transport.test.js.map +1 -1
  46. package/dist/cli/share.d.ts +19 -0
  47. package/dist/cli/share.d.ts.map +1 -1
  48. package/dist/cli/share.js +57 -37
  49. package/dist/cli/share.js.map +1 -1
  50. package/dist/cli/share.test.js +87 -93
  51. package/dist/cli/share.test.js.map +1 -1
  52. package/dist/cli/streaming-pull-planner.bench.test.d.ts +15 -0
  53. package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +1 -0
  54. package/dist/cli/streaming-pull-planner.bench.test.js +273 -0
  55. package/dist/cli/streaming-pull-planner.bench.test.js.map +1 -0
  56. package/dist/cli/sync-scope.test.js +8 -0
  57. package/dist/cli/sync-scope.test.js.map +1 -1
  58. package/dist/cli/sync.d.ts +12 -0
  59. package/dist/cli/sync.d.ts.map +1 -1
  60. package/dist/cli/sync.js +226 -118
  61. package/dist/cli/sync.js.map +1 -1
  62. package/dist/cli/sync.test.js +215 -2
  63. package/dist/cli/sync.test.js.map +1 -1
  64. package/dist/cli/watch-event-push-conflict.test.js +0 -2
  65. package/dist/cli/watch-event-push-conflict.test.js.map +1 -1
  66. package/dist/cognito-auth.d.ts +0 -30
  67. package/dist/cognito-auth.d.ts.map +1 -1
  68. package/dist/cognito-auth.js +0 -175
  69. package/dist/cognito-auth.js.map +1 -1
  70. package/dist/index.d.ts +2 -2
  71. package/dist/index.d.ts.map +1 -1
  72. package/dist/index.js +1 -1
  73. package/dist/index.js.map +1 -1
  74. package/dist/journal-store.test.js +130 -7
  75. package/dist/journal-store.test.js.map +1 -1
  76. package/dist/journal.d.ts +9 -1
  77. package/dist/journal.d.ts.map +1 -1
  78. package/dist/journal.js +30 -21
  79. package/dist/journal.js.map +1 -1
  80. package/dist/lib/exit-codes.d.ts +15 -0
  81. package/dist/lib/exit-codes.d.ts.map +1 -1
  82. package/dist/lib/exit-codes.js +15 -0
  83. package/dist/lib/exit-codes.js.map +1 -1
  84. package/dist/machine-auth.test.js +0 -295
  85. package/dist/machine-auth.test.js.map +1 -1
  86. package/dist/object-io.d.ts +2 -2
  87. package/dist/object-io.d.ts.map +1 -1
  88. package/dist/object-io.js +3 -18
  89. package/dist/object-io.js.map +1 -1
  90. package/dist/personal-vault.d.ts.map +1 -1
  91. package/dist/personal-vault.js +10 -3
  92. package/dist/personal-vault.js.map +1 -1
  93. package/dist/personal-vault.test.js +35 -0
  94. package/dist/personal-vault.test.js.map +1 -1
  95. package/dist/remote-pull.d.ts.map +1 -1
  96. package/dist/remote-pull.js +9 -2
  97. package/dist/remote-pull.js.map +1 -1
  98. package/dist/remote-pull.test.js +59 -2
  99. package/dist/remote-pull.test.js.map +1 -1
  100. package/dist/s3.d.ts +1 -18
  101. package/dist/s3.d.ts.map +1 -1
  102. package/dist/s3.js +24 -47
  103. package/dist/s3.js.map +1 -1
  104. package/dist/s3.list-cost.test.d.ts +2 -0
  105. package/dist/s3.list-cost.test.d.ts.map +1 -0
  106. package/dist/s3.list-cost.test.js +51 -0
  107. package/dist/s3.list-cost.test.js.map +1 -0
  108. package/dist/s3.test.js +1 -27
  109. package/dist/s3.test.js.map +1 -1
  110. package/dist/sync/realtime-telemetry.d.ts +1 -1
  111. package/dist/sync/realtime-telemetry.d.ts.map +1 -1
  112. package/dist/sync/realtime-telemetry.js +14 -6
  113. package/dist/sync/realtime-telemetry.js.map +1 -1
  114. package/dist/sync/realtime-telemetry.test.js +136 -16
  115. package/dist/sync/realtime-telemetry.test.js.map +1 -1
  116. package/dist/sync/remote-file-spool.d.ts +16 -0
  117. package/dist/sync/remote-file-spool.d.ts.map +1 -0
  118. package/dist/sync/remote-file-spool.js +91 -0
  119. package/dist/sync/remote-file-spool.js.map +1 -0
  120. package/dist/sync/remote-file-spool.test.d.ts +2 -0
  121. package/dist/sync/remote-file-spool.test.d.ts.map +1 -0
  122. package/dist/sync/remote-file-spool.test.js +77 -0
  123. package/dist/sync/remote-file-spool.test.js.map +1 -0
  124. package/dist/sync-core.d.ts +4 -6
  125. package/dist/sync-core.d.ts.map +1 -1
  126. package/dist/sync-core.js +4 -10
  127. package/dist/sync-core.js.map +1 -1
  128. package/dist/sync-core.test.js +1 -25
  129. package/dist/sync-core.test.js.map +1 -1
  130. package/package.json +2 -1
  131. package/dist/cli/rescue-breaker.test.d.ts +0 -2
  132. package/dist/cli/rescue-breaker.test.d.ts.map +0 -1
  133. package/dist/cli/rescue-breaker.test.js +0 -314
  134. package/dist/cli/rescue-breaker.test.js.map +0 -1
  135. package/dist/cli/rescue-realpath.test.d.ts +0 -2
  136. package/dist/cli/rescue-realpath.test.d.ts.map +0 -1
  137. package/dist/cli/rescue-realpath.test.js +0 -109
  138. package/dist/cli/rescue-realpath.test.js.map +0 -1
  139. package/dist/cli/rescue-source-switch-floor.test.d.ts +0 -2
  140. package/dist/cli/rescue-source-switch-floor.test.d.ts.map +0 -1
  141. package/dist/cli/rescue-source-switch-floor.test.js +0 -210
  142. package/dist/cli/rescue-source-switch-floor.test.js.map +0 -1
@@ -95,6 +95,15 @@ vi.mock("../s3.js", async (importOriginal) => {
95
95
  { key: "docs/handoff.md", size: 42, lastModified: new Date(), etag: '"abc123"' },
96
96
  { key: "knowledge/readme.md", size: 100, lastModified: new Date(), etag: '"def456"' },
97
97
  ];
98
+ const listRemoteFiles = innerVi.fn().mockResolvedValue(remoteFiles);
99
+ // Each pull consumes one independent LIST snapshot; never reuse a prior
100
+ // pull's rows, so watcher/list freshness tests observe the actual call.
101
+ async function* streamRemoteFiles(...args) {
102
+ const files = await listRemoteFiles(...args);
103
+ for (const remoteFile of files) {
104
+ yield remoteFile;
105
+ }
106
+ }
98
107
  return {
99
108
  ...actual,
100
109
  toPosixKey: (key) => key.split("\\").join("/"),
@@ -108,7 +117,8 @@ vi.mock("../s3.js", async (importOriginal) => {
108
117
  innerFs.writeFileSync(localPath, "mock file content");
109
118
  return { metadata: {} };
110
119
  }),
111
- listRemoteFiles: innerVi.fn().mockResolvedValue(remoteFiles),
120
+ listRemoteFiles,
121
+ streamRemoteFiles,
112
122
  deleteRemoteFile: innerVi.fn().mockResolvedValue(undefined),
113
123
  headRemoteFile: innerVi.fn().mockResolvedValue(null),
114
124
  primeObjectTransport: innerVi.fn().mockResolvedValue(undefined),
@@ -125,11 +135,12 @@ vi.mock("../remote-pull.js", async (importOriginal) => {
125
135
  const { vi: innerVi } = await import("vitest");
126
136
  return { ...actual, listRemoteForScope: innerVi.fn(actual.listRemoteForScope) };
127
137
  });
128
- import { sync, reportNewFilesToNotify, applyRealtimeDeltaPage, isSyncMutationStdinCommand, runSyncMutationStdinJson, SYNC_MUTATION_STDIN_COMMAND, targetedJournalSessionCensusSizes, win32SymlinkFlavorBroken, } from "./sync.js";
138
+ import { sync, reportNewFilesToNotify, applyRealtimeDeltaPage, isSyncMutationStdinCommand, runSyncMutationStdinJson, SYNC_MUTATION_STDIN_COMMAND, FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE, targetedJournalSessionCensusSizes, win32SymlinkFlavorBroken, } from "./sync.js";
129
139
  import { share } from "./share.js";
130
140
  import * as s3Module from "../s3.js";
131
141
  import * as conflictModule from "./conflict.js";
132
142
  import * as remotePullModule from "../remote-pull.js";
143
+ import { CooperativePassScheduler } from "../bin/sync-runner-watch-loop.js";
133
144
  import { reindex } from "./reindex.js";
134
145
  import { readSyncProgress } from "../sync-progress.js";
135
146
  import { VaultAuthError } from "../vault-client.js";
@@ -620,6 +631,208 @@ describe("sync", () => {
620
631
  },
621
632
  });
622
633
  }, 60_000);
634
+ it("hands queued realtime work the durable boundary inside a large full pull", async () => {
635
+ const rowCount = 59_020;
636
+ // Keep the transfer count fixed while tuning checkpoint cadence below; the
637
+ // fixture remains a 59,020-row streamed journal at every candidate size.
638
+ const changedRowCount = 2_000;
639
+ const fixedNow = new Date("2026-09-17T12:00:00.000Z");
640
+ const materializedHash = "5633d479dfae75ba7a78914ee380fa202bd6126e7c6b7c22e3ebc9e1a6ddc871";
641
+ const remoteFiles = Array.from({ length: rowCount }, (_value, index) => ({
642
+ key: `docs/full-pass/${index}.md`,
643
+ size: 17,
644
+ lastModified: fixedNow,
645
+ etag: `"${index < changedRowCount ? "remote" : "stable"}-${index}"`,
646
+ }));
647
+ writeJournal("acme", {
648
+ version: "2",
649
+ lastSync: "",
650
+ pulls: [],
651
+ files: Object.fromEntries(remoteFiles.map((remoteFile, index) => [
652
+ remoteFile.key,
653
+ {
654
+ hash: materializedHash,
655
+ size: 17,
656
+ syncedAt: "2026-09-16T00:00:00.000Z",
657
+ direction: "down",
658
+ remoteEtag: `${index < changedRowCount ? "stale" : "stable"}-${index}`,
659
+ mtimeMs: fixedNow.getTime(),
660
+ ctimeMs: fixedNow.getTime(),
661
+ },
662
+ ])),
663
+ });
664
+ const listRemoteFiles = vi.mocked(s3Module.listRemoteFiles);
665
+ const originalListRemoteFiles = listRemoteFiles.getMockImplementation();
666
+ listRemoteFiles.mockResolvedValue(remoteFiles);
667
+ const materializedPaths = new Set(remoteFiles.map((remoteFile) => path.join(tmpDir, "companies", "acme", remoteFile.key)));
668
+ const originalLstatSync = fs.lstatSync;
669
+ const lstatSync = vi.spyOn(fs, "lstatSync").mockImplementation((filePath) => {
670
+ if (typeof filePath === "string" && materializedPaths.has(filePath)) {
671
+ return {
672
+ isSymbolicLink: () => false,
673
+ isFile: () => true,
674
+ isDirectory: () => false,
675
+ size: 17,
676
+ mtimeMs: fixedNow.getTime(),
677
+ ctimeMs: fixedNow.getTime(),
678
+ };
679
+ }
680
+ return originalLstatSync(filePath);
681
+ });
682
+ const originalReadFileSync = fs.readFileSync;
683
+ const readFileSync = vi.spyOn(fs, "readFileSync").mockImplementation((filePath, ...args) => {
684
+ if (typeof filePath === "string" && materializedPaths.has(filePath)) {
685
+ return Buffer.from("mock file content");
686
+ }
687
+ return Reflect.apply(originalReadFileSync, fs, [filePath, ...args]);
688
+ });
689
+ const downloadFile = vi.mocked(s3Module.downloadFile);
690
+ const originalDownload = downloadFile.getMockImplementation();
691
+ let downloadsCompleted = 0;
692
+ let inFlightDownloads = 0;
693
+ let maxInFlightDownloads = 0;
694
+ let checkpointCalls = 0;
695
+ let realtimeQueued = false;
696
+ let realtimeDispatchedAt = -1;
697
+ const scheduler = new CooperativePassScheduler(() => 0);
698
+ let realtimePass;
699
+ downloadFile.mockImplementation(async (_ctx, _key, localPath, options) => {
700
+ inFlightDownloads += 1;
701
+ maxInFlightDownloads = Math.max(maxInFlightDownloads, inFlightDownloads);
702
+ await Promise.resolve();
703
+ options?.beforeReplace?.();
704
+ materializedPaths.add(localPath);
705
+ downloadsCompleted += 1;
706
+ if (downloadsCompleted === 1) {
707
+ realtimeQueued = true;
708
+ realtimePass = scheduler.submit(async () => {
709
+ // The chore drained its transfer unit and committed it before the
710
+ // scheduler admitted this realtime writer.
711
+ expect(inFlightDownloads).toBe(0);
712
+ realtimeDispatchedAt = downloadsCompleted;
713
+ remoteFiles.push({
714
+ key: "docs/realtime/during-full-pull.md",
715
+ size: 17,
716
+ lastModified: fixedNow,
717
+ etag: '"realtime"',
718
+ });
719
+ materializedPaths.add(path.join(tmpDir, "companies", "acme", "docs", "realtime", "during-full-pull.md"));
720
+ const realtimeJournal = readJournal("acme");
721
+ realtimeJournal.files["docs/realtime/during-full-pull.md"] = {
722
+ hash: materializedHash,
723
+ size: 17,
724
+ syncedAt: fixedNow.toISOString(),
725
+ direction: "up",
726
+ remoteEtag: "realtime",
727
+ mtimeMs: fixedNow.getTime(),
728
+ ctimeMs: fixedNow.getTime(),
729
+ };
730
+ writeJournal("acme", realtimeJournal);
731
+ }, "realtime");
732
+ }
733
+ inFlightDownloads -= 1;
734
+ return {
735
+ metadata: {},
736
+ contentHash: materializedHash,
737
+ contentSize: 17,
738
+ };
739
+ });
740
+ try {
741
+ await scheduler.submit(async ({ checkpoint }) => {
742
+ await sync({
743
+ company: "acme",
744
+ vaultConfig: mockConfig,
745
+ hqRoot: tmpDir,
746
+ skipReindex: true,
747
+ onEvent: () => undefined,
748
+ cooperativeCheckpoint: async () => {
749
+ checkpointCalls += 1;
750
+ return checkpoint();
751
+ },
752
+ });
753
+ }, "chore");
754
+ await realtimePass;
755
+ }
756
+ finally {
757
+ readFileSync.mockRestore();
758
+ lstatSync.mockRestore();
759
+ if (originalDownload)
760
+ downloadFile.mockImplementation(originalDownload);
761
+ else
762
+ downloadFile.mockReset();
763
+ if (originalListRemoteFiles)
764
+ listRemoteFiles.mockImplementation(originalListRemoteFiles);
765
+ else
766
+ listRemoteFiles.mockReset();
767
+ }
768
+ expect(maxInFlightDownloads).toBeGreaterThan(1);
769
+ expect(realtimeQueued).toBe(true);
770
+ expect(checkpointCalls).toBeGreaterThan(0);
771
+ expect(realtimeDispatchedAt).toBeGreaterThan(0);
772
+ expect(realtimeDispatchedAt).toBeLessThanOrEqual(FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE);
773
+ const finalJournal = readJournal("acme");
774
+ expect(finalJournal.files["docs/realtime/during-full-pull.md"]).toBeDefined();
775
+ expect(Object.keys(finalJournal.files)).toHaveLength(rowCount + 1);
776
+ expect(remoteFiles.every((remoteFile) => finalJournal.files[remoteFile.key] !== undefined)).toBe(true);
777
+ }, 120_000);
778
+ it("keeps every no-op full-pull journal row and emits no local deletes", async () => {
779
+ const fixedNow = new Date("2026-09-17T12:00:00.000Z");
780
+ const materialized = "mock file content";
781
+ const materializedHash = "5633d479dfae75ba7a78914ee380fa202bd6126e7c6b7c22e3ebc9e1a6ddc871";
782
+ const remoteFiles = Array.from({ length: 8 }, (_value, index) => ({
783
+ key: `docs/noop-full-pull/${index}.md`,
784
+ size: Buffer.byteLength(materialized),
785
+ lastModified: fixedNow,
786
+ etag: `"stable-${index}"`,
787
+ }));
788
+ for (const remoteFile of remoteFiles) {
789
+ const localPath = path.join(tmpDir, "companies", "acme", remoteFile.key);
790
+ fs.mkdirSync(path.dirname(localPath), { recursive: true });
791
+ fs.writeFileSync(localPath, materialized);
792
+ }
793
+ writeJournal("acme", {
794
+ version: "2",
795
+ lastSync: "",
796
+ pulls: [],
797
+ files: Object.fromEntries(remoteFiles.map((remoteFile) => [
798
+ remoteFile.key,
799
+ {
800
+ hash: materializedHash,
801
+ size: remoteFile.size,
802
+ syncedAt: "2026-09-16T00:00:00.000Z",
803
+ direction: "down",
804
+ remoteEtag: remoteFile.etag.slice(1, -1),
805
+ },
806
+ ])),
807
+ });
808
+ const listRemoteFiles = vi.mocked(s3Module.listRemoteFiles);
809
+ const originalListRemoteFiles = listRemoteFiles.getMockImplementation();
810
+ listRemoteFiles.mockResolvedValue(remoteFiles);
811
+ const unlinkSync = vi.spyOn(fs, "unlinkSync");
812
+ try {
813
+ await sync({
814
+ company: "acme",
815
+ vaultConfig: mockConfig,
816
+ hqRoot: tmpDir,
817
+ skipReindex: true,
818
+ onEvent: () => undefined,
819
+ });
820
+ const companyRoot = path.join(tmpDir, "companies", "acme") + path.sep;
821
+ const localDeletes = unlinkSync.mock.calls.filter(([candidate]) => typeof candidate === "string" && candidate.startsWith(companyRoot));
822
+ expect(localDeletes).toEqual([]);
823
+ }
824
+ finally {
825
+ unlinkSync.mockRestore();
826
+ if (originalListRemoteFiles)
827
+ listRemoteFiles.mockImplementation(originalListRemoteFiles);
828
+ else
829
+ listRemoteFiles.mockReset();
830
+ }
831
+ // Re-open through the on-disk journal reader rather than the in-memory
832
+ // object that planned this pass: omitted rows would otherwise look safe.
833
+ const reopened = readJournal("acme");
834
+ expect(Object.keys(reopened.files).sort()).toEqual(remoteFiles.map((file) => file.key).sort());
835
+ });
623
836
  it("retains targeted-pull baselines as row fingerprints rather than entry objects", async () => {
624
837
  writeJournal("acme", {
625
838
  version: "2",