@indigoai-us/hq-cloud 6.18.1-alpha.6 → 6.18.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/dist/bandwidth.d.ts +11 -45
  2. package/dist/bandwidth.d.ts.map +1 -1
  3. package/dist/bandwidth.js +18 -231
  4. package/dist/bandwidth.js.map +1 -1
  5. package/dist/bandwidth.test.js +4 -124
  6. package/dist/bandwidth.test.js.map +1 -1
  7. package/dist/bin/sync-runner-company.d.ts +0 -2
  8. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  9. package/dist/bin/sync-runner-company.js +0 -6
  10. package/dist/bin/sync-runner-company.js.map +1 -1
  11. package/dist/bin/sync-runner-manifest.d.ts +2 -0
  12. package/dist/bin/sync-runner-manifest.d.ts.map +1 -1
  13. package/dist/bin/sync-runner-manifest.js +439 -1
  14. package/dist/bin/sync-runner-manifest.js.map +1 -1
  15. package/dist/bin/sync-runner-manifest.test.js +374 -16
  16. package/dist/bin/sync-runner-manifest.test.js.map +1 -1
  17. package/dist/bin/sync-runner-watch-loop.d.ts +1 -26
  18. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  19. package/dist/bin/sync-runner-watch-loop.js +3 -217
  20. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  21. package/dist/bin/sync-runner.d.ts +2 -26
  22. package/dist/bin/sync-runner.d.ts.map +1 -1
  23. package/dist/bin/sync-runner.js +5 -14
  24. package/dist/bin/sync-runner.js.map +1 -1
  25. package/dist/bin/sync-runner.test.js +1 -107
  26. package/dist/bin/sync-runner.test.js.map +1 -1
  27. package/dist/cli/share.d.ts +6 -18
  28. package/dist/cli/share.d.ts.map +1 -1
  29. package/dist/cli/share.js +47 -92
  30. package/dist/cli/share.js.map +1 -1
  31. package/dist/cli/share.test.js +3 -3
  32. package/dist/cli/share.test.js.map +1 -1
  33. package/dist/cli/sync.d.ts +4 -17
  34. package/dist/cli/sync.d.ts.map +1 -1
  35. package/dist/cli/sync.js +27 -155
  36. package/dist/cli/sync.js.map +1 -1
  37. package/dist/cli/sync.test.js +1 -315
  38. package/dist/cli/sync.test.js.map +1 -1
  39. package/dist/manifest/upload-manifest.d.ts +6 -0
  40. package/dist/manifest/upload-manifest.d.ts.map +1 -1
  41. package/dist/manifest/upload-manifest.js +18 -3
  42. package/dist/manifest/upload-manifest.js.map +1 -1
  43. package/dist/object-io.d.ts +7 -33
  44. package/dist/object-io.d.ts.map +1 -1
  45. package/dist/object-io.js +140 -326
  46. package/dist/object-io.js.map +1 -1
  47. package/dist/object-io.test.js +1 -377
  48. package/dist/object-io.test.js.map +1 -1
  49. package/dist/s3.d.ts +3 -8
  50. package/dist/s3.d.ts.map +1 -1
  51. package/dist/s3.js +27 -304
  52. package/dist/s3.js.map +1 -1
  53. package/dist/s3.test.js +0 -163
  54. package/dist/s3.test.js.map +1 -1
  55. package/dist/sync/inbound-event-admission.d.ts.map +1 -1
  56. package/dist/sync/inbound-event-admission.js +0 -3
  57. package/dist/sync/inbound-event-admission.js.map +1 -1
  58. package/dist/sync/inbound-event-admission.test.js +0 -6
  59. package/dist/sync/inbound-event-admission.test.js.map +1 -1
  60. package/dist/sync/push-receiver.d.ts.map +1 -1
  61. package/dist/sync/push-receiver.js +1 -30
  62. package/dist/sync/push-receiver.js.map +1 -1
  63. package/dist/sync/push-receiver.test.js +0 -97
  64. package/dist/sync/push-receiver.test.js.map +1 -1
  65. package/dist/sync-core.d.ts +1 -2
  66. package/dist/sync-core.d.ts.map +1 -1
  67. package/dist/sync-core.js +10 -5
  68. package/dist/sync-core.js.map +1 -1
  69. package/dist/sync-core.test.js +0 -16
  70. package/dist/sync-core.test.js.map +1 -1
  71. package/package.json +1 -1
  72. package/dist/s3-download-stream-hash.test.d.ts +0 -2
  73. package/dist/s3-download-stream-hash.test.d.ts.map +0 -1
  74. package/dist/s3-download-stream-hash.test.js +0 -70
  75. package/dist/s3-download-stream-hash.test.js.map +0 -1
@@ -163,7 +163,7 @@ import { readJournal, writeJournal, getEntry, listJournals, } from "../journal.j
163
163
  import { SqsPushReceiver } from "../sync/push-receiver.js";
164
164
  import { buildTargetedPullArgv } from "../bin/sync-runner-watch-routes.js";
165
165
  import { ReconcileCursorStore } from "../sync/reconcile-cursor.js";
166
- import { ObjectBodyIdleTimeoutError, RateLimitedError, setObjectIOFactory, setObjectIORequestTimeoutMsForTesting, } from "../object-io.js";
166
+ import { RateLimitedError, setObjectIOFactory, setObjectIORequestTimeoutMsForTesting, } from "../object-io.js";
167
167
  import { isTransientNetworkError } from "../lib/net-errors.js";
168
168
  /**
169
169
  * Read a journal the way production does.
@@ -790,320 +790,6 @@ describe("sync", () => {
790
790
  expect(Object.keys(finalJournal.files)).toHaveLength(rowCount + 1);
791
791
  expect(remoteFiles.every((remoteFile) => finalJournal.files[remoteFile.key] !== undefined)).toBe(true);
792
792
  }, 120_000);
793
- it("admits queued realtime work while a two-second chore download is still in flight", async () => {
794
- const fixedNow = new Date("2026-09-24T12:00:00.000Z");
795
- const remoteFiles = [
796
- { key: "docs/slow-1.md", size: 4, lastModified: fixedNow, etag: '"slow-1"' },
797
- { key: "docs/slow-2.md", size: 4, lastModified: fixedNow, etag: '"slow-2"' },
798
- ];
799
- writeJournal("acme", { version: "2", lastSync: "", pulls: [], files: {} });
800
- vi.stubEnv("HQ_SYNC_TRANSFER_CONCURRENCY", "1");
801
- vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce(remoteFiles);
802
- const scheduler = new CooperativePassScheduler(() => 0);
803
- const downloadFile = vi.mocked(s3Module.downloadFile);
804
- const originalDownload = downloadFile.getMockImplementation();
805
- let downloadsCompleted = 0;
806
- let inFlightDownloads = 0;
807
- let realtimeQueuedAt = -1;
808
- let realtimeStartedAt = -1;
809
- let completedAtHandoff = -1;
810
- let realtimePass;
811
- downloadFile.mockImplementation(async (_context, key, localPath, options) => {
812
- inFlightDownloads += 1;
813
- if (downloadsCompleted === 0 && realtimeQueuedAt < 0) {
814
- setTimeout(() => {
815
- realtimeQueuedAt = performance.now();
816
- realtimePass = scheduler.submit(async () => {
817
- realtimeStartedAt = performance.now();
818
- completedAtHandoff = downloadsCompleted;
819
- expect(inFlightDownloads).toBe(1);
820
- }, "realtime");
821
- }, 1_000);
822
- }
823
- await new Promise((resolve) => setTimeout(resolve, 2_000));
824
- options?.beforeReplace?.();
825
- fs.mkdirSync(path.dirname(localPath), { recursive: true });
826
- fs.writeFileSync(localPath, "slow");
827
- inFlightDownloads -= 1;
828
- downloadsCompleted += 1;
829
- return {
830
- metadata: {},
831
- contentHash: "5633d479dfae75ba7a78914ee380fa202bd6126e7c6b7c22e3ebc9e1a6ddc871",
832
- contentSize: 4,
833
- };
834
- });
835
- try {
836
- await scheduler.submit(async ({ checkpoint }) => sync({
837
- company: "acme",
838
- vaultConfig: mockConfig,
839
- hqRoot: tmpDir,
840
- skipReindex: true,
841
- onEvent: () => undefined,
842
- cooperativeCheckpoint: checkpoint,
843
- }), "chore");
844
- await realtimePass;
845
- }
846
- finally {
847
- if (originalDownload)
848
- downloadFile.mockImplementation(originalDownload);
849
- else
850
- downloadFile.mockReset();
851
- vi.unstubAllEnvs();
852
- }
853
- expect(realtimeQueuedAt).toBeGreaterThanOrEqual(0);
854
- expect(realtimeStartedAt - realtimeQueuedAt).toBeLessThan(1_500);
855
- expect(completedAtHandoff).toBe(0);
856
- expect(downloadsCompleted).toBe(remoteFiles.length);
857
- }, 15_000);
858
- it("discards a stale staged download when realtime commits the same path", async () => {
859
- const fixedNow = new Date("2026-09-24T12:00:00.000Z");
860
- const key = "docs/same-path.md";
861
- const remoteFiles = [
862
- { key, size: 11, lastModified: fixedNow, etag: '"chore-version"' },
863
- ];
864
- writeJournal("acme", { version: "2", lastSync: "", pulls: [], files: {} });
865
- vi.stubEnv("HQ_SYNC_TRANSFER_CONCURRENCY", "1");
866
- vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce(remoteFiles);
867
- const scheduler = new CooperativePassScheduler(() => 0);
868
- const downloadFile = vi.mocked(s3Module.downloadFile);
869
- const originalDownload = downloadFile.getMockImplementation();
870
- const localPath = path.join(tmpDir, "companies", "acme", key);
871
- let queuedAt = -1;
872
- let realtimeStartedAt = -1;
873
- let transferFinishedAt = -1;
874
- let realtimePass;
875
- downloadFile.mockImplementation(async (_context, _key, stagePath, options) => {
876
- setTimeout(() => {
877
- queuedAt = performance.now();
878
- realtimePass = scheduler.submit(async () => {
879
- realtimeStartedAt = performance.now();
880
- fs.mkdirSync(path.dirname(localPath), { recursive: true });
881
- fs.writeFileSync(localPath, "realtime version");
882
- const journal = readJournal("acme");
883
- const stat = fs.statSync(localPath);
884
- journal.files[key] = {
885
- hash: crypto.createHash("sha256").update("realtime version").digest("hex"),
886
- size: stat.size,
887
- syncedAt: fixedNow.toISOString(),
888
- direction: "up",
889
- remoteEtag: "realtime-version",
890
- mtimeMs: stat.mtimeMs,
891
- ctimeMs: stat.ctimeMs,
892
- };
893
- writeJournal("acme", journal);
894
- }, "realtime");
895
- }, 50);
896
- await new Promise((resolve) => setTimeout(resolve, 2_000));
897
- options?.beforeReplace?.();
898
- fs.writeFileSync(stagePath, "stale chore");
899
- transferFinishedAt = performance.now();
900
- return {
901
- metadata: {},
902
- contentHash: crypto.createHash("sha256").update("stale chore").digest("hex"),
903
- contentSize: 11,
904
- };
905
- });
906
- let result;
907
- try {
908
- await scheduler.submit(async ({ checkpoint }) => {
909
- result = await sync({
910
- company: "acme",
911
- vaultConfig: mockConfig,
912
- hqRoot: tmpDir,
913
- skipReindex: true,
914
- onEvent: () => undefined,
915
- cooperativeCheckpoint: checkpoint,
916
- });
917
- }, "chore");
918
- await realtimePass;
919
- }
920
- finally {
921
- if (originalDownload)
922
- downloadFile.mockImplementation(originalDownload);
923
- else
924
- downloadFile.mockReset();
925
- vi.unstubAllEnvs();
926
- }
927
- expect(queuedAt).toBeGreaterThanOrEqual(0);
928
- expect(realtimeStartedAt - queuedAt).toBeLessThan(1_500);
929
- expect(realtimeStartedAt).toBeLessThan(transferFinishedAt);
930
- expect(fs.readFileSync(localPath, "utf8")).toBe("realtime version");
931
- expect(result?.filesDownloaded).toBe(0);
932
- expect(result?.conflicts).toBe(1);
933
- expect(readJournal("acme").files[key]?.remoteEtag).toBe("realtime-version");
934
- expect(fs.readdirSync(path.dirname(localPath)).some((name) => name.startsWith(".hq-tmp-stage-"))).toBe(false);
935
- }, 15_000);
936
- it("discards a staged download when realtime updates an existing journal row", async () => {
937
- const fixedNow = new Date("2026-09-24T12:00:00.000Z");
938
- const key = "docs/existing-row.md";
939
- const baseline = "preexisting local version";
940
- const baselineHash = crypto.createHash("sha256").update(baseline).digest("hex");
941
- const localPath = path.join(tmpDir, "companies", "acme", key);
942
- fs.mkdirSync(path.dirname(localPath), { recursive: true });
943
- fs.writeFileSync(localPath, baseline);
944
- writeJournal("acme", {
945
- version: "2",
946
- lastSync: "",
947
- pulls: [],
948
- files: {
949
- [key]: {
950
- hash: baselineHash,
951
- size: baseline.length,
952
- syncedAt: new Date(fixedNow.getTime() - 60_000).toISOString(),
953
- direction: "down",
954
- kind: "file",
955
- remoteEtag: "old-etag",
956
- },
957
- },
958
- });
959
- vi.stubEnv("HQ_SYNC_TRANSFER_CONCURRENCY", "1");
960
- vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
961
- { key, size: 11, lastModified: fixedNow, etag: '"chore-version"' },
962
- ]);
963
- const scheduler = new CooperativePassScheduler(() => 0);
964
- const downloadFile = vi.mocked(s3Module.downloadFile);
965
- const originalDownload = downloadFile.getMockImplementation();
966
- let realtimeStartedAt = -1;
967
- let transferFinishedAt = -1;
968
- let realtimePass;
969
- downloadFile.mockImplementation(async (_context, _key, stagePath, options) => {
970
- setTimeout(() => {
971
- realtimePass = scheduler.submit(async () => {
972
- realtimeStartedAt = performance.now();
973
- const journal = readJournal("acme");
974
- journal.files[key] = {
975
- ...journal.files[key],
976
- syncedAt: fixedNow.toISOString(),
977
- remoteEtag: "realtime-version",
978
- };
979
- writeJournal("acme", journal);
980
- }, "realtime");
981
- }, 50);
982
- await new Promise((resolve) => setTimeout(resolve, 2_000));
983
- options?.beforeReplace?.();
984
- fs.writeFileSync(stagePath, "stale chore");
985
- transferFinishedAt = performance.now();
986
- return {
987
- metadata: {},
988
- contentHash: crypto.createHash("sha256").update("stale chore").digest("hex"),
989
- contentSize: 11,
990
- };
991
- });
992
- let result;
993
- try {
994
- await scheduler.submit(async ({ checkpoint }) => {
995
- result = await sync({
996
- company: "acme",
997
- vaultConfig: mockConfig,
998
- hqRoot: tmpDir,
999
- skipReindex: true,
1000
- onEvent: () => undefined,
1001
- cooperativeCheckpoint: checkpoint,
1002
- });
1003
- }, "chore");
1004
- await realtimePass;
1005
- }
1006
- finally {
1007
- if (originalDownload)
1008
- downloadFile.mockImplementation(originalDownload);
1009
- else
1010
- downloadFile.mockReset();
1011
- vi.unstubAllEnvs();
1012
- }
1013
- expect(realtimeStartedAt).toBeGreaterThanOrEqual(0);
1014
- expect(realtimeStartedAt).toBeLessThan(transferFinishedAt);
1015
- expect(fs.readFileSync(localPath, "utf8")).toBe(baseline);
1016
- expect(result?.filesDownloaded).toBe(0);
1017
- expect(result?.conflicts).toBe(1);
1018
- expect(readJournal("acme").files[key]?.remoteEtag).toBe("realtime-version");
1019
- expect(fs.readdirSync(path.dirname(localPath)).some((name) => name.startsWith(".hq-tmp-stage-"))).toBe(false);
1020
- }, 15_000);
1021
- it("releases a timed-out full-pull path before admitting the same key to realtime", async () => {
1022
- const scheduler = new CooperativePassScheduler(() => 0);
1023
- const downloadFile = vi.mocked(s3Module.downloadFile);
1024
- const originalDownload = downloadFile.getMockImplementation();
1025
- let stalledPathActive = false;
1026
- let timedOutAt = -1;
1027
- let realtimeStartedAt = -1;
1028
- let realtimePass;
1029
- downloadFile.mockImplementation(async (context, key, localPath, options) => {
1030
- if (key === "docs/handoff.md" && timedOutAt < 0) {
1031
- stalledPathActive = true;
1032
- const startedAt = performance.now();
1033
- realtimePass = scheduler.submit(async () => {
1034
- realtimeStartedAt = performance.now();
1035
- expect(stalledPathActive).toBe(false);
1036
- expect(key).toBe("docs/handoff.md");
1037
- }, "realtime");
1038
- await new Promise((resolve) => setTimeout(resolve, 30));
1039
- stalledPathActive = false;
1040
- timedOutAt = performance.now();
1041
- expect(timedOutAt - startedAt).toBeLessThan(1_000);
1042
- throw new ObjectBodyIdleTimeoutError(key, 30);
1043
- }
1044
- return originalDownload(context, key, localPath, options);
1045
- });
1046
- try {
1047
- await scheduler.submit(async ({ checkpoint }) => sync({
1048
- company: "acme",
1049
- vaultConfig: mockConfig,
1050
- hqRoot: tmpDir,
1051
- skipReindex: true,
1052
- onEvent: () => undefined,
1053
- cooperativeCheckpoint: () => checkpoint(),
1054
- }), "chore");
1055
- await realtimePass;
1056
- }
1057
- finally {
1058
- if (originalDownload)
1059
- downloadFile.mockImplementation(originalDownload);
1060
- else
1061
- downloadFile.mockReset();
1062
- }
1063
- expect(timedOutAt).toBeGreaterThanOrEqual(0);
1064
- expect(realtimeStartedAt).toBeGreaterThan(timedOutAt);
1065
- expect(realtimeStartedAt - timedOutAt).toBeLessThan(1_000);
1066
- expect(stalledPathActive).toBe(false);
1067
- });
1068
- it("continues an ordinary pull after one object body times out", async () => {
1069
- const listRemoteFiles = vi.mocked(s3Module.listRemoteFiles);
1070
- const originalListRemoteFiles = listRemoteFiles.getMockImplementation();
1071
- listRemoteFiles.mockResolvedValue([
1072
- { key: "docs/stalled.md", size: 7, lastModified: new Date(), etag: '"stalled"' },
1073
- { key: "docs/later.md", size: 7, lastModified: new Date(), etag: '"later"' },
1074
- ]);
1075
- writeJournal("acme", { version: "2", lastSync: "", pulls: [], files: {} });
1076
- const downloadFile = vi.mocked(s3Module.downloadFile);
1077
- const originalDownload = downloadFile.getMockImplementation();
1078
- const attemptedKeys = [];
1079
- downloadFile.mockImplementation(async (context, key, localPath, options) => {
1080
- attemptedKeys.push(key);
1081
- if (key === "docs/stalled.md")
1082
- throw new ObjectBodyIdleTimeoutError(key, 30);
1083
- return originalDownload(context, key, localPath, options);
1084
- });
1085
- try {
1086
- await sync({
1087
- company: "acme",
1088
- vaultConfig: mockConfig,
1089
- hqRoot: tmpDir,
1090
- skipReindex: true,
1091
- onEvent: () => undefined,
1092
- });
1093
- }
1094
- finally {
1095
- if (originalDownload)
1096
- downloadFile.mockImplementation(originalDownload);
1097
- else
1098
- downloadFile.mockReset();
1099
- if (originalListRemoteFiles)
1100
- listRemoteFiles.mockImplementation(originalListRemoteFiles);
1101
- else
1102
- listRemoteFiles.mockReset();
1103
- }
1104
- expect(attemptedKeys).toEqual(["docs/stalled.md", "docs/later.md"]);
1105
- expect(fs.existsSync(path.join(tmpDir, "companies", "acme", "docs", "later.md"))).toBe(true);
1106
- });
1107
793
  /**
1108
794
  * A cooperative hand-off yields the root lock then continues the same run,
1109
795
  * so downloads completed before the hand-off stay on this pass's counters