@indigoai-us/hq-cloud 6.16.68-alpha.2 → 6.16.68-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/sync.d.ts +7 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +42 -6
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +96 -2
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/lib/net-errors.d.ts +6 -0
- package/dist/lib/net-errors.d.ts.map +1 -1
- package/dist/lib/net-errors.js +10 -0
- package/dist/lib/net-errors.js.map +1 -1
- package/dist/lib/net-errors.test.js +4 -0
- package/dist/lib/net-errors.test.js.map +1 -1
- package/dist/object-io.d.ts +8 -0
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +41 -12
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +44 -1
- package/dist/object-io.test.js.map +1 -1
- package/dist/s3.d.ts +8 -1
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +5 -9
- package/dist/s3.js.map +1 -1
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +12 -10
- package/dist/vault-client.js.map +1 -1
- package/package.json +1 -1
package/dist/cli/sync.test.js
CHANGED
|
@@ -104,6 +104,19 @@ vi.mock("../s3.js", async (importOriginal) => {
|
|
|
104
104
|
yield remoteFile;
|
|
105
105
|
}
|
|
106
106
|
}
|
|
107
|
+
async function* streamRemoteFilePages(ctx, prefix, waitPage) {
|
|
108
|
+
const pending = listRemoteFiles(ctx, prefix);
|
|
109
|
+
const files = await (waitPage ? waitPage(pending) : pending);
|
|
110
|
+
const rows = Array.isArray(files) ? files : [];
|
|
111
|
+
if (rows.length === 0) {
|
|
112
|
+
yield [];
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
const pageSize = 1000;
|
|
116
|
+
for (let i = 0; i < rows.length; i += pageSize) {
|
|
117
|
+
yield rows.slice(i, i + pageSize);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
107
120
|
return {
|
|
108
121
|
...actual,
|
|
109
122
|
toPosixKey: (key) => key.split("\\").join("/"),
|
|
@@ -119,6 +132,7 @@ vi.mock("../s3.js", async (importOriginal) => {
|
|
|
119
132
|
}),
|
|
120
133
|
listRemoteFiles,
|
|
121
134
|
streamRemoteFiles,
|
|
135
|
+
streamRemoteFilePages,
|
|
122
136
|
deleteRemoteFile: innerVi.fn().mockResolvedValue(undefined),
|
|
123
137
|
headRemoteFile: innerVi.fn().mockResolvedValue(null),
|
|
124
138
|
primeObjectTransport: innerVi.fn().mockResolvedValue(undefined),
|
|
@@ -135,7 +149,7 @@ vi.mock("../remote-pull.js", async (importOriginal) => {
|
|
|
135
149
|
const { vi: innerVi } = await import("vitest");
|
|
136
150
|
return { ...actual, listRemoteForScope: innerVi.fn(actual.listRemoteForScope) };
|
|
137
151
|
});
|
|
138
|
-
import { sync, reportNewFilesToNotify, applyRealtimeDeltaPage, isSyncMutationStdinCommand, runSyncMutationStdinJson, SYNC_MUTATION_STDIN_COMMAND, FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE, FULL_PULL_LIST_PAGE_CHECKPOINT_SIZE, targetedJournalSessionCensusSizes, win32SymlinkFlavorBroken, _testing as syncTesting, } from "./sync.js";
|
|
152
|
+
import { sync, reportNewFilesToNotify, applyRealtimeDeltaPage, isSyncMutationStdinCommand, runSyncMutationStdinJson, SYNC_MUTATION_STDIN_COMMAND, FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE, FULL_PULL_LIST_PAGE_CHECKPOINT_SIZE, setListCooperativeYieldMsForTesting, targetedJournalSessionCensusSizes, win32SymlinkFlavorBroken, _testing as syncTesting, } from "./sync.js";
|
|
139
153
|
import { share } from "./share.js";
|
|
140
154
|
import * as s3Module from "../s3.js";
|
|
141
155
|
import * as conflictModule from "./conflict.js";
|
|
@@ -149,7 +163,8 @@ import { readJournal, writeJournal, getEntry, listJournals, } from "../journal.j
|
|
|
149
163
|
import { SqsPushReceiver } from "../sync/push-receiver.js";
|
|
150
164
|
import { buildTargetedPullArgv } from "../bin/sync-runner-watch-routes.js";
|
|
151
165
|
import { ReconcileCursorStore } from "../sync/reconcile-cursor.js";
|
|
152
|
-
import { RateLimitedError, setObjectIOFactory } from "../object-io.js";
|
|
166
|
+
import { RateLimitedError, setObjectIOFactory, setObjectIORequestTimeoutMsForTesting, } from "../object-io.js";
|
|
167
|
+
import { isTransientNetworkError } from "../lib/net-errors.js";
|
|
153
168
|
/**
|
|
154
169
|
* Read a journal the way production does.
|
|
155
170
|
*
|
|
@@ -969,6 +984,85 @@ describe("sync", () => {
|
|
|
969
984
|
{ type: "cooperative-handoff", direction: "pull", resumedAt: FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE },
|
|
970
985
|
]);
|
|
971
986
|
}, 120_000);
|
|
987
|
+
it("fails a LIST page that never responds as transient within the request timeout", async () => {
|
|
988
|
+
setObjectIORequestTimeoutMsForTesting(50);
|
|
989
|
+
const listRemoteFiles = vi.mocked(s3Module.listRemoteFiles);
|
|
990
|
+
const originalListRemoteFiles = listRemoteFiles.getMockImplementation();
|
|
991
|
+
listRemoteFiles.mockImplementation(() => new Promise((_, reject) => {
|
|
992
|
+
const signal = AbortSignal.timeout(50);
|
|
993
|
+
signal.addEventListener("abort", () => {
|
|
994
|
+
reject(Object.assign(new Error("The operation was aborted due to timeout"), {
|
|
995
|
+
name: "TimeoutError",
|
|
996
|
+
}));
|
|
997
|
+
});
|
|
998
|
+
}));
|
|
999
|
+
const started = Date.now();
|
|
1000
|
+
let caught;
|
|
1001
|
+
try {
|
|
1002
|
+
await sync({
|
|
1003
|
+
company: "acme",
|
|
1004
|
+
vaultConfig: mockConfig,
|
|
1005
|
+
hqRoot: tmpDir,
|
|
1006
|
+
skipReindex: true,
|
|
1007
|
+
onEvent: () => undefined,
|
|
1008
|
+
cooperativeCheckpoint: async () => false,
|
|
1009
|
+
});
|
|
1010
|
+
}
|
|
1011
|
+
catch (err) {
|
|
1012
|
+
caught = err;
|
|
1013
|
+
}
|
|
1014
|
+
finally {
|
|
1015
|
+
setObjectIORequestTimeoutMsForTesting(undefined);
|
|
1016
|
+
if (originalListRemoteFiles)
|
|
1017
|
+
listRemoteFiles.mockImplementation(originalListRemoteFiles);
|
|
1018
|
+
else
|
|
1019
|
+
listRemoteFiles.mockReset();
|
|
1020
|
+
}
|
|
1021
|
+
expect(isTransientNetworkError(caught)).toBe(true);
|
|
1022
|
+
expect(Date.now() - started).toBeLessThan(2_000);
|
|
1023
|
+
}, 10_000);
|
|
1024
|
+
it("completes queued realtime while a full-pass LIST page is still outstanding", async () => {
|
|
1025
|
+
setListCooperativeYieldMsForTesting(20);
|
|
1026
|
+
const listRemoteFiles = vi.mocked(s3Module.listRemoteFiles);
|
|
1027
|
+
const originalListRemoteFiles = listRemoteFiles.getMockImplementation();
|
|
1028
|
+
let releaseList = () => undefined;
|
|
1029
|
+
let listReleased = false;
|
|
1030
|
+
listRemoteFiles.mockImplementation(() => new Promise((resolve) => {
|
|
1031
|
+
releaseList = resolve;
|
|
1032
|
+
}));
|
|
1033
|
+
const scheduler = new CooperativePassScheduler(() => 0);
|
|
1034
|
+
let realtimePass;
|
|
1035
|
+
const chore = scheduler.submit(async ({ checkpoint }) => {
|
|
1036
|
+
realtimePass = scheduler.submit(async () => {
|
|
1037
|
+
expect(listReleased).toBe(false);
|
|
1038
|
+
}, "realtime");
|
|
1039
|
+
return sync({
|
|
1040
|
+
company: "acme",
|
|
1041
|
+
vaultConfig: mockConfig,
|
|
1042
|
+
hqRoot: tmpDir,
|
|
1043
|
+
skipReindex: true,
|
|
1044
|
+
onEvent: () => undefined,
|
|
1045
|
+
cooperativeCheckpoint: async () => checkpoint(),
|
|
1046
|
+
});
|
|
1047
|
+
}, "chore");
|
|
1048
|
+
try {
|
|
1049
|
+
await vi.waitFor(() => {
|
|
1050
|
+
expect(realtimePass).toBeDefined();
|
|
1051
|
+
});
|
|
1052
|
+
await realtimePass;
|
|
1053
|
+
expect(listReleased).toBe(false);
|
|
1054
|
+
listReleased = true;
|
|
1055
|
+
releaseList([]);
|
|
1056
|
+
await chore;
|
|
1057
|
+
}
|
|
1058
|
+
finally {
|
|
1059
|
+
setListCooperativeYieldMsForTesting(undefined);
|
|
1060
|
+
if (originalListRemoteFiles)
|
|
1061
|
+
listRemoteFiles.mockImplementation(originalListRemoteFiles);
|
|
1062
|
+
else
|
|
1063
|
+
listRemoteFiles.mockReset();
|
|
1064
|
+
}
|
|
1065
|
+
}, 10_000);
|
|
972
1066
|
it("keeps every no-op full-pull journal row and emits no local deletes", async () => {
|
|
973
1067
|
const fixedNow = new Date("2026-09-17T12:00:00.000Z");
|
|
974
1068
|
const materialized = "mock file content";
|