@indigoai-us/hq-cloud 6.14.7 → 6.14.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backup-prune.d.ts +68 -0
- package/dist/backup-prune.d.ts.map +1 -0
- package/dist/backup-prune.js +196 -0
- package/dist/backup-prune.js.map +1 -0
- package/dist/backup-prune.test.d.ts +2 -0
- package/dist/backup-prune.test.d.ts.map +1 -0
- package/dist/backup-prune.test.js +89 -0
- package/dist/backup-prune.test.js.map +1 -0
- package/dist/bin/backup-prune-runner.d.ts +3 -0
- package/dist/bin/backup-prune-runner.d.ts.map +1 -0
- package/dist/bin/backup-prune-runner.js +50 -0
- package/dist/bin/backup-prune-runner.js.map +1 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +28 -0
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +2 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/cli/rescue-classify-ordering.test.js +105 -0
- package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
- package/dist/cli/rescue-core.d.ts +1 -0
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +478 -300
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-snapshot.d.ts +14 -0
- package/dist/cli/rescue-snapshot.d.ts.map +1 -0
- package/dist/cli/rescue-snapshot.js +39 -0
- package/dist/cli/rescue-snapshot.js.map +1 -0
- package/dist/cli/rescue-snapshot.test.d.ts +2 -0
- package/dist/cli/rescue-snapshot.test.d.ts.map +1 -0
- package/dist/cli/rescue-snapshot.test.js +46 -0
- package/dist/cli/rescue-snapshot.test.js.map +1 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +13 -0
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +51 -0
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/object-io.d.ts +19 -2
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +168 -32
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +157 -1
- package/dist/object-io.test.js.map +1 -1
- package/dist/signals/get.test.js +21 -1
- package/dist/signals/get.test.js.map +1 -1
- package/dist/signals/list.test.js +21 -1
- package/dist/signals/list.test.js.map +1 -1
- package/dist/sources/get.test.js +21 -1
- package/dist/sources/get.test.js.map +1 -1
- package/dist/sources/list.test.js +21 -1
- package/dist/sources/list.test.js.map +1 -1
- package/package.json +3 -2
- package/src/backup-prune.test.ts +98 -0
- package/src/backup-prune.ts +182 -0
- package/src/bin/backup-prune-runner.ts +33 -0
- package/src/bin/sync-runner-watch-loop.ts +18 -0
- package/src/bin/sync-runner.ts +2 -0
- package/src/cli/rescue-classify-ordering.test.ts +121 -0
- package/src/cli/rescue-core.ts +261 -86
- package/src/cli/rescue-snapshot.test.ts +57 -0
- package/src/cli/rescue-snapshot.ts +51 -0
- package/src/cli/sync.test.ts +62 -0
- package/src/cli/sync.ts +18 -0
- package/src/object-io.test.ts +175 -0
- package/src/object-io.ts +213 -32
- package/src/signals/get.test.ts +26 -2
- package/src/signals/list.test.ts +26 -2
- package/src/sources/get.test.ts +26 -2
- package/src/sources/list.test.ts +26 -2
package/src/cli/sync.test.ts
CHANGED
|
@@ -2566,6 +2566,68 @@ describe("sync", () => {
|
|
|
2566
2566
|
expect(journal.files["knowledge/readme.md"].remoteEtag).toBe("def456");
|
|
2567
2567
|
});
|
|
2568
2568
|
|
|
2569
|
+
it("checkpoints completed downloads so a failed bulk pass resumes only its tail", async () => {
|
|
2570
|
+
process.env.HQ_SYNC_TRANSFER_CONCURRENCY = "1";
|
|
2571
|
+
const remoteFiles = Array.from({ length: 9 }, (_, index) => ({
|
|
2572
|
+
key: `bulk/file-${index}.md`,
|
|
2573
|
+
size: 10,
|
|
2574
|
+
lastModified: new Date(),
|
|
2575
|
+
etag: `"bulk-${index}"`,
|
|
2576
|
+
}));
|
|
2577
|
+
const defaultDownload = vi.mocked(s3Module.downloadFile).getMockImplementation();
|
|
2578
|
+
if (!defaultDownload) throw new Error("missing default download mock");
|
|
2579
|
+
|
|
2580
|
+
try {
|
|
2581
|
+
vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce(remoteFiles);
|
|
2582
|
+
vi.mocked(s3Module.downloadFile).mockImplementation(
|
|
2583
|
+
async (ctx, key, localPath) => {
|
|
2584
|
+
if (key === "bulk/file-8.md") {
|
|
2585
|
+
// This models the next transfer crashing the process. The first
|
|
2586
|
+
// eight entries must already be durable before the final writer.
|
|
2587
|
+
const checkpoint = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
|
|
2588
|
+
expect(Object.keys(checkpoint.files)).toHaveLength(8);
|
|
2589
|
+
throw new Error("simulated crash after checkpoint");
|
|
2590
|
+
}
|
|
2591
|
+
return defaultDownload(ctx, key, localPath);
|
|
2592
|
+
},
|
|
2593
|
+
);
|
|
2594
|
+
|
|
2595
|
+
const first = await sync({
|
|
2596
|
+
company: "acme",
|
|
2597
|
+
vaultConfig: mockConfig,
|
|
2598
|
+
hqRoot: tmpDir,
|
|
2599
|
+
skipReindex: true,
|
|
2600
|
+
onEvent: () => undefined,
|
|
2601
|
+
});
|
|
2602
|
+
expect(first.filesDownloaded).toBe(8);
|
|
2603
|
+
|
|
2604
|
+
const checkpoint = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
|
|
2605
|
+
expect(Object.keys(checkpoint.files)).toHaveLength(8);
|
|
2606
|
+
|
|
2607
|
+
vi.mocked(s3Module.downloadFile).mockImplementation(defaultDownload);
|
|
2608
|
+
vi.mocked(s3Module.downloadFile).mockClear();
|
|
2609
|
+
vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce(remoteFiles);
|
|
2610
|
+
|
|
2611
|
+
await sync({
|
|
2612
|
+
company: "acme",
|
|
2613
|
+
vaultConfig: mockConfig,
|
|
2614
|
+
hqRoot: tmpDir,
|
|
2615
|
+
skipReindex: true,
|
|
2616
|
+
onEvent: () => undefined,
|
|
2617
|
+
});
|
|
2618
|
+
|
|
2619
|
+
expect(vi.mocked(s3Module.downloadFile)).toHaveBeenCalledTimes(1);
|
|
2620
|
+
expect(vi.mocked(s3Module.downloadFile)).toHaveBeenCalledWith(
|
|
2621
|
+
expect.anything(),
|
|
2622
|
+
"bulk/file-8.md",
|
|
2623
|
+
expect.any(String),
|
|
2624
|
+
);
|
|
2625
|
+
} finally {
|
|
2626
|
+
vi.mocked(s3Module.downloadFile).mockImplementation(defaultDownload);
|
|
2627
|
+
delete process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
|
|
2628
|
+
}
|
|
2629
|
+
});
|
|
2630
|
+
|
|
2569
2631
|
// ── Stage-1 plan event ─────────────────────────────────────────────────
|
|
2570
2632
|
|
|
2571
2633
|
it("emits a plan event before any progress events", async () => {
|
package/src/cli/sync.ts
CHANGED
|
@@ -602,6 +602,8 @@ interface PullRunContext {
|
|
|
602
602
|
*/
|
|
603
603
|
currentExcludeSet: string[];
|
|
604
604
|
fileTombstones: ReadonlyMap<string, CompanyTombstone>;
|
|
605
|
+
/** Successful downloads since the last durable journal checkpoint. */
|
|
606
|
+
downloadsSinceJournalCheckpoint: number;
|
|
605
607
|
}
|
|
606
608
|
|
|
607
609
|
interface PullCounters {
|
|
@@ -647,6 +649,8 @@ export function resolveAutoPruneCap(): number {
|
|
|
647
649
|
|
|
648
650
|
/** Max time to wait on the best-effort new-files notification POST. */
|
|
649
651
|
const NOTIFY_FILE_ADDED_TIMEOUT_MS = 5000;
|
|
652
|
+
/** Bound crash recovery to at most this many completed pull downloads. */
|
|
653
|
+
const DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE = 8;
|
|
650
654
|
|
|
651
655
|
/**
|
|
652
656
|
* Server cap on files per `/v1/notify/file-added` report. The endpoint rejects
|
|
@@ -915,6 +919,7 @@ async function buildPullContext(options: SyncOptions): Promise<PullRunContext> {
|
|
|
915
919
|
currentPrefixSet,
|
|
916
920
|
currentExcludeSet,
|
|
917
921
|
fileTombstones,
|
|
922
|
+
downloadsSinceJournalCheckpoint: 0,
|
|
918
923
|
};
|
|
919
924
|
}
|
|
920
925
|
|
|
@@ -1534,6 +1539,19 @@ async function downloadOne(
|
|
|
1534
1539
|
message: err instanceof Error ? err.message : String(err),
|
|
1535
1540
|
});
|
|
1536
1541
|
}
|
|
1542
|
+
return;
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1545
|
+
run.downloadsSinceJournalCheckpoint++;
|
|
1546
|
+
if (
|
|
1547
|
+
run.downloadsSinceJournalCheckpoint >=
|
|
1548
|
+
DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE
|
|
1549
|
+
) {
|
|
1550
|
+
// writeJournal is already fsync + rename atomic. Persist completed work
|
|
1551
|
+
// before scheduling more objects so an abrupt process exit only replays a
|
|
1552
|
+
// small tail, not an entire bulk download.
|
|
1553
|
+
writeJournal(run.journalSlug, run.journal);
|
|
1554
|
+
run.downloadsSinceJournalCheckpoint = 0;
|
|
1537
1555
|
}
|
|
1538
1556
|
}
|
|
1539
1557
|
|
package/src/object-io.test.ts
CHANGED
|
@@ -9,12 +9,17 @@
|
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
|
12
|
+
import * as http from "node:http";
|
|
13
|
+
import { once } from "node:events";
|
|
14
|
+
import { spawn } from "node:child_process";
|
|
12
15
|
import {
|
|
13
16
|
PresignObjectIO,
|
|
14
17
|
S3SdkObjectIO,
|
|
15
18
|
setObjectIOFactory,
|
|
19
|
+
setPresignedGetTransportForTesting,
|
|
16
20
|
resolveObjectIO,
|
|
17
21
|
presignObjectIOFactory,
|
|
22
|
+
type PresignedGetTransport,
|
|
18
23
|
type PresignTransportClient,
|
|
19
24
|
} from "./object-io.js";
|
|
20
25
|
import type { EntityContext } from "./types.js";
|
|
@@ -23,6 +28,55 @@ import { isAccessDenied } from "./sync-core.js";
|
|
|
23
28
|
|
|
24
29
|
const COMPANY = "cmp_test";
|
|
25
30
|
|
|
31
|
+
function fetchBackedPresignedGetTransport(): PresignedGetTransport {
|
|
32
|
+
return {
|
|
33
|
+
async get(url, headers) {
|
|
34
|
+
const response = await fetch(url, { method: "GET", headers });
|
|
35
|
+
const body = response.body;
|
|
36
|
+
return {
|
|
37
|
+
status: response.status,
|
|
38
|
+
headers: response.headers,
|
|
39
|
+
body: body
|
|
40
|
+
? (async function* (): AsyncIterable<Uint8Array> {
|
|
41
|
+
const reader = body.getReader();
|
|
42
|
+
let exhausted = false;
|
|
43
|
+
try {
|
|
44
|
+
while (true) {
|
|
45
|
+
const { done, value } = await reader.read();
|
|
46
|
+
if (done) {
|
|
47
|
+
exhausted = true;
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
if (value.byteLength > 0) yield value;
|
|
51
|
+
}
|
|
52
|
+
} finally {
|
|
53
|
+
if (!exhausted) {
|
|
54
|
+
try {
|
|
55
|
+
await reader.cancel();
|
|
56
|
+
} catch {
|
|
57
|
+
// The response may already have been cancelled by destroy.
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
reader.releaseLock();
|
|
61
|
+
}
|
|
62
|
+
})()
|
|
63
|
+
: undefined,
|
|
64
|
+
destroy: () => {
|
|
65
|
+
void body?.cancel().catch(() => undefined);
|
|
66
|
+
},
|
|
67
|
+
};
|
|
68
|
+
},
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
beforeEach(() => {
|
|
73
|
+
setPresignedGetTransportForTesting(fetchBackedPresignedGetTransport());
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
afterEach(() => {
|
|
77
|
+
setPresignedGetTransportForTesting(null);
|
|
78
|
+
});
|
|
79
|
+
|
|
26
80
|
function ctx(): EntityContext {
|
|
27
81
|
return {
|
|
28
82
|
uid: COMPANY,
|
|
@@ -506,6 +560,27 @@ describe("PresignObjectIO.getObject", () => {
|
|
|
506
560
|
});
|
|
507
561
|
});
|
|
508
562
|
|
|
563
|
+
it("drains the injected transport body instead of returning empty content", async () => {
|
|
564
|
+
const { vault, setPresign } = makeVault();
|
|
565
|
+
setPresign([{ key: "shared/a.md", op: "get", url: "https://s3/get" }]);
|
|
566
|
+
const transportGet = vi.fn(async () => ({
|
|
567
|
+
status: 200,
|
|
568
|
+
headers: new Headers({ "x-amz-meta-source": "transport" }),
|
|
569
|
+
body: (async function* (): AsyncIterable<Uint8Array> {
|
|
570
|
+
yield Buffer.from("transport-bytes");
|
|
571
|
+
})(),
|
|
572
|
+
destroy: () => undefined,
|
|
573
|
+
}));
|
|
574
|
+
setPresignedGetTransportForTesting({ get: transportGet });
|
|
575
|
+
|
|
576
|
+
const io = new PresignObjectIO(vault, COMPANY);
|
|
577
|
+
const res = await io.getObject("shared/a.md");
|
|
578
|
+
|
|
579
|
+
expect(res.body.toString("utf-8")).toBe("transport-bytes");
|
|
580
|
+
expect(res.metadata).toEqual({ source: "transport" });
|
|
581
|
+
expect(transportGet).toHaveBeenCalledWith("https://s3/get", undefined);
|
|
582
|
+
});
|
|
583
|
+
|
|
509
584
|
it("throws a NotFound-named error on 404 (so s3.ts catch sites match)", async () => {
|
|
510
585
|
const { vault, setPresign } = makeVault();
|
|
511
586
|
setPresign([{ key: "gone", op: "get", url: "https://s3/get" }]);
|
|
@@ -1044,3 +1119,103 @@ describe("presignObjectIOFactory — personal vaults route to S3 SDK", () => {
|
|
|
1044
1119
|
expect(personal).toBeInstanceOf(S3SdkObjectIO);
|
|
1045
1120
|
});
|
|
1046
1121
|
});
|
|
1122
|
+
|
|
1123
|
+
describe("PresignObjectIO — core HTTP presigned GET", () => {
|
|
1124
|
+
function streamingVault(baseUrl: string): PresignTransportClient {
|
|
1125
|
+
return {
|
|
1126
|
+
presign: async (input) => ({
|
|
1127
|
+
results: input.keys.map((key) => ({
|
|
1128
|
+
key: key.key,
|
|
1129
|
+
op: "get" as const,
|
|
1130
|
+
url: `${baseUrl}/${key.key}`,
|
|
1131
|
+
})),
|
|
1132
|
+
expiresAt: "2099-01-01T00:00:00.000Z",
|
|
1133
|
+
}),
|
|
1134
|
+
listFiles: async () => ({ objects: [], cursor: null, truncated: false }),
|
|
1135
|
+
};
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
it("destroys an early-exit response and rejects a FIN-short body", async () => {
|
|
1139
|
+
setPresignedGetTransportForTesting(null);
|
|
1140
|
+
let responseClosed: Promise<unknown> | undefined;
|
|
1141
|
+
const server = http.createServer((req, res) => {
|
|
1142
|
+
if (req.url === "/short") {
|
|
1143
|
+
res.writeHead(200, { "content-length": "10" });
|
|
1144
|
+
res.end("short");
|
|
1145
|
+
return;
|
|
1146
|
+
}
|
|
1147
|
+
|
|
1148
|
+
res.writeHead(200, { "content-length": "131072" });
|
|
1149
|
+
res.write(Buffer.alloc(65_536, 1));
|
|
1150
|
+
responseClosed = once(res, "close");
|
|
1151
|
+
const finish = setTimeout(() => res.end(Buffer.alloc(65_536, 2)), 1_000);
|
|
1152
|
+
res.once("close", () => clearTimeout(finish));
|
|
1153
|
+
});
|
|
1154
|
+
server.listen(0, "127.0.0.1");
|
|
1155
|
+
await once(server, "listening");
|
|
1156
|
+
const address = server.address();
|
|
1157
|
+
if (!address || typeof address === "string") {
|
|
1158
|
+
throw new Error("test server did not bind a TCP port");
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
try {
|
|
1162
|
+
const io = new PresignObjectIO(
|
|
1163
|
+
streamingVault(`http://127.0.0.1:${address.port}`),
|
|
1164
|
+
COMPANY,
|
|
1165
|
+
);
|
|
1166
|
+
const streamed = await io.getObjectStream("large");
|
|
1167
|
+
const iterator = streamed.body[Symbol.asyncIterator]();
|
|
1168
|
+
expect((await iterator.next()).value).toBeInstanceOf(Uint8Array);
|
|
1169
|
+
await iterator.return?.();
|
|
1170
|
+
if (!responseClosed) throw new Error("response did not start streaming");
|
|
1171
|
+
await expect(responseClosed).resolves.toBeDefined();
|
|
1172
|
+
|
|
1173
|
+
await expect(io.getObject("short")).rejects.toThrow(
|
|
1174
|
+
/content-length mismatch/,
|
|
1175
|
+
);
|
|
1176
|
+
} finally {
|
|
1177
|
+
server.close();
|
|
1178
|
+
await once(server, "close");
|
|
1179
|
+
}
|
|
1180
|
+
});
|
|
1181
|
+
|
|
1182
|
+
it.runIf(
|
|
1183
|
+
process.platform === "win32" && /^(22|24)\./.test(process.versions.node),
|
|
1184
|
+
)("keeps a child FIN safe after the consumer has applied backpressure", async () => {
|
|
1185
|
+
setPresignedGetTransportForTesting(null);
|
|
1186
|
+
const child = spawn(process.execPath, [
|
|
1187
|
+
"-e",
|
|
1188
|
+
[
|
|
1189
|
+
"const http = require('node:http');",
|
|
1190
|
+
"const body = Buffer.alloc(524288, 7);",
|
|
1191
|
+
"const server = http.createServer((_req, res) => {",
|
|
1192
|
+
" res.writeHead(200, { 'content-length': String(body.length) });",
|
|
1193
|
+
" for (let offset = 0; offset < body.length; offset += 16384) res.write(body.subarray(offset, offset + 16384));",
|
|
1194
|
+
" res.end();",
|
|
1195
|
+
"});",
|
|
1196
|
+
"server.listen(0, '127.0.0.1', () => process.stdout.write(String(server.address().port) + '\\n'));",
|
|
1197
|
+
"process.on('SIGTERM', () => server.close(() => process.exit(0)));",
|
|
1198
|
+
].join("\n"),
|
|
1199
|
+
], { stdio: ["ignore", "pipe", "inherit"] });
|
|
1200
|
+
|
|
1201
|
+
const port = await new Promise<number>((resolve, reject) => {
|
|
1202
|
+
child.once("error", reject);
|
|
1203
|
+
child.stdout?.once("data", (data: Buffer) => resolve(Number(data.toString())));
|
|
1204
|
+
child.once("exit", (code) => reject(new Error(`child server exited (${code})`)));
|
|
1205
|
+
});
|
|
1206
|
+
|
|
1207
|
+
try {
|
|
1208
|
+
const io = new PresignObjectIO(
|
|
1209
|
+
streamingVault(`http://127.0.0.1:${port}`),
|
|
1210
|
+
COMPANY,
|
|
1211
|
+
);
|
|
1212
|
+
const streamed = await io.getObjectStream("bulk");
|
|
1213
|
+
await new Promise((resolve) => setTimeout(resolve, 40));
|
|
1214
|
+
let bytes = 0;
|
|
1215
|
+
for await (const chunk of streamed.body) bytes += chunk.byteLength;
|
|
1216
|
+
expect(bytes).toBe(524_288);
|
|
1217
|
+
} finally {
|
|
1218
|
+
child.kill("SIGTERM");
|
|
1219
|
+
}
|
|
1220
|
+
});
|
|
1221
|
+
});
|
package/src/object-io.ts
CHANGED
|
@@ -13,8 +13,8 @@
|
|
|
13
13
|
* - `PresignObjectIO` — the presigned-URL path. The vault-service decides
|
|
14
14
|
* access as a runtime DDB check (no IAM policy ceiling) and hands back
|
|
15
15
|
* short-lived presigned GET/PUT/DELETE URLs + (for PUT) the exact headers
|
|
16
|
-
* to replay. The client never holds AWS credentials — it
|
|
17
|
-
*
|
|
16
|
+
* to replay. The client never holds AWS credentials — it requests the
|
|
17
|
+
* signed URLs directly.
|
|
18
18
|
*
|
|
19
19
|
* The seam is a per-EntityContext factory resolved INSIDE s3.ts, so every
|
|
20
20
|
* existing call site (`uploadFile(ctx, …)`, `downloadFile(ctx, …)`, …) keeps
|
|
@@ -31,6 +31,9 @@ import {
|
|
|
31
31
|
DeleteObjectCommand,
|
|
32
32
|
HeadObjectCommand,
|
|
33
33
|
} from "@aws-sdk/client-s3";
|
|
34
|
+
import * as http from "node:http";
|
|
35
|
+
import * as https from "node:https";
|
|
36
|
+
import type { IncomingMessage } from "node:http";
|
|
34
37
|
import type { EntityContext } from "./types.js";
|
|
35
38
|
import type {
|
|
36
39
|
PresignOp,
|
|
@@ -344,6 +347,141 @@ function metaFromHeaders(headers: Headers): Record<string, string> {
|
|
|
344
347
|
return meta;
|
|
345
348
|
}
|
|
346
349
|
|
|
350
|
+
/**
|
|
351
|
+
* Response shape shared by every presigned GET consumer. Unlike the runtime
|
|
352
|
+
* `fetch` response, its body is backed by Node's core HTTP client, so it never
|
|
353
|
+
* enters the runtime-bundled undici parser path.
|
|
354
|
+
*/
|
|
355
|
+
export interface PresignedGetResponse {
|
|
356
|
+
status: number;
|
|
357
|
+
headers: Headers;
|
|
358
|
+
body?: AsyncIterable<Uint8Array>;
|
|
359
|
+
destroy(): void;
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
/** Transport seam retained for deterministic wire-level regression tests. */
|
|
363
|
+
export interface PresignedGetTransport {
|
|
364
|
+
get(
|
|
365
|
+
url: string,
|
|
366
|
+
headers: Record<string, string> | undefined,
|
|
367
|
+
): Promise<PresignedGetResponse>;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
function headersFromNode(
|
|
371
|
+
headers: IncomingMessage["headers"],
|
|
372
|
+
): Headers {
|
|
373
|
+
const out = new Headers();
|
|
374
|
+
for (const [name, value] of Object.entries(headers)) {
|
|
375
|
+
if (value === undefined) continue;
|
|
376
|
+
out.set(name, Array.isArray(value) ? value.join(", ") : value);
|
|
377
|
+
}
|
|
378
|
+
return out;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
function expectedContentLength(headers: Headers): number | undefined {
|
|
382
|
+
const value = headers.get("content-length");
|
|
383
|
+
if (value === null) return undefined;
|
|
384
|
+
if (!/^\d+$/.test(value)) {
|
|
385
|
+
throw new Error(`invalid presigned GET content-length: ${value}`);
|
|
386
|
+
}
|
|
387
|
+
const length = Number(value);
|
|
388
|
+
if (!Number.isSafeInteger(length)) {
|
|
389
|
+
throw new Error(`invalid presigned GET content-length: ${value}`);
|
|
390
|
+
}
|
|
391
|
+
return length;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
function contentLengthError(expected: number, actual: number): Error {
|
|
395
|
+
return new Error(
|
|
396
|
+
`presigned GET content-length mismatch: expected ${expected} bytes, received ${actual}`,
|
|
397
|
+
);
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
async function* checkedNodeBody(
|
|
401
|
+
message: IncomingMessage,
|
|
402
|
+
expected: number | undefined,
|
|
403
|
+
): AsyncIterable<Uint8Array> {
|
|
404
|
+
let received = 0;
|
|
405
|
+
let exhausted = false;
|
|
406
|
+
try {
|
|
407
|
+
try {
|
|
408
|
+
for await (const chunk of message) {
|
|
409
|
+
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
410
|
+
received += bytes.byteLength;
|
|
411
|
+
if (expected !== undefined && received > expected) {
|
|
412
|
+
throw contentLengthError(expected, received);
|
|
413
|
+
}
|
|
414
|
+
yield bytes;
|
|
415
|
+
}
|
|
416
|
+
} catch (err) {
|
|
417
|
+
// A FIN before Node has received the declared bytes normally surfaces as
|
|
418
|
+
// an "aborted" stream error. Preserve the useful data-integrity cause.
|
|
419
|
+
if (expected !== undefined && received !== expected) {
|
|
420
|
+
throw contentLengthError(expected, received);
|
|
421
|
+
}
|
|
422
|
+
throw err;
|
|
423
|
+
}
|
|
424
|
+
exhausted = true;
|
|
425
|
+
if (expected !== undefined && received !== expected) {
|
|
426
|
+
throw contentLengthError(expected, received);
|
|
427
|
+
}
|
|
428
|
+
} finally {
|
|
429
|
+
// Consumers such as HEAD only inspect headers, and a caller can stop after
|
|
430
|
+
// a prefix. Destroying in both cases releases the socket immediately.
|
|
431
|
+
if (!exhausted) message.destroy();
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
function nodePresignedGet(
|
|
436
|
+
url: string,
|
|
437
|
+
headers: Record<string, string> | undefined,
|
|
438
|
+
): Promise<PresignedGetResponse> {
|
|
439
|
+
const target = new URL(url);
|
|
440
|
+
return new Promise((resolve, reject) => {
|
|
441
|
+
const handleResponse = (message: IncomingMessage): void => {
|
|
442
|
+
const responseHeaders = headersFromNode(message.headers);
|
|
443
|
+
let expected: number | undefined;
|
|
444
|
+
try {
|
|
445
|
+
expected = expectedContentLength(responseHeaders);
|
|
446
|
+
} catch (err) {
|
|
447
|
+
message.destroy();
|
|
448
|
+
reject(err);
|
|
449
|
+
return;
|
|
450
|
+
}
|
|
451
|
+
resolve({
|
|
452
|
+
status: message.statusCode ?? 0,
|
|
453
|
+
headers: responseHeaders,
|
|
454
|
+
body: checkedNodeBody(message, expected),
|
|
455
|
+
destroy: () => message.destroy(),
|
|
456
|
+
});
|
|
457
|
+
};
|
|
458
|
+
const request = target.protocol === "https:"
|
|
459
|
+
? https.request(target, { method: "GET", headers }, handleResponse)
|
|
460
|
+
: target.protocol === "http:"
|
|
461
|
+
? http.request(target, { method: "GET", headers }, handleResponse)
|
|
462
|
+
: undefined;
|
|
463
|
+
if (!request) {
|
|
464
|
+
reject(new Error(`unsupported presigned GET protocol: ${target.protocol}`));
|
|
465
|
+
return;
|
|
466
|
+
}
|
|
467
|
+
request.once("error", reject);
|
|
468
|
+
request.end();
|
|
469
|
+
});
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
const NODE_PRESIGNED_GET_TRANSPORT: PresignedGetTransport = {
|
|
473
|
+
get: nodePresignedGet,
|
|
474
|
+
};
|
|
475
|
+
|
|
476
|
+
let presignedGetTransport: PresignedGetTransport = NODE_PRESIGNED_GET_TRANSPORT;
|
|
477
|
+
|
|
478
|
+
/** Test-only override; production always uses Node's core HTTP(S) transport. */
|
|
479
|
+
export function setPresignedGetTransportForTesting(
|
|
480
|
+
transport: PresignedGetTransport | null,
|
|
481
|
+
): void {
|
|
482
|
+
presignedGetTransport = transport ?? NODE_PRESIGNED_GET_TRANSPORT;
|
|
483
|
+
}
|
|
484
|
+
|
|
347
485
|
/**
|
|
348
486
|
* Per-key presign denial codes that are EXPECTED and must be reclassified as
|
|
349
487
|
* skip-with-log — throwing the `name: "Forbidden"` shape (see
|
|
@@ -825,20 +963,21 @@ export class PresignObjectIO implements ObjectIO {
|
|
|
825
963
|
|
|
826
964
|
async getObjectStream(key: string): Promise<GetObjectStreamResult> {
|
|
827
965
|
const row = await this.resolveGetUrlForBody(key);
|
|
828
|
-
const res = await
|
|
966
|
+
const res = await presignedGetWithRetry(
|
|
967
|
+
row.url,
|
|
968
|
+
row.headers,
|
|
969
|
+
`presigned GET ${key}`,
|
|
970
|
+
);
|
|
829
971
|
if (res.status === 404) {
|
|
830
|
-
|
|
972
|
+
res.destroy();
|
|
831
973
|
throw notFoundError(key);
|
|
832
974
|
}
|
|
833
|
-
if (
|
|
834
|
-
const detail = await
|
|
975
|
+
if (res.status < 200 || res.status >= 300) {
|
|
976
|
+
const detail = await safePresignedGetText(res);
|
|
835
977
|
throw new Error(`presigned GET failed for ${key}: ${res.status} ${detail}`);
|
|
836
978
|
}
|
|
837
|
-
if (!res.body) {
|
|
838
|
-
return { body: (async function* () {})(), metadata: metaFromHeaders(res.headers) };
|
|
839
|
-
}
|
|
840
979
|
return {
|
|
841
|
-
body:
|
|
980
|
+
body: res.body ?? (async function* () {})(),
|
|
842
981
|
metadata: metaFromHeaders(res.headers),
|
|
843
982
|
};
|
|
844
983
|
}
|
|
@@ -933,9 +1072,13 @@ export class PresignObjectIO implements ObjectIO {
|
|
|
933
1072
|
}
|
|
934
1073
|
url = row.url;
|
|
935
1074
|
}
|
|
936
|
-
const res = await
|
|
1075
|
+
const res = await presignedGetWithRetry(
|
|
1076
|
+
url,
|
|
1077
|
+
undefined,
|
|
1078
|
+
`presigned HEAD ${key}`,
|
|
1079
|
+
);
|
|
937
1080
|
if (res.status === 404) {
|
|
938
|
-
|
|
1081
|
+
res.destroy();
|
|
939
1082
|
return null;
|
|
940
1083
|
}
|
|
941
1084
|
if (res.status === 403) {
|
|
@@ -944,12 +1087,11 @@ export class PresignObjectIO implements ObjectIO {
|
|
|
944
1087
|
// denial branch above. The SDK transport throws name:"Forbidden" here;
|
|
945
1088
|
// mirror it so both transports agree and no caller mistakes a denial
|
|
946
1089
|
// for a missing object.
|
|
947
|
-
|
|
1090
|
+
res.destroy();
|
|
948
1091
|
throw accessDeniedError(key, "presigned HEAD returned 403");
|
|
949
1092
|
}
|
|
950
|
-
if (
|
|
951
|
-
await
|
|
952
|
-
const detail = await safeText(res);
|
|
1093
|
+
if (res.status < 200 || res.status >= 300) {
|
|
1094
|
+
const detail = await safePresignedGetText(res);
|
|
953
1095
|
throw new Error(`presigned HEAD failed for ${key}: ${res.status} ${detail}`);
|
|
954
1096
|
}
|
|
955
1097
|
const result: HeadObjectResult = {
|
|
@@ -958,7 +1100,7 @@ export class PresignObjectIO implements ObjectIO {
|
|
|
958
1100
|
size: Number(res.headers.get("content-length") ?? "0"),
|
|
959
1101
|
metadata: metaFromHeaders(res.headers),
|
|
960
1102
|
};
|
|
961
|
-
|
|
1103
|
+
res.destroy();
|
|
962
1104
|
return result;
|
|
963
1105
|
}
|
|
964
1106
|
}
|
|
@@ -979,21 +1121,6 @@ async function cancelBody(res: Response): Promise<void> {
|
|
|
979
1121
|
}
|
|
980
1122
|
}
|
|
981
1123
|
|
|
982
|
-
async function* webReadableToAsyncIterable(
|
|
983
|
-
stream: ReadableStream<Uint8Array>,
|
|
984
|
-
): AsyncIterable<Uint8Array> {
|
|
985
|
-
const reader = stream.getReader();
|
|
986
|
-
try {
|
|
987
|
-
while (true) {
|
|
988
|
-
const { done, value } = await reader.read();
|
|
989
|
-
if (done) return;
|
|
990
|
-
if (value && value.byteLength > 0) yield value;
|
|
991
|
-
}
|
|
992
|
-
} finally {
|
|
993
|
-
reader.releaseLock();
|
|
994
|
-
}
|
|
995
|
-
}
|
|
996
|
-
|
|
997
1124
|
function parseLastModified(value: string | null): Date {
|
|
998
1125
|
if (!value) return new Date();
|
|
999
1126
|
const d = new Date(value);
|
|
@@ -1023,6 +1150,60 @@ function sleep(ms: number): Promise<void> {
|
|
|
1023
1150
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
1024
1151
|
}
|
|
1025
1152
|
|
|
1153
|
+
/**
|
|
1154
|
+
* Shared presigned-GET path for streaming reads and header-only probes.
|
|
1155
|
+
*
|
|
1156
|
+
* GET deliberately uses Node core HTTP(S), not global fetch: on affected
|
|
1157
|
+
* Node/Windows builds the bundled undici parser can assert after a FIN arrives
|
|
1158
|
+
* while the consumer is backpressured. Keep the retry budget identical to the
|
|
1159
|
+
* PUT/DELETE fetch path below.
|
|
1160
|
+
*/
|
|
1161
|
+
async function presignedGetWithRetry(
|
|
1162
|
+
url: string,
|
|
1163
|
+
headers: Record<string, string> | undefined,
|
|
1164
|
+
what: string,
|
|
1165
|
+
): Promise<PresignedGetResponse> {
|
|
1166
|
+
let lastError: unknown;
|
|
1167
|
+
for (let attempt = 0; attempt <= FETCH_MAX_RETRIES; attempt++) {
|
|
1168
|
+
if (attempt > 0) {
|
|
1169
|
+
const backoff = FETCH_BASE_DELAY_MS * 2 ** (attempt - 1);
|
|
1170
|
+
const jitter = Math.floor(Math.random() * FETCH_BASE_DELAY_MS);
|
|
1171
|
+
await sleep(backoff + jitter);
|
|
1172
|
+
}
|
|
1173
|
+
let res: PresignedGetResponse;
|
|
1174
|
+
try {
|
|
1175
|
+
res = await presignedGetTransport.get(url, headers);
|
|
1176
|
+
} catch (err) {
|
|
1177
|
+
lastError = err;
|
|
1178
|
+
continue;
|
|
1179
|
+
}
|
|
1180
|
+
if (isTransientStatus(res.status) && attempt < FETCH_MAX_RETRIES) {
|
|
1181
|
+
res.destroy();
|
|
1182
|
+
lastError = new Error(`${what}: transient ${res.status}`);
|
|
1183
|
+
continue;
|
|
1184
|
+
}
|
|
1185
|
+
return res;
|
|
1186
|
+
}
|
|
1187
|
+
throw lastError instanceof Error
|
|
1188
|
+
? lastError
|
|
1189
|
+
: new Error(`${what}: failed after ${FETCH_MAX_RETRIES} retries`);
|
|
1190
|
+
}
|
|
1191
|
+
|
|
1192
|
+
async function safePresignedGetText(res: PresignedGetResponse): Promise<string> {
|
|
1193
|
+
try {
|
|
1194
|
+
let text = "";
|
|
1195
|
+
for await (const chunk of res.body ?? []) {
|
|
1196
|
+
text += Buffer.from(chunk).toString("utf-8");
|
|
1197
|
+
if (text.length >= 200) break;
|
|
1198
|
+
}
|
|
1199
|
+
return text.slice(0, 200);
|
|
1200
|
+
} catch {
|
|
1201
|
+
return "";
|
|
1202
|
+
} finally {
|
|
1203
|
+
res.destroy();
|
|
1204
|
+
}
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1026
1207
|
/**
|
|
1027
1208
|
* `fetch` with bounded retry on network errors + transient 5xx. The presigned
|
|
1028
1209
|
* URL is reusable until expiry and the bodies are in-memory Buffers, so a
|
package/src/signals/get.test.ts
CHANGED
|
@@ -3,13 +3,18 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
|
6
|
+
import { Readable } from "node:stream";
|
|
6
7
|
import { GetObjectCommand, type S3Client } from "@aws-sdk/client-s3";
|
|
7
8
|
import { getSignal, SignalNotFoundError } from "./get.js";
|
|
8
9
|
import {
|
|
9
10
|
_setSignalsS3Factory,
|
|
10
11
|
_resetSignalsS3Factory,
|
|
11
12
|
} from "./internals.js";
|
|
12
|
-
import
|
|
13
|
+
import {
|
|
14
|
+
setPresignedGetTransportForTesting,
|
|
15
|
+
type PresignedGetTransport,
|
|
16
|
+
type PresignTransportClient,
|
|
17
|
+
} from "../object-io.js";
|
|
13
18
|
import type { EntityContext } from "../types.js";
|
|
14
19
|
import { InvalidSignalTypeError } from "../schemas/signal-types.js";
|
|
15
20
|
|
|
@@ -208,6 +213,21 @@ Body.
|
|
|
208
213
|
// Presigned transport (vault client + company vault) — HQ-59
|
|
209
214
|
// ---------------------------------------------------------------------------
|
|
210
215
|
|
|
216
|
+
function fetchBackedPresignedGetTransport(): PresignedGetTransport {
|
|
217
|
+
return {
|
|
218
|
+
async get(url, headers) {
|
|
219
|
+
const response = await fetch(url, { method: "GET", headers });
|
|
220
|
+
const body = response.body ? Readable.fromWeb(response.body) : undefined;
|
|
221
|
+
return {
|
|
222
|
+
status: response.status,
|
|
223
|
+
headers: response.headers,
|
|
224
|
+
body,
|
|
225
|
+
destroy: () => body?.destroy(),
|
|
226
|
+
};
|
|
227
|
+
},
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
|
|
211
231
|
function makePresignVault(objects: Record<string, string>): PresignTransportClient {
|
|
212
232
|
vi.stubGlobal(
|
|
213
233
|
"fetch",
|
|
@@ -218,6 +238,7 @@ function makePresignVault(objects: Record<string, string>): PresignTransportClie
|
|
|
218
238
|
return new Response(Buffer.from(content, "utf-8"), { status: 200 });
|
|
219
239
|
}),
|
|
220
240
|
);
|
|
241
|
+
setPresignedGetTransportForTesting(fetchBackedPresignedGetTransport());
|
|
221
242
|
return {
|
|
222
243
|
presign: async (input) => ({
|
|
223
244
|
results: input.keys.map((k) => ({
|
|
@@ -232,7 +253,10 @@ function makePresignVault(objects: Record<string, string>): PresignTransportClie
|
|
|
232
253
|
}
|
|
233
254
|
|
|
234
255
|
describe("getSignal — presigned transport (vault + cmp_)", () => {
|
|
235
|
-
afterEach(() =>
|
|
256
|
+
afterEach(() => {
|
|
257
|
+
setPresignedGetTransportForTesting(null);
|
|
258
|
+
vi.unstubAllGlobals();
|
|
259
|
+
});
|
|
236
260
|
|
|
237
261
|
it("reads via presign (no S3) and surfaces typed cross-refs", async () => {
|
|
238
262
|
_setSignalsS3Factory(
|