@indigoai-us/hq-cloud 6.18.3 → 6.18.4-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bandwidth.d.ts +56 -10
- package/dist/bandwidth.d.ts.map +1 -1
- package/dist/bandwidth.js +253 -17
- package/dist/bandwidth.js.map +1 -1
- package/dist/bandwidth.test.js +103 -4
- package/dist/bandwidth.test.js.map +1 -1
- package/dist/bin/sync-runner-company.d.ts +2 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +15 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +56 -0
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-events.test.js +15 -0
- package/dist/bin/sync-runner-events.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +31 -5
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +228 -10
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +31 -2
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +14 -5
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +120 -39
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts +26 -8
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +227 -62
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +252 -3
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +21 -4
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +140 -19
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +274 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/lib/machine-id.d.ts +7 -0
- package/dist/lib/machine-id.d.ts.map +1 -1
- package/dist/lib/machine-id.js +54 -7
- package/dist/lib/machine-id.js.map +1 -1
- package/dist/lib/machine-id.test.js +20 -1
- package/dist/lib/machine-id.test.js.map +1 -1
- package/dist/lib/ontology-entity-push.d.ts +51 -0
- package/dist/lib/ontology-entity-push.d.ts.map +1 -0
- package/dist/lib/ontology-entity-push.js +125 -0
- package/dist/lib/ontology-entity-push.js.map +1 -0
- package/dist/lib/ontology-entity-push.test.d.ts +2 -0
- package/dist/lib/ontology-entity-push.test.d.ts.map +1 -0
- package/dist/lib/ontology-entity-push.test.js +149 -0
- package/dist/lib/ontology-entity-push.test.js.map +1 -0
- package/dist/object-io.d.ts +38 -7
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +336 -140
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +423 -1
- package/dist/object-io.test.js.map +1 -1
- package/dist/s3-download-stream-hash.test.d.ts +2 -0
- package/dist/s3-download-stream-hash.test.d.ts.map +1 -0
- package/dist/s3-download-stream-hash.test.js +70 -0
- package/dist/s3-download-stream-hash.test.js.map +1 -0
- package/dist/s3.d.ts +8 -3
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +330 -27
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +304 -0
- package/dist/s3.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts +14 -4
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +145 -14
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +266 -1
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/inbound-event-admission.d.ts.map +1 -1
- package/dist/sync/inbound-event-admission.js +3 -0
- package/dist/sync/inbound-event-admission.js.map +1 -1
- package/dist/sync/inbound-event-admission.test.js +6 -0
- package/dist/sync/inbound-event-admission.test.js.map +1 -1
- package/dist/sync/push-transport.d.ts +15 -0
- package/dist/sync/push-transport.d.ts.map +1 -1
- package/dist/sync/push-transport.js +27 -0
- package/dist/sync/push-transport.js.map +1 -1
- package/dist/sync/push-transport.test.js +14 -1
- package/dist/sync/push-transport.test.js.map +1 -1
- package/dist/sync-core.d.ts +2 -1
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +5 -10
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +16 -0
- package/dist/sync-core.test.js.map +1 -1
- package/dist/vault-client.d.ts +5 -0
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +7 -0
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-client.test.js +39 -0
- package/dist/vault-client.test.js.map +1 -1
- package/package.json +1 -1
package/dist/object-io.test.js
CHANGED
|
@@ -10,11 +10,13 @@
|
|
|
10
10
|
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
|
11
11
|
import { createHash } from "node:crypto";
|
|
12
12
|
import * as http from "node:http";
|
|
13
|
+
import { PassThrough } from "node:stream";
|
|
13
14
|
import { once } from "node:events";
|
|
14
15
|
import { spawn } from "node:child_process";
|
|
15
|
-
import { PresignObjectIO, S3SdkObjectIO, setObjectIOFactory, setObjectIORequestTimeoutMsForTesting, setPresignedGetTransportForTesting, resolveObjectIO, presignObjectIOFactory, OBJECT_IO_REQUEST_TIMEOUT_MS, ObjectLockChecksumRequiredError, } from "./object-io.js";
|
|
16
|
+
import { PresignObjectIO, S3SdkObjectIO, setObjectIOFactory, setObjectIOBodyIdleTimeoutMsForTesting, setObjectIORequestTimeoutMsForTesting, setPresignedGetTransportForTesting, resolveObjectIO, presignObjectIOFactory, OBJECT_IO_REQUEST_TIMEOUT_MS, ObjectBodyIdleTimeoutError, ObjectLockChecksumRequiredError, } from "./object-io.js";
|
|
16
17
|
import * as objectIoModule from "./object-io.js";
|
|
17
18
|
import { isTransientNetworkError } from "./lib/net-errors.js";
|
|
19
|
+
import { BandwidthGovernor, setBandwidthGovernorForTesting } from "./bandwidth.js";
|
|
18
20
|
import { contentMd5Base64 } from "./lib/s3-content-checksum.js";
|
|
19
21
|
import { isAccessDenied } from "./sync-core.js";
|
|
20
22
|
const COMPANY = "cmp_test";
|
|
@@ -67,6 +69,321 @@ beforeEach(() => {
|
|
|
67
69
|
afterEach(() => {
|
|
68
70
|
setPresignedGetTransportForTesting(null);
|
|
69
71
|
});
|
|
72
|
+
describe("realtime ObjectIO payload priority", () => {
|
|
73
|
+
afterEach(() => {
|
|
74
|
+
setBandwidthGovernorForTesting(null);
|
|
75
|
+
vi.unstubAllGlobals();
|
|
76
|
+
});
|
|
77
|
+
it("bypasses SDK GET and PUT pacing while retaining ordinary chore pacing", async () => {
|
|
78
|
+
const governor = new BandwidthGovernor({ maxBytesPerSec: 1 });
|
|
79
|
+
const acquire = vi.spyOn(governor, "acquire").mockResolvedValue();
|
|
80
|
+
setBandwidthGovernorForTesting(governor);
|
|
81
|
+
const body = (async function* () {
|
|
82
|
+
yield Buffer.from("payload");
|
|
83
|
+
})();
|
|
84
|
+
const io = new S3SdkObjectIO(ctx());
|
|
85
|
+
io.client = {
|
|
86
|
+
send: async (command) => command.constructor.name === "GetObjectCommand"
|
|
87
|
+
? { Body: body, Metadata: {} }
|
|
88
|
+
: { ETag: '"uploaded"' },
|
|
89
|
+
};
|
|
90
|
+
await io.putObject({
|
|
91
|
+
key: "realtime-put",
|
|
92
|
+
body: Buffer.from("realtime"),
|
|
93
|
+
contentType: "text/plain",
|
|
94
|
+
bypassBandwidthGovernor: true,
|
|
95
|
+
});
|
|
96
|
+
const realtimeGet = await io.getObjectStream("realtime-get", {
|
|
97
|
+
bypassBandwidthGovernor: true,
|
|
98
|
+
});
|
|
99
|
+
await realtimeGet.body[Symbol.asyncIterator]().next();
|
|
100
|
+
expect(acquire).not.toHaveBeenCalled();
|
|
101
|
+
await io.putObject({
|
|
102
|
+
key: "chore-put",
|
|
103
|
+
body: Buffer.from("chore"),
|
|
104
|
+
contentType: "text/plain",
|
|
105
|
+
});
|
|
106
|
+
expect(acquire).toHaveBeenCalledOnce();
|
|
107
|
+
});
|
|
108
|
+
it("bypasses presigned GET and PUT pacing for realtime transfers", async () => {
|
|
109
|
+
const governor = new BandwidthGovernor({ maxBytesPerSec: 1 });
|
|
110
|
+
const acquire = vi.spyOn(governor, "acquire").mockResolvedValue();
|
|
111
|
+
setBandwidthGovernorForTesting(governor);
|
|
112
|
+
const { vault, setPresign } = makeVault();
|
|
113
|
+
const fetchMock = vi.fn();
|
|
114
|
+
vi.stubGlobal("fetch", fetchMock);
|
|
115
|
+
setPresign([
|
|
116
|
+
{ key: "realtime", op: "put", url: "https://s3/realtime-put" },
|
|
117
|
+
{ key: "realtime", op: "get", url: "https://s3/realtime-get" },
|
|
118
|
+
{ key: "chore", op: "put", url: "https://s3/chore-put" },
|
|
119
|
+
]);
|
|
120
|
+
fetchMock.mockResolvedValue(new Response(null, { status: 200, headers: { etag: '"ok"' } }));
|
|
121
|
+
setPresignedGetTransportForTesting({
|
|
122
|
+
get: async () => ({
|
|
123
|
+
status: 200,
|
|
124
|
+
headers: new Headers(),
|
|
125
|
+
body: (async function* () { yield Buffer.from("payload"); })(),
|
|
126
|
+
destroy: () => undefined,
|
|
127
|
+
}),
|
|
128
|
+
});
|
|
129
|
+
const io = new PresignObjectIO(vault, COMPANY);
|
|
130
|
+
await io.putObject({
|
|
131
|
+
key: "realtime",
|
|
132
|
+
body: Buffer.from("realtime"),
|
|
133
|
+
contentType: "text/plain",
|
|
134
|
+
bypassBandwidthGovernor: true,
|
|
135
|
+
});
|
|
136
|
+
const realtimeGet = await io.getObjectStream("realtime", {
|
|
137
|
+
bypassBandwidthGovernor: true,
|
|
138
|
+
});
|
|
139
|
+
await realtimeGet.body[Symbol.asyncIterator]().next();
|
|
140
|
+
expect(acquire).not.toHaveBeenCalled();
|
|
141
|
+
await io.putObject({
|
|
142
|
+
key: "chore",
|
|
143
|
+
body: Buffer.from("chore"),
|
|
144
|
+
contentType: "text/plain",
|
|
145
|
+
});
|
|
146
|
+
expect(acquire).toHaveBeenCalledOnce();
|
|
147
|
+
});
|
|
148
|
+
});
|
|
149
|
+
describe("ObjectIO idle timeout and bandwidth pacing", () => {
|
|
150
|
+
afterEach(() => {
|
|
151
|
+
setBandwidthGovernorForTesting(null);
|
|
152
|
+
setObjectIOBodyIdleTimeoutMsForTesting(undefined);
|
|
153
|
+
});
|
|
154
|
+
it("does not count governor wait time as socket idle time", async () => {
|
|
155
|
+
setObjectIOBodyIdleTimeoutMsForTesting(30);
|
|
156
|
+
const governor = new BandwidthGovernor({ maxBytesPerSec: 10 }, {
|
|
157
|
+
sleep: async (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
158
|
+
});
|
|
159
|
+
setBandwidthGovernorForTesting(governor);
|
|
160
|
+
const body = (async function* () {
|
|
161
|
+
yield Buffer.alloc(1);
|
|
162
|
+
})();
|
|
163
|
+
const io = new S3SdkObjectIO(ctx());
|
|
164
|
+
io.client = {
|
|
165
|
+
send: async () => ({ Body: body, Metadata: {} }),
|
|
166
|
+
};
|
|
167
|
+
const response = await io.getObjectStream("docs/paced.md");
|
|
168
|
+
const chunks = [];
|
|
169
|
+
for await (const chunk of response.body)
|
|
170
|
+
chunks.push(chunk);
|
|
171
|
+
expect(Buffer.concat(chunks.map((chunk) => Buffer.from(chunk)))).toEqual(Buffer.alloc(1));
|
|
172
|
+
}, 5_000);
|
|
173
|
+
});
|
|
174
|
+
describe("ObjectIO adaptive bandwidth sampling", () => {
|
|
175
|
+
afterEach(() => {
|
|
176
|
+
setBandwidthGovernorForTesting(null);
|
|
177
|
+
vi.unstubAllGlobals();
|
|
178
|
+
});
|
|
179
|
+
for (const transport of ["S3 SDK", "presigned"]) {
|
|
180
|
+
it(`learns raw upload capacity from an upload-only ${transport} run`, async () => {
|
|
181
|
+
let now = 0;
|
|
182
|
+
const governor = new BandwidthGovernor({ percent: 50 }, { now: () => now, sleep: async (ms) => { now += ms; } });
|
|
183
|
+
setBandwidthGovernorForTesting(governor);
|
|
184
|
+
if (transport === "S3 SDK") {
|
|
185
|
+
const io = new S3SdkObjectIO(ctx());
|
|
186
|
+
io.client = {
|
|
187
|
+
send: async () => {
|
|
188
|
+
now += 1000;
|
|
189
|
+
return { ETag: '"uploaded"' };
|
|
190
|
+
},
|
|
191
|
+
};
|
|
192
|
+
await io.putObject({
|
|
193
|
+
key: "upload-only.bin",
|
|
194
|
+
body: Buffer.alloc(1_000_000),
|
|
195
|
+
contentType: "application/octet-stream",
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
else {
|
|
199
|
+
const { vault, setPresign } = makeVault();
|
|
200
|
+
setPresign([{ key: "upload-only.bin", op: "put", url: "https://s3/upload-only" }]);
|
|
201
|
+
const fetchMock = vi.fn(async () => {
|
|
202
|
+
now += 1000;
|
|
203
|
+
return new Response(null, { status: 200, headers: { etag: '"uploaded"' } });
|
|
204
|
+
});
|
|
205
|
+
vi.stubGlobal("fetch", fetchMock);
|
|
206
|
+
await new PresignObjectIO(vault, COMPANY).putObject({
|
|
207
|
+
key: "upload-only.bin",
|
|
208
|
+
body: Buffer.alloc(1_000_000),
|
|
209
|
+
contentType: "application/octet-stream",
|
|
210
|
+
});
|
|
211
|
+
expect(fetchMock).toHaveBeenCalledOnce();
|
|
212
|
+
}
|
|
213
|
+
expect(governor.effectiveRateBps()).toBe(500_000);
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
it("samples transport arrivals before buffered GET chunks are pulled", async () => {
|
|
217
|
+
let now = 1000;
|
|
218
|
+
const governor = new BandwidthGovernor({ percent: 60 }, { now: () => now, sleep: async (ms) => { now += ms; } });
|
|
219
|
+
setBandwidthGovernorForTesting(governor);
|
|
220
|
+
const body = new PassThrough();
|
|
221
|
+
const io = new S3SdkObjectIO(ctx());
|
|
222
|
+
io.client = {
|
|
223
|
+
send: async () => ({ Body: body, Metadata: {} }),
|
|
224
|
+
};
|
|
225
|
+
const response = await io.getObjectStream("buffered.bin");
|
|
226
|
+
body.write(Buffer.alloc(4096));
|
|
227
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
228
|
+
now += 40;
|
|
229
|
+
body.write(Buffer.alloc(4096));
|
|
230
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
231
|
+
now += 40;
|
|
232
|
+
body.end(Buffer.alloc(4096));
|
|
233
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
234
|
+
expect(governor.effectiveRateBps()).toBeGreaterThanOrEqual(60_000);
|
|
235
|
+
expect(governor.effectiveRateBps()).toBeLessThanOrEqual(70_000);
|
|
236
|
+
let received = 0;
|
|
237
|
+
for await (const chunk of response.body)
|
|
238
|
+
received += chunk.byteLength;
|
|
239
|
+
expect(received).toBe(12_288);
|
|
240
|
+
});
|
|
241
|
+
});
|
|
242
|
+
describe("ObjectIO chore transfer admission", () => {
|
|
243
|
+
afterEach(() => {
|
|
244
|
+
setBandwidthGovernorForTesting(null);
|
|
245
|
+
});
|
|
246
|
+
it("waits for a download slot before opening the next SDK GET", async () => {
|
|
247
|
+
setBandwidthGovernorForTesting(new BandwidthGovernor({ maxConcurrency: 1 }));
|
|
248
|
+
const opened = [];
|
|
249
|
+
const io = new S3SdkObjectIO(ctx());
|
|
250
|
+
io.client = {
|
|
251
|
+
send: async (command) => {
|
|
252
|
+
opened.push(command.input.Key);
|
|
253
|
+
return {
|
|
254
|
+
Body: (async function* () { yield Buffer.from(command.input.Key); })(),
|
|
255
|
+
Metadata: {},
|
|
256
|
+
};
|
|
257
|
+
},
|
|
258
|
+
};
|
|
259
|
+
const first = await io.getObjectStream("first.bin");
|
|
260
|
+
const secondPromise = io.getObjectStream("second.bin");
|
|
261
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
262
|
+
expect(opened).toEqual(["first.bin"]);
|
|
263
|
+
for await (const _chunk of first.body) { /* drain the first transfer */ }
|
|
264
|
+
const second = await secondPromise;
|
|
265
|
+
expect(opened).toEqual(["first.bin", "second.bin"]);
|
|
266
|
+
for await (const _chunk of second.body) { /* drain the second transfer */ }
|
|
267
|
+
});
|
|
268
|
+
it("waits for a download slot before opening the next presigned GET", async () => {
|
|
269
|
+
setBandwidthGovernorForTesting(new BandwidthGovernor({ maxConcurrency: 1 }));
|
|
270
|
+
const { vault, setPresign } = makeVault();
|
|
271
|
+
setPresign([
|
|
272
|
+
{ key: "first.bin", op: "get", url: "https://s3/first" },
|
|
273
|
+
{ key: "second.bin", op: "get", url: "https://s3/second" },
|
|
274
|
+
]);
|
|
275
|
+
const opened = [];
|
|
276
|
+
setPresignedGetTransportForTesting({
|
|
277
|
+
get: async (url) => {
|
|
278
|
+
opened.push(url);
|
|
279
|
+
return {
|
|
280
|
+
status: 200,
|
|
281
|
+
headers: new Headers(),
|
|
282
|
+
body: (async function* () { yield Buffer.from(url); })(),
|
|
283
|
+
destroy: () => undefined,
|
|
284
|
+
};
|
|
285
|
+
},
|
|
286
|
+
});
|
|
287
|
+
const io = new PresignObjectIO(vault, COMPANY);
|
|
288
|
+
const first = await io.getObjectStream("first.bin");
|
|
289
|
+
const secondPromise = io.getObjectStream("second.bin");
|
|
290
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
291
|
+
expect(opened).toEqual(["https://s3/first"]);
|
|
292
|
+
for await (const _chunk of first.body) { /* drain the first transfer */ }
|
|
293
|
+
const second = await secondPromise;
|
|
294
|
+
expect(opened).toHaveLength(2);
|
|
295
|
+
for await (const _chunk of second.body) { /* drain the second transfer */ }
|
|
296
|
+
});
|
|
297
|
+
});
|
|
298
|
+
describe("ObjectIO ranged GET validators", () => {
|
|
299
|
+
afterEach(() => {
|
|
300
|
+
setObjectIOBodyIdleTimeoutMsForTesting(undefined);
|
|
301
|
+
setBandwidthGovernorForTesting(null);
|
|
302
|
+
});
|
|
303
|
+
it("sends Range and If-Match through the S3 SDK and exposes response identity", async () => {
|
|
304
|
+
const sent = [];
|
|
305
|
+
const body = (async function* () {
|
|
306
|
+
yield Buffer.from("suffix");
|
|
307
|
+
})();
|
|
308
|
+
const io = new S3SdkObjectIO(ctx());
|
|
309
|
+
io.client = {
|
|
310
|
+
send: async (command) => {
|
|
311
|
+
sent.push(command.input);
|
|
312
|
+
return {
|
|
313
|
+
Body: body,
|
|
314
|
+
Metadata: {},
|
|
315
|
+
ETag: '"etag-a"',
|
|
316
|
+
VersionId: "version-a",
|
|
317
|
+
ContentLength: 6,
|
|
318
|
+
ContentRange: "bytes 20-25/26",
|
|
319
|
+
$metadata: { httpStatusCode: 206 },
|
|
320
|
+
};
|
|
321
|
+
},
|
|
322
|
+
};
|
|
323
|
+
const streamed = await io.getObjectStream("resume.bin", {
|
|
324
|
+
rangeStart: 20,
|
|
325
|
+
ifMatch: '"etag-a"',
|
|
326
|
+
});
|
|
327
|
+
const identity = streamed;
|
|
328
|
+
expect(sent[0]).toMatchObject({ Range: "bytes=20-", IfMatch: '"etag-a"' });
|
|
329
|
+
expect(identity).toMatchObject({
|
|
330
|
+
etag: '"etag-a"',
|
|
331
|
+
versionId: "version-a",
|
|
332
|
+
contentLength: 6,
|
|
333
|
+
contentRange: "bytes 20-25/26",
|
|
334
|
+
statusCode: 206,
|
|
335
|
+
});
|
|
336
|
+
});
|
|
337
|
+
it("sends Range and If-Match through presigned GET and exposes response identity", async () => {
|
|
338
|
+
const { vault, setPresign } = makeVault();
|
|
339
|
+
setPresign([{ key: "resume.bin", op: "get", url: "https://s3/resume" }]);
|
|
340
|
+
const get = vi.fn(async () => ({
|
|
341
|
+
status: 206,
|
|
342
|
+
headers: new Headers({
|
|
343
|
+
etag: '"etag-a"',
|
|
344
|
+
"x-amz-version-id": "version-a",
|
|
345
|
+
"content-length": "6",
|
|
346
|
+
"content-range": "bytes 20-25/26",
|
|
347
|
+
}),
|
|
348
|
+
body: (async function* () { yield Buffer.from("suffix"); })(),
|
|
349
|
+
destroy: vi.fn(),
|
|
350
|
+
}));
|
|
351
|
+
setPresignedGetTransportForTesting({ get });
|
|
352
|
+
const io = new PresignObjectIO(vault, COMPANY);
|
|
353
|
+
const streamed = await io.getObjectStream("resume.bin", {
|
|
354
|
+
rangeStart: 20,
|
|
355
|
+
ifMatch: '"etag-a"',
|
|
356
|
+
});
|
|
357
|
+
const identity = streamed;
|
|
358
|
+
expect(get).toHaveBeenCalledWith("https://s3/resume", expect.objectContaining({ Range: "bytes=20-", "If-Match": '"etag-a"' }));
|
|
359
|
+
expect(identity).toMatchObject({
|
|
360
|
+
etag: '"etag-a"',
|
|
361
|
+
versionId: "version-a",
|
|
362
|
+
contentLength: 6,
|
|
363
|
+
contentRange: "bytes 20-25/26",
|
|
364
|
+
statusCode: 206,
|
|
365
|
+
});
|
|
366
|
+
});
|
|
367
|
+
it("reports a ranged presigned 412 as a precondition failure", async () => {
|
|
368
|
+
const { vault, setPresign } = makeVault();
|
|
369
|
+
setPresign([{ key: "resume.bin", op: "get", url: "https://s3/resume" }]);
|
|
370
|
+
const destroy = vi.fn();
|
|
371
|
+
setPresignedGetTransportForTesting({
|
|
372
|
+
get: async () => ({
|
|
373
|
+
status: 412,
|
|
374
|
+
headers: new Headers(),
|
|
375
|
+
body: (async function* () { yield Buffer.from("stale etag"); })(),
|
|
376
|
+
destroy,
|
|
377
|
+
}),
|
|
378
|
+
});
|
|
379
|
+
const io = new PresignObjectIO(vault, COMPANY);
|
|
380
|
+
await expect(io.getObjectStream("resume.bin", {
|
|
381
|
+
rangeStart: 20,
|
|
382
|
+
ifMatch: '"etag-a"',
|
|
383
|
+
})).rejects.toMatchObject({ name: "PreconditionFailed" });
|
|
384
|
+
expect(destroy).toHaveBeenCalledOnce();
|
|
385
|
+
});
|
|
386
|
+
});
|
|
70
387
|
function ctx() {
|
|
71
388
|
return {
|
|
72
389
|
uid: COMPANY,
|
|
@@ -172,6 +489,7 @@ describe("factory selection", () => {
|
|
|
172
489
|
describe("S3SdkObjectIO request timeout", () => {
|
|
173
490
|
afterEach(() => {
|
|
174
491
|
setObjectIORequestTimeoutMsForTesting(undefined);
|
|
492
|
+
setObjectIOBodyIdleTimeoutMsForTesting(undefined);
|
|
175
493
|
});
|
|
176
494
|
it("aborts a hung ListObjectsV2 within OBJECT_IO_REQUEST_TIMEOUT_MS as transient", async () => {
|
|
177
495
|
expect(OBJECT_IO_REQUEST_TIMEOUT_MS).toBe(60_000);
|
|
@@ -210,6 +528,64 @@ describe("S3SdkObjectIO request timeout", () => {
|
|
|
210
528
|
expect(isTransientNetworkError(caught)).toBe(true);
|
|
211
529
|
expect(Date.now() - started).toBeLessThan(1_000);
|
|
212
530
|
}, 5_000);
|
|
531
|
+
it("cancels a GET body that stops yielding after its headers arrive", async () => {
|
|
532
|
+
setObjectIOBodyIdleTimeoutMsForTesting(40);
|
|
533
|
+
let cancelled = false;
|
|
534
|
+
const body = {
|
|
535
|
+
[Symbol.asyncIterator]: () => ({
|
|
536
|
+
next: () => new Promise(() => { }),
|
|
537
|
+
return: async () => ({ done: true, value: undefined }),
|
|
538
|
+
}),
|
|
539
|
+
destroy: () => { cancelled = true; },
|
|
540
|
+
};
|
|
541
|
+
const io = new S3SdkObjectIO(ctx());
|
|
542
|
+
io.client = {
|
|
543
|
+
send: async () => ({ Body: body, Metadata: {} }),
|
|
544
|
+
};
|
|
545
|
+
const response = await io.getObjectStream("docs/stalled.md");
|
|
546
|
+
const started = Date.now();
|
|
547
|
+
let caught;
|
|
548
|
+
try {
|
|
549
|
+
await response.body[Symbol.asyncIterator]().next();
|
|
550
|
+
}
|
|
551
|
+
catch (error) {
|
|
552
|
+
caught = error;
|
|
553
|
+
}
|
|
554
|
+
expect(caught).toBeInstanceOf(ObjectBodyIdleTimeoutError);
|
|
555
|
+
expect(cancelled).toBe(true);
|
|
556
|
+
expect(Date.now() - started).toBeLessThan(1_000);
|
|
557
|
+
}, 5_000);
|
|
558
|
+
it("cancels a stalled presigned GET body", async () => {
|
|
559
|
+
setObjectIOBodyIdleTimeoutMsForTesting(40);
|
|
560
|
+
const { vault, setPresign } = makeVault();
|
|
561
|
+
setPresign([{ key: "docs/stalled.md", op: "get", url: "https://example.test/stalled" }]);
|
|
562
|
+
let destroyed = false;
|
|
563
|
+
const body = {
|
|
564
|
+
[Symbol.asyncIterator]: () => ({
|
|
565
|
+
next: () => new Promise(() => { }),
|
|
566
|
+
return: async () => ({ done: true, value: undefined }),
|
|
567
|
+
}),
|
|
568
|
+
};
|
|
569
|
+
setPresignedGetTransportForTesting({
|
|
570
|
+
get: async () => ({
|
|
571
|
+
status: 200,
|
|
572
|
+
headers: new Headers(),
|
|
573
|
+
body,
|
|
574
|
+
destroy: () => { destroyed = true; },
|
|
575
|
+
}),
|
|
576
|
+
});
|
|
577
|
+
const io = new PresignObjectIO(vault, COMPANY);
|
|
578
|
+
const response = await io.getObjectStream("docs/stalled.md");
|
|
579
|
+
let caught;
|
|
580
|
+
try {
|
|
581
|
+
await response.body[Symbol.asyncIterator]().next();
|
|
582
|
+
}
|
|
583
|
+
catch (error) {
|
|
584
|
+
caught = error;
|
|
585
|
+
}
|
|
586
|
+
expect(caught).toBeInstanceOf(ObjectBodyIdleTimeoutError);
|
|
587
|
+
expect(destroyed).toBe(true);
|
|
588
|
+
}, 5_000);
|
|
213
589
|
});
|
|
214
590
|
describe("S3SdkObjectIO.putObject — conditional-write fence", () => {
|
|
215
591
|
it("maps ifMatch (re-quoted) and ifNoneMatch onto the PutObjectCommand", async () => {
|
|
@@ -296,6 +672,29 @@ describe("PresignObjectIO.putObject", () => {
|
|
|
296
672
|
// ETag returned with quotes stripped.
|
|
297
673
|
expect(res.etag).toBe("abc123");
|
|
298
674
|
});
|
|
675
|
+
it("keeps the server upgrade URL on a plan-limit PUT denial", async () => {
|
|
676
|
+
const { vault, setPresign } = makeVault();
|
|
677
|
+
const upgradeUrl = "https://billing.example.test/company/acme";
|
|
678
|
+
setPresign([
|
|
679
|
+
{
|
|
680
|
+
key: "shared/paused.md",
|
|
681
|
+
op: "put",
|
|
682
|
+
error: "Plan limit reached",
|
|
683
|
+
code: "PLAN_LIMIT_REACHED",
|
|
684
|
+
upgradeUrl,
|
|
685
|
+
},
|
|
686
|
+
]);
|
|
687
|
+
const io = new PresignObjectIO(vault, COMPANY);
|
|
688
|
+
await expect(io.putObject({
|
|
689
|
+
key: "shared/paused.md",
|
|
690
|
+
body: Buffer.from("hello"),
|
|
691
|
+
contentType: "text/markdown",
|
|
692
|
+
})).rejects.toMatchObject({
|
|
693
|
+
statusCode: 402,
|
|
694
|
+
code: "PLAN_LIMIT_REACHED",
|
|
695
|
+
upgradeUrl,
|
|
696
|
+
});
|
|
697
|
+
});
|
|
299
698
|
it("sends Content-MD5 (base64 of raw MD5) for a known body when the presign did not sign a checksum", async () => {
|
|
300
699
|
const body = Buffer.from("hello");
|
|
301
700
|
expect(contentMd5Base64(body)).toBe("XUFAKrxLKna5cZ2REBfFkg==");
|
|
@@ -1630,4 +2029,27 @@ describe("PresignObjectIO — core HTTP presigned GET", () => {
|
|
|
1630
2029
|
}
|
|
1631
2030
|
});
|
|
1632
2031
|
});
|
|
2032
|
+
describe("presigned GET node body wrapper", () => {
|
|
2033
|
+
it("keeps a socket error raised before iteration for the reader instead of throwing it", async () => {
|
|
2034
|
+
const { PassThrough } = await import("node:stream");
|
|
2035
|
+
const { checkedNodeBodyForTesting } = await import("./object-io.js");
|
|
2036
|
+
const message = new PassThrough();
|
|
2037
|
+
const unhandled = [];
|
|
2038
|
+
const onUncaught = (error) => { unhandled.push(error); };
|
|
2039
|
+
process.on("uncaughtException", onUncaught);
|
|
2040
|
+
try {
|
|
2041
|
+
const body = checkedNodeBodyForTesting(message, undefined);
|
|
2042
|
+
message.emit("error", new Error("socket reset before iteration"));
|
|
2043
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
2044
|
+
expect(unhandled).toEqual([]);
|
|
2045
|
+
const read = (async () => {
|
|
2046
|
+
for await (const _chunk of body) { /* drain */ }
|
|
2047
|
+
})();
|
|
2048
|
+
await expect(read).rejects.toThrow("socket reset before iteration");
|
|
2049
|
+
}
|
|
2050
|
+
finally {
|
|
2051
|
+
process.off("uncaughtException", onUncaught);
|
|
2052
|
+
}
|
|
2053
|
+
});
|
|
2054
|
+
});
|
|
1633
2055
|
//# sourceMappingURL=object-io.test.js.map
|