@c9up/archive 0.1.20 → 0.1.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +96 -3
- package/dist/ArchiveProvider.d.ts +6 -1
- package/dist/ArchiveProvider.d.ts.map +1 -1
- package/dist/ArchiveProvider.js +73 -1
- package/dist/ArchiveProvider.js.map +1 -1
- package/dist/DriveManager.d.ts +10 -3
- package/dist/DriveManager.d.ts.map +1 -1
- package/dist/DriveManager.js +23 -6
- package/dist/DriveManager.js.map +1 -1
- package/dist/GcsDriver.d.ts +14 -9
- package/dist/GcsDriver.d.ts.map +1 -1
- package/dist/GcsDriver.js +202 -98
- package/dist/GcsDriver.js.map +1 -1
- package/dist/S3Driver.d.ts +11 -7
- package/dist/S3Driver.d.ts.map +1 -1
- package/dist/S3Driver.js +73 -110
- package/dist/S3Driver.js.map +1 -1
- package/dist/StaticMiddleware.d.ts +5 -1
- package/dist/StaticMiddleware.d.ts.map +1 -1
- package/dist/StaticMiddleware.js +114 -13
- package/dist/StaticMiddleware.js.map +1 -1
- package/dist/StorageManager.d.ts +28 -3
- package/dist/StorageManager.d.ts.map +1 -1
- package/dist/StorageManager.js +333 -132
- package/dist/StorageManager.js.map +1 -1
- package/dist/augmentations.d.ts +17 -0
- package/dist/augmentations.d.ts.map +1 -1
- package/dist/errno.d.ts +6 -0
- package/dist/errno.d.ts.map +1 -0
- package/dist/errno.js +11 -0
- package/dist/errno.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/keys.d.ts +34 -0
- package/dist/keys.d.ts.map +1 -0
- package/dist/keys.js +88 -0
- package/dist/keys.js.map +1 -0
- package/dist/local-signature.d.ts +32 -0
- package/dist/local-signature.d.ts.map +1 -0
- package/dist/local-signature.js +58 -0
- package/dist/local-signature.js.map +1 -0
- package/dist/move-to-disk.d.ts +24 -0
- package/dist/move-to-disk.d.ts.map +1 -0
- package/dist/move-to-disk.js +167 -0
- package/dist/move-to-disk.js.map +1 -0
- package/dist/objects.d.ts.map +1 -1
- package/dist/objects.js +5 -2
- package/dist/objects.js.map +1 -1
- package/dist/provider-fetch.d.ts +36 -0
- package/dist/provider-fetch.d.ts.map +1 -0
- package/dist/provider-fetch.js +158 -0
- package/dist/provider-fetch.js.map +1 -0
- package/dist/services/main.d.ts +1 -1
- package/dist/services/main.d.ts.map +1 -1
- package/dist/services/main.js +14 -1
- package/dist/services/main.js.map +1 -1
- package/dist/signed-route.d.ts +18 -6
- package/dist/signed-route.d.ts.map +1 -1
- package/dist/signed-route.js +186 -31
- package/dist/signed-route.js.map +1 -1
- package/dist/streams.d.ts +36 -0
- package/dist/streams.d.ts.map +1 -0
- package/dist/streams.js +108 -0
- package/dist/streams.js.map +1 -0
- package/dist/testing/FakeStorage.d.ts.map +1 -1
- package/dist/testing/FakeStorage.js +2 -1
- package/dist/testing/FakeStorage.js.map +1 -1
- package/package.json +7 -3
- package/src/ArchiveProvider.ts +105 -4
- package/src/DriveManager.ts +32 -6
- package/src/GcsDriver.ts +256 -139
- package/src/S3Driver.ts +87 -130
- package/src/StaticMiddleware.ts +124 -13
- package/src/StorageManager.ts +398 -141
- package/src/augmentations.ts +24 -1
- package/src/errno.ts +10 -0
- package/src/index.ts +1 -0
- package/src/keys.ts +104 -0
- package/src/local-signature.ts +76 -0
- package/src/move-to-disk.ts +234 -0
- package/src/objects.ts +5 -2
- package/src/provider-fetch.ts +171 -0
- package/src/services/main.ts +15 -2
- package/src/signed-route.ts +216 -36
- package/src/streams.ts +118 -0
- package/src/testing/FakeStorage.ts +2 -3
package/src/signed-route.ts
CHANGED
|
@@ -3,17 +3,31 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Verifies the HMAC produced by {@link LocalDriver.getSignedUrl} and,
|
|
5
5
|
* on success, serves the file from the driver. Tampered signatures →
|
|
6
|
-
* 403 `
|
|
7
|
-
*
|
|
8
|
-
*
|
|
6
|
+
* 403 `E_ARCHIVE_INVALID_SIGNATURE`. Expired `exp` → 403
|
|
7
|
+
* `E_ARCHIVE_EXPIRED_SIGNATURE`. A valid signature for a file that is not
|
|
8
|
+
* stored → 404 `E_ARCHIVE_NOT_FOUND`.
|
|
9
|
+
* No signature at all → the file's visibility decides: a public file is
|
|
10
|
+
* served, a private one refused with 403 `E_ARCHIVE_ACCESS_DENIED`, and a
|
|
11
|
+
* name that is not stored falls through to `next()`.
|
|
9
12
|
*
|
|
10
13
|
* @implements Story 43.2 AC 5
|
|
11
14
|
*/
|
|
12
15
|
|
|
13
16
|
import { createHmac, timingSafeEqual } from "node:crypto";
|
|
14
|
-
import
|
|
15
|
-
import {
|
|
16
|
-
import
|
|
17
|
+
import { ArchiveError } from "./errors.js";
|
|
18
|
+
import { matchesIfNoneMatch } from "./etag.js";
|
|
19
|
+
import { normalizeKey } from "./keys.js";
|
|
20
|
+
import {
|
|
21
|
+
HEADER_OVERRIDES,
|
|
22
|
+
type HeaderOverride,
|
|
23
|
+
isHeaderValue,
|
|
24
|
+
signedPayload,
|
|
25
|
+
} from "./local-signature.js";
|
|
26
|
+
import type {
|
|
27
|
+
LocalDriver,
|
|
28
|
+
Metadata,
|
|
29
|
+
StorageManager,
|
|
30
|
+
} from "./StorageManager.js";
|
|
17
31
|
|
|
18
32
|
/**
|
|
19
33
|
* Minimal request/response contract the signed-route handler needs.
|
|
@@ -25,7 +39,11 @@ import type { LocalDriver, StorageManager } from "./StorageManager.js";
|
|
|
25
39
|
export interface SignedRouteResponse {
|
|
26
40
|
status(code: number): SignedRouteResponse;
|
|
27
41
|
header(name: string, value: string): SignedRouteResponse;
|
|
28
|
-
|
|
42
|
+
/**
|
|
43
|
+
* Send a readable as the body. The framework's response streams it to
|
|
44
|
+
* the client; the file is never held in memory whole.
|
|
45
|
+
*/
|
|
46
|
+
stream(body: NodeJS.ReadableStream): Promise<void> | void;
|
|
29
47
|
json(data: unknown): void;
|
|
30
48
|
}
|
|
31
49
|
|
|
@@ -54,8 +72,13 @@ export interface SignedRouteOptions {
|
|
|
54
72
|
* recompute the expected signature.
|
|
55
73
|
*/
|
|
56
74
|
driver: LocalDriver;
|
|
57
|
-
/**
|
|
58
|
-
*
|
|
75
|
+
/**
|
|
76
|
+
* URL prefix for file serving. Defaults to — and must equal — the
|
|
77
|
+
* driver's `routeBasePath` (`driver.getRouteBasePath()`), which is what
|
|
78
|
+
* its URLs are built on. Configure the path on the driver; a different
|
|
79
|
+
* value here is refused at construction (`E_ARCHIVE_PREFIX_MISMATCH`)
|
|
80
|
+
* rather than producing a route that can never match a signed URL.
|
|
81
|
+
*/
|
|
59
82
|
prefix?: string;
|
|
60
83
|
}
|
|
61
84
|
|
|
@@ -71,10 +94,17 @@ type SignedRouteMiddleware = (
|
|
|
71
94
|
export function createSignedRouteHandler(
|
|
72
95
|
options: SignedRouteOptions,
|
|
73
96
|
): SignedRouteMiddleware {
|
|
74
|
-
const prefix = options.
|
|
75
|
-
if (
|
|
76
|
-
|
|
77
|
-
|
|
97
|
+
const prefix = options.driver.getRouteBasePath();
|
|
98
|
+
if (
|
|
99
|
+
options.prefix !== undefined &&
|
|
100
|
+
options.prefix.replace(/\/+$/, "") !== prefix
|
|
101
|
+
) {
|
|
102
|
+
throw new ArchiveError(
|
|
103
|
+
"E_ARCHIVE_PREFIX_MISMATCH",
|
|
104
|
+
`createSignedRouteHandler: prefix '${options.prefix}' does not match the driver's routeBasePath '${prefix}', so no URL the driver signs would ever reach this route.`,
|
|
105
|
+
{
|
|
106
|
+
hint: "Set routeBasePath on the LocalDriver (or services.fs) and leave prefix out.",
|
|
107
|
+
},
|
|
78
108
|
);
|
|
79
109
|
}
|
|
80
110
|
const secret = options.driver.getSigningSecret();
|
|
@@ -100,25 +130,80 @@ export function createSignedRouteHandler(
|
|
|
100
130
|
// path segment; we decode here before recomputing the HMAC.
|
|
101
131
|
const encodedPath = reqPath.slice(prefix.length).replace(/^\//, "");
|
|
102
132
|
if (encodedPath === "") return next();
|
|
103
|
-
let
|
|
133
|
+
let decoded: string;
|
|
104
134
|
try {
|
|
105
|
-
|
|
135
|
+
decoded = decodeSegments(encodedPath);
|
|
106
136
|
} catch {
|
|
107
137
|
ctx.response.status(403).json({
|
|
108
|
-
error: {
|
|
138
|
+
error: {
|
|
139
|
+
code: "E_ARCHIVE_INVALID_SIGNATURE",
|
|
140
|
+
message: "Malformed path",
|
|
141
|
+
},
|
|
109
142
|
});
|
|
110
143
|
return;
|
|
111
144
|
}
|
|
145
|
+
// The key the signer signed is the normalised one; a path that does
|
|
146
|
+
// not normalise (`..`, an encoded separator, a control character) names
|
|
147
|
+
// nothing this route will serve.
|
|
148
|
+
let filePath: string;
|
|
149
|
+
try {
|
|
150
|
+
filePath = normalizeKey(decoded);
|
|
151
|
+
} catch {
|
|
152
|
+
invalidKey(ctx);
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
112
155
|
|
|
113
156
|
// Pull `exp`/`sig` from the parsed query object — matches
|
|
114
157
|
// Ream's `Request.qs()` contract (`Record<string, unknown>`).
|
|
115
158
|
const qs = readQueryObject(ctx);
|
|
159
|
+
if (qs.exp === undefined && qs.sig === undefined) {
|
|
160
|
+
// No signature: this route still owns the prefix. Handing every
|
|
161
|
+
// unsigned request on to `next()` let whatever was mounted behind —
|
|
162
|
+
// a static server over the storage root, the obvious companion —
|
|
163
|
+
// serve a PRIVATE file to anyone who left the query off. Visibility
|
|
164
|
+
// is decided here: a public file is served, a private one refused,
|
|
165
|
+
// and only a name that is not stored goes on to the next handler.
|
|
166
|
+
//
|
|
167
|
+
// NAMED DEVIATION: upstream answers this refusal with 401. It is a
|
|
168
|
+
// 403 here, as every other refusal of this route already was: no
|
|
169
|
+
// credential the client could add makes a private file servable
|
|
170
|
+
// without a signature.
|
|
171
|
+
let visibility: "public" | "private" | undefined;
|
|
172
|
+
try {
|
|
173
|
+
if (await storage.exists(filePath)) {
|
|
174
|
+
visibility = await storage.getVisibility(filePath);
|
|
175
|
+
}
|
|
176
|
+
} catch (err) {
|
|
177
|
+
// A name the disk refuses (a visibility marker's) is answered,
|
|
178
|
+
// not thrown into the application's error handler.
|
|
179
|
+
if (isKeyError(err)) {
|
|
180
|
+
invalidKey(ctx);
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
throw err;
|
|
184
|
+
}
|
|
185
|
+
if (visibility === undefined) return next();
|
|
186
|
+
if (visibility !== "public") {
|
|
187
|
+
ctx.response.status(403).json({
|
|
188
|
+
error: { code: "E_ARCHIVE_ACCESS_DENIED", message: "Access denied" },
|
|
189
|
+
});
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
return serve(ctx, filePath);
|
|
193
|
+
}
|
|
116
194
|
const expRaw = readStringParam(qs, "exp");
|
|
117
195
|
const sigHex = readStringParam(qs, "sig");
|
|
118
196
|
if (expRaw === null || sigHex === null) {
|
|
119
|
-
//
|
|
120
|
-
//
|
|
121
|
-
|
|
197
|
+
// Half a signature, or one repeated into an array: a request that
|
|
198
|
+
// tries to use a signature and cannot be verified is refused, never
|
|
199
|
+
// downgraded to the unsigned path.
|
|
200
|
+
ctx.response.status(403).json({
|
|
201
|
+
error: {
|
|
202
|
+
code: "E_ARCHIVE_INVALID_SIGNATURE",
|
|
203
|
+
message: "Malformed signature",
|
|
204
|
+
},
|
|
205
|
+
});
|
|
206
|
+
return;
|
|
122
207
|
}
|
|
123
208
|
|
|
124
209
|
// Validate sig shape up-front: 64 hex chars (sha256). Rejects
|
|
@@ -128,7 +213,7 @@ export function createSignedRouteHandler(
|
|
|
128
213
|
if (!/^[a-f0-9]{64}$/.test(sigHex)) {
|
|
129
214
|
ctx.response.status(403).json({
|
|
130
215
|
error: {
|
|
131
|
-
code: "
|
|
216
|
+
code: "E_ARCHIVE_INVALID_SIGNATURE",
|
|
132
217
|
message: "Malformed signature",
|
|
133
218
|
},
|
|
134
219
|
});
|
|
@@ -141,51 +226,138 @@ export function createSignedRouteHandler(
|
|
|
141
226
|
const exp = Number.parseInt(expRaw, 10);
|
|
142
227
|
if (!/^\d+$/.test(expRaw) || !Number.isFinite(exp)) {
|
|
143
228
|
ctx.response.status(403).json({
|
|
144
|
-
error: {
|
|
229
|
+
error: {
|
|
230
|
+
code: "E_ARCHIVE_INVALID_SIGNATURE",
|
|
231
|
+
message: "Malformed expiry",
|
|
232
|
+
},
|
|
145
233
|
});
|
|
146
234
|
return;
|
|
147
235
|
}
|
|
148
236
|
|
|
237
|
+
// Response-header overrides travel in the query and are covered by
|
|
238
|
+
// the signature. One present in a shape that cannot be verified
|
|
239
|
+
// (repeated, or carrying a control character) is refused outright.
|
|
240
|
+
const overrides: HeaderOverride[] = [];
|
|
241
|
+
for (const [param] of HEADER_OVERRIDES) {
|
|
242
|
+
const raw = qs[param];
|
|
243
|
+
if (raw === undefined) continue;
|
|
244
|
+
if (typeof raw !== "string" || !isHeaderValue(raw)) {
|
|
245
|
+
ctx.response.status(403).json({
|
|
246
|
+
error: {
|
|
247
|
+
code: "E_ARCHIVE_INVALID_SIGNATURE",
|
|
248
|
+
message: "Malformed header override",
|
|
249
|
+
},
|
|
250
|
+
});
|
|
251
|
+
return;
|
|
252
|
+
}
|
|
253
|
+
if (raw !== "") overrides.push([param, raw]);
|
|
254
|
+
}
|
|
255
|
+
|
|
149
256
|
// Constant-time HMAC verification on the decoded path. The regex
|
|
150
257
|
// check above already guarantees 32 bytes, so `timingSafeEqual`
|
|
151
258
|
// cannot throw on a length mismatch here. Run BEFORE the expiry
|
|
152
259
|
// check so the response time cannot leak whether a sig was valid
|
|
153
260
|
// for an expired URL versus invalid for a fresh one.
|
|
154
261
|
const expected = createHmac("sha256", secret)
|
|
155
|
-
.update(
|
|
262
|
+
.update(signedPayload(filePath, exp, overrides))
|
|
156
263
|
.digest();
|
|
157
264
|
const provided = Buffer.from(sigHex, "hex");
|
|
158
265
|
if (!timingSafeEqual(provided, expected)) {
|
|
159
266
|
ctx.response.status(403).json({
|
|
160
|
-
error: {
|
|
267
|
+
error: {
|
|
268
|
+
code: "E_ARCHIVE_INVALID_SIGNATURE",
|
|
269
|
+
message: "Invalid signature",
|
|
270
|
+
},
|
|
161
271
|
});
|
|
162
272
|
return;
|
|
163
273
|
}
|
|
164
274
|
|
|
165
275
|
if (exp * 1000 <= Date.now()) {
|
|
166
276
|
ctx.response.status(403).json({
|
|
167
|
-
error: {
|
|
277
|
+
error: {
|
|
278
|
+
code: "E_ARCHIVE_EXPIRED_SIGNATURE",
|
|
279
|
+
message: "Signed URL expired",
|
|
280
|
+
},
|
|
168
281
|
});
|
|
169
282
|
return;
|
|
170
283
|
}
|
|
171
284
|
|
|
172
285
|
// Signature verified — serve the file.
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
286
|
+
return serve(ctx, filePath, overrides);
|
|
287
|
+
};
|
|
288
|
+
|
|
289
|
+
async function serve(
|
|
290
|
+
ctx: SignedRouteHttpContext,
|
|
291
|
+
filePath: string,
|
|
292
|
+
overrides: readonly HeaderOverride[] = [],
|
|
293
|
+
): Promise<void> {
|
|
294
|
+
// Metadata first, the bytes only once they are known to be wanted: the
|
|
295
|
+
// whole file used to be read into memory for every request, and a
|
|
296
|
+
// client revalidating its copy got a full transfer each time.
|
|
297
|
+
let metadata: Metadata;
|
|
298
|
+
try {
|
|
299
|
+
metadata = await storage.getMetadata(filePath);
|
|
300
|
+
} catch (err) {
|
|
301
|
+
if (isNotFound(err)) return notFound(ctx);
|
|
302
|
+
throw err;
|
|
303
|
+
}
|
|
304
|
+
ctx.response.header("ETag", metadata.etag);
|
|
305
|
+
ctx.response.header("Last-Modified", metadata.lastModified.toUTCString());
|
|
306
|
+
ctx.response.header("Content-Type", metadata.contentType);
|
|
307
|
+
// Signed overrides win over what the extension implies.
|
|
308
|
+
for (const [param, value] of overrides) {
|
|
309
|
+
const header = HEADER_OVERRIDES.find(([name]) => name === param)?.[1];
|
|
310
|
+
if (header !== undefined) ctx.response.header(header, value);
|
|
311
|
+
}
|
|
312
|
+
if (
|
|
313
|
+
matchesIfNoneMatch(ctx.request.header("if-none-match"), metadata.etag)
|
|
314
|
+
) {
|
|
315
|
+
ctx.response.status(304);
|
|
178
316
|
return;
|
|
179
317
|
}
|
|
180
|
-
|
|
181
|
-
ctx.response.header("Content-Type", inferMimeType(ext));
|
|
182
|
-
ctx.response.header("Content-Length", String(content.length));
|
|
318
|
+
ctx.response.header("Content-Length", String(metadata.contentLength));
|
|
183
319
|
if (ctx.request.method() === "HEAD") {
|
|
184
320
|
ctx.response.status(200);
|
|
185
321
|
return;
|
|
186
322
|
}
|
|
187
|
-
|
|
188
|
-
|
|
323
|
+
let body: NodeJS.ReadableStream;
|
|
324
|
+
try {
|
|
325
|
+
body = await storage.getStream(filePath);
|
|
326
|
+
} catch (err) {
|
|
327
|
+
// Deleted between the metadata read and the open.
|
|
328
|
+
if (isNotFound(err)) return notFound(ctx);
|
|
329
|
+
throw err;
|
|
330
|
+
}
|
|
331
|
+
ctx.response.status(200);
|
|
332
|
+
await ctx.response.stream(body);
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/** The refusals a disk raises for a key it will not address. */
|
|
337
|
+
const KEY_ERRORS = new Set([
|
|
338
|
+
"E_ARCHIVE_INVALID_KEY",
|
|
339
|
+
"E_ARCHIVE_PATH_TRAVERSAL_DETECTED",
|
|
340
|
+
"E_ARCHIVE_UNALLOWED_CHARACTERS",
|
|
341
|
+
]);
|
|
342
|
+
|
|
343
|
+
function isNotFound(err: unknown): boolean {
|
|
344
|
+
return err instanceof ArchiveError && err.code === "E_ARCHIVE_NOT_FOUND";
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
function notFound(ctx: SignedRouteHttpContext): void {
|
|
348
|
+
ctx.response.status(404).json({
|
|
349
|
+
error: { code: "E_ARCHIVE_NOT_FOUND", message: "File not found" },
|
|
350
|
+
});
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
function isKeyError(err: unknown): boolean {
|
|
354
|
+
return err instanceof ArchiveError && KEY_ERRORS.has(err.code);
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
function invalidKey(ctx: SignedRouteHttpContext): void {
|
|
358
|
+
ctx.response.status(403).json({
|
|
359
|
+
error: { code: "E_ARCHIVE_INVALID_KEY", message: "Invalid path" },
|
|
360
|
+
});
|
|
189
361
|
}
|
|
190
362
|
|
|
191
363
|
/**
|
|
@@ -213,11 +385,19 @@ function readStringParam(
|
|
|
213
385
|
/**
|
|
214
386
|
* Decode each URL-encoded path segment. Throws if any segment
|
|
215
387
|
* contains invalid percent-escapes (`decodeURIComponent` throws on
|
|
216
|
-
* malformed input)
|
|
388
|
+
* malformed input), or decodes to a separator: the signer encodes each
|
|
389
|
+
* segment on its own, so an encoded `/` or `\` never comes from a URL it
|
|
390
|
+
* built, and would let one request segment pose as two.
|
|
217
391
|
*/
|
|
218
392
|
function decodeSegments(encodedPath: string): string {
|
|
219
393
|
return encodedPath
|
|
220
394
|
.split("/")
|
|
221
|
-
.map((seg) =>
|
|
395
|
+
.map((seg) => {
|
|
396
|
+
const decoded = decodeURIComponent(seg);
|
|
397
|
+
if (/[/\\]/.test(decoded)) {
|
|
398
|
+
throw new Error("encoded separator in a path segment");
|
|
399
|
+
}
|
|
400
|
+
return decoded;
|
|
401
|
+
})
|
|
222
402
|
.join("/");
|
|
223
403
|
}
|
package/src/streams.ts
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/// <reference lib="dom" />
|
|
2
|
+
// Uses the WHATWG streams globals, which live in the DOM lib even on Node.
|
|
3
|
+
/**
|
|
4
|
+
* Stream helpers shared by the cloud drivers.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { Readable } from "node:stream";
|
|
8
|
+
|
|
9
|
+
/** A chunk of a Node readable as bytes (string chunks are UTF-8). */
|
|
10
|
+
export function toBuffer(chunk: unknown): Buffer {
|
|
11
|
+
if (Buffer.isBuffer(chunk)) return chunk;
|
|
12
|
+
if (chunk instanceof Uint8Array) return Buffer.from(chunk);
|
|
13
|
+
return Buffer.from(String(chunk));
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Type guard for any readable that exposes `destroy()` (Node streams do; a
|
|
18
|
+
* generic iterable does not), so file descriptors and sockets can be released
|
|
19
|
+
* without a cast.
|
|
20
|
+
*/
|
|
21
|
+
function hasDestroy(x: unknown): x is { destroy(err?: Error): void } {
|
|
22
|
+
return (
|
|
23
|
+
typeof x === "object" &&
|
|
24
|
+
x !== null &&
|
|
25
|
+
"destroy" in x &&
|
|
26
|
+
typeof x.destroy === "function"
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Drain a readable into a single Buffer, capped at `maxBytes`. Throws if the
|
|
32
|
+
* stream yields more than that — the caller declared a length and we refuse
|
|
33
|
+
* to silently over-buffer — after destroying the upstream so its descriptor
|
|
34
|
+
* or socket does not leak.
|
|
35
|
+
*/
|
|
36
|
+
export async function collectToBuffer(
|
|
37
|
+
readable: NodeJS.ReadableStream,
|
|
38
|
+
maxBytes: number,
|
|
39
|
+
): Promise<Buffer> {
|
|
40
|
+
const chunks: Buffer[] = [];
|
|
41
|
+
let total = 0;
|
|
42
|
+
for await (const chunk of readable) {
|
|
43
|
+
const buf = toBuffer(chunk);
|
|
44
|
+
total += buf.length;
|
|
45
|
+
if (total > maxBytes) {
|
|
46
|
+
if (hasDestroy(readable)) readable.destroy();
|
|
47
|
+
throw new Error(
|
|
48
|
+
`putStream: readable exceeded declared contentLength (${total} > ${maxBytes})`,
|
|
49
|
+
);
|
|
50
|
+
}
|
|
51
|
+
chunks.push(buf);
|
|
52
|
+
}
|
|
53
|
+
return Buffer.concat(chunks, total);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Async-iterable that yields Buffers of exactly `partSize` bytes (the last
|
|
58
|
+
* one may be smaller). Used to slice a stream into multipart parts or
|
|
59
|
+
* resumable-upload chunks.
|
|
60
|
+
*
|
|
61
|
+
* **Backpressure.** Memory usage is bounded by `partSize + readable`'s own
|
|
62
|
+
* `highWaterMark`. The `for await` loop pulls from `readable` only when the
|
|
63
|
+
* consumer has finished awaiting the previous yield, so a slow consumer
|
|
64
|
+
* naturally throttles a fast producer through Node's stream pause/resume
|
|
65
|
+
* protocol. Pre-buffered streams (e.g. `Readable.from(buffer)`) bypass this
|
|
66
|
+
* and surface their entire payload in one tick; for those, expect peak ≈
|
|
67
|
+
* `payload + partSize`.
|
|
68
|
+
*/
|
|
69
|
+
export async function* chunkStream(
|
|
70
|
+
readable: NodeJS.ReadableStream,
|
|
71
|
+
partSize: number,
|
|
72
|
+
): AsyncIterable<Buffer> {
|
|
73
|
+
let pending: Buffer[] = [];
|
|
74
|
+
let pendingSize = 0;
|
|
75
|
+
for await (const chunk of readable) {
|
|
76
|
+
const buf = toBuffer(chunk);
|
|
77
|
+
pending.push(buf);
|
|
78
|
+
pendingSize += buf.length;
|
|
79
|
+
while (pendingSize >= partSize) {
|
|
80
|
+
const joined = Buffer.concat(pending, pendingSize);
|
|
81
|
+
yield joined.subarray(0, partSize);
|
|
82
|
+
const leftover = joined.subarray(partSize);
|
|
83
|
+
pending = leftover.length > 0 ? [leftover] : [];
|
|
84
|
+
pendingSize = leftover.length;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
if (pendingSize > 0) {
|
|
88
|
+
yield Buffer.concat(pending, pendingSize);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* A WHATWG response body as a Node readable, without a cast between the DOM
|
|
94
|
+
* and `node:stream/web` typings of the same runtime object.
|
|
95
|
+
*
|
|
96
|
+
* Destroying the readable before the end cancels the body, so the
|
|
97
|
+
* connection behind it is released instead of left to the collector.
|
|
98
|
+
*/
|
|
99
|
+
export function nodeReadable(body: ReadableStream<Uint8Array>): Readable {
|
|
100
|
+
async function* chunks(): AsyncGenerator<Buffer> {
|
|
101
|
+
const reader = body.getReader();
|
|
102
|
+
let finished = false;
|
|
103
|
+
try {
|
|
104
|
+
for (;;) {
|
|
105
|
+
const { done, value } = await reader.read();
|
|
106
|
+
if (done) {
|
|
107
|
+
finished = true;
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
yield Buffer.from(value.buffer, value.byteOffset, value.byteLength);
|
|
111
|
+
}
|
|
112
|
+
} finally {
|
|
113
|
+
if (!finished) await reader.cancel().catch(() => {});
|
|
114
|
+
reader.releaseLock();
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
return Readable.from(chunks(), { objectMode: false });
|
|
118
|
+
}
|
|
@@ -37,6 +37,7 @@ import type {
|
|
|
37
37
|
WriteOptions,
|
|
38
38
|
} from "../StorageManager.js";
|
|
39
39
|
import { StorageManager } from "../StorageManager.js";
|
|
40
|
+
import { toBuffer } from "../streams.js";
|
|
40
41
|
|
|
41
42
|
interface StoredObject {
|
|
42
43
|
content: Buffer;
|
|
@@ -90,9 +91,7 @@ export class FakeStorage implements StorageDriver {
|
|
|
90
91
|
): Promise<void> {
|
|
91
92
|
const chunks: Buffer[] = [];
|
|
92
93
|
for await (const chunk of readable) {
|
|
93
|
-
chunks.push(
|
|
94
|
-
Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as string),
|
|
95
|
-
);
|
|
94
|
+
chunks.push(toBuffer(chunk));
|
|
96
95
|
}
|
|
97
96
|
const buf = Buffer.concat(chunks);
|
|
98
97
|
this.#store.set(filePath, {
|