@c9up/archive 0.1.19 → 0.1.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/README.md +96 -3
  2. package/dist/ArchiveProvider.d.ts +8 -2
  3. package/dist/ArchiveProvider.d.ts.map +1 -1
  4. package/dist/ArchiveProvider.js +73 -1
  5. package/dist/ArchiveProvider.js.map +1 -1
  6. package/dist/DriveManager.d.ts +16 -6
  7. package/dist/DriveManager.d.ts.map +1 -1
  8. package/dist/DriveManager.js +23 -6
  9. package/dist/DriveManager.js.map +1 -1
  10. package/dist/GcsDriver.d.ts +14 -9
  11. package/dist/GcsDriver.d.ts.map +1 -1
  12. package/dist/GcsDriver.js +202 -98
  13. package/dist/GcsDriver.js.map +1 -1
  14. package/dist/S3Driver.d.ts +11 -7
  15. package/dist/S3Driver.d.ts.map +1 -1
  16. package/dist/S3Driver.js +73 -110
  17. package/dist/S3Driver.js.map +1 -1
  18. package/dist/StaticMiddleware.d.ts +5 -1
  19. package/dist/StaticMiddleware.d.ts.map +1 -1
  20. package/dist/StaticMiddleware.js +114 -13
  21. package/dist/StaticMiddleware.js.map +1 -1
  22. package/dist/StorageManager.d.ts +36 -4
  23. package/dist/StorageManager.d.ts.map +1 -1
  24. package/dist/StorageManager.js +360 -137
  25. package/dist/StorageManager.js.map +1 -1
  26. package/dist/augmentations.d.ts +17 -0
  27. package/dist/augmentations.d.ts.map +1 -1
  28. package/dist/errno.d.ts +6 -0
  29. package/dist/errno.d.ts.map +1 -0
  30. package/dist/errno.js +11 -0
  31. package/dist/errno.js.map +1 -0
  32. package/dist/index.d.ts +2 -1
  33. package/dist/index.d.ts.map +1 -1
  34. package/dist/index.js.map +1 -1
  35. package/dist/keys.d.ts +34 -0
  36. package/dist/keys.d.ts.map +1 -0
  37. package/dist/keys.js +88 -0
  38. package/dist/keys.js.map +1 -0
  39. package/dist/local-signature.d.ts +32 -0
  40. package/dist/local-signature.d.ts.map +1 -0
  41. package/dist/local-signature.js +58 -0
  42. package/dist/local-signature.js.map +1 -0
  43. package/dist/move-to-disk.d.ts +24 -0
  44. package/dist/move-to-disk.d.ts.map +1 -0
  45. package/dist/move-to-disk.js +167 -0
  46. package/dist/move-to-disk.js.map +1 -0
  47. package/dist/objects.d.ts.map +1 -1
  48. package/dist/objects.js +5 -2
  49. package/dist/objects.js.map +1 -1
  50. package/dist/provider-fetch.d.ts +36 -0
  51. package/dist/provider-fetch.d.ts.map +1 -0
  52. package/dist/provider-fetch.js +158 -0
  53. package/dist/provider-fetch.js.map +1 -0
  54. package/dist/services/main.d.ts +1 -1
  55. package/dist/services/main.d.ts.map +1 -1
  56. package/dist/services/main.js +14 -1
  57. package/dist/services/main.js.map +1 -1
  58. package/dist/signed-route.d.ts +18 -6
  59. package/dist/signed-route.d.ts.map +1 -1
  60. package/dist/signed-route.js +186 -31
  61. package/dist/signed-route.js.map +1 -1
  62. package/dist/streams.d.ts +36 -0
  63. package/dist/streams.d.ts.map +1 -0
  64. package/dist/streams.js +108 -0
  65. package/dist/streams.js.map +1 -0
  66. package/dist/testing/FakeStorage.d.ts.map +1 -1
  67. package/dist/testing/FakeStorage.js +2 -1
  68. package/dist/testing/FakeStorage.js.map +1 -1
  69. package/package.json +7 -3
  70. package/src/ArchiveProvider.ts +106 -4
  71. package/src/DriveManager.ts +38 -8
  72. package/src/GcsDriver.ts +256 -139
  73. package/src/S3Driver.ts +87 -130
  74. package/src/StaticMiddleware.ts +124 -13
  75. package/src/StorageManager.ts +433 -147
  76. package/src/augmentations.ts +24 -1
  77. package/src/errno.ts +10 -0
  78. package/src/index.ts +2 -0
  79. package/src/keys.ts +104 -0
  80. package/src/local-signature.ts +76 -0
  81. package/src/move-to-disk.ts +234 -0
  82. package/src/objects.ts +5 -2
  83. package/src/provider-fetch.ts +171 -0
  84. package/src/services/main.ts +15 -2
  85. package/src/signed-route.ts +216 -36
  86. package/src/streams.ts +118 -0
  87. package/src/testing/FakeStorage.ts +2 -3
@@ -10,7 +10,7 @@
10
10
  * Populated by `ArchiveProvider.boot()`.
11
11
  */
12
12
 
13
- import type { StorageManager } from "../StorageManager.js";
13
+ import { StorageManager } from "../StorageManager.js";
14
14
 
15
15
  let instance: StorageManager | undefined;
16
16
  let resolve: (() => StorageManager) | undefined;
@@ -53,7 +53,20 @@ export function clearStorage(): void {
53
53
  instance = undefined;
54
54
  }
55
55
 
56
- const storage: StorageManager = new Proxy({} as StorageManager, {
56
+ /**
57
+ * The proxy's target: an object of the right type that is never used —
58
+ * every read goes to the bound instance through the `get` trap. Built from
59
+ * the prototype and proven by `instanceof`, so no cast stands in for it.
60
+ */
61
+ function inertTarget(): StorageManager {
62
+ const target: unknown = Object.create(StorageManager.prototype);
63
+ if (!(target instanceof StorageManager)) {
64
+ throw new Error("[archive] could not build the storage accessor");
65
+ }
66
+ return target;
67
+ }
68
+
69
+ const storage: StorageManager = new Proxy(inertTarget(), {
57
70
  get(_target, prop) {
58
71
  // A module loader inspects what it imports before anyone uses it: it reads
59
72
  // `then` to decide whether the namespace is thenable, and various symbols
@@ -3,17 +3,31 @@
3
3
  *
4
4
  * Verifies the HMAC produced by {@link LocalDriver.getSignedUrl} and,
5
5
  * on success, serves the file from the driver. Tampered signatures →
6
- * 403 `E_INVALID_SIGNATURE`. Expired `exp` → 403 `E_EXPIRED_SIGNATURE`.
7
- * Missing query params → falls through to `next()` so unsigned routes
8
- * can coexist on the same prefix.
6
+ * 403 `E_ARCHIVE_INVALID_SIGNATURE`. Expired `exp` → 403
7
+ * `E_ARCHIVE_EXPIRED_SIGNATURE`. A valid signature for a file that is not
8
+ * stored → 404 `E_ARCHIVE_NOT_FOUND`.
9
+ * No signature at all → the file's visibility decides: a public file is
10
+ * served, a private one refused with 403 `E_ARCHIVE_ACCESS_DENIED`, and a
11
+ * name that is not stored falls through to `next()`.
9
12
  *
10
13
  * @implements Story 43.2 AC 5
11
14
  */
12
15
 
13
16
  import { createHmac, timingSafeEqual } from "node:crypto";
14
- import * as path from "node:path";
15
- import { inferMimeType } from "./mime-types.js";
16
- import type { LocalDriver, StorageManager } from "./StorageManager.js";
17
+ import { ArchiveError } from "./errors.js";
18
+ import { matchesIfNoneMatch } from "./etag.js";
19
+ import { normalizeKey } from "./keys.js";
20
+ import {
21
+ HEADER_OVERRIDES,
22
+ type HeaderOverride,
23
+ isHeaderValue,
24
+ signedPayload,
25
+ } from "./local-signature.js";
26
+ import type {
27
+ LocalDriver,
28
+ Metadata,
29
+ StorageManager,
30
+ } from "./StorageManager.js";
17
31
 
18
32
  /**
19
33
  * Minimal request/response contract the signed-route handler needs.
@@ -25,7 +39,11 @@ import type { LocalDriver, StorageManager } from "./StorageManager.js";
25
39
  export interface SignedRouteResponse {
26
40
  status(code: number): SignedRouteResponse;
27
41
  header(name: string, value: string): SignedRouteResponse;
28
- sendBuffer(buf: Buffer): void;
42
+ /**
43
+ * Send a readable as the body. The framework's response streams it to
44
+ * the client; the file is never held in memory whole.
45
+ */
46
+ stream(body: NodeJS.ReadableStream): Promise<void> | void;
29
47
  json(data: unknown): void;
30
48
  }
31
49
 
@@ -54,8 +72,13 @@ export interface SignedRouteOptions {
54
72
  * recompute the expected signature.
55
73
  */
56
74
  driver: LocalDriver;
57
- /** URL prefix for signed file serving (default: `/storage`). Must
58
- * match the prefix used by `LocalDriver.url()`. */
75
+ /**
76
+ * URL prefix for file serving. Defaults to — and must equal — the
77
+ * driver's `routeBasePath` (`driver.getRouteBasePath()`), which is what
78
+ * its URLs are built on. Configure the path on the driver; a different
79
+ * value here is refused at construction (`E_ARCHIVE_PREFIX_MISMATCH`)
80
+ * rather than producing a route that can never match a signed URL.
81
+ */
59
82
  prefix?: string;
60
83
  }
61
84
 
@@ -71,10 +94,17 @@ type SignedRouteMiddleware = (
71
94
  export function createSignedRouteHandler(
72
95
  options: SignedRouteOptions,
73
96
  ): SignedRouteMiddleware {
74
- const prefix = options.prefix ?? "/storage";
75
- if (prefix === "") {
76
- throw new Error(
77
- "createSignedRouteHandler: prefix cannot be empty. Use '/storage' (default) or a specific path.",
97
+ const prefix = options.driver.getRouteBasePath();
98
+ if (
99
+ options.prefix !== undefined &&
100
+ options.prefix.replace(/\/+$/, "") !== prefix
101
+ ) {
102
+ throw new ArchiveError(
103
+ "E_ARCHIVE_PREFIX_MISMATCH",
104
+ `createSignedRouteHandler: prefix '${options.prefix}' does not match the driver's routeBasePath '${prefix}', so no URL the driver signs would ever reach this route.`,
105
+ {
106
+ hint: "Set routeBasePath on the LocalDriver (or services.fs) and leave prefix out.",
107
+ },
78
108
  );
79
109
  }
80
110
  const secret = options.driver.getSigningSecret();
@@ -100,25 +130,80 @@ export function createSignedRouteHandler(
100
130
  // path segment; we decode here before recomputing the HMAC.
101
131
  const encodedPath = reqPath.slice(prefix.length).replace(/^\//, "");
102
132
  if (encodedPath === "") return next();
103
- let filePath: string;
133
+ let decoded: string;
104
134
  try {
105
- filePath = decodeSegments(encodedPath);
135
+ decoded = decodeSegments(encodedPath);
106
136
  } catch {
107
137
  ctx.response.status(403).json({
108
- error: { code: "E_INVALID_SIGNATURE", message: "Malformed path" },
138
+ error: {
139
+ code: "E_ARCHIVE_INVALID_SIGNATURE",
140
+ message: "Malformed path",
141
+ },
109
142
  });
110
143
  return;
111
144
  }
145
+ // The key the signer signed is the normalised one; a path that does
146
+ // not normalise (`..`, an encoded separator, a control character) names
147
+ // nothing this route will serve.
148
+ let filePath: string;
149
+ try {
150
+ filePath = normalizeKey(decoded);
151
+ } catch {
152
+ invalidKey(ctx);
153
+ return;
154
+ }
112
155
 
113
156
  // Pull `exp`/`sig` from the parsed query object — matches
114
157
  // Ream's `Request.qs()` contract (`Record<string, unknown>`).
115
158
  const qs = readQueryObject(ctx);
159
+ if (qs.exp === undefined && qs.sig === undefined) {
160
+ // No signature: this route still owns the prefix. Handing every
161
+ // unsigned request on to `next()` let whatever was mounted behind —
162
+ // a static server over the storage root, the obvious companion —
163
+ // serve a PRIVATE file to anyone who left the query off. Visibility
164
+ // is decided here: a public file is served, a private one refused,
165
+ // and only a name that is not stored goes on to the next handler.
166
+ //
167
+ // NAMED DEVIATION: upstream answers this refusal with 401. It is a
168
+ // 403 here, as every other refusal of this route already was: no
169
+ // credential the client could add makes a private file servable
170
+ // without a signature.
171
+ let visibility: "public" | "private" | undefined;
172
+ try {
173
+ if (await storage.exists(filePath)) {
174
+ visibility = await storage.getVisibility(filePath);
175
+ }
176
+ } catch (err) {
177
+ // A name the disk refuses (a visibility marker's) is answered,
178
+ // not thrown into the application's error handler.
179
+ if (isKeyError(err)) {
180
+ invalidKey(ctx);
181
+ return;
182
+ }
183
+ throw err;
184
+ }
185
+ if (visibility === undefined) return next();
186
+ if (visibility !== "public") {
187
+ ctx.response.status(403).json({
188
+ error: { code: "E_ARCHIVE_ACCESS_DENIED", message: "Access denied" },
189
+ });
190
+ return;
191
+ }
192
+ return serve(ctx, filePath);
193
+ }
116
194
  const expRaw = readStringParam(qs, "exp");
117
195
  const sigHex = readStringParam(qs, "sig");
118
196
  if (expRaw === null || sigHex === null) {
119
- // No signature present — hand off to the next middleware so
120
- // unsigned routes (e.g. StaticMiddleware) can handle it.
121
- return next();
197
+ // Half a signature, or one repeated into an array: a request that
198
+ // tries to use a signature and cannot be verified is refused, never
199
+ // downgraded to the unsigned path.
200
+ ctx.response.status(403).json({
201
+ error: {
202
+ code: "E_ARCHIVE_INVALID_SIGNATURE",
203
+ message: "Malformed signature",
204
+ },
205
+ });
206
+ return;
122
207
  }
123
208
 
124
209
  // Validate sig shape up-front: 64 hex chars (sha256). Rejects
@@ -128,7 +213,7 @@ export function createSignedRouteHandler(
128
213
  if (!/^[a-f0-9]{64}$/.test(sigHex)) {
129
214
  ctx.response.status(403).json({
130
215
  error: {
131
- code: "E_INVALID_SIGNATURE",
216
+ code: "E_ARCHIVE_INVALID_SIGNATURE",
132
217
  message: "Malformed signature",
133
218
  },
134
219
  });
@@ -141,51 +226,138 @@ export function createSignedRouteHandler(
141
226
  const exp = Number.parseInt(expRaw, 10);
142
227
  if (!/^\d+$/.test(expRaw) || !Number.isFinite(exp)) {
143
228
  ctx.response.status(403).json({
144
- error: { code: "E_INVALID_SIGNATURE", message: "Malformed expiry" },
229
+ error: {
230
+ code: "E_ARCHIVE_INVALID_SIGNATURE",
231
+ message: "Malformed expiry",
232
+ },
145
233
  });
146
234
  return;
147
235
  }
148
236
 
237
+ // Response-header overrides travel in the query and are covered by
238
+ // the signature. One present in a shape that cannot be verified
239
+ // (repeated, or carrying a control character) is refused outright.
240
+ const overrides: HeaderOverride[] = [];
241
+ for (const [param] of HEADER_OVERRIDES) {
242
+ const raw = qs[param];
243
+ if (raw === undefined) continue;
244
+ if (typeof raw !== "string" || !isHeaderValue(raw)) {
245
+ ctx.response.status(403).json({
246
+ error: {
247
+ code: "E_ARCHIVE_INVALID_SIGNATURE",
248
+ message: "Malformed header override",
249
+ },
250
+ });
251
+ return;
252
+ }
253
+ if (raw !== "") overrides.push([param, raw]);
254
+ }
255
+
149
256
  // Constant-time HMAC verification on the decoded path. The regex
150
257
  // check above already guarantees 32 bytes, so `timingSafeEqual`
151
258
  // cannot throw on a length mismatch here. Run BEFORE the expiry
152
259
  // check so the response time cannot leak whether a sig was valid
153
260
  // for an expired URL versus invalid for a fresh one.
154
261
  const expected = createHmac("sha256", secret)
155
- .update(`${filePath}\n${exp}`)
262
+ .update(signedPayload(filePath, exp, overrides))
156
263
  .digest();
157
264
  const provided = Buffer.from(sigHex, "hex");
158
265
  if (!timingSafeEqual(provided, expected)) {
159
266
  ctx.response.status(403).json({
160
- error: { code: "E_INVALID_SIGNATURE", message: "Invalid signature" },
267
+ error: {
268
+ code: "E_ARCHIVE_INVALID_SIGNATURE",
269
+ message: "Invalid signature",
270
+ },
161
271
  });
162
272
  return;
163
273
  }
164
274
 
165
275
  if (exp * 1000 <= Date.now()) {
166
276
  ctx.response.status(403).json({
167
- error: { code: "E_EXPIRED_SIGNATURE", message: "Signed URL expired" },
277
+ error: {
278
+ code: "E_ARCHIVE_EXPIRED_SIGNATURE",
279
+ message: "Signed URL expired",
280
+ },
168
281
  });
169
282
  return;
170
283
  }
171
284
 
172
285
  // Signature verified — serve the file.
173
- const content = await storage.get(filePath);
174
- if (content === null) {
175
- ctx.response
176
- .status(404)
177
- .json({ error: { code: "E_NOT_FOUND", message: "File not found" } });
286
+ return serve(ctx, filePath, overrides);
287
+ };
288
+
289
+ async function serve(
290
+ ctx: SignedRouteHttpContext,
291
+ filePath: string,
292
+ overrides: readonly HeaderOverride[] = [],
293
+ ): Promise<void> {
294
+ // Metadata first, the bytes only once they are known to be wanted: the
295
+ // whole file used to be read into memory for every request, and a
296
+ // client revalidating its copy got a full transfer each time.
297
+ let metadata: Metadata;
298
+ try {
299
+ metadata = await storage.getMetadata(filePath);
300
+ } catch (err) {
301
+ if (isNotFound(err)) return notFound(ctx);
302
+ throw err;
303
+ }
304
+ ctx.response.header("ETag", metadata.etag);
305
+ ctx.response.header("Last-Modified", metadata.lastModified.toUTCString());
306
+ ctx.response.header("Content-Type", metadata.contentType);
307
+ // Signed overrides win over what the extension implies.
308
+ for (const [param, value] of overrides) {
309
+ const header = HEADER_OVERRIDES.find(([name]) => name === param)?.[1];
310
+ if (header !== undefined) ctx.response.header(header, value);
311
+ }
312
+ if (
313
+ matchesIfNoneMatch(ctx.request.header("if-none-match"), metadata.etag)
314
+ ) {
315
+ ctx.response.status(304);
178
316
  return;
179
317
  }
180
- const ext = path.extname(filePath).toLowerCase();
181
- ctx.response.header("Content-Type", inferMimeType(ext));
182
- ctx.response.header("Content-Length", String(content.length));
318
+ ctx.response.header("Content-Length", String(metadata.contentLength));
183
319
  if (ctx.request.method() === "HEAD") {
184
320
  ctx.response.status(200);
185
321
  return;
186
322
  }
187
- ctx.response.status(200).sendBuffer(content);
188
- };
323
+ let body: NodeJS.ReadableStream;
324
+ try {
325
+ body = await storage.getStream(filePath);
326
+ } catch (err) {
327
+ // Deleted between the metadata read and the open.
328
+ if (isNotFound(err)) return notFound(ctx);
329
+ throw err;
330
+ }
331
+ ctx.response.status(200);
332
+ await ctx.response.stream(body);
333
+ }
334
+ }
335
+
336
+ /** The refusals a disk raises for a key it will not address. */
337
+ const KEY_ERRORS = new Set([
338
+ "E_ARCHIVE_INVALID_KEY",
339
+ "E_ARCHIVE_PATH_TRAVERSAL_DETECTED",
340
+ "E_ARCHIVE_UNALLOWED_CHARACTERS",
341
+ ]);
342
+
343
+ function isNotFound(err: unknown): boolean {
344
+ return err instanceof ArchiveError && err.code === "E_ARCHIVE_NOT_FOUND";
345
+ }
346
+
347
+ function notFound(ctx: SignedRouteHttpContext): void {
348
+ ctx.response.status(404).json({
349
+ error: { code: "E_ARCHIVE_NOT_FOUND", message: "File not found" },
350
+ });
351
+ }
352
+
353
+ function isKeyError(err: unknown): boolean {
354
+ return err instanceof ArchiveError && KEY_ERRORS.has(err.code);
355
+ }
356
+
357
+ function invalidKey(ctx: SignedRouteHttpContext): void {
358
+ ctx.response.status(403).json({
359
+ error: { code: "E_ARCHIVE_INVALID_KEY", message: "Invalid path" },
360
+ });
189
361
  }
190
362
 
191
363
  /**
@@ -213,11 +385,19 @@ function readStringParam(
213
385
  /**
214
386
  * Decode each URL-encoded path segment. Throws if any segment
215
387
  * contains invalid percent-escapes (`decodeURIComponent` throws on
216
- * malformed input).
388
+ * malformed input), or decodes to a separator: the signer encodes each
389
+ * segment on its own, so an encoded `/` or `\` never comes from a URL it
390
+ * built, and would let one request segment pose as two.
217
391
  */
218
392
  function decodeSegments(encodedPath: string): string {
219
393
  return encodedPath
220
394
  .split("/")
221
- .map((seg) => decodeURIComponent(seg))
395
+ .map((seg) => {
396
+ const decoded = decodeURIComponent(seg);
397
+ if (/[/\\]/.test(decoded)) {
398
+ throw new Error("encoded separator in a path segment");
399
+ }
400
+ return decoded;
401
+ })
222
402
  .join("/");
223
403
  }
package/src/streams.ts ADDED
@@ -0,0 +1,118 @@
1
+ /// <reference lib="dom" />
2
+ // Uses the WHATWG streams globals, which live in the DOM lib even on Node.
3
+ /**
4
+ * Stream helpers shared by the cloud drivers.
5
+ */
6
+
7
+ import { Readable } from "node:stream";
8
+
9
+ /** A chunk of a Node readable as bytes (string chunks are UTF-8). */
10
+ export function toBuffer(chunk: unknown): Buffer {
11
+ if (Buffer.isBuffer(chunk)) return chunk;
12
+ if (chunk instanceof Uint8Array) return Buffer.from(chunk);
13
+ return Buffer.from(String(chunk));
14
+ }
15
+
16
+ /**
17
+ * Type guard for any readable that exposes `destroy()` (Node streams do; a
18
+ * generic iterable does not), so file descriptors and sockets can be released
19
+ * without a cast.
20
+ */
21
+ function hasDestroy(x: unknown): x is { destroy(err?: Error): void } {
22
+ return (
23
+ typeof x === "object" &&
24
+ x !== null &&
25
+ "destroy" in x &&
26
+ typeof x.destroy === "function"
27
+ );
28
+ }
29
+
30
+ /**
31
+ * Drain a readable into a single Buffer, capped at `maxBytes`. Throws if the
32
+ * stream yields more than that — the caller declared a length and we refuse
33
+ * to silently over-buffer — after destroying the upstream so its descriptor
34
+ * or socket does not leak.
35
+ */
36
+ export async function collectToBuffer(
37
+ readable: NodeJS.ReadableStream,
38
+ maxBytes: number,
39
+ ): Promise<Buffer> {
40
+ const chunks: Buffer[] = [];
41
+ let total = 0;
42
+ for await (const chunk of readable) {
43
+ const buf = toBuffer(chunk);
44
+ total += buf.length;
45
+ if (total > maxBytes) {
46
+ if (hasDestroy(readable)) readable.destroy();
47
+ throw new Error(
48
+ `putStream: readable exceeded declared contentLength (${total} > ${maxBytes})`,
49
+ );
50
+ }
51
+ chunks.push(buf);
52
+ }
53
+ return Buffer.concat(chunks, total);
54
+ }
55
+
56
+ /**
57
+ * Async-iterable that yields Buffers of exactly `partSize` bytes (the last
58
+ * one may be smaller). Used to slice a stream into multipart parts or
59
+ * resumable-upload chunks.
60
+ *
61
+ * **Backpressure.** Memory usage is bounded by `partSize + readable`'s own
62
+ * `highWaterMark`. The `for await` loop pulls from `readable` only when the
63
+ * consumer has finished awaiting the previous yield, so a slow consumer
64
+ * naturally throttles a fast producer through Node's stream pause/resume
65
+ * protocol. Pre-buffered streams (e.g. `Readable.from(buffer)`) bypass this
66
+ * and surface their entire payload in one tick; for those, expect peak ≈
67
+ * `payload + partSize`.
68
+ */
69
+ export async function* chunkStream(
70
+ readable: NodeJS.ReadableStream,
71
+ partSize: number,
72
+ ): AsyncIterable<Buffer> {
73
+ let pending: Buffer[] = [];
74
+ let pendingSize = 0;
75
+ for await (const chunk of readable) {
76
+ const buf = toBuffer(chunk);
77
+ pending.push(buf);
78
+ pendingSize += buf.length;
79
+ while (pendingSize >= partSize) {
80
+ const joined = Buffer.concat(pending, pendingSize);
81
+ yield joined.subarray(0, partSize);
82
+ const leftover = joined.subarray(partSize);
83
+ pending = leftover.length > 0 ? [leftover] : [];
84
+ pendingSize = leftover.length;
85
+ }
86
+ }
87
+ if (pendingSize > 0) {
88
+ yield Buffer.concat(pending, pendingSize);
89
+ }
90
+ }
91
+
92
+ /**
93
+ * A WHATWG response body as a Node readable, without a cast between the DOM
94
+ * and `node:stream/web` typings of the same runtime object.
95
+ *
96
+ * Destroying the readable before the end cancels the body, so the
97
+ * connection behind it is released instead of left to the collector.
98
+ */
99
+ export function nodeReadable(body: ReadableStream<Uint8Array>): Readable {
100
+ async function* chunks(): AsyncGenerator<Buffer> {
101
+ const reader = body.getReader();
102
+ let finished = false;
103
+ try {
104
+ for (;;) {
105
+ const { done, value } = await reader.read();
106
+ if (done) {
107
+ finished = true;
108
+ return;
109
+ }
110
+ yield Buffer.from(value.buffer, value.byteOffset, value.byteLength);
111
+ }
112
+ } finally {
113
+ if (!finished) await reader.cancel().catch(() => {});
114
+ reader.releaseLock();
115
+ }
116
+ }
117
+ return Readable.from(chunks(), { objectMode: false });
118
+ }
@@ -37,6 +37,7 @@ import type {
37
37
  WriteOptions,
38
38
  } from "../StorageManager.js";
39
39
  import { StorageManager } from "../StorageManager.js";
40
+ import { toBuffer } from "../streams.js";
40
41
 
41
42
  interface StoredObject {
42
43
  content: Buffer;
@@ -90,9 +91,7 @@ export class FakeStorage implements StorageDriver {
90
91
  ): Promise<void> {
91
92
  const chunks: Buffer[] = [];
92
93
  for await (const chunk of readable) {
93
- chunks.push(
94
- Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as string),
95
- );
94
+ chunks.push(toBuffer(chunk));
96
95
  }
97
96
  const buf = Buffer.concat(chunks);
98
97
  this.#store.set(filePath, {