@c9up/archive 0.1.20 → 0.1.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/README.md +96 -3
  2. package/dist/ArchiveProvider.d.ts +6 -1
  3. package/dist/ArchiveProvider.d.ts.map +1 -1
  4. package/dist/ArchiveProvider.js +73 -1
  5. package/dist/ArchiveProvider.js.map +1 -1
  6. package/dist/DriveManager.d.ts +10 -3
  7. package/dist/DriveManager.d.ts.map +1 -1
  8. package/dist/DriveManager.js +23 -6
  9. package/dist/DriveManager.js.map +1 -1
  10. package/dist/GcsDriver.d.ts +14 -9
  11. package/dist/GcsDriver.d.ts.map +1 -1
  12. package/dist/GcsDriver.js +202 -98
  13. package/dist/GcsDriver.js.map +1 -1
  14. package/dist/S3Driver.d.ts +11 -7
  15. package/dist/S3Driver.d.ts.map +1 -1
  16. package/dist/S3Driver.js +73 -110
  17. package/dist/S3Driver.js.map +1 -1
  18. package/dist/StaticMiddleware.d.ts +5 -1
  19. package/dist/StaticMiddleware.d.ts.map +1 -1
  20. package/dist/StaticMiddleware.js +114 -13
  21. package/dist/StaticMiddleware.js.map +1 -1
  22. package/dist/StorageManager.d.ts +28 -3
  23. package/dist/StorageManager.d.ts.map +1 -1
  24. package/dist/StorageManager.js +333 -132
  25. package/dist/StorageManager.js.map +1 -1
  26. package/dist/augmentations.d.ts +17 -0
  27. package/dist/augmentations.d.ts.map +1 -1
  28. package/dist/errno.d.ts +6 -0
  29. package/dist/errno.d.ts.map +1 -0
  30. package/dist/errno.js +11 -0
  31. package/dist/errno.js.map +1 -0
  32. package/dist/index.d.ts +1 -0
  33. package/dist/index.d.ts.map +1 -1
  34. package/dist/index.js.map +1 -1
  35. package/dist/keys.d.ts +34 -0
  36. package/dist/keys.d.ts.map +1 -0
  37. package/dist/keys.js +88 -0
  38. package/dist/keys.js.map +1 -0
  39. package/dist/local-signature.d.ts +32 -0
  40. package/dist/local-signature.d.ts.map +1 -0
  41. package/dist/local-signature.js +58 -0
  42. package/dist/local-signature.js.map +1 -0
  43. package/dist/move-to-disk.d.ts +24 -0
  44. package/dist/move-to-disk.d.ts.map +1 -0
  45. package/dist/move-to-disk.js +167 -0
  46. package/dist/move-to-disk.js.map +1 -0
  47. package/dist/objects.d.ts.map +1 -1
  48. package/dist/objects.js +5 -2
  49. package/dist/objects.js.map +1 -1
  50. package/dist/provider-fetch.d.ts +36 -0
  51. package/dist/provider-fetch.d.ts.map +1 -0
  52. package/dist/provider-fetch.js +158 -0
  53. package/dist/provider-fetch.js.map +1 -0
  54. package/dist/services/main.d.ts +1 -1
  55. package/dist/services/main.d.ts.map +1 -1
  56. package/dist/services/main.js +14 -1
  57. package/dist/services/main.js.map +1 -1
  58. package/dist/signed-route.d.ts +18 -6
  59. package/dist/signed-route.d.ts.map +1 -1
  60. package/dist/signed-route.js +186 -31
  61. package/dist/signed-route.js.map +1 -1
  62. package/dist/streams.d.ts +36 -0
  63. package/dist/streams.d.ts.map +1 -0
  64. package/dist/streams.js +108 -0
  65. package/dist/streams.js.map +1 -0
  66. package/dist/testing/FakeStorage.d.ts.map +1 -1
  67. package/dist/testing/FakeStorage.js +2 -1
  68. package/dist/testing/FakeStorage.js.map +1 -1
  69. package/package.json +7 -3
  70. package/src/ArchiveProvider.ts +105 -4
  71. package/src/DriveManager.ts +32 -6
  72. package/src/GcsDriver.ts +256 -139
  73. package/src/S3Driver.ts +87 -130
  74. package/src/StaticMiddleware.ts +124 -13
  75. package/src/StorageManager.ts +398 -141
  76. package/src/augmentations.ts +24 -1
  77. package/src/errno.ts +10 -0
  78. package/src/index.ts +1 -0
  79. package/src/keys.ts +104 -0
  80. package/src/local-signature.ts +76 -0
  81. package/src/move-to-disk.ts +234 -0
  82. package/src/objects.ts +5 -2
  83. package/src/provider-fetch.ts +171 -0
  84. package/src/services/main.ts +15 -2
  85. package/src/signed-route.ts +216 -36
  86. package/src/streams.ts +118 -0
  87. package/src/testing/FakeStorage.ts +2 -3
package/src/S3Driver.ts CHANGED
@@ -13,14 +13,15 @@
13
13
 
14
14
  import { createHash, createHmac } from "node:crypto";
15
15
  import { Readable } from "node:stream";
16
- import type { ReadableStream as NodeReadableStream } from "node:stream/web";
17
16
  import { ArchiveError } from "./errors.js";
18
17
  import {
19
18
  assertValidExpiry,
20
19
  DEFAULT_EXPIRES_IN,
21
20
  parseExpiry,
22
21
  } from "./expiry.js";
22
+ import { normalizeKey, normalizePrefix } from "./keys.js";
23
23
  import { DriveDirectory, DriveFile } from "./objects.js";
24
+ import { fetchWithIdleTimeout, requestBody } from "./provider-fetch.js";
24
25
  import type {
25
26
  ListAllOptions,
26
27
  ListAllResult,
@@ -32,6 +33,7 @@ import type {
32
33
  Visibility,
33
34
  WriteOptions,
34
35
  } from "./StorageManager.js";
36
+ import { chunkStream, collectToBuffer, nodeReadable } from "./streams.js";
35
37
 
36
38
  /** AWS minimum for a non-final multipart part. */
37
39
  const S3_MULTIPART_PART_SIZE = 5 * 1024 * 1024;
@@ -54,9 +56,14 @@ function s3StrictEncode(str: string): string {
54
56
  * URI-encode an S3 object key per AWS SigV4 rules: encode each path segment
55
57
  * individually but preserve `/` separators. Handles spaces, unicode, `#`, `?`, `%`
56
58
  * as well as the strict AWS set (`! * ' ( )`).
59
+ *
60
+ * Every object URL is built through here, so this is where a key is
61
+ * normalised (see `keys.ts`) when the driver is used without a manager in
62
+ * front: `..` never reaches a URL, where it would resolve into another
63
+ * bucket on the same host.
57
64
  */
58
65
  function s3EncodeKey(key: string): string {
59
- return key.split("/").map(s3StrictEncode).join("/");
66
+ return normalizeKey(key).split("/").map(s3StrictEncode).join("/");
60
67
  }
61
68
 
62
69
  export interface S3Config {
@@ -67,39 +74,32 @@ export interface S3Config {
67
74
  secretAccessKey: string;
68
75
  publicUrl?: string;
69
76
  /**
70
- * How long one request to the provider may take, in ms. Default 30 000.
77
+ * How long the provider may stay silent during one request, in ms.
78
+ * Default 30 000.
71
79
  *
72
- * Without a bound a hung connection never settles: the handler awaiting it
73
- * waits forever, and enough of them stop the server from serving. Set 0 to
74
- * disable, which only makes sense behind an outer timeout.
80
+ * An idle bound, not a deadline: it runs while the provider is awaited
81
+ * (taking the request body, sending headers, sending the next chunk), so a
82
+ * long transfer that keeps moving is never cut. Without a bound a hung
83
+ * connection never settles: the handler awaiting it waits forever, and
84
+ * enough of them stop the server from serving. Set 0 to disable, which
85
+ * only makes sense behind an outer timeout.
75
86
  */
76
87
  requestTimeoutMs?: number;
77
88
  }
78
89
 
79
90
  export class S3Driver implements StorageDriver {
80
91
  /**
81
- * One request to the provider, bounded by {@link S3Config.requestTimeoutMs}.
82
- *
83
- * `AbortSignal.timeout` aborts the socket, so a hung connection surfaces as
84
- * an error the caller can retry instead of a promise that never settles.
92
+ * One request to the provider, bounded by {@link S3Config.requestTimeoutMs}
93
+ * as an idle timeout (see `provider-fetch.ts`): a hung connection surfaces
94
+ * as an error the caller can retry, a long transfer that keeps moving does
95
+ * not.
85
96
  */
86
- async #fetch(url: string, init: RequestInit = {}): Promise<Response> {
87
- const ms = this.#config.requestTimeoutMs ?? 30_000;
88
- if (ms <= 0) return fetch(url, init);
89
- try {
90
- return await fetch(url, { ...init, signal: AbortSignal.timeout(ms) });
91
- } catch (err) {
92
- if (err instanceof Error && err.name === "TimeoutError") {
93
- throw new ArchiveError(
94
- "E_ARCHIVE_REQUEST_TIMEOUT",
95
- `The storage provider did not answer within ${ms}ms.`,
96
- {
97
- hint: "Raise `requestTimeoutMs` for large objects, or check connectivity to the bucket.",
98
- },
99
- );
100
- }
101
- throw err;
102
- }
97
+ #fetch(url: string, init: RequestInit = {}): Promise<Response> {
98
+ return fetchWithIdleTimeout(
99
+ url,
100
+ init,
101
+ this.#config.requestTimeoutMs ?? 30_000,
102
+ );
103
103
  }
104
104
 
105
105
  #config: S3Config;
@@ -154,7 +154,7 @@ export class S3Driver implements StorageDriver {
154
154
  const res = await this.#fetch(url, {
155
155
  method: "PUT",
156
156
  headers,
157
- body: body as BodyInit,
157
+ body: requestBody(body),
158
158
  });
159
159
  if (!res.ok) {
160
160
  throw new Error(`S3 PUT failed (${res.status}): ${await res.text()}`);
@@ -183,7 +183,7 @@ export class S3Driver implements StorageDriver {
183
183
  const res = await this.#fetch(url, {
184
184
  method: "PUT",
185
185
  headers,
186
- body: body as BodyInit,
186
+ body: requestBody(body),
187
187
  });
188
188
  if (!res.ok) {
189
189
  throw new Error(`S3 PUT failed (${res.status}): ${await res.text()}`);
@@ -248,11 +248,7 @@ export class S3Driver implements StorageDriver {
248
248
  // Edge case: 200 with no body. Return an empty readable.
249
249
  return Readable.from(Buffer.alloc(0));
250
250
  }
251
- // Node's `Readable.fromWeb` expects the `node:stream/web` variant,
252
- // while `this.#fetch().body` gives the lib.dom variant. Runtime shape is
253
- // identical — the difference is only in TS lib typings. A single
254
- // cast to the node-side type is enough (no `unknown` pivot).
255
- return Readable.fromWeb(res.body as NodeReadableStream<Uint8Array>);
251
+ return nodeReadable(res.body);
256
252
  }
257
253
 
258
254
  async get(filePath: string): Promise<Buffer | null> {
@@ -570,7 +566,8 @@ export class S3Driver implements StorageDriver {
570
566
  await this.delete(from);
571
567
  }
572
568
 
573
- async deleteAll(prefix: string): Promise<void> {
569
+ async deleteAll(given: string): Promise<void> {
570
+ const prefix = normalizePrefix(given);
574
571
  // List one page at a time and batch-delete (up to 1000 keys per
575
572
  // DeleteObjects call) until the bucket is drained of the prefix.
576
573
  let continuationToken: string | undefined;
@@ -606,7 +603,7 @@ export class S3Driver implements StorageDriver {
606
603
  /<Contents>[\s\S]*?<Key>([^<]+)<\/Key>[\s\S]*?<\/Contents>/g;
607
604
  let m: RegExpExecArray | null = keyRe.exec(xml);
608
605
  while (m !== null) {
609
- if (m[1] !== undefined) keys.push(m[1]);
606
+ if (m[1] !== undefined) keys.push(xmlUnescape(m[1]));
610
607
  m = keyRe.exec(xml);
611
608
  }
612
609
  if (keys.length > 0) await this.#deleteObjects(keys);
@@ -614,7 +611,10 @@ export class S3Driver implements StorageDriver {
614
611
  const tokenMatch = xml.match(
615
612
  /<NextContinuationToken>([^<]+)<\/NextContinuationToken>/,
616
613
  );
617
- continuationToken = truncated && tokenMatch ? tokenMatch[1] : undefined;
614
+ continuationToken =
615
+ truncated && tokenMatch?.[1] !== undefined
616
+ ? xmlUnescape(tokenMatch[1])
617
+ : undefined;
618
618
  } while (continuationToken);
619
619
  }
620
620
 
@@ -637,7 +637,7 @@ export class S3Driver implements StorageDriver {
637
637
  const res = await this.#fetch(url, {
638
638
  method: "POST",
639
639
  headers,
640
- body: body as BodyInit,
640
+ body: requestBody(body),
641
641
  });
642
642
  if (!res.ok) {
643
643
  throw new Error(
@@ -647,12 +647,13 @@ export class S3Driver implements StorageDriver {
647
647
  }
648
648
 
649
649
  async listAll(
650
- prefix: string,
650
+ given: string,
651
651
  options?: ListAllOptions,
652
652
  ): Promise<ListAllResult> {
653
653
  const recursive = options?.recursive ?? false;
654
+ const prefix = normalizePrefix(given);
654
655
  const normalizedPrefix =
655
- prefix === "" || prefix === "/"
656
+ prefix === ""
656
657
  ? ""
657
658
  : !recursive && !prefix.endsWith("/")
658
659
  ? `${prefix}/`
@@ -688,7 +689,7 @@ export class S3Driver implements StorageDriver {
688
689
  let pm: RegExpExecArray | null = prefixRe.exec(xml);
689
690
  while (pm !== null) {
690
691
  if (pm[1] !== undefined) {
691
- objects.push(new DriveDirectory(pm[1].replace(/\/$/, "")));
692
+ objects.push(new DriveDirectory(xmlUnescape(pm[1]).replace(/\/$/, "")));
692
693
  }
693
694
  pm = prefixRe.exec(xml);
694
695
  }
@@ -696,7 +697,7 @@ export class S3Driver implements StorageDriver {
696
697
  let bm: RegExpExecArray | null = blockRe.exec(xml);
697
698
  while (bm !== null) {
698
699
  const block = bm[1] ?? "";
699
- const key = block.match(/<Key>([^<]+)<\/Key>/)?.[1] ?? "";
700
+ const key = xmlUnescape(block.match(/<Key>([^<]+)<\/Key>/)?.[1] ?? "");
700
701
  const sizeRaw = block.match(/<Size>([^<]+)<\/Size>/)?.[1] ?? "0";
701
702
  const lm = block.match(/<LastModified>([^<]+)<\/LastModified>/)?.[1];
702
703
  const size = Number.parseInt(sizeRaw, 10);
@@ -707,8 +708,9 @@ export class S3Driver implements StorageDriver {
707
708
  mimeType: "application/octet-stream",
708
709
  contentType: "application/octet-stream",
709
710
  lastModified: lm ? new Date(lm) : new Date(0),
710
- etag:
711
- block.match(/<ETag>([^<]+)<\/ETag>/)?.[1]?.replace(/"/g, "") ?? "",
711
+ etag: xmlUnescape(
712
+ block.match(/<ETag>([^<]+)<\/ETag>/)?.[1] ?? "",
713
+ ).replace(/"/g, ""),
712
714
  visibility: "private",
713
715
  }),
714
716
  );
@@ -719,12 +721,16 @@ export class S3Driver implements StorageDriver {
719
721
  /<NextContinuationToken>([^<]+)<\/NextContinuationToken>/,
720
722
  );
721
723
  return {
722
- paginationToken: truncated && tokenMatch ? tokenMatch[1] : undefined,
724
+ paginationToken:
725
+ truncated && tokenMatch?.[1] !== undefined
726
+ ? xmlUnescape(tokenMatch[1])
727
+ : undefined,
723
728
  objects,
724
729
  };
725
730
  }
726
731
 
727
- async *list(prefix: string): AsyncIterable<StorageEntry> {
732
+ async *list(given: string): AsyncIterable<StorageEntry> {
733
+ const prefix = normalizePrefix(given);
728
734
  let continuationToken: string | undefined;
729
735
  do {
730
736
  const qp: Array<[string, string]> = [
@@ -756,7 +762,7 @@ export class S3Driver implements StorageDriver {
756
762
  let blockMatch: RegExpExecArray | null = blockRe.exec(xml);
757
763
  while (blockMatch !== null) {
758
764
  const block = blockMatch[1] ?? "";
759
- const key = block.match(/<Key>([^<]+)<\/Key>/)?.[1] ?? "";
765
+ const key = xmlUnescape(block.match(/<Key>([^<]+)<\/Key>/)?.[1] ?? "");
760
766
  const sizeRaw = block.match(/<Size>([^<]+)<\/Size>/)?.[1] ?? "0";
761
767
  const lm = block.match(/<LastModified>([^<]+)<\/LastModified>/)?.[1];
762
768
  yield {
@@ -770,7 +776,10 @@ export class S3Driver implements StorageDriver {
770
776
  const tokenMatch = xml.match(
771
777
  /<NextContinuationToken>([^<]+)<\/NextContinuationToken>/,
772
778
  );
773
- continuationToken = truncated && tokenMatch ? tokenMatch[1] : undefined;
779
+ continuationToken =
780
+ truncated && tokenMatch?.[1] !== undefined
781
+ ? xmlUnescape(tokenMatch[1])
782
+ : undefined;
774
783
  } while (continuationToken);
775
784
  }
776
785
 
@@ -833,7 +842,8 @@ export class S3Driver implements StorageDriver {
833
842
  .join("&");
834
843
  const canonicalRequest = [
835
844
  method,
836
- `/${this.#config.bucket}/${s3EncodeKey(key)}`,
845
+ // `key` is empty for a bucket-level request (list, batch delete).
846
+ `/${this.#config.bucket}/${key === "" ? "" : s3EncodeKey(key)}`,
837
847
  canonicalQuery,
838
848
  canonicalHeaders,
839
849
  signedHeaders,
@@ -890,7 +900,7 @@ export class S3Driver implements StorageDriver {
890
900
  `S3 multipart initiate: UploadId missing from response: ${xml}`,
891
901
  );
892
902
  }
893
- return match[1];
903
+ return xmlUnescape(match[1]);
894
904
  }
895
905
 
896
906
  /** Upload a single part. Returns the server-reported ETag (quotes kept). */
@@ -919,7 +929,7 @@ export class S3Driver implements StorageDriver {
919
929
  const res = await this.#fetch(url, {
920
930
  method: "PUT",
921
931
  headers,
922
- body: body as BodyInit,
932
+ body: requestBody(body),
923
933
  });
924
934
  if (!res.ok) {
925
935
  throw new Error(
@@ -961,7 +971,7 @@ export class S3Driver implements StorageDriver {
961
971
  const res = await this.#fetch(url, {
962
972
  method: "POST",
963
973
  headers: signed,
964
- body: xmlBody as BodyInit,
974
+ body: requestBody(xmlBody),
965
975
  });
966
976
  if (!res.ok) {
967
977
  throw new Error(
@@ -1032,6 +1042,31 @@ function unsignedMetadataHeaders(
1032
1042
  return headers;
1033
1043
  }
1034
1044
 
1045
+ /**
1046
+ * Decode the text of an XML element as S3 sends it: the five predefined
1047
+ * entities and numeric character references. Every text node read from a
1048
+ * response goes through here — a listed key `a&amp;b` names the object
1049
+ * `a&b`, and taking it verbatim misreported the key and made `deleteAll`
1050
+ * send `a&amp;amp;b`, leaving the real object behind without an error.
1051
+ */
1052
+ function xmlUnescape(text: string): string {
1053
+ return text.replace(
1054
+ /&(amp|lt|gt|quot|apos|#[0-9]+|#x[0-9a-f]+);/gi,
1055
+ (entity, name: string) => {
1056
+ const lower = name.toLowerCase();
1057
+ if (lower === "amp") return "&";
1058
+ if (lower === "lt") return "<";
1059
+ if (lower === "gt") return ">";
1060
+ if (lower === "quot") return '"';
1061
+ if (lower === "apos") return "'";
1062
+ const code = lower.startsWith("#x")
1063
+ ? Number.parseInt(lower.slice(2), 16)
1064
+ : Number.parseInt(lower.slice(1), 10);
1065
+ return code <= 0x10ffff ? String.fromCodePoint(code) : entity;
1066
+ },
1067
+ );
1068
+ }
1069
+
1035
1070
  /** Minimal XML text escaping for object keys inside a DeleteObjects body. */
1036
1071
  function xmlEscape(value: string): string {
1037
1072
  return value
@@ -1040,84 +1075,6 @@ function xmlEscape(value: string): string {
1040
1075
  .replace(/>/g, "&gt;");
1041
1076
  }
1042
1077
 
1043
- /**
1044
- * Drain a readable into a single Buffer, capped at `maxBytes`. Throws
1045
- * if the stream yields more than that — caller declared a contentLength
1046
- * and we refuse to silently over-buffer.
1047
- */
1048
- /**
1049
- * Type guard for any readable that exposes `destroy()` (Node streams
1050
- * do; a generic `ReadableStream` iterable doesn't). Lets us release
1051
- * file descriptors / sockets without an `as unknown as T` cast.
1052
- */
1053
- function hasDestroy(x: unknown): x is { destroy(err?: Error): void } {
1054
- return (
1055
- typeof x === "object" &&
1056
- x !== null &&
1057
- typeof (x as { destroy?: unknown }).destroy === "function"
1058
- );
1059
- }
1060
-
1061
- async function collectToBuffer(
1062
- readable: NodeJS.ReadableStream,
1063
- maxBytes: number,
1064
- ): Promise<Buffer> {
1065
- const chunks: Buffer[] = [];
1066
- let total = 0;
1067
- for await (const chunk of readable) {
1068
- const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as string);
1069
- total += buf.length;
1070
- if (total > maxBytes) {
1071
- // Destroy the upstream so FDs / sockets don't leak. Without
1072
- // this, an `fs.createReadStream` or `IncomingMessage` would
1073
- // stay open for the lifetime of the process.
1074
- if (hasDestroy(readable)) readable.destroy();
1075
- throw new Error(
1076
- `putStream: readable exceeded declared contentLength (${total} > ${maxBytes})`,
1077
- );
1078
- }
1079
- chunks.push(buf);
1080
- }
1081
- return Buffer.concat(chunks, total);
1082
- }
1083
-
1084
- /**
1085
- * Async-iterable that yields Buffers of exactly `partSize` bytes (last
1086
- * one may be smaller). Used to slice a stream into S3 multipart parts.
1087
- *
1088
- * **Backpressure.** Memory usage is bounded by `partSize + readable`'s
1089
- * own `highWaterMark`. The `for await` loop pulls from `readable` only
1090
- * when the consumer has finished awaiting the previous yield, so a slow
1091
- * consumer naturally throttles a fast producer through Node's stream
1092
- * pause/resume protocol — `pending` is drained to a single yield before
1093
- * the loop pulls the next chunk. Pass a `Readable` with a tighter
1094
- * `highWaterMark` to lower the peak. Pre-buffered streams (e.g.
1095
- * `Readable.from(buffer)`) bypass this and surface their entire payload
1096
- * in one tick; for those, expect peak ≈ `payload + partSize`.
1097
- */
1098
- async function* chunkStream(
1099
- readable: NodeJS.ReadableStream,
1100
- partSize: number,
1101
- ): AsyncIterable<Buffer> {
1102
- let pending: Buffer[] = [];
1103
- let pendingSize = 0;
1104
- for await (const chunk of readable) {
1105
- const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as string);
1106
- pending.push(buf);
1107
- pendingSize += buf.length;
1108
- while (pendingSize >= partSize) {
1109
- const joined = Buffer.concat(pending, pendingSize);
1110
- yield joined.subarray(0, partSize);
1111
- const leftover = joined.subarray(partSize);
1112
- pending = leftover.length > 0 ? [leftover] : [];
1113
- pendingSize = leftover.length;
1114
- }
1115
- }
1116
- if (pendingSize > 0) {
1117
- yield Buffer.concat(pending, pendingSize);
1118
- }
1119
- }
1120
-
1121
1078
  /**
1122
1079
  * Quick-and-narrow ACL XML classification: treat the object as public
1123
1080
  * iff the ACL grants `READ` to the `AllUsers` group. Anything else
@@ -13,6 +13,7 @@
13
13
  import * as fs from "node:fs";
14
14
  import * as fsp from "node:fs/promises";
15
15
  import * as path from "node:path";
16
+ import { Readable } from "node:stream";
16
17
  import { matchesIfNoneMatch, statTag } from "./etag.js";
17
18
  import { DEFAULT_EXTENSIONS, inferMimeType } from "./mime-types.js";
18
19
  import { VISIBILITY_SIDECAR_SUFFIX } from "./StorageManager.js";
@@ -26,7 +27,11 @@ import { VISIBILITY_SIDECAR_SUFFIX } from "./StorageManager.js";
26
27
  export interface StaticMiddlewareResponse {
27
28
  status(code: number): StaticMiddlewareResponse;
28
29
  header(name: string, value: string): StaticMiddlewareResponse;
29
- sendBuffer(buf: Buffer): void;
30
+ /**
31
+ * Send a readable as the body. The framework's response streams it to
32
+ * the client, so a request never holds the whole file in memory.
33
+ */
34
+ stream(body: NodeJS.ReadableStream): Promise<void> | void;
30
35
  }
31
36
 
32
37
  export interface StaticMiddlewareHttpContext {
@@ -64,6 +69,99 @@ export interface StaticConfig {
64
69
  extensions?: string[];
65
70
  }
66
71
 
72
+ /**
73
+ * Whether a file may be served unsigned, by its local-disk visibility marker.
74
+ *
75
+ * No marker is the local disk's default, public. A marker that says exactly
76
+ * `public` is public. Anything else — private, unreadable, malformed, or a
77
+ * value that is neither — is not served: an undetermined visibility must
78
+ * never be read as public.
79
+ */
80
+ async function declaredPublic(file: string): Promise<boolean> {
81
+ let raw: string;
82
+ try {
83
+ const handle = await fsp.open(
84
+ file + VISIBILITY_SIDECAR_SUFFIX,
85
+ fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0),
86
+ );
87
+ try {
88
+ raw = await handle.readFile("utf8");
89
+ } finally {
90
+ await handle.close();
91
+ }
92
+ } catch (err) {
93
+ return isErrnoCode(err, "ENOENT");
94
+ }
95
+ try {
96
+ const parsed: unknown = JSON.parse(raw);
97
+ return (
98
+ typeof parsed === "object" &&
99
+ parsed !== null &&
100
+ "visibility" in parsed &&
101
+ parsed.visibility === "public"
102
+ );
103
+ } catch {
104
+ return false;
105
+ }
106
+ }
107
+
108
+ /**
109
+ * Percent-decode a request path segment by segment, or `undefined` when it
110
+ * cannot be served: a malformed escape, or a segment that decodes to a
111
+ * separator, a control character or a dot-only name. Those only appear once
112
+ * decoded, so a check on the raw path would miss them — and a dot-only
113
+ * segment is refused even when it would stay inside the root, as a static
114
+ * file server refuses `..` outright.
115
+ */
116
+ function decodePath(raw: string): string | undefined {
117
+ const segments: string[] = [];
118
+ for (const segment of raw.split("/")) {
119
+ let decoded: string;
120
+ try {
121
+ decoded = decodeURIComponent(segment);
122
+ } catch {
123
+ return undefined;
124
+ }
125
+ if (!isPlainSegment(decoded)) return undefined;
126
+ if (/^\.{2,}$/.test(decoded)) return undefined;
127
+ segments.push(decoded);
128
+ }
129
+ return segments.join("/");
130
+ }
131
+
132
+ /** No separator and no control character in one decoded segment. */
133
+ function isPlainSegment(segment: string): boolean {
134
+ for (let i = 0; i < segment.length; i++) {
135
+ const code = segment.charCodeAt(i);
136
+ if (code < 0x20 || code === 0x7f) return false;
137
+ if (segment[i] === "/" || segment[i] === "\\") return false;
138
+ }
139
+ return true;
140
+ }
141
+
142
+ /** Whether `err` is a filesystem error carrying `code`. */
143
+ function isErrnoCode(err: unknown, code: string): boolean {
144
+ return (
145
+ typeof err === "object" &&
146
+ err !== null &&
147
+ "code" in err &&
148
+ err.code === code
149
+ );
150
+ }
151
+
152
+ /**
153
+ * The first `size` bytes of an open file, as a stream that owns the
154
+ * descriptor. An empty file gets an empty stream (a read range cannot end
155
+ * before it starts), and its descriptor is closed here.
156
+ */
157
+ function bodyOf(handle: fsp.FileHandle, size: number): NodeJS.ReadableStream {
158
+ if (size === 0) {
159
+ void handle.close().catch(() => {});
160
+ return Readable.from([]);
161
+ }
162
+ return handle.createReadStream({ start: 0, end: size - 1 });
163
+ }
164
+
67
165
  export class StaticMiddleware {
68
166
  #root: string;
69
167
  #prefix: string;
@@ -110,8 +208,15 @@ export class StaticMiddleware {
110
208
  return next();
111
209
  }
112
210
 
113
- // Strip prefix and resolve file path
114
- const relativePath = reqPath.slice(this.#prefix.length) || "/index.html";
211
+ // Strip the prefix and percent-decode. The framework hands over the raw
212
+ // request target, so `my%20logo.png` used to be looked up literally
213
+ // and every name with a space, `+`, `#` or a non-ASCII letter was
214
+ // unreachable. Every check below runs on the DECODED path, which is
215
+ // the one the filesystem sees.
216
+ const relativePath = decodePath(
217
+ reqPath.slice(this.#prefix.length) || "/index.html",
218
+ );
219
+ if (relativePath === undefined) return next();
115
220
 
116
221
  // Visibility sidecars are an implementation detail of LocalDriver
117
222
  // — never serve them, even if the static root happens to overlap
@@ -157,6 +262,11 @@ export class StaticMiddleware {
157
262
  return next();
158
263
  }
159
264
 
265
+ // The static root may overlap a local disk's root, where a file made
266
+ // private carries a visibility marker beside it. Serving it here,
267
+ // unsigned, would make `setVisibility('private')` protect nothing.
268
+ if (!(await declaredPublic(realPath))) return next();
269
+
160
270
  // Quoted and weak (RFC 9110 §8.8.3): the bare md5 hex this used to send
161
271
  // is not a valid entity-tag, and a strict cache may ignore it outright.
162
272
  const etag = statTag(stat);
@@ -194,22 +304,23 @@ export class StaticMiddleware {
194
304
  //
195
305
  // A parent directory swapped in the same window is not covered; that
196
306
  // needs openat-relative-to-a-descriptor, which Node does not expose.
197
- let content: Buffer;
307
+ let handle: fsp.FileHandle;
198
308
  try {
199
- const handle = await fsp.open(
309
+ handle = await fsp.open(
200
310
  realPath,
201
311
  fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW,
202
312
  );
203
- try {
204
- content = await handle.readFile();
205
- } finally {
206
- await handle.close();
207
- }
208
313
  } catch {
209
314
  return next();
210
315
  }
211
- // Send the raw Buffer — NOT content.toString() which would destroy binary
212
- // data (PNG, PDF, ZIP, etc.) by interpreting it as UTF-8.
213
- ctx.response.status(200).sendBuffer(content);
316
+ // Streamed from the descriptor, never read whole: a buffered read held
317
+ // one full copy of the file per request until the response flushed, and
318
+ // a file past Node's 2 GiB read cap could not be served at all. The
319
+ // stream owns the descriptor and closes it when it ends, fails or is
320
+ // abandoned by a client that left. It stops at the size the
321
+ // Content-Length announced, so a file that grows meanwhile cannot
322
+ // overrun it.
323
+ ctx.response.status(200);
324
+ await ctx.response.stream(bodyOf(handle, stat.size));
214
325
  }
215
326
  }