@c9up/archive 0.1.20 → 0.1.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +96 -3
- package/dist/ArchiveProvider.d.ts +6 -1
- package/dist/ArchiveProvider.d.ts.map +1 -1
- package/dist/ArchiveProvider.js +73 -1
- package/dist/ArchiveProvider.js.map +1 -1
- package/dist/DriveManager.d.ts +10 -3
- package/dist/DriveManager.d.ts.map +1 -1
- package/dist/DriveManager.js +23 -6
- package/dist/DriveManager.js.map +1 -1
- package/dist/GcsDriver.d.ts +14 -9
- package/dist/GcsDriver.d.ts.map +1 -1
- package/dist/GcsDriver.js +202 -98
- package/dist/GcsDriver.js.map +1 -1
- package/dist/S3Driver.d.ts +11 -7
- package/dist/S3Driver.d.ts.map +1 -1
- package/dist/S3Driver.js +73 -110
- package/dist/S3Driver.js.map +1 -1
- package/dist/StaticMiddleware.d.ts +5 -1
- package/dist/StaticMiddleware.d.ts.map +1 -1
- package/dist/StaticMiddleware.js +114 -13
- package/dist/StaticMiddleware.js.map +1 -1
- package/dist/StorageManager.d.ts +28 -3
- package/dist/StorageManager.d.ts.map +1 -1
- package/dist/StorageManager.js +333 -132
- package/dist/StorageManager.js.map +1 -1
- package/dist/augmentations.d.ts +17 -0
- package/dist/augmentations.d.ts.map +1 -1
- package/dist/errno.d.ts +6 -0
- package/dist/errno.d.ts.map +1 -0
- package/dist/errno.js +11 -0
- package/dist/errno.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/keys.d.ts +34 -0
- package/dist/keys.d.ts.map +1 -0
- package/dist/keys.js +88 -0
- package/dist/keys.js.map +1 -0
- package/dist/local-signature.d.ts +32 -0
- package/dist/local-signature.d.ts.map +1 -0
- package/dist/local-signature.js +58 -0
- package/dist/local-signature.js.map +1 -0
- package/dist/move-to-disk.d.ts +24 -0
- package/dist/move-to-disk.d.ts.map +1 -0
- package/dist/move-to-disk.js +167 -0
- package/dist/move-to-disk.js.map +1 -0
- package/dist/objects.d.ts.map +1 -1
- package/dist/objects.js +5 -2
- package/dist/objects.js.map +1 -1
- package/dist/provider-fetch.d.ts +36 -0
- package/dist/provider-fetch.d.ts.map +1 -0
- package/dist/provider-fetch.js +158 -0
- package/dist/provider-fetch.js.map +1 -0
- package/dist/services/main.d.ts +1 -1
- package/dist/services/main.d.ts.map +1 -1
- package/dist/services/main.js +14 -1
- package/dist/services/main.js.map +1 -1
- package/dist/signed-route.d.ts +18 -6
- package/dist/signed-route.d.ts.map +1 -1
- package/dist/signed-route.js +186 -31
- package/dist/signed-route.js.map +1 -1
- package/dist/streams.d.ts +36 -0
- package/dist/streams.d.ts.map +1 -0
- package/dist/streams.js +108 -0
- package/dist/streams.js.map +1 -0
- package/dist/testing/FakeStorage.d.ts.map +1 -1
- package/dist/testing/FakeStorage.js +2 -1
- package/dist/testing/FakeStorage.js.map +1 -1
- package/package.json +7 -3
- package/src/ArchiveProvider.ts +105 -4
- package/src/DriveManager.ts +32 -6
- package/src/GcsDriver.ts +256 -139
- package/src/S3Driver.ts +87 -130
- package/src/StaticMiddleware.ts +124 -13
- package/src/StorageManager.ts +398 -141
- package/src/augmentations.ts +24 -1
- package/src/errno.ts +10 -0
- package/src/index.ts +1 -0
- package/src/keys.ts +104 -0
- package/src/local-signature.ts +76 -0
- package/src/move-to-disk.ts +234 -0
- package/src/objects.ts +5 -2
- package/src/provider-fetch.ts +171 -0
- package/src/services/main.ts +15 -2
- package/src/signed-route.ts +216 -36
- package/src/streams.ts +118 -0
- package/src/testing/FakeStorage.ts +2 -3
package/src/S3Driver.ts
CHANGED
|
@@ -13,14 +13,15 @@
|
|
|
13
13
|
|
|
14
14
|
import { createHash, createHmac } from "node:crypto";
|
|
15
15
|
import { Readable } from "node:stream";
|
|
16
|
-
import type { ReadableStream as NodeReadableStream } from "node:stream/web";
|
|
17
16
|
import { ArchiveError } from "./errors.js";
|
|
18
17
|
import {
|
|
19
18
|
assertValidExpiry,
|
|
20
19
|
DEFAULT_EXPIRES_IN,
|
|
21
20
|
parseExpiry,
|
|
22
21
|
} from "./expiry.js";
|
|
22
|
+
import { normalizeKey, normalizePrefix } from "./keys.js";
|
|
23
23
|
import { DriveDirectory, DriveFile } from "./objects.js";
|
|
24
|
+
import { fetchWithIdleTimeout, requestBody } from "./provider-fetch.js";
|
|
24
25
|
import type {
|
|
25
26
|
ListAllOptions,
|
|
26
27
|
ListAllResult,
|
|
@@ -32,6 +33,7 @@ import type {
|
|
|
32
33
|
Visibility,
|
|
33
34
|
WriteOptions,
|
|
34
35
|
} from "./StorageManager.js";
|
|
36
|
+
import { chunkStream, collectToBuffer, nodeReadable } from "./streams.js";
|
|
35
37
|
|
|
36
38
|
/** AWS minimum for a non-final multipart part. */
|
|
37
39
|
const S3_MULTIPART_PART_SIZE = 5 * 1024 * 1024;
|
|
@@ -54,9 +56,14 @@ function s3StrictEncode(str: string): string {
|
|
|
54
56
|
* URI-encode an S3 object key per AWS SigV4 rules: encode each path segment
|
|
55
57
|
* individually but preserve `/` separators. Handles spaces, unicode, `#`, `?`, `%`
|
|
56
58
|
* as well as the strict AWS set (`! * ' ( )`).
|
|
59
|
+
*
|
|
60
|
+
* Every object URL is built through here, so this is where a key is
|
|
61
|
+
* normalised (see `keys.ts`) when the driver is used without a manager in
|
|
62
|
+
* front: `..` never reaches a URL, where it would resolve into another
|
|
63
|
+
* bucket on the same host.
|
|
57
64
|
*/
|
|
58
65
|
function s3EncodeKey(key: string): string {
|
|
59
|
-
return key.split("/").map(s3StrictEncode).join("/");
|
|
66
|
+
return normalizeKey(key).split("/").map(s3StrictEncode).join("/");
|
|
60
67
|
}
|
|
61
68
|
|
|
62
69
|
export interface S3Config {
|
|
@@ -67,39 +74,32 @@ export interface S3Config {
|
|
|
67
74
|
secretAccessKey: string;
|
|
68
75
|
publicUrl?: string;
|
|
69
76
|
/**
|
|
70
|
-
* How long
|
|
77
|
+
* How long the provider may stay silent during one request, in ms.
|
|
78
|
+
* Default 30 000.
|
|
71
79
|
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
*
|
|
80
|
+
* An idle bound, not a deadline: it runs while the provider is awaited
|
|
81
|
+
* (taking the request body, sending headers, sending the next chunk), so a
|
|
82
|
+
* long transfer that keeps moving is never cut. Without a bound a hung
|
|
83
|
+
* connection never settles: the handler awaiting it waits forever, and
|
|
84
|
+
* enough of them stop the server from serving. Set 0 to disable, which
|
|
85
|
+
* only makes sense behind an outer timeout.
|
|
75
86
|
*/
|
|
76
87
|
requestTimeoutMs?: number;
|
|
77
88
|
}
|
|
78
89
|
|
|
79
90
|
export class S3Driver implements StorageDriver {
|
|
80
91
|
/**
|
|
81
|
-
* One request to the provider, bounded by {@link S3Config.requestTimeoutMs}
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
92
|
+
* One request to the provider, bounded by {@link S3Config.requestTimeoutMs}
|
|
93
|
+
* as an idle timeout (see `provider-fetch.ts`): a hung connection surfaces
|
|
94
|
+
* as an error the caller can retry, a long transfer that keeps moving does
|
|
95
|
+
* not.
|
|
85
96
|
*/
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
if (err instanceof Error && err.name === "TimeoutError") {
|
|
93
|
-
throw new ArchiveError(
|
|
94
|
-
"E_ARCHIVE_REQUEST_TIMEOUT",
|
|
95
|
-
`The storage provider did not answer within ${ms}ms.`,
|
|
96
|
-
{
|
|
97
|
-
hint: "Raise `requestTimeoutMs` for large objects, or check connectivity to the bucket.",
|
|
98
|
-
},
|
|
99
|
-
);
|
|
100
|
-
}
|
|
101
|
-
throw err;
|
|
102
|
-
}
|
|
97
|
+
#fetch(url: string, init: RequestInit = {}): Promise<Response> {
|
|
98
|
+
return fetchWithIdleTimeout(
|
|
99
|
+
url,
|
|
100
|
+
init,
|
|
101
|
+
this.#config.requestTimeoutMs ?? 30_000,
|
|
102
|
+
);
|
|
103
103
|
}
|
|
104
104
|
|
|
105
105
|
#config: S3Config;
|
|
@@ -154,7 +154,7 @@ export class S3Driver implements StorageDriver {
|
|
|
154
154
|
const res = await this.#fetch(url, {
|
|
155
155
|
method: "PUT",
|
|
156
156
|
headers,
|
|
157
|
-
body: body
|
|
157
|
+
body: requestBody(body),
|
|
158
158
|
});
|
|
159
159
|
if (!res.ok) {
|
|
160
160
|
throw new Error(`S3 PUT failed (${res.status}): ${await res.text()}`);
|
|
@@ -183,7 +183,7 @@ export class S3Driver implements StorageDriver {
|
|
|
183
183
|
const res = await this.#fetch(url, {
|
|
184
184
|
method: "PUT",
|
|
185
185
|
headers,
|
|
186
|
-
body: body
|
|
186
|
+
body: requestBody(body),
|
|
187
187
|
});
|
|
188
188
|
if (!res.ok) {
|
|
189
189
|
throw new Error(`S3 PUT failed (${res.status}): ${await res.text()}`);
|
|
@@ -248,11 +248,7 @@ export class S3Driver implements StorageDriver {
|
|
|
248
248
|
// Edge case: 200 with no body. Return an empty readable.
|
|
249
249
|
return Readable.from(Buffer.alloc(0));
|
|
250
250
|
}
|
|
251
|
-
|
|
252
|
-
// while `this.#fetch().body` gives the lib.dom variant. Runtime shape is
|
|
253
|
-
// identical — the difference is only in TS lib typings. A single
|
|
254
|
-
// cast to the node-side type is enough (no `unknown` pivot).
|
|
255
|
-
return Readable.fromWeb(res.body as NodeReadableStream<Uint8Array>);
|
|
251
|
+
return nodeReadable(res.body);
|
|
256
252
|
}
|
|
257
253
|
|
|
258
254
|
async get(filePath: string): Promise<Buffer | null> {
|
|
@@ -570,7 +566,8 @@ export class S3Driver implements StorageDriver {
|
|
|
570
566
|
await this.delete(from);
|
|
571
567
|
}
|
|
572
568
|
|
|
573
|
-
async deleteAll(
|
|
569
|
+
async deleteAll(given: string): Promise<void> {
|
|
570
|
+
const prefix = normalizePrefix(given);
|
|
574
571
|
// List one page at a time and batch-delete (up to 1000 keys per
|
|
575
572
|
// DeleteObjects call) until the bucket is drained of the prefix.
|
|
576
573
|
let continuationToken: string | undefined;
|
|
@@ -606,7 +603,7 @@ export class S3Driver implements StorageDriver {
|
|
|
606
603
|
/<Contents>[\s\S]*?<Key>([^<]+)<\/Key>[\s\S]*?<\/Contents>/g;
|
|
607
604
|
let m: RegExpExecArray | null = keyRe.exec(xml);
|
|
608
605
|
while (m !== null) {
|
|
609
|
-
if (m[1] !== undefined) keys.push(m[1]);
|
|
606
|
+
if (m[1] !== undefined) keys.push(xmlUnescape(m[1]));
|
|
610
607
|
m = keyRe.exec(xml);
|
|
611
608
|
}
|
|
612
609
|
if (keys.length > 0) await this.#deleteObjects(keys);
|
|
@@ -614,7 +611,10 @@ export class S3Driver implements StorageDriver {
|
|
|
614
611
|
const tokenMatch = xml.match(
|
|
615
612
|
/<NextContinuationToken>([^<]+)<\/NextContinuationToken>/,
|
|
616
613
|
);
|
|
617
|
-
continuationToken =
|
|
614
|
+
continuationToken =
|
|
615
|
+
truncated && tokenMatch?.[1] !== undefined
|
|
616
|
+
? xmlUnescape(tokenMatch[1])
|
|
617
|
+
: undefined;
|
|
618
618
|
} while (continuationToken);
|
|
619
619
|
}
|
|
620
620
|
|
|
@@ -637,7 +637,7 @@ export class S3Driver implements StorageDriver {
|
|
|
637
637
|
const res = await this.#fetch(url, {
|
|
638
638
|
method: "POST",
|
|
639
639
|
headers,
|
|
640
|
-
body: body
|
|
640
|
+
body: requestBody(body),
|
|
641
641
|
});
|
|
642
642
|
if (!res.ok) {
|
|
643
643
|
throw new Error(
|
|
@@ -647,12 +647,13 @@ export class S3Driver implements StorageDriver {
|
|
|
647
647
|
}
|
|
648
648
|
|
|
649
649
|
async listAll(
|
|
650
|
-
|
|
650
|
+
given: string,
|
|
651
651
|
options?: ListAllOptions,
|
|
652
652
|
): Promise<ListAllResult> {
|
|
653
653
|
const recursive = options?.recursive ?? false;
|
|
654
|
+
const prefix = normalizePrefix(given);
|
|
654
655
|
const normalizedPrefix =
|
|
655
|
-
prefix === ""
|
|
656
|
+
prefix === ""
|
|
656
657
|
? ""
|
|
657
658
|
: !recursive && !prefix.endsWith("/")
|
|
658
659
|
? `${prefix}/`
|
|
@@ -688,7 +689,7 @@ export class S3Driver implements StorageDriver {
|
|
|
688
689
|
let pm: RegExpExecArray | null = prefixRe.exec(xml);
|
|
689
690
|
while (pm !== null) {
|
|
690
691
|
if (pm[1] !== undefined) {
|
|
691
|
-
objects.push(new DriveDirectory(pm[1].replace(/\/$/, "")));
|
|
692
|
+
objects.push(new DriveDirectory(xmlUnescape(pm[1]).replace(/\/$/, "")));
|
|
692
693
|
}
|
|
693
694
|
pm = prefixRe.exec(xml);
|
|
694
695
|
}
|
|
@@ -696,7 +697,7 @@ export class S3Driver implements StorageDriver {
|
|
|
696
697
|
let bm: RegExpExecArray | null = blockRe.exec(xml);
|
|
697
698
|
while (bm !== null) {
|
|
698
699
|
const block = bm[1] ?? "";
|
|
699
|
-
const key = block.match(/<Key>([^<]+)<\/Key>/)?.[1] ?? "";
|
|
700
|
+
const key = xmlUnescape(block.match(/<Key>([^<]+)<\/Key>/)?.[1] ?? "");
|
|
700
701
|
const sizeRaw = block.match(/<Size>([^<]+)<\/Size>/)?.[1] ?? "0";
|
|
701
702
|
const lm = block.match(/<LastModified>([^<]+)<\/LastModified>/)?.[1];
|
|
702
703
|
const size = Number.parseInt(sizeRaw, 10);
|
|
@@ -707,8 +708,9 @@ export class S3Driver implements StorageDriver {
|
|
|
707
708
|
mimeType: "application/octet-stream",
|
|
708
709
|
contentType: "application/octet-stream",
|
|
709
710
|
lastModified: lm ? new Date(lm) : new Date(0),
|
|
710
|
-
etag:
|
|
711
|
-
block.match(/<ETag>([^<]+)<\/ETag>/)?.[1]
|
|
711
|
+
etag: xmlUnescape(
|
|
712
|
+
block.match(/<ETag>([^<]+)<\/ETag>/)?.[1] ?? "",
|
|
713
|
+
).replace(/"/g, ""),
|
|
712
714
|
visibility: "private",
|
|
713
715
|
}),
|
|
714
716
|
);
|
|
@@ -719,12 +721,16 @@ export class S3Driver implements StorageDriver {
|
|
|
719
721
|
/<NextContinuationToken>([^<]+)<\/NextContinuationToken>/,
|
|
720
722
|
);
|
|
721
723
|
return {
|
|
722
|
-
paginationToken:
|
|
724
|
+
paginationToken:
|
|
725
|
+
truncated && tokenMatch?.[1] !== undefined
|
|
726
|
+
? xmlUnescape(tokenMatch[1])
|
|
727
|
+
: undefined,
|
|
723
728
|
objects,
|
|
724
729
|
};
|
|
725
730
|
}
|
|
726
731
|
|
|
727
|
-
async *list(
|
|
732
|
+
async *list(given: string): AsyncIterable<StorageEntry> {
|
|
733
|
+
const prefix = normalizePrefix(given);
|
|
728
734
|
let continuationToken: string | undefined;
|
|
729
735
|
do {
|
|
730
736
|
const qp: Array<[string, string]> = [
|
|
@@ -756,7 +762,7 @@ export class S3Driver implements StorageDriver {
|
|
|
756
762
|
let blockMatch: RegExpExecArray | null = blockRe.exec(xml);
|
|
757
763
|
while (blockMatch !== null) {
|
|
758
764
|
const block = blockMatch[1] ?? "";
|
|
759
|
-
const key = block.match(/<Key>([^<]+)<\/Key>/)?.[1] ?? "";
|
|
765
|
+
const key = xmlUnescape(block.match(/<Key>([^<]+)<\/Key>/)?.[1] ?? "");
|
|
760
766
|
const sizeRaw = block.match(/<Size>([^<]+)<\/Size>/)?.[1] ?? "0";
|
|
761
767
|
const lm = block.match(/<LastModified>([^<]+)<\/LastModified>/)?.[1];
|
|
762
768
|
yield {
|
|
@@ -770,7 +776,10 @@ export class S3Driver implements StorageDriver {
|
|
|
770
776
|
const tokenMatch = xml.match(
|
|
771
777
|
/<NextContinuationToken>([^<]+)<\/NextContinuationToken>/,
|
|
772
778
|
);
|
|
773
|
-
continuationToken =
|
|
779
|
+
continuationToken =
|
|
780
|
+
truncated && tokenMatch?.[1] !== undefined
|
|
781
|
+
? xmlUnescape(tokenMatch[1])
|
|
782
|
+
: undefined;
|
|
774
783
|
} while (continuationToken);
|
|
775
784
|
}
|
|
776
785
|
|
|
@@ -833,7 +842,8 @@ export class S3Driver implements StorageDriver {
|
|
|
833
842
|
.join("&");
|
|
834
843
|
const canonicalRequest = [
|
|
835
844
|
method,
|
|
836
|
-
|
|
845
|
+
// `key` is empty for a bucket-level request (list, batch delete).
|
|
846
|
+
`/${this.#config.bucket}/${key === "" ? "" : s3EncodeKey(key)}`,
|
|
837
847
|
canonicalQuery,
|
|
838
848
|
canonicalHeaders,
|
|
839
849
|
signedHeaders,
|
|
@@ -890,7 +900,7 @@ export class S3Driver implements StorageDriver {
|
|
|
890
900
|
`S3 multipart initiate: UploadId missing from response: ${xml}`,
|
|
891
901
|
);
|
|
892
902
|
}
|
|
893
|
-
return match[1];
|
|
903
|
+
return xmlUnescape(match[1]);
|
|
894
904
|
}
|
|
895
905
|
|
|
896
906
|
/** Upload a single part. Returns the server-reported ETag (quotes kept). */
|
|
@@ -919,7 +929,7 @@ export class S3Driver implements StorageDriver {
|
|
|
919
929
|
const res = await this.#fetch(url, {
|
|
920
930
|
method: "PUT",
|
|
921
931
|
headers,
|
|
922
|
-
body: body
|
|
932
|
+
body: requestBody(body),
|
|
923
933
|
});
|
|
924
934
|
if (!res.ok) {
|
|
925
935
|
throw new Error(
|
|
@@ -961,7 +971,7 @@ export class S3Driver implements StorageDriver {
|
|
|
961
971
|
const res = await this.#fetch(url, {
|
|
962
972
|
method: "POST",
|
|
963
973
|
headers: signed,
|
|
964
|
-
body: xmlBody
|
|
974
|
+
body: requestBody(xmlBody),
|
|
965
975
|
});
|
|
966
976
|
if (!res.ok) {
|
|
967
977
|
throw new Error(
|
|
@@ -1032,6 +1042,31 @@ function unsignedMetadataHeaders(
|
|
|
1032
1042
|
return headers;
|
|
1033
1043
|
}
|
|
1034
1044
|
|
|
1045
|
+
/**
|
|
1046
|
+
* Decode the text of an XML element as S3 sends it: the five predefined
|
|
1047
|
+
* entities and numeric character references. Every text node read from a
|
|
1048
|
+
* response goes through here — a listed key `a&b` names the object
|
|
1049
|
+
* `a&b`, and taking it verbatim misreported the key and made `deleteAll`
|
|
1050
|
+
* send `a&amp;b`, leaving the real object behind without an error.
|
|
1051
|
+
*/
|
|
1052
|
+
function xmlUnescape(text: string): string {
|
|
1053
|
+
return text.replace(
|
|
1054
|
+
/&(amp|lt|gt|quot|apos|#[0-9]+|#x[0-9a-f]+);/gi,
|
|
1055
|
+
(entity, name: string) => {
|
|
1056
|
+
const lower = name.toLowerCase();
|
|
1057
|
+
if (lower === "amp") return "&";
|
|
1058
|
+
if (lower === "lt") return "<";
|
|
1059
|
+
if (lower === "gt") return ">";
|
|
1060
|
+
if (lower === "quot") return '"';
|
|
1061
|
+
if (lower === "apos") return "'";
|
|
1062
|
+
const code = lower.startsWith("#x")
|
|
1063
|
+
? Number.parseInt(lower.slice(2), 16)
|
|
1064
|
+
: Number.parseInt(lower.slice(1), 10);
|
|
1065
|
+
return code <= 0x10ffff ? String.fromCodePoint(code) : entity;
|
|
1066
|
+
},
|
|
1067
|
+
);
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1035
1070
|
/** Minimal XML text escaping for object keys inside a DeleteObjects body. */
|
|
1036
1071
|
function xmlEscape(value: string): string {
|
|
1037
1072
|
return value
|
|
@@ -1040,84 +1075,6 @@ function xmlEscape(value: string): string {
|
|
|
1040
1075
|
.replace(/>/g, ">");
|
|
1041
1076
|
}
|
|
1042
1077
|
|
|
1043
|
-
/**
|
|
1044
|
-
* Drain a readable into a single Buffer, capped at `maxBytes`. Throws
|
|
1045
|
-
* if the stream yields more than that — caller declared a contentLength
|
|
1046
|
-
* and we refuse to silently over-buffer.
|
|
1047
|
-
*/
|
|
1048
|
-
/**
|
|
1049
|
-
* Type guard for any readable that exposes `destroy()` (Node streams
|
|
1050
|
-
* do; a generic `ReadableStream` iterable doesn't). Lets us release
|
|
1051
|
-
* file descriptors / sockets without an `as unknown as T` cast.
|
|
1052
|
-
*/
|
|
1053
|
-
function hasDestroy(x: unknown): x is { destroy(err?: Error): void } {
|
|
1054
|
-
return (
|
|
1055
|
-
typeof x === "object" &&
|
|
1056
|
-
x !== null &&
|
|
1057
|
-
typeof (x as { destroy?: unknown }).destroy === "function"
|
|
1058
|
-
);
|
|
1059
|
-
}
|
|
1060
|
-
|
|
1061
|
-
async function collectToBuffer(
|
|
1062
|
-
readable: NodeJS.ReadableStream,
|
|
1063
|
-
maxBytes: number,
|
|
1064
|
-
): Promise<Buffer> {
|
|
1065
|
-
const chunks: Buffer[] = [];
|
|
1066
|
-
let total = 0;
|
|
1067
|
-
for await (const chunk of readable) {
|
|
1068
|
-
const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as string);
|
|
1069
|
-
total += buf.length;
|
|
1070
|
-
if (total > maxBytes) {
|
|
1071
|
-
// Destroy the upstream so FDs / sockets don't leak. Without
|
|
1072
|
-
// this, an `fs.createReadStream` or `IncomingMessage` would
|
|
1073
|
-
// stay open for the lifetime of the process.
|
|
1074
|
-
if (hasDestroy(readable)) readable.destroy();
|
|
1075
|
-
throw new Error(
|
|
1076
|
-
`putStream: readable exceeded declared contentLength (${total} > ${maxBytes})`,
|
|
1077
|
-
);
|
|
1078
|
-
}
|
|
1079
|
-
chunks.push(buf);
|
|
1080
|
-
}
|
|
1081
|
-
return Buffer.concat(chunks, total);
|
|
1082
|
-
}
|
|
1083
|
-
|
|
1084
|
-
/**
|
|
1085
|
-
* Async-iterable that yields Buffers of exactly `partSize` bytes (last
|
|
1086
|
-
* one may be smaller). Used to slice a stream into S3 multipart parts.
|
|
1087
|
-
*
|
|
1088
|
-
* **Backpressure.** Memory usage is bounded by `partSize + readable`'s
|
|
1089
|
-
* own `highWaterMark`. The `for await` loop pulls from `readable` only
|
|
1090
|
-
* when the consumer has finished awaiting the previous yield, so a slow
|
|
1091
|
-
* consumer naturally throttles a fast producer through Node's stream
|
|
1092
|
-
* pause/resume protocol — `pending` is drained to a single yield before
|
|
1093
|
-
* the loop pulls the next chunk. Pass a `Readable` with a tighter
|
|
1094
|
-
* `highWaterMark` to lower the peak. Pre-buffered streams (e.g.
|
|
1095
|
-
* `Readable.from(buffer)`) bypass this and surface their entire payload
|
|
1096
|
-
* in one tick; for those, expect peak ≈ `payload + partSize`.
|
|
1097
|
-
*/
|
|
1098
|
-
async function* chunkStream(
|
|
1099
|
-
readable: NodeJS.ReadableStream,
|
|
1100
|
-
partSize: number,
|
|
1101
|
-
): AsyncIterable<Buffer> {
|
|
1102
|
-
let pending: Buffer[] = [];
|
|
1103
|
-
let pendingSize = 0;
|
|
1104
|
-
for await (const chunk of readable) {
|
|
1105
|
-
const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as string);
|
|
1106
|
-
pending.push(buf);
|
|
1107
|
-
pendingSize += buf.length;
|
|
1108
|
-
while (pendingSize >= partSize) {
|
|
1109
|
-
const joined = Buffer.concat(pending, pendingSize);
|
|
1110
|
-
yield joined.subarray(0, partSize);
|
|
1111
|
-
const leftover = joined.subarray(partSize);
|
|
1112
|
-
pending = leftover.length > 0 ? [leftover] : [];
|
|
1113
|
-
pendingSize = leftover.length;
|
|
1114
|
-
}
|
|
1115
|
-
}
|
|
1116
|
-
if (pendingSize > 0) {
|
|
1117
|
-
yield Buffer.concat(pending, pendingSize);
|
|
1118
|
-
}
|
|
1119
|
-
}
|
|
1120
|
-
|
|
1121
1078
|
/**
|
|
1122
1079
|
* Quick-and-narrow ACL XML classification: treat the object as public
|
|
1123
1080
|
* iff the ACL grants `READ` to the `AllUsers` group. Anything else
|
package/src/StaticMiddleware.ts
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
import * as fs from "node:fs";
|
|
14
14
|
import * as fsp from "node:fs/promises";
|
|
15
15
|
import * as path from "node:path";
|
|
16
|
+
import { Readable } from "node:stream";
|
|
16
17
|
import { matchesIfNoneMatch, statTag } from "./etag.js";
|
|
17
18
|
import { DEFAULT_EXTENSIONS, inferMimeType } from "./mime-types.js";
|
|
18
19
|
import { VISIBILITY_SIDECAR_SUFFIX } from "./StorageManager.js";
|
|
@@ -26,7 +27,11 @@ import { VISIBILITY_SIDECAR_SUFFIX } from "./StorageManager.js";
|
|
|
26
27
|
export interface StaticMiddlewareResponse {
|
|
27
28
|
status(code: number): StaticMiddlewareResponse;
|
|
28
29
|
header(name: string, value: string): StaticMiddlewareResponse;
|
|
29
|
-
|
|
30
|
+
/**
|
|
31
|
+
* Send a readable as the body. The framework's response streams it to
|
|
32
|
+
* the client, so a request never holds the whole file in memory.
|
|
33
|
+
*/
|
|
34
|
+
stream(body: NodeJS.ReadableStream): Promise<void> | void;
|
|
30
35
|
}
|
|
31
36
|
|
|
32
37
|
export interface StaticMiddlewareHttpContext {
|
|
@@ -64,6 +69,99 @@ export interface StaticConfig {
|
|
|
64
69
|
extensions?: string[];
|
|
65
70
|
}
|
|
66
71
|
|
|
72
|
+
/**
|
|
73
|
+
* Whether a file may be served unsigned, by its local-disk visibility marker.
|
|
74
|
+
*
|
|
75
|
+
* No marker is the local disk's default, public. A marker that says exactly
|
|
76
|
+
* `public` is public. Anything else — private, unreadable, malformed, or a
|
|
77
|
+
* value that is neither — is not served: an undetermined visibility must
|
|
78
|
+
* never be read as public.
|
|
79
|
+
*/
|
|
80
|
+
async function declaredPublic(file: string): Promise<boolean> {
|
|
81
|
+
let raw: string;
|
|
82
|
+
try {
|
|
83
|
+
const handle = await fsp.open(
|
|
84
|
+
file + VISIBILITY_SIDECAR_SUFFIX,
|
|
85
|
+
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0),
|
|
86
|
+
);
|
|
87
|
+
try {
|
|
88
|
+
raw = await handle.readFile("utf8");
|
|
89
|
+
} finally {
|
|
90
|
+
await handle.close();
|
|
91
|
+
}
|
|
92
|
+
} catch (err) {
|
|
93
|
+
return isErrnoCode(err, "ENOENT");
|
|
94
|
+
}
|
|
95
|
+
try {
|
|
96
|
+
const parsed: unknown = JSON.parse(raw);
|
|
97
|
+
return (
|
|
98
|
+
typeof parsed === "object" &&
|
|
99
|
+
parsed !== null &&
|
|
100
|
+
"visibility" in parsed &&
|
|
101
|
+
parsed.visibility === "public"
|
|
102
|
+
);
|
|
103
|
+
} catch {
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Percent-decode a request path segment by segment, or `undefined` when it
|
|
110
|
+
* cannot be served: a malformed escape, or a segment that decodes to a
|
|
111
|
+
* separator, a control character or a dot-only name. Those only appear once
|
|
112
|
+
* decoded, so a check on the raw path would miss them — and a dot-only
|
|
113
|
+
* segment is refused even when it would stay inside the root, as a static
|
|
114
|
+
* file server refuses `..` outright.
|
|
115
|
+
*/
|
|
116
|
+
function decodePath(raw: string): string | undefined {
|
|
117
|
+
const segments: string[] = [];
|
|
118
|
+
for (const segment of raw.split("/")) {
|
|
119
|
+
let decoded: string;
|
|
120
|
+
try {
|
|
121
|
+
decoded = decodeURIComponent(segment);
|
|
122
|
+
} catch {
|
|
123
|
+
return undefined;
|
|
124
|
+
}
|
|
125
|
+
if (!isPlainSegment(decoded)) return undefined;
|
|
126
|
+
if (/^\.{2,}$/.test(decoded)) return undefined;
|
|
127
|
+
segments.push(decoded);
|
|
128
|
+
}
|
|
129
|
+
return segments.join("/");
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** No separator and no control character in one decoded segment. */
|
|
133
|
+
function isPlainSegment(segment: string): boolean {
|
|
134
|
+
for (let i = 0; i < segment.length; i++) {
|
|
135
|
+
const code = segment.charCodeAt(i);
|
|
136
|
+
if (code < 0x20 || code === 0x7f) return false;
|
|
137
|
+
if (segment[i] === "/" || segment[i] === "\\") return false;
|
|
138
|
+
}
|
|
139
|
+
return true;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** Whether `err` is a filesystem error carrying `code`. */
|
|
143
|
+
function isErrnoCode(err: unknown, code: string): boolean {
|
|
144
|
+
return (
|
|
145
|
+
typeof err === "object" &&
|
|
146
|
+
err !== null &&
|
|
147
|
+
"code" in err &&
|
|
148
|
+
err.code === code
|
|
149
|
+
);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* The first `size` bytes of an open file, as a stream that owns the
|
|
154
|
+
* descriptor. An empty file gets an empty stream (a read range cannot end
|
|
155
|
+
* before it starts), and its descriptor is closed here.
|
|
156
|
+
*/
|
|
157
|
+
function bodyOf(handle: fsp.FileHandle, size: number): NodeJS.ReadableStream {
|
|
158
|
+
if (size === 0) {
|
|
159
|
+
void handle.close().catch(() => {});
|
|
160
|
+
return Readable.from([]);
|
|
161
|
+
}
|
|
162
|
+
return handle.createReadStream({ start: 0, end: size - 1 });
|
|
163
|
+
}
|
|
164
|
+
|
|
67
165
|
export class StaticMiddleware {
|
|
68
166
|
#root: string;
|
|
69
167
|
#prefix: string;
|
|
@@ -110,8 +208,15 @@ export class StaticMiddleware {
|
|
|
110
208
|
return next();
|
|
111
209
|
}
|
|
112
210
|
|
|
113
|
-
// Strip prefix and
|
|
114
|
-
|
|
211
|
+
// Strip the prefix and percent-decode. The framework hands over the raw
|
|
212
|
+
// request target, so `my%20logo.png` used to be looked up literally
|
|
213
|
+
// and every name with a space, `+`, `#` or a non-ASCII letter was
|
|
214
|
+
// unreachable. Every check below runs on the DECODED path, which is
|
|
215
|
+
// the one the filesystem sees.
|
|
216
|
+
const relativePath = decodePath(
|
|
217
|
+
reqPath.slice(this.#prefix.length) || "/index.html",
|
|
218
|
+
);
|
|
219
|
+
if (relativePath === undefined) return next();
|
|
115
220
|
|
|
116
221
|
// Visibility sidecars are an implementation detail of LocalDriver
|
|
117
222
|
// — never serve them, even if the static root happens to overlap
|
|
@@ -157,6 +262,11 @@ export class StaticMiddleware {
|
|
|
157
262
|
return next();
|
|
158
263
|
}
|
|
159
264
|
|
|
265
|
+
// The static root may overlap a local disk's root, where a file made
|
|
266
|
+
// private carries a visibility marker beside it. Serving it here,
|
|
267
|
+
// unsigned, would make `setVisibility('private')` protect nothing.
|
|
268
|
+
if (!(await declaredPublic(realPath))) return next();
|
|
269
|
+
|
|
160
270
|
// Quoted and weak (RFC 9110 §8.8.3): the bare md5 hex this used to send
|
|
161
271
|
// is not a valid entity-tag, and a strict cache may ignore it outright.
|
|
162
272
|
const etag = statTag(stat);
|
|
@@ -194,22 +304,23 @@ export class StaticMiddleware {
|
|
|
194
304
|
//
|
|
195
305
|
// A parent directory swapped in the same window is not covered; that
|
|
196
306
|
// needs openat-relative-to-a-descriptor, which Node does not expose.
|
|
197
|
-
let
|
|
307
|
+
let handle: fsp.FileHandle;
|
|
198
308
|
try {
|
|
199
|
-
|
|
309
|
+
handle = await fsp.open(
|
|
200
310
|
realPath,
|
|
201
311
|
fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW,
|
|
202
312
|
);
|
|
203
|
-
try {
|
|
204
|
-
content = await handle.readFile();
|
|
205
|
-
} finally {
|
|
206
|
-
await handle.close();
|
|
207
|
-
}
|
|
208
313
|
} catch {
|
|
209
314
|
return next();
|
|
210
315
|
}
|
|
211
|
-
//
|
|
212
|
-
//
|
|
213
|
-
|
|
316
|
+
// Streamed from the descriptor, never read whole: a buffered read held
|
|
317
|
+
// one full copy of the file per request until the response flushed, and
|
|
318
|
+
// a file past Node's 2 GiB read cap could not be served at all. The
|
|
319
|
+
// stream owns the descriptor and closes it when it ends, fails or is
|
|
320
|
+
// abandoned by a client that left. It stops at the size the
|
|
321
|
+
// Content-Length announced, so a file that grows meanwhile cannot
|
|
322
|
+
// overrun it.
|
|
323
|
+
ctx.response.status(200);
|
|
324
|
+
await ctx.response.stream(bodyOf(handle, stat.size));
|
|
214
325
|
}
|
|
215
326
|
}
|