@c9up/archive 0.1.19 → 0.1.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +96 -3
- package/dist/ArchiveProvider.d.ts +8 -2
- package/dist/ArchiveProvider.d.ts.map +1 -1
- package/dist/ArchiveProvider.js +73 -1
- package/dist/ArchiveProvider.js.map +1 -1
- package/dist/DriveManager.d.ts +16 -6
- package/dist/DriveManager.d.ts.map +1 -1
- package/dist/DriveManager.js +23 -6
- package/dist/DriveManager.js.map +1 -1
- package/dist/GcsDriver.d.ts +14 -9
- package/dist/GcsDriver.d.ts.map +1 -1
- package/dist/GcsDriver.js +202 -98
- package/dist/GcsDriver.js.map +1 -1
- package/dist/S3Driver.d.ts +11 -7
- package/dist/S3Driver.d.ts.map +1 -1
- package/dist/S3Driver.js +73 -110
- package/dist/S3Driver.js.map +1 -1
- package/dist/StaticMiddleware.d.ts +5 -1
- package/dist/StaticMiddleware.d.ts.map +1 -1
- package/dist/StaticMiddleware.js +114 -13
- package/dist/StaticMiddleware.js.map +1 -1
- package/dist/StorageManager.d.ts +36 -4
- package/dist/StorageManager.d.ts.map +1 -1
- package/dist/StorageManager.js +360 -137
- package/dist/StorageManager.js.map +1 -1
- package/dist/augmentations.d.ts +17 -0
- package/dist/augmentations.d.ts.map +1 -1
- package/dist/errno.d.ts +6 -0
- package/dist/errno.d.ts.map +1 -0
- package/dist/errno.js +11 -0
- package/dist/errno.js.map +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/keys.d.ts +34 -0
- package/dist/keys.d.ts.map +1 -0
- package/dist/keys.js +88 -0
- package/dist/keys.js.map +1 -0
- package/dist/local-signature.d.ts +32 -0
- package/dist/local-signature.d.ts.map +1 -0
- package/dist/local-signature.js +58 -0
- package/dist/local-signature.js.map +1 -0
- package/dist/move-to-disk.d.ts +24 -0
- package/dist/move-to-disk.d.ts.map +1 -0
- package/dist/move-to-disk.js +167 -0
- package/dist/move-to-disk.js.map +1 -0
- package/dist/objects.d.ts.map +1 -1
- package/dist/objects.js +5 -2
- package/dist/objects.js.map +1 -1
- package/dist/provider-fetch.d.ts +36 -0
- package/dist/provider-fetch.d.ts.map +1 -0
- package/dist/provider-fetch.js +158 -0
- package/dist/provider-fetch.js.map +1 -0
- package/dist/services/main.d.ts +1 -1
- package/dist/services/main.d.ts.map +1 -1
- package/dist/services/main.js +14 -1
- package/dist/services/main.js.map +1 -1
- package/dist/signed-route.d.ts +18 -6
- package/dist/signed-route.d.ts.map +1 -1
- package/dist/signed-route.js +186 -31
- package/dist/signed-route.js.map +1 -1
- package/dist/streams.d.ts +36 -0
- package/dist/streams.d.ts.map +1 -0
- package/dist/streams.js +108 -0
- package/dist/streams.js.map +1 -0
- package/dist/testing/FakeStorage.d.ts.map +1 -1
- package/dist/testing/FakeStorage.js +2 -1
- package/dist/testing/FakeStorage.js.map +1 -1
- package/package.json +7 -3
- package/src/ArchiveProvider.ts +106 -4
- package/src/DriveManager.ts +38 -8
- package/src/GcsDriver.ts +256 -139
- package/src/S3Driver.ts +87 -130
- package/src/StaticMiddleware.ts +124 -13
- package/src/StorageManager.ts +433 -147
- package/src/augmentations.ts +24 -1
- package/src/errno.ts +10 -0
- package/src/index.ts +2 -0
- package/src/keys.ts +104 -0
- package/src/local-signature.ts +76 -0
- package/src/move-to-disk.ts +234 -0
- package/src/objects.ts +5 -2
- package/src/provider-fetch.ts +171 -0
- package/src/services/main.ts +15 -2
- package/src/signed-route.ts +216 -36
- package/src/streams.ts +118 -0
- package/src/testing/FakeStorage.ts +2 -3
package/src/GcsDriver.ts
CHANGED
|
@@ -15,15 +15,16 @@
|
|
|
15
15
|
|
|
16
16
|
import { createHash, createSign } from "node:crypto";
|
|
17
17
|
import { Readable } from "node:stream";
|
|
18
|
-
import type { ReadableStream as NodeReadableStream } from "node:stream/web";
|
|
19
18
|
import { ArchiveError } from "./errors.js";
|
|
20
19
|
import {
|
|
21
20
|
assertValidExpiry,
|
|
22
21
|
DEFAULT_EXPIRES_IN,
|
|
23
22
|
parseExpiry,
|
|
24
23
|
} from "./expiry.js";
|
|
24
|
+
import { normalizeKey, normalizePrefix } from "./keys.js";
|
|
25
25
|
import { inferMimeType } from "./mime-types.js";
|
|
26
26
|
import { DriveDirectory, DriveFile } from "./objects.js";
|
|
27
|
+
import { fetchWithIdleTimeout, requestBody } from "./provider-fetch.js";
|
|
27
28
|
import type {
|
|
28
29
|
ListAllOptions,
|
|
29
30
|
ListAllResult,
|
|
@@ -35,6 +36,7 @@ import type {
|
|
|
35
36
|
Visibility,
|
|
36
37
|
WriteOptions,
|
|
37
38
|
} from "./StorageManager.js";
|
|
39
|
+
import { chunkStream, collectToBuffer, nodeReadable } from "./streams.js";
|
|
38
40
|
|
|
39
41
|
/** GCS V4 strict URL encoding — identical to AWS's RFC3986-unreserved
|
|
40
42
|
* set. Duplicated from `S3Driver.ts` on purpose: per cerebrum rule,
|
|
@@ -54,7 +56,7 @@ function gcsStrictEncode(str: string): string {
|
|
|
54
56
|
* `/` is not among them, so encoding it there would break every signature.
|
|
55
57
|
*/
|
|
56
58
|
function gcsEncodeKey(key: string): string {
|
|
57
|
-
return key.split("/").map(gcsStrictEncode).join("/");
|
|
59
|
+
return normalizeKey(key).split("/").map(gcsStrictEncode).join("/");
|
|
58
60
|
}
|
|
59
61
|
|
|
60
62
|
/**
|
|
@@ -65,19 +67,13 @@ function gcsEncodeKey(key: string): string {
|
|
|
65
67
|
* makes the request address a different resource, which answers 404. Since
|
|
66
68
|
* `exists()` and `delete()` read a 404 as "not there", every object inside a
|
|
67
69
|
* prefix would silently look absent.
|
|
70
|
+
*
|
|
71
|
+
* Both encoders normalise the key first (see `keys.ts`): every object URL is
|
|
72
|
+
* built through one of them, so a `..` never reaches the provider even when
|
|
73
|
+
* the driver is used without a manager in front.
|
|
68
74
|
*/
|
|
69
75
|
function gcsEncodeObjectName(key: string): string {
|
|
70
|
-
return gcsStrictEncode(key);
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
/** Type guard matching the same pattern as `S3Driver.hasDestroy` —
|
|
74
|
-
* duplicated on purpose (package-local). */
|
|
75
|
-
function hasDestroy(x: unknown): x is { destroy(err?: Error): void } {
|
|
76
|
-
return (
|
|
77
|
-
typeof x === "object" &&
|
|
78
|
-
x !== null &&
|
|
79
|
-
typeof (x as { destroy?: unknown }).destroy === "function"
|
|
80
|
-
);
|
|
76
|
+
return gcsStrictEncode(normalizeKey(key));
|
|
81
77
|
}
|
|
82
78
|
|
|
83
79
|
/** Base64url encode a Buffer (RFC 4648 §5) — no `=` padding, `-` / `_`
|
|
@@ -100,13 +96,18 @@ export interface GcsConfig {
|
|
|
100
96
|
bucket: string;
|
|
101
97
|
serviceAccount: GcsServiceAccount;
|
|
102
98
|
/** Optional CDN prefix for public URLs (e.g. `https://cdn.example.com`). */
|
|
103
|
-
publicUrl?: string
|
|
104
|
-
|
|
99
|
+
publicUrl?: string;
|
|
100
|
+
/**
|
|
101
|
+
* How long the provider may stay silent during one request, in ms.
|
|
102
|
+
* Default 30 000.
|
|
105
103
|
*
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
|
|
104
|
+
* An idle bound, not a deadline: it runs while the provider is awaited
|
|
105
|
+
* (taking the request body, sending headers, sending the next chunk), so a
|
|
106
|
+
* long transfer that keeps moving is never cut. Without a bound a hung
|
|
107
|
+
* connection never settles: the handler awaiting it waits forever, and
|
|
108
|
+
* enough of them stop the server from serving. Set 0 to disable, which
|
|
109
|
+
* only makes sense behind an outer timeout.
|
|
110
|
+
*/
|
|
110
111
|
requestTimeoutMs?: number;
|
|
111
112
|
}
|
|
112
113
|
|
|
@@ -116,30 +117,25 @@ const GCS_META_ROOT = "https://storage.googleapis.com/storage/v1";
|
|
|
116
117
|
const GCS_OAUTH_URL = "https://oauth2.googleapis.com/token";
|
|
117
118
|
const GCS_SCOPE = "https://www.googleapis.com/auth/devstorage.read_write";
|
|
118
119
|
|
|
120
|
+
/**
|
|
121
|
+
* Size of one resumable-upload chunk: 8 MiB. Every chunk but the last must be
|
|
122
|
+
* a multiple of 256 KiB, and the upload holds at most two in memory.
|
|
123
|
+
*/
|
|
124
|
+
const GCS_RESUMABLE_CHUNK = 32 * 256 * 1024;
|
|
125
|
+
|
|
119
126
|
export class GcsDriver implements StorageDriver {
|
|
120
127
|
/**
|
|
121
|
-
* One request to the provider, bounded by {@link GcsConfig.requestTimeoutMs}
|
|
122
|
-
*
|
|
123
|
-
*
|
|
124
|
-
*
|
|
128
|
+
* One request to the provider, bounded by {@link GcsConfig.requestTimeoutMs}
|
|
129
|
+
* as an idle timeout (see `provider-fetch.ts`): a hung connection surfaces
|
|
130
|
+
* as an error the caller can retry, a long transfer that keeps moving does
|
|
131
|
+
* not.
|
|
125
132
|
*/
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
if (err instanceof Error && err.name === "TimeoutError") {
|
|
133
|
-
throw new ArchiveError(
|
|
134
|
-
"E_ARCHIVE_REQUEST_TIMEOUT",
|
|
135
|
-
`The storage provider did not answer within ${ms}ms.`,
|
|
136
|
-
{
|
|
137
|
-
hint: "Raise `requestTimeoutMs` for large objects, or check connectivity to the bucket.",
|
|
138
|
-
},
|
|
139
|
-
);
|
|
140
|
-
}
|
|
141
|
-
throw err;
|
|
142
|
-
}
|
|
133
|
+
#fetch(url: string, init: RequestInit = {}): Promise<Response> {
|
|
134
|
+
return fetchWithIdleTimeout(
|
|
135
|
+
url,
|
|
136
|
+
init,
|
|
137
|
+
this.#config.requestTimeoutMs ?? 30_000,
|
|
138
|
+
);
|
|
143
139
|
}
|
|
144
140
|
|
|
145
141
|
#config: GcsConfig;
|
|
@@ -166,8 +162,8 @@ export class GcsDriver implements StorageDriver {
|
|
|
166
162
|
options?: WriteOptions,
|
|
167
163
|
): Promise<void> {
|
|
168
164
|
const body = typeof content === "string" ? Buffer.from(content) : content;
|
|
165
|
+
const url = `${GCS_UPLOAD_ROOT}/b/${this.#config.bucket}/o?uploadType=media&name=${gcsEncodeObjectName(filePath)}`;
|
|
169
166
|
const token = await this.#getAccessToken();
|
|
170
|
-
const url = `${GCS_UPLOAD_ROOT}/b/${this.#config.bucket}/o?uploadType=media&name=${gcsStrictEncode(filePath)}`;
|
|
171
167
|
const mime =
|
|
172
168
|
options?.contentType ??
|
|
173
169
|
(inferMimeType(extFromPath(filePath)) || "application/octet-stream");
|
|
@@ -182,7 +178,7 @@ export class GcsDriver implements StorageDriver {
|
|
|
182
178
|
const res = await this.#fetch(url, {
|
|
183
179
|
method: "POST",
|
|
184
180
|
headers,
|
|
185
|
-
body: body
|
|
181
|
+
body: requestBody(body),
|
|
186
182
|
});
|
|
187
183
|
if (!res.ok) {
|
|
188
184
|
throw new Error(`GCS PUT failed (${res.status}): ${await res.text()}`);
|
|
@@ -212,32 +208,139 @@ export class GcsDriver implements StorageDriver {
|
|
|
212
208
|
readable: NodeJS.ReadableStream,
|
|
213
209
|
options?: PutStreamOptions,
|
|
214
210
|
): Promise<void> {
|
|
215
|
-
|
|
216
|
-
// deferred — document upgrade path here when a real large-file
|
|
217
|
-
// case appears.
|
|
218
|
-
const maxBytes = options?.contentLength ?? Number.MAX_SAFE_INTEGER;
|
|
219
|
-
const buf = await collectToBuffer(readable, maxBytes);
|
|
220
|
-
const token = await this.#getAccessToken();
|
|
221
|
-
const url = `${GCS_UPLOAD_ROOT}/b/${this.#config.bucket}/o?uploadType=media&name=${gcsStrictEncode(filePath)}`;
|
|
211
|
+
const name = gcsEncodeObjectName(filePath);
|
|
222
212
|
const mime =
|
|
223
213
|
options?.contentType ??
|
|
224
214
|
(inferMimeType(extFromPath(filePath)) || "application/octet-stream");
|
|
215
|
+
const declared = options?.contentLength;
|
|
216
|
+
// A declared length that fits in one chunk: one media upload.
|
|
217
|
+
if (declared !== undefined && declared <= GCS_RESUMABLE_CHUNK) {
|
|
218
|
+
const url = `${GCS_UPLOAD_ROOT}/b/${this.#config.bucket}/o?uploadType=media&name=${name}`;
|
|
219
|
+
const buf = await collectToBuffer(readable, declared);
|
|
220
|
+
const token = await this.#getAccessToken();
|
|
221
|
+
const res = await this.#fetch(url, {
|
|
222
|
+
method: "POST",
|
|
223
|
+
headers: {
|
|
224
|
+
authorization: `Bearer ${token}`,
|
|
225
|
+
"content-type": mime,
|
|
226
|
+
},
|
|
227
|
+
body: requestBody(buf),
|
|
228
|
+
});
|
|
229
|
+
if (!res.ok) {
|
|
230
|
+
throw new Error(`GCS PUT failed (${res.status}): ${await res.text()}`);
|
|
231
|
+
}
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
// Anything else goes through a resumable session, one bounded chunk at
|
|
235
|
+
// a time. Collecting the stream first made a 4 GB temp file a 4 GB
|
|
236
|
+
// allocation before a byte left — and copyFromFs/moveFromFs never
|
|
237
|
+
// declare a length.
|
|
238
|
+
const session = await this.#openResumableSession(name, mime);
|
|
239
|
+
const chunks = chunkStream(readable, GCS_RESUMABLE_CHUNK)[
|
|
240
|
+
Symbol.asyncIterator
|
|
241
|
+
]();
|
|
242
|
+
let offset = 0;
|
|
243
|
+
try {
|
|
244
|
+
// One chunk of look-ahead: the request that carries the last chunk
|
|
245
|
+
// must say it is the last, with the total.
|
|
246
|
+
let current = await chunks.next();
|
|
247
|
+
for (;;) {
|
|
248
|
+
const following = current.done ? current : await chunks.next();
|
|
249
|
+
const chunk = current.done ? Buffer.alloc(0) : current.value;
|
|
250
|
+
const last = following.done === true;
|
|
251
|
+
await this.#uploadChunk(session, chunk, offset, last);
|
|
252
|
+
offset += chunk.length;
|
|
253
|
+
if (last) return;
|
|
254
|
+
current = following;
|
|
255
|
+
}
|
|
256
|
+
} catch (err) {
|
|
257
|
+
// Best-effort: release the session's staged bytes. The original
|
|
258
|
+
// error is the one that explains the failure.
|
|
259
|
+
await this.#fetch(session, { method: "DELETE" }).catch(() => {});
|
|
260
|
+
throw err;
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* Start a resumable upload and return its session URI.
|
|
266
|
+
*
|
|
267
|
+
* The URI comes from the provider's `Location` header and receives the
|
|
268
|
+
* object's bytes, so one that does not point at the provider's upload
|
|
269
|
+
* endpoint is refused rather than followed.
|
|
270
|
+
*/
|
|
271
|
+
async #openResumableSession(name: string, mime: string): Promise<string> {
|
|
272
|
+
const url = `${GCS_UPLOAD_ROOT}/b/${this.#config.bucket}/o?uploadType=resumable&name=${name}`;
|
|
273
|
+
const token = await this.#getAccessToken();
|
|
225
274
|
const res = await this.#fetch(url, {
|
|
226
275
|
method: "POST",
|
|
227
276
|
headers: {
|
|
228
277
|
authorization: `Bearer ${token}`,
|
|
229
|
-
"content-type":
|
|
278
|
+
"content-type": "application/json; charset=UTF-8",
|
|
279
|
+
"x-upload-content-type": mime,
|
|
230
280
|
},
|
|
231
|
-
body:
|
|
281
|
+
body: "{}",
|
|
232
282
|
});
|
|
233
283
|
if (!res.ok) {
|
|
234
|
-
throw new Error(
|
|
284
|
+
throw new Error(
|
|
285
|
+
`GCS resumable upload could not start (${res.status}): ${await res.text()}`,
|
|
286
|
+
);
|
|
287
|
+
}
|
|
288
|
+
const session = res.headers.get("location");
|
|
289
|
+
if (session === null || !session.startsWith(`${GCS_UPLOAD_ROOT}/`)) {
|
|
290
|
+
throw new ArchiveError(
|
|
291
|
+
"E_ARCHIVE_UNEXPECTED_RESPONSE",
|
|
292
|
+
`GCS answered a resumable upload with an unusable session URI (${JSON.stringify(session)}).`,
|
|
293
|
+
{
|
|
294
|
+
hint: "The upload was not sent anywhere. Retry, or check for a proxy rewriting responses.",
|
|
295
|
+
},
|
|
296
|
+
);
|
|
297
|
+
}
|
|
298
|
+
return session;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/**
|
|
302
|
+
* Send one chunk of a resumable upload. A chunk that is not the last is
|
|
303
|
+
* a multiple of 256 KiB (see {@link GCS_RESUMABLE_CHUNK}) and must be
|
|
304
|
+
* persisted whole before the next is sent.
|
|
305
|
+
*/
|
|
306
|
+
async #uploadChunk(
|
|
307
|
+
session: string,
|
|
308
|
+
chunk: Buffer,
|
|
309
|
+
offset: number,
|
|
310
|
+
last: boolean,
|
|
311
|
+
): Promise<void> {
|
|
312
|
+
const end = offset + chunk.length - 1;
|
|
313
|
+
const total = offset + chunk.length;
|
|
314
|
+
const range = last
|
|
315
|
+
? chunk.length === 0
|
|
316
|
+
? `bytes */${total}`
|
|
317
|
+
: `bytes ${offset}-${end}/${total}`
|
|
318
|
+
: `bytes ${offset}-${end}/*`;
|
|
319
|
+
const res = await this.#fetch(session, {
|
|
320
|
+
method: "PUT",
|
|
321
|
+
headers: { "content-range": range },
|
|
322
|
+
body: requestBody(chunk),
|
|
323
|
+
});
|
|
324
|
+
if (last) {
|
|
325
|
+
if (!res.ok) {
|
|
326
|
+
throw new Error(
|
|
327
|
+
`GCS resumable upload failed to finish (${res.status}): ${await res.text()}`,
|
|
328
|
+
);
|
|
329
|
+
}
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
332
|
+
// 308 Resume Incomplete, reporting what the provider has kept.
|
|
333
|
+
const kept = res.headers.get("range");
|
|
334
|
+
if (res.status !== 308 || kept !== `bytes=0-${end}`) {
|
|
335
|
+
throw new Error(
|
|
336
|
+
`GCS resumable upload: chunk at ${offset} not persisted (${res.status}, range ${JSON.stringify(kept)}).`,
|
|
337
|
+
);
|
|
235
338
|
}
|
|
236
339
|
}
|
|
237
340
|
|
|
238
341
|
async get(filePath: string): Promise<Buffer | null> {
|
|
239
|
-
const token = await this.#getAccessToken();
|
|
240
342
|
const url = `${GCS_META_ROOT}/b/${this.#config.bucket}/o/${gcsEncodeObjectName(filePath)}?alt=media`;
|
|
343
|
+
const token = await this.#getAccessToken();
|
|
241
344
|
const res = await this.#fetch(url, {
|
|
242
345
|
method: "GET",
|
|
243
346
|
headers: { authorization: `Bearer ${token}` },
|
|
@@ -248,8 +351,8 @@ export class GcsDriver implements StorageDriver {
|
|
|
248
351
|
}
|
|
249
352
|
|
|
250
353
|
async getStream(filePath: string): Promise<NodeJS.ReadableStream> {
|
|
251
|
-
const token = await this.#getAccessToken();
|
|
252
354
|
const url = `${GCS_META_ROOT}/b/${this.#config.bucket}/o/${gcsEncodeObjectName(filePath)}?alt=media`;
|
|
355
|
+
const token = await this.#getAccessToken();
|
|
253
356
|
const res = await this.#fetch(url, {
|
|
254
357
|
method: "GET",
|
|
255
358
|
headers: { authorization: `Bearer ${token}` },
|
|
@@ -265,12 +368,12 @@ export class GcsDriver implements StorageDriver {
|
|
|
265
368
|
}
|
|
266
369
|
if (!res.ok) throw new Error(`GCS GET failed (${res.status})`);
|
|
267
370
|
if (res.body === null) return Readable.from(Buffer.alloc(0));
|
|
268
|
-
return
|
|
371
|
+
return nodeReadable(res.body);
|
|
269
372
|
}
|
|
270
373
|
|
|
271
374
|
async delete(filePath: string): Promise<boolean> {
|
|
272
|
-
const token = await this.#getAccessToken();
|
|
273
375
|
const url = `${GCS_META_ROOT}/b/${this.#config.bucket}/o/${gcsEncodeObjectName(filePath)}`;
|
|
376
|
+
const token = await this.#getAccessToken();
|
|
274
377
|
const res = await this.#fetch(url, {
|
|
275
378
|
method: "DELETE",
|
|
276
379
|
headers: { authorization: `Bearer ${token}` },
|
|
@@ -280,8 +383,8 @@ export class GcsDriver implements StorageDriver {
|
|
|
280
383
|
}
|
|
281
384
|
|
|
282
385
|
async exists(filePath: string): Promise<boolean> {
|
|
283
|
-
const token = await this.#getAccessToken();
|
|
284
386
|
const url = `${GCS_META_ROOT}/b/${this.#config.bucket}/o/${gcsEncodeObjectName(filePath)}`;
|
|
387
|
+
const token = await this.#getAccessToken();
|
|
285
388
|
const res = await this.#fetch(url, {
|
|
286
389
|
method: "GET",
|
|
287
390
|
headers: { authorization: `Bearer ${token}` },
|
|
@@ -310,8 +413,8 @@ export class GcsDriver implements StorageDriver {
|
|
|
310
413
|
}
|
|
311
414
|
|
|
312
415
|
async getMetadata(filePath: string): Promise<Metadata> {
|
|
313
|
-
const token = await this.#getAccessToken();
|
|
314
416
|
const url = `${GCS_META_ROOT}/b/${this.#config.bucket}/o/${gcsEncodeObjectName(filePath)}`;
|
|
417
|
+
const token = await this.#getAccessToken();
|
|
315
418
|
const res = await this.#fetch(url, {
|
|
316
419
|
method: "GET",
|
|
317
420
|
headers: { authorization: `Bearer ${token}` },
|
|
@@ -326,31 +429,26 @@ export class GcsDriver implements StorageDriver {
|
|
|
326
429
|
if (!res.ok) {
|
|
327
430
|
throw new Error(`GCS HEAD failed (${res.status}): ${await res.text()}`);
|
|
328
431
|
}
|
|
329
|
-
const json =
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
?
|
|
338
|
-
(a) =>
|
|
339
|
-
a.entity === "allUsers" &&
|
|
340
|
-
(a.role === "READER" || a.role === "OWNER"),
|
|
341
|
-
)
|
|
342
|
-
? "public"
|
|
343
|
-
: "private"
|
|
432
|
+
const json = await jsonObject(res, "an object metadata request");
|
|
433
|
+
// Public only on an explicit allUsers grant: an ACL that is missing or
|
|
434
|
+
// shaped unexpectedly reads as private.
|
|
435
|
+
const visibility: Visibility = records(json, "acl").some(
|
|
436
|
+
(grant) =>
|
|
437
|
+
text(grant, "entity") === "allUsers" &&
|
|
438
|
+
(text(grant, "role") === "READER" || text(grant, "role") === "OWNER"),
|
|
439
|
+
)
|
|
440
|
+
? "public"
|
|
344
441
|
: "private";
|
|
345
|
-
const size = Number.parseInt(json
|
|
346
|
-
const mimeType = json
|
|
442
|
+
const size = Number.parseInt(text(json, "size") ?? "0", 10);
|
|
443
|
+
const mimeType = text(json, "contentType") ?? "application/octet-stream";
|
|
444
|
+
const updated = text(json, "updated");
|
|
347
445
|
return {
|
|
348
446
|
size,
|
|
349
447
|
contentLength: size,
|
|
350
448
|
mimeType,
|
|
351
449
|
contentType: mimeType,
|
|
352
|
-
lastModified:
|
|
353
|
-
etag: (json
|
|
450
|
+
lastModified: updated ? new Date(updated) : new Date(0),
|
|
451
|
+
etag: (text(json, "etag") ?? "").replace(/^(W\/)?"(.*)"$/, "$1$2"),
|
|
354
452
|
visibility,
|
|
355
453
|
};
|
|
356
454
|
}
|
|
@@ -360,8 +458,8 @@ export class GcsDriver implements StorageDriver {
|
|
|
360
458
|
}
|
|
361
459
|
|
|
362
460
|
async setVisibility(filePath: string, visibility: Visibility): Promise<void> {
|
|
363
|
-
const token = await this.#getAccessToken();
|
|
364
461
|
const base = `${GCS_META_ROOT}/b/${this.#config.bucket}/o/${gcsEncodeObjectName(filePath)}/acl`;
|
|
462
|
+
const token = await this.#getAccessToken();
|
|
365
463
|
if (visibility === "public") {
|
|
366
464
|
const res = await this.#fetch(base, {
|
|
367
465
|
method: "POST",
|
|
@@ -391,8 +489,8 @@ export class GcsDriver implements StorageDriver {
|
|
|
391
489
|
}
|
|
392
490
|
|
|
393
491
|
async copy(from: string, to: string, options?: WriteOptions): Promise<void> {
|
|
394
|
-
const token = await this.#getAccessToken();
|
|
395
492
|
const url = `${GCS_META_ROOT}/b/${this.#config.bucket}/o/${gcsEncodeObjectName(from)}/copyTo/b/${this.#config.bucket}/o/${gcsEncodeObjectName(to)}`;
|
|
493
|
+
const token = await this.#getAccessToken();
|
|
396
494
|
const res = await this.#fetch(url, {
|
|
397
495
|
method: "POST",
|
|
398
496
|
headers: { authorization: `Bearer ${token}` },
|
|
@@ -418,7 +516,8 @@ export class GcsDriver implements StorageDriver {
|
|
|
418
516
|
await this.delete(from);
|
|
419
517
|
}
|
|
420
518
|
|
|
421
|
-
async deleteAll(
|
|
519
|
+
async deleteAll(given: string): Promise<void> {
|
|
520
|
+
const prefix = normalizePrefix(given);
|
|
422
521
|
// GCS has no batch-delete endpoint; drain the listing one object at
|
|
423
522
|
// a time. `list()` already paginates lazily.
|
|
424
523
|
const keys: string[] = [];
|
|
@@ -431,12 +530,13 @@ export class GcsDriver implements StorageDriver {
|
|
|
431
530
|
}
|
|
432
531
|
|
|
433
532
|
async listAll(
|
|
434
|
-
|
|
533
|
+
given: string,
|
|
435
534
|
options?: ListAllOptions,
|
|
436
535
|
): Promise<ListAllResult> {
|
|
437
536
|
const recursive = options?.recursive ?? false;
|
|
537
|
+
const prefix = normalizePrefix(given);
|
|
438
538
|
const normalizedPrefix =
|
|
439
|
-
prefix === ""
|
|
539
|
+
prefix === ""
|
|
440
540
|
? ""
|
|
441
541
|
: !recursive && !prefix.endsWith("/")
|
|
442
542
|
? `${prefix}/`
|
|
@@ -455,40 +555,32 @@ export class GcsDriver implements StorageDriver {
|
|
|
455
555
|
if (!res.ok) {
|
|
456
556
|
throw new Error(`GCS LIST failed (${res.status}): ${await res.text()}`);
|
|
457
557
|
}
|
|
458
|
-
const json =
|
|
459
|
-
items?: Array<{
|
|
460
|
-
name?: string;
|
|
461
|
-
size?: string;
|
|
462
|
-
updated?: string;
|
|
463
|
-
contentType?: string;
|
|
464
|
-
etag?: string;
|
|
465
|
-
}>;
|
|
466
|
-
prefixes?: string[];
|
|
467
|
-
nextPageToken?: string;
|
|
468
|
-
};
|
|
558
|
+
const json = await jsonObject(res, "a listing");
|
|
469
559
|
const objects: Array<DriveFile | DriveDirectory> = [];
|
|
470
|
-
for (const dir of json
|
|
560
|
+
for (const dir of texts(json, "prefixes")) {
|
|
471
561
|
objects.push(new DriveDirectory(dir.replace(/\/$/, "")));
|
|
472
562
|
}
|
|
473
|
-
for (const item of json
|
|
474
|
-
const size = Number.parseInt(item
|
|
475
|
-
const mimeType = item
|
|
563
|
+
for (const item of records(json, "items")) {
|
|
564
|
+
const size = Number.parseInt(text(item, "size") ?? "0", 10);
|
|
565
|
+
const mimeType = text(item, "contentType") ?? "application/octet-stream";
|
|
566
|
+
const updated = text(item, "updated");
|
|
476
567
|
objects.push(
|
|
477
|
-
new DriveFile(item
|
|
568
|
+
new DriveFile(text(item, "name") ?? "", this, {
|
|
478
569
|
size,
|
|
479
570
|
contentLength: size,
|
|
480
571
|
mimeType,
|
|
481
572
|
contentType: mimeType,
|
|
482
|
-
lastModified:
|
|
483
|
-
etag: (item
|
|
573
|
+
lastModified: updated ? new Date(updated) : new Date(0),
|
|
574
|
+
etag: (text(item, "etag") ?? "").replace(/^(W\/)?"(.*)"$/, "$1$2"),
|
|
484
575
|
visibility: "private",
|
|
485
576
|
}),
|
|
486
577
|
);
|
|
487
578
|
}
|
|
488
|
-
return { paginationToken: json
|
|
579
|
+
return { paginationToken: text(json, "nextPageToken"), objects };
|
|
489
580
|
}
|
|
490
581
|
|
|
491
|
-
async *list(
|
|
582
|
+
async *list(given: string): AsyncIterable<StorageEntry> {
|
|
583
|
+
const prefix = normalizePrefix(given);
|
|
492
584
|
let pageToken: string | undefined;
|
|
493
585
|
do {
|
|
494
586
|
const params = new URLSearchParams({ prefix });
|
|
@@ -502,18 +594,16 @@ export class GcsDriver implements StorageDriver {
|
|
|
502
594
|
if (!res.ok) {
|
|
503
595
|
throw new Error(`GCS LIST failed (${res.status}): ${await res.text()}`);
|
|
504
596
|
}
|
|
505
|
-
const json =
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
};
|
|
509
|
-
for (const item of json.items ?? []) {
|
|
597
|
+
const json = await jsonObject(res, "a listing");
|
|
598
|
+
for (const item of records(json, "items")) {
|
|
599
|
+
const updated = text(item, "updated");
|
|
510
600
|
yield {
|
|
511
|
-
path: item
|
|
512
|
-
size: Number.parseInt(item
|
|
513
|
-
lastModified:
|
|
601
|
+
path: text(item, "name") ?? "",
|
|
602
|
+
size: Number.parseInt(text(item, "size") ?? "0", 10),
|
|
603
|
+
lastModified: updated ? new Date(updated) : new Date(0),
|
|
514
604
|
};
|
|
515
605
|
}
|
|
516
|
-
pageToken = json
|
|
606
|
+
pageToken = text(json, "nextPageToken");
|
|
517
607
|
} while (pageToken);
|
|
518
608
|
}
|
|
519
609
|
|
|
@@ -645,18 +735,18 @@ export class GcsDriver implements StorageDriver {
|
|
|
645
735
|
`GCS token exchange failed (${res.status}): ${await res.text()}`,
|
|
646
736
|
);
|
|
647
737
|
}
|
|
648
|
-
const json =
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
};
|
|
652
|
-
if (!json.access_token) {
|
|
738
|
+
const json = await jsonObject(res, "the token exchange");
|
|
739
|
+
const accessToken = text(json, "access_token");
|
|
740
|
+
if (!accessToken) {
|
|
653
741
|
throw new Error("GCS token exchange: access_token missing from response");
|
|
654
742
|
}
|
|
743
|
+
const expiresIn =
|
|
744
|
+
typeof json.expires_in === "number" ? json.expires_in : 3600;
|
|
655
745
|
this.#cachedToken = {
|
|
656
|
-
value:
|
|
657
|
-
expiresAt: now +
|
|
746
|
+
value: accessToken,
|
|
747
|
+
expiresAt: now + expiresIn * 1000,
|
|
658
748
|
};
|
|
659
|
-
return
|
|
749
|
+
return accessToken;
|
|
660
750
|
}
|
|
661
751
|
}
|
|
662
752
|
|
|
@@ -672,25 +762,52 @@ function extFromPath(filePath: string): string {
|
|
|
672
762
|
return filePath.slice(idx).toLowerCase();
|
|
673
763
|
}
|
|
674
764
|
|
|
675
|
-
/**
|
|
676
|
-
*
|
|
677
|
-
*
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
);
|
|
692
|
-
}
|
|
693
|
-
chunks.push(buf);
|
|
765
|
+
/**
|
|
766
|
+
* A provider response body that must be a JSON object. Each field is then
|
|
767
|
+
* read through {@link text}/{@link records}/{@link texts}, which check its
|
|
768
|
+
* type instead of asserting the whole shape.
|
|
769
|
+
*/
|
|
770
|
+
async function jsonObject(
|
|
771
|
+
res: Response,
|
|
772
|
+
what: string,
|
|
773
|
+
): Promise<Record<string, unknown>> {
|
|
774
|
+
const json: unknown = await res.json();
|
|
775
|
+
if (!isRecord(json)) {
|
|
776
|
+
throw new ArchiveError(
|
|
777
|
+
"E_ARCHIVE_UNEXPECTED_RESPONSE",
|
|
778
|
+
`GCS answered ${what} with something other than a JSON object.`,
|
|
779
|
+
{ hint: "Check for a proxy rewriting responses to the storage API." },
|
|
780
|
+
);
|
|
694
781
|
}
|
|
695
|
-
return
|
|
782
|
+
return json;
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
786
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
/** A string field, or `undefined` when absent or of another type. */
|
|
790
|
+
function text(
|
|
791
|
+
record: Record<string, unknown>,
|
|
792
|
+
key: string,
|
|
793
|
+
): string | undefined {
|
|
794
|
+
const value = record[key];
|
|
795
|
+
return typeof value === "string" ? value : undefined;
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
/** The object entries of an array field; anything else is left out. */
|
|
799
|
+
function records(
|
|
800
|
+
record: Record<string, unknown>,
|
|
801
|
+
key: string,
|
|
802
|
+
): Record<string, unknown>[] {
|
|
803
|
+
const value = record[key];
|
|
804
|
+
return Array.isArray(value) ? value.filter(isRecord) : [];
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
/** The string entries of an array field; anything else is left out. */
|
|
808
|
+
function texts(record: Record<string, unknown>, key: string): string[] {
|
|
809
|
+
const value = record[key];
|
|
810
|
+
return Array.isArray(value)
|
|
811
|
+
? value.filter((entry): entry is string => typeof entry === "string")
|
|
812
|
+
: [];
|
|
696
813
|
}
|