@cloudbitmaps/gcs 0.18.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -9
- package/dist/backend.d.ts +10 -5
- package/dist/bucket-presence.d.ts +37 -0
- package/dist/crc32c.d.ts +5 -0
- package/dist/gcs-errors.d.ts +8 -1
- package/dist/index.js +350 -130
- package/dist/index.js.map +4 -4
- package/dist/read-client.d.ts +6 -0
- package/dist/registry.d.ts +1 -0
- package/dist/storage.d.ts +3 -0
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
// src/backend.ts
|
|
2
|
-
import { ValidationError as
|
|
2
|
+
import { ValidationError as ValidationError6, brandAsBackend } from "@cloudbitmaps/core/driver-kit";
|
|
3
3
|
import { Storage as GcsClient } from "@google-cloud/storage";
|
|
4
4
|
|
|
5
5
|
// src/storage.ts
|
|
6
6
|
import {
|
|
7
7
|
NotFoundError,
|
|
8
|
-
TransientError,
|
|
8
|
+
TransientError as TransientError2,
|
|
9
9
|
ValidationError as ValidationError3,
|
|
10
10
|
WriteConflictError,
|
|
11
11
|
isNotFoundError,
|
|
@@ -15,33 +15,8 @@ import {
|
|
|
15
15
|
import { createHash, randomBytes } from "node:crypto";
|
|
16
16
|
import { once as once2 } from "node:events";
|
|
17
17
|
|
|
18
|
-
// src/
|
|
19
|
-
import {
|
|
20
|
-
ValidationError,
|
|
21
|
-
encodeNameForKey,
|
|
22
|
-
namespaceKeyPart,
|
|
23
|
-
prefixPart,
|
|
24
|
-
validateSegmentRef
|
|
25
|
-
} from "@cloudbitmaps/core/driver-kit";
|
|
26
|
-
import { normalizeObjectPrefix } from "@cloudbitmaps/core/driver-kit";
|
|
27
|
-
var SUFFIX = ".crbm";
|
|
28
|
-
function segmentObjectPrefix(prefix, ref) {
|
|
29
|
-
validateSegmentRef(ref);
|
|
30
|
-
return `${prefixPart(prefix)}${namespaceKeyPart(ref.namespace)}/segments/${encodeNameForKey(ref.segment)}.`;
|
|
31
|
-
}
|
|
32
|
-
function storageObjectName(prefix, key) {
|
|
33
|
-
if (!Number.isInteger(key.generation) || key.generation < 0) {
|
|
34
|
-
throw new ValidationError(`generation must be a non-negative integer; got ${key.generation}`);
|
|
35
|
-
}
|
|
36
|
-
return `${segmentObjectPrefix(prefix, key)}${key.generation}${SUFFIX}`;
|
|
37
|
-
}
|
|
38
|
-
function parseGenerationFromName(segmentPrefix, objectName) {
|
|
39
|
-
if (!objectName.startsWith(segmentPrefix) || !objectName.endsWith(SUFFIX)) return null;
|
|
40
|
-
const middle = objectName.slice(segmentPrefix.length, objectName.length - SUFFIX.length);
|
|
41
|
-
if (!/^(0|[1-9]\d*)$/.test(middle)) return null;
|
|
42
|
-
const generation = Number(middle);
|
|
43
|
-
return Number.isSafeInteger(generation) ? generation : null;
|
|
44
|
-
}
|
|
18
|
+
// src/bucket-presence.ts
|
|
19
|
+
import { TransientError } from "@cloudbitmaps/core/driver-kit";
|
|
45
20
|
|
|
46
21
|
// src/gcs-errors.ts
|
|
47
22
|
function httpStatus(err) {
|
|
@@ -62,7 +37,14 @@ function isThrottle(err) {
|
|
|
62
37
|
return status === 429 || status === 503;
|
|
63
38
|
}
|
|
64
39
|
function isNotFound(err) {
|
|
65
|
-
return httpStatus(err) === 404;
|
|
40
|
+
return httpStatus(err) === 404 && !isMissingBucket(err);
|
|
41
|
+
}
|
|
42
|
+
function isMissingBucket(err) {
|
|
43
|
+
if (httpStatus(err) !== 404) return false;
|
|
44
|
+
const e = err;
|
|
45
|
+
const says = (m) => typeof m === "string" && /bucket does not exist/i.test(m);
|
|
46
|
+
if (says(e.message)) return true;
|
|
47
|
+
return Array.isArray(e.errors) && e.errors.some((inner) => says(inner?.message));
|
|
66
48
|
}
|
|
67
49
|
function isInvalidRange(err) {
|
|
68
50
|
return httpStatus(err) === 416;
|
|
@@ -111,7 +93,160 @@ function isTransient(err) {
|
|
|
111
93
|
if (status === 408 || status === 429 || status !== void 0 && status >= 500 && status < 600)
|
|
112
94
|
return true;
|
|
113
95
|
const net = networkCode(err);
|
|
114
|
-
return net
|
|
96
|
+
return net !== void 0 && TRANSPORT_CODES.has(net);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// src/scrub-error.ts
|
|
100
|
+
var DROPPED = /* @__PURE__ */ new Set([
|
|
101
|
+
"request",
|
|
102
|
+
"agent",
|
|
103
|
+
"socket",
|
|
104
|
+
"_httpMessage",
|
|
105
|
+
"connection",
|
|
106
|
+
"req",
|
|
107
|
+
"res",
|
|
108
|
+
"config"
|
|
109
|
+
]);
|
|
110
|
+
var CREDENTIAL_NAME = /^(?:proxy-)?authorization$|^cookie$|^set-cookie$|^x-api-key$/i;
|
|
111
|
+
var CREDENTIAL_LINES = /^((?:proxy-)?authorization|cookie|set-cookie|x-api-key):[^\r\n]*/gim;
|
|
112
|
+
var REDACTED = "[redacted]";
|
|
113
|
+
var isPlain = (value) => {
|
|
114
|
+
const proto = Object.getPrototypeOf(value);
|
|
115
|
+
return proto === Object.prototype || proto === null;
|
|
116
|
+
};
|
|
117
|
+
function responseSummary(response) {
|
|
118
|
+
const out = {};
|
|
119
|
+
for (const key of ["status", "statusText"]) {
|
|
120
|
+
const value = response[key];
|
|
121
|
+
if (typeof value === "number" || typeof value === "string") out[key] = value;
|
|
122
|
+
}
|
|
123
|
+
return out;
|
|
124
|
+
}
|
|
125
|
+
function clean(value, copies) {
|
|
126
|
+
if (typeof value === "string") return value.replace(CREDENTIAL_LINES, `$1: ${REDACTED}`);
|
|
127
|
+
if (typeof value !== "object" || value === null) {
|
|
128
|
+
return typeof value === "function" || typeof value === "symbol" ? void 0 : value;
|
|
129
|
+
}
|
|
130
|
+
if (copies.has(value)) return copies.get(value);
|
|
131
|
+
if (Array.isArray(value)) {
|
|
132
|
+
const list = [];
|
|
133
|
+
copies.set(value, list);
|
|
134
|
+
for (const item of value) list.push(clean(item, copies));
|
|
135
|
+
return list;
|
|
136
|
+
}
|
|
137
|
+
const isError = value instanceof Error;
|
|
138
|
+
if (!isError && !isPlain(value)) return void 0;
|
|
139
|
+
const copy = Object.create(Object.getPrototypeOf(value));
|
|
140
|
+
copies.set(value, copy);
|
|
141
|
+
for (const key of Reflect.ownKeys(value)) {
|
|
142
|
+
if (typeof key === "string" && DROPPED.has(key)) continue;
|
|
143
|
+
let held;
|
|
144
|
+
try {
|
|
145
|
+
held = value[key];
|
|
146
|
+
} catch {
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, key);
|
|
150
|
+
let kept;
|
|
151
|
+
if (typeof key === "string" && CREDENTIAL_NAME.test(key)) kept = held == null ? held : REDACTED;
|
|
152
|
+
else if (key === "response" && typeof held === "object" && held !== null) {
|
|
153
|
+
kept = responseSummary(held);
|
|
154
|
+
} else kept = clean(held, copies);
|
|
155
|
+
if (kept === void 0 && held !== void 0) continue;
|
|
156
|
+
Object.defineProperty(copy, key, {
|
|
157
|
+
value: kept,
|
|
158
|
+
writable: true,
|
|
159
|
+
configurable: true,
|
|
160
|
+
enumerable: descriptor?.enumerable ?? true
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
return copy;
|
|
164
|
+
}
|
|
165
|
+
function holdsTransport(err, seen) {
|
|
166
|
+
if (seen.has(err)) return false;
|
|
167
|
+
seen.add(err);
|
|
168
|
+
if (Object.getOwnPropertyNames(err).some((key) => DROPPED.has(key) || key === "response")) {
|
|
169
|
+
return true;
|
|
170
|
+
}
|
|
171
|
+
const cause = err.cause;
|
|
172
|
+
return typeof cause === "object" && cause !== null && holdsTransport(cause, seen);
|
|
173
|
+
}
|
|
174
|
+
function scrubCredentials(err) {
|
|
175
|
+
if (typeof err !== "object" || err === null) return err;
|
|
176
|
+
return holdsTransport(err, /* @__PURE__ */ new Set()) ? clean(err, /* @__PURE__ */ new WeakMap()) : err;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// src/bucket-presence.ts
|
|
180
|
+
var BucketPresence = class {
|
|
181
|
+
constructor(storage, bucket) {
|
|
182
|
+
this.storage = storage;
|
|
183
|
+
this.bucket = bucket;
|
|
184
|
+
}
|
|
185
|
+
storage;
|
|
186
|
+
bucket;
|
|
187
|
+
settled = false;
|
|
188
|
+
asking;
|
|
189
|
+
/** Resolve when a 404 can be read as an object's; reject when the bucket does not exist or the listing failed in transit. */
|
|
190
|
+
async confirm() {
|
|
191
|
+
if (this.settled) return;
|
|
192
|
+
this.asking ??= this.ask().finally(() => {
|
|
193
|
+
this.asking = void 0;
|
|
194
|
+
});
|
|
195
|
+
return this.asking;
|
|
196
|
+
}
|
|
197
|
+
async ask() {
|
|
198
|
+
let failure;
|
|
199
|
+
try {
|
|
200
|
+
await this.storage.bucket(this.bucket).getFiles({ maxResults: 1, autoPaginate: false });
|
|
201
|
+
} catch (raised) {
|
|
202
|
+
failure = { raised };
|
|
203
|
+
}
|
|
204
|
+
if (failure !== void 0) {
|
|
205
|
+
const err = scrubCredentials(failure.raised);
|
|
206
|
+
const status = err?.code;
|
|
207
|
+
if (status === 404) {
|
|
208
|
+
throw new Error(`the GCS bucket does not exist: ${this.bucket}`, { cause: err });
|
|
209
|
+
}
|
|
210
|
+
if (isTransient(err)) {
|
|
211
|
+
throw new TransientError(
|
|
212
|
+
`transient GCS fault confirming the bucket exists: ${String(status)}`,
|
|
213
|
+
{
|
|
214
|
+
cause: err
|
|
215
|
+
}
|
|
216
|
+
);
|
|
217
|
+
}
|
|
218
|
+
if (status !== 403) return;
|
|
219
|
+
}
|
|
220
|
+
this.settled = true;
|
|
221
|
+
}
|
|
222
|
+
};
|
|
223
|
+
|
|
224
|
+
// src/keys.ts
|
|
225
|
+
import {
|
|
226
|
+
ValidationError,
|
|
227
|
+
encodeNameForKey,
|
|
228
|
+
namespaceKeyPart,
|
|
229
|
+
prefixPart,
|
|
230
|
+
validateSegmentRef
|
|
231
|
+
} from "@cloudbitmaps/core/driver-kit";
|
|
232
|
+
import { normalizeObjectPrefix } from "@cloudbitmaps/core/driver-kit";
|
|
233
|
+
var SUFFIX = ".crbm";
|
|
234
|
+
function segmentObjectPrefix(prefix, ref) {
|
|
235
|
+
validateSegmentRef(ref);
|
|
236
|
+
return `${prefixPart(prefix)}${namespaceKeyPart(ref.namespace)}/segments/${encodeNameForKey(ref.segment)}.`;
|
|
237
|
+
}
|
|
238
|
+
function storageObjectName(prefix, key) {
|
|
239
|
+
if (!Number.isInteger(key.generation) || key.generation < 0) {
|
|
240
|
+
throw new ValidationError(`generation must be a non-negative integer; got ${key.generation}`);
|
|
241
|
+
}
|
|
242
|
+
return `${segmentObjectPrefix(prefix, key)}${key.generation}${SUFFIX}`;
|
|
243
|
+
}
|
|
244
|
+
function parseGenerationFromName(segmentPrefix, objectName) {
|
|
245
|
+
if (!objectName.startsWith(segmentPrefix) || !objectName.endsWith(SUFFIX)) return null;
|
|
246
|
+
const middle = objectName.slice(segmentPrefix.length, objectName.length - SUFFIX.length);
|
|
247
|
+
if (!/^(0|[1-9]\d*)$/.test(middle)) return null;
|
|
248
|
+
const generation = Number(middle);
|
|
249
|
+
return Number.isSafeInteger(generation) ? generation : null;
|
|
115
250
|
}
|
|
116
251
|
|
|
117
252
|
// src/read-timeout.ts
|
|
@@ -196,86 +331,6 @@ async function retryDownload(download, deadline) {
|
|
|
196
331
|
}
|
|
197
332
|
}
|
|
198
333
|
|
|
199
|
-
// src/scrub-error.ts
|
|
200
|
-
var DROPPED = /* @__PURE__ */ new Set([
|
|
201
|
-
"request",
|
|
202
|
-
"agent",
|
|
203
|
-
"socket",
|
|
204
|
-
"_httpMessage",
|
|
205
|
-
"connection",
|
|
206
|
-
"req",
|
|
207
|
-
"res",
|
|
208
|
-
"config"
|
|
209
|
-
]);
|
|
210
|
-
var CREDENTIAL_NAME = /^(?:proxy-)?authorization$|^cookie$|^set-cookie$|^x-api-key$/i;
|
|
211
|
-
var CREDENTIAL_LINES = /^((?:proxy-)?authorization|cookie|set-cookie|x-api-key):[^\r\n]*/gim;
|
|
212
|
-
var REDACTED = "[redacted]";
|
|
213
|
-
var isPlain = (value) => {
|
|
214
|
-
const proto = Object.getPrototypeOf(value);
|
|
215
|
-
return proto === Object.prototype || proto === null;
|
|
216
|
-
};
|
|
217
|
-
function responseSummary(response) {
|
|
218
|
-
const out = {};
|
|
219
|
-
for (const key of ["status", "statusText"]) {
|
|
220
|
-
const value = response[key];
|
|
221
|
-
if (typeof value === "number" || typeof value === "string") out[key] = value;
|
|
222
|
-
}
|
|
223
|
-
return out;
|
|
224
|
-
}
|
|
225
|
-
function clean(value, copies) {
|
|
226
|
-
if (typeof value === "string") return value.replace(CREDENTIAL_LINES, `$1: ${REDACTED}`);
|
|
227
|
-
if (typeof value !== "object" || value === null) {
|
|
228
|
-
return typeof value === "function" || typeof value === "symbol" ? void 0 : value;
|
|
229
|
-
}
|
|
230
|
-
if (copies.has(value)) return copies.get(value);
|
|
231
|
-
if (Array.isArray(value)) {
|
|
232
|
-
const list = [];
|
|
233
|
-
copies.set(value, list);
|
|
234
|
-
for (const item of value) list.push(clean(item, copies));
|
|
235
|
-
return list;
|
|
236
|
-
}
|
|
237
|
-
const isError = value instanceof Error;
|
|
238
|
-
if (!isError && !isPlain(value)) return void 0;
|
|
239
|
-
const copy = Object.create(Object.getPrototypeOf(value));
|
|
240
|
-
copies.set(value, copy);
|
|
241
|
-
for (const key of Reflect.ownKeys(value)) {
|
|
242
|
-
if (typeof key === "string" && DROPPED.has(key)) continue;
|
|
243
|
-
let held;
|
|
244
|
-
try {
|
|
245
|
-
held = value[key];
|
|
246
|
-
} catch {
|
|
247
|
-
continue;
|
|
248
|
-
}
|
|
249
|
-
const descriptor = Object.getOwnPropertyDescriptor(value, key);
|
|
250
|
-
let kept;
|
|
251
|
-
if (typeof key === "string" && CREDENTIAL_NAME.test(key)) kept = held == null ? held : REDACTED;
|
|
252
|
-
else if (key === "response" && typeof held === "object" && held !== null) {
|
|
253
|
-
kept = responseSummary(held);
|
|
254
|
-
} else kept = clean(held, copies);
|
|
255
|
-
if (kept === void 0 && held !== void 0) continue;
|
|
256
|
-
Object.defineProperty(copy, key, {
|
|
257
|
-
value: kept,
|
|
258
|
-
writable: true,
|
|
259
|
-
configurable: true,
|
|
260
|
-
enumerable: descriptor?.enumerable ?? true
|
|
261
|
-
});
|
|
262
|
-
}
|
|
263
|
-
return copy;
|
|
264
|
-
}
|
|
265
|
-
function holdsTransport(err, seen) {
|
|
266
|
-
if (seen.has(err)) return false;
|
|
267
|
-
seen.add(err);
|
|
268
|
-
if (Object.getOwnPropertyNames(err).some((key) => DROPPED.has(key) || key === "response")) {
|
|
269
|
-
return true;
|
|
270
|
-
}
|
|
271
|
-
const cause = err.cause;
|
|
272
|
-
return typeof cause === "object" && cause !== null && holdsTransport(cause, seen);
|
|
273
|
-
}
|
|
274
|
-
function scrubCredentials(err) {
|
|
275
|
-
if (typeof err !== "object" || err === null) return err;
|
|
276
|
-
return holdsTransport(err, /* @__PURE__ */ new Set()) ? clean(err, /* @__PURE__ */ new WeakMap()) : err;
|
|
277
|
-
}
|
|
278
|
-
|
|
279
334
|
// src/send-once.ts
|
|
280
335
|
import { once } from "node:events";
|
|
281
336
|
async function saveOnce(file, body, options) {
|
|
@@ -373,6 +428,7 @@ var GcsStorageDriver = class {
|
|
|
373
428
|
threshold;
|
|
374
429
|
readTimeoutMs;
|
|
375
430
|
clock;
|
|
431
|
+
presence;
|
|
376
432
|
constructor(options) {
|
|
377
433
|
this.storage = options.storage;
|
|
378
434
|
this.clock = options.clock ?? REAL_TIME;
|
|
@@ -387,9 +443,15 @@ var GcsStorageDriver = class {
|
|
|
387
443
|
throw new ValidationError3(`${name} must be a positive safe integer; got ${value}`);
|
|
388
444
|
}
|
|
389
445
|
}
|
|
446
|
+
if (options.maxObjectBytes !== void 0 && options.maxObjectBytes > 5 * 1024 ** 4) {
|
|
447
|
+
throw new ValidationError3(
|
|
448
|
+
`maxObjectBytes must be at most GCS's 5 TiB object limit (${5 * 1024 ** 4}); got ${options.maxObjectBytes}`
|
|
449
|
+
);
|
|
450
|
+
}
|
|
390
451
|
this.maxObjectBytes = options.maxObjectBytes ?? DEFAULT_MAX_OBJECT_BYTES;
|
|
391
452
|
this.threshold = options.simpleUploadThresholdBytes ?? DEFAULT_UPLOAD_THRESHOLD_BYTES;
|
|
392
453
|
this.readTimeoutMs = resolveReadTimeoutMs(options.readTimeoutMs);
|
|
454
|
+
this.presence = new BucketPresence(this.storage, this.bucket);
|
|
393
455
|
}
|
|
394
456
|
capabilities() {
|
|
395
457
|
return { rangeRead: true, maxObjectBytes: this.maxObjectBytes, conditionalPut: true };
|
|
@@ -448,7 +510,7 @@ var GcsStorageDriver = class {
|
|
|
448
510
|
this.checkRange(res, key, offset, length);
|
|
449
511
|
return res.bytes;
|
|
450
512
|
} catch (err) {
|
|
451
|
-
throw this.
|
|
513
|
+
throw await this.readFailure(err, key);
|
|
452
514
|
}
|
|
453
515
|
}
|
|
454
516
|
/**
|
|
@@ -524,7 +586,7 @@ var GcsStorageDriver = class {
|
|
|
524
586
|
}
|
|
525
587
|
return { bytes: res.bytes, size: this.tailSize(res, maxBytes, key) };
|
|
526
588
|
} catch (err) {
|
|
527
|
-
throw this.
|
|
589
|
+
throw await this.readFailure(err, key);
|
|
528
590
|
}
|
|
529
591
|
}
|
|
530
592
|
/**
|
|
@@ -575,8 +637,9 @@ var GcsStorageDriver = class {
|
|
|
575
637
|
}
|
|
576
638
|
async delete(key) {
|
|
577
639
|
try {
|
|
578
|
-
await this.file(storageObjectName(this.prefix, key)).delete(
|
|
640
|
+
await this.file(storageObjectName(this.prefix, key)).delete();
|
|
579
641
|
} catch (err) {
|
|
642
|
+
if (isNotFound(err)) return this.presence.confirm();
|
|
580
643
|
throw this.mapError(err);
|
|
581
644
|
}
|
|
582
645
|
}
|
|
@@ -595,6 +658,11 @@ var GcsStorageDriver = class {
|
|
|
595
658
|
}
|
|
596
659
|
}
|
|
597
660
|
}
|
|
661
|
+
/** A read's error, once a 404 is known to be the object's and not the bucket's ({@link BucketPresence}). */
|
|
662
|
+
async readFailure(err, key) {
|
|
663
|
+
if (isNotFound(err)) await this.presence.confirm();
|
|
664
|
+
return this.mapReadError(err, key);
|
|
665
|
+
}
|
|
598
666
|
/** Map GCS read errors to the driver vocabulary; pass everything else through {@link mapError}. */
|
|
599
667
|
mapReadError(err, key) {
|
|
600
668
|
err = scrubCredentials(err);
|
|
@@ -604,9 +672,9 @@ var GcsStorageDriver = class {
|
|
|
604
672
|
if (isInvalidRange(err)) {
|
|
605
673
|
return new ValidationError3(`range out of bounds for ${key.segment}.${key.generation}`);
|
|
606
674
|
}
|
|
607
|
-
if (err instanceof ReadTimedOut) return new
|
|
675
|
+
if (err instanceof ReadTimedOut) return new TransientError2(err.message, { cause: err });
|
|
608
676
|
if (isTransportFault(err)) {
|
|
609
|
-
return new
|
|
677
|
+
return new TransientError2(
|
|
610
678
|
`transient GCS fault: ${String(err.code)}`,
|
|
611
679
|
{
|
|
612
680
|
cause: err
|
|
@@ -624,7 +692,7 @@ var GcsStorageDriver = class {
|
|
|
624
692
|
mapError(err) {
|
|
625
693
|
err = scrubCredentials(err);
|
|
626
694
|
if (isTransient(err)) {
|
|
627
|
-
return new
|
|
695
|
+
return new TransientError2(
|
|
628
696
|
`transient GCS fault: ${err?.code ?? "unknown"}`,
|
|
629
697
|
{ cause: err }
|
|
630
698
|
);
|
|
@@ -689,10 +757,10 @@ var GcsUploadSink = class _GcsUploadSink {
|
|
|
689
757
|
}
|
|
690
758
|
this.hash.update(bytes);
|
|
691
759
|
if (this.stream !== void 0) {
|
|
692
|
-
if (!this.stream.write(bytes)) await once2(this.stream, "drain");
|
|
760
|
+
if (!this.stream.write(Buffer.from(bytes))) await once2(this.stream, "drain");
|
|
693
761
|
return;
|
|
694
762
|
}
|
|
695
|
-
this.buffered.push(bytes);
|
|
763
|
+
this.buffered.push(new Uint8Array(bytes));
|
|
696
764
|
this.bufferedLen += bytes.length;
|
|
697
765
|
if (this.bufferedLen > this.threshold) this.startResumable();
|
|
698
766
|
}
|
|
@@ -744,7 +812,7 @@ var GcsUploadSink = class _GcsUploadSink {
|
|
|
744
812
|
}
|
|
745
813
|
if (!isThrottle(err)) throw err;
|
|
746
814
|
if (resent >= THROTTLE_RESENDS) {
|
|
747
|
-
throw new
|
|
815
|
+
throw new TransientError2(
|
|
748
816
|
`GCS upload was throttled on each of its ${resent + 1} sends; the object may or may not exist`,
|
|
749
817
|
{ cause: err }
|
|
750
818
|
);
|
|
@@ -773,11 +841,32 @@ import {
|
|
|
773
841
|
IntegrityError,
|
|
774
842
|
MAX_ROW_BYTES,
|
|
775
843
|
ObjectStoreRegistry,
|
|
776
|
-
TransientError as
|
|
844
|
+
TransientError as TransientError3,
|
|
777
845
|
ValidationError as ValidationError4,
|
|
778
846
|
WriteConflictError as WriteConflictError2,
|
|
779
847
|
normalizeObjectPrefix as normalizeObjectPrefix2
|
|
780
848
|
} from "@cloudbitmaps/core/driver-kit";
|
|
849
|
+
|
|
850
|
+
// src/crc32c.ts
|
|
851
|
+
var TABLE = (() => {
|
|
852
|
+
const table = new Uint32Array(256);
|
|
853
|
+
for (let n = 0; n < 256; n++) {
|
|
854
|
+
let c = n;
|
|
855
|
+
for (let k = 0; k < 8; k++) c = c & 1 ? 2197175160 ^ c >>> 1 : c >>> 1;
|
|
856
|
+
table[n] = c >>> 0;
|
|
857
|
+
}
|
|
858
|
+
return table;
|
|
859
|
+
})();
|
|
860
|
+
function crc32cBase64(bytes) {
|
|
861
|
+
let crc = 4294967295;
|
|
862
|
+
for (const byte of bytes) crc = TABLE[(crc ^ byte) & 255] ^ crc >>> 8;
|
|
863
|
+
const value = (crc ^ 4294967295) >>> 0;
|
|
864
|
+
const out = Buffer.alloc(4);
|
|
865
|
+
out.writeUInt32BE(value);
|
|
866
|
+
return out.toString("base64");
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
// src/registry.ts
|
|
781
870
|
var GcsRegistryStore = class {
|
|
782
871
|
constructor(storage, readStorage, bucket, readTimeoutMs, conditionalDelete) {
|
|
783
872
|
this.storage = storage;
|
|
@@ -785,6 +874,7 @@ var GcsRegistryStore = class {
|
|
|
785
874
|
this.bucket = bucket;
|
|
786
875
|
this.readTimeoutMs = readTimeoutMs;
|
|
787
876
|
this.conditionalDelete = conditionalDelete;
|
|
877
|
+
this.presence = new BucketPresence(storage, bucket);
|
|
788
878
|
}
|
|
789
879
|
storage;
|
|
790
880
|
readStorage;
|
|
@@ -792,6 +882,7 @@ var GcsRegistryStore = class {
|
|
|
792
882
|
readTimeoutMs;
|
|
793
883
|
conditionalDelete;
|
|
794
884
|
label = "GCS";
|
|
885
|
+
presence;
|
|
795
886
|
/** A handle on one registry object. */
|
|
796
887
|
file(name) {
|
|
797
888
|
return this.storage.bucket(this.bucket).file(name);
|
|
@@ -817,7 +908,10 @@ var GcsRegistryStore = class {
|
|
|
817
908
|
deadline
|
|
818
909
|
);
|
|
819
910
|
} catch (err) {
|
|
820
|
-
if (isNotFound(err))
|
|
911
|
+
if (isNotFound(err)) {
|
|
912
|
+
await this.presence.confirm();
|
|
913
|
+
return null;
|
|
914
|
+
}
|
|
821
915
|
throw mapReadError(err);
|
|
822
916
|
}
|
|
823
917
|
if (res.status !== 200) {
|
|
@@ -833,6 +927,11 @@ var GcsRegistryStore = class {
|
|
|
833
927
|
try {
|
|
834
928
|
await saveOnce(this.file(key), body, {
|
|
835
929
|
contentType: "application/json",
|
|
930
|
+
// The checksum goes with the upload, for GCS to check before it stores anything. The SDK's own check runs
|
|
931
|
+
// after the upload instead, and on a mismatch, or an answer that names no checksum, deletes the object by
|
|
932
|
+
// name with no precondition: for a row, that is the live row, another writer's newer one included.
|
|
933
|
+
validation: false,
|
|
934
|
+
metadata: { crc32c: crc32cBase64(body) },
|
|
836
935
|
preconditionOpts: {
|
|
837
936
|
ifGenerationMatch: expect === "absent" ? 0 : generationFence(expect.version, key)
|
|
838
937
|
}
|
|
@@ -849,6 +948,7 @@ var GcsRegistryStore = class {
|
|
|
849
948
|
await this.file(key).delete({ ifGenerationMatch: generationFence(expect.version, key) });
|
|
850
949
|
} catch (err) {
|
|
851
950
|
if (isPreconditionFailed(err) || isNotFound(err)) {
|
|
951
|
+
if (isNotFound(err)) await this.presence.confirm();
|
|
852
952
|
throw new WriteConflictError2(`registry OCC conflict deleting ${key}`);
|
|
853
953
|
}
|
|
854
954
|
throw mapError(err);
|
|
@@ -865,6 +965,9 @@ var GcsRegistryStore = class {
|
|
|
865
965
|
throw mapError(err);
|
|
866
966
|
}
|
|
867
967
|
for (const f of files ?? []) yield f.name;
|
|
968
|
+
if (next?.pageToken !== void 0 && next.pageToken === pageToken) {
|
|
969
|
+
throw new IntegrityError("GCS listing returned the page token it was given");
|
|
970
|
+
}
|
|
868
971
|
pageToken = next?.pageToken;
|
|
869
972
|
} while (pageToken !== void 0);
|
|
870
973
|
}
|
|
@@ -878,9 +981,9 @@ function generationFence(version, key) {
|
|
|
878
981
|
}
|
|
879
982
|
function mapReadError(err) {
|
|
880
983
|
err = scrubCredentials(err);
|
|
881
|
-
if (err instanceof ReadTimedOut) return new
|
|
984
|
+
if (err instanceof ReadTimedOut) return new TransientError3(err.message, { cause: err });
|
|
882
985
|
if (isTransportFault(err)) {
|
|
883
|
-
return new
|
|
986
|
+
return new TransientError3(`transient GCS fault: ${String(err.code)}`, {
|
|
884
987
|
cause: err
|
|
885
988
|
});
|
|
886
989
|
}
|
|
@@ -889,7 +992,7 @@ function mapReadError(err) {
|
|
|
889
992
|
function mapError(err) {
|
|
890
993
|
err = scrubCredentials(err);
|
|
891
994
|
if (isTransient(err)) {
|
|
892
|
-
return new
|
|
995
|
+
return new TransientError3(
|
|
893
996
|
`transient GCS fault: ${err?.code ?? err?.name ?? "unknown"}`,
|
|
894
997
|
{ cause: err }
|
|
895
998
|
);
|
|
@@ -921,6 +1024,115 @@ var GcsRegistryDriver = class extends ObjectStoreRegistry {
|
|
|
921
1024
|
}
|
|
922
1025
|
};
|
|
923
1026
|
|
|
1027
|
+
// src/read-client.ts
|
|
1028
|
+
import { ValidationError as ValidationError5 } from "@cloudbitmaps/core/driver-kit";
|
|
1029
|
+
function downloadClient(client, supplied) {
|
|
1030
|
+
if (!isSdkClient(client)) {
|
|
1031
|
+
if (client.retryOptions?.autoRetry === false) return client;
|
|
1032
|
+
throw supplied ? refusal(
|
|
1033
|
+
true,
|
|
1034
|
+
"it is not a `Storage` client (a test double, say)",
|
|
1035
|
+
"Give it `retryOptions: { autoRetry: false }` to have it used as it is"
|
|
1036
|
+
) : refusal(
|
|
1037
|
+
false,
|
|
1038
|
+
"the `Storage` it was built from is not the SDK's (a mocked module, say)",
|
|
1039
|
+
"Pass the mock as `client`, with `retryOptions: { autoRetry: false }`"
|
|
1040
|
+
);
|
|
1041
|
+
}
|
|
1042
|
+
const instead = "Stub `Storage.prototype.bucket` instead, which reaches that client too, or pass a test double that does not extend `Storage`, with `retryOptions: { autoRetry: false }`";
|
|
1043
|
+
if (Object.hasOwn(client, "bucket")) {
|
|
1044
|
+
throw refusal(
|
|
1045
|
+
supplied,
|
|
1046
|
+
"its `bucket` method is replaced on the instance, as by a test stub, which a client built from its class would not have",
|
|
1047
|
+
instead
|
|
1048
|
+
);
|
|
1049
|
+
}
|
|
1050
|
+
if (overridesBucket(client)) {
|
|
1051
|
+
throw refusal(
|
|
1052
|
+
supplied,
|
|
1053
|
+
"its class overrides `bucket`, as a test double built on `Storage` does, so a client built from its class need not hold what this one holds",
|
|
1054
|
+
instead
|
|
1055
|
+
);
|
|
1056
|
+
}
|
|
1057
|
+
const second = supplied ? "a second client built from its class with its settings" : "a second client built from the same settings";
|
|
1058
|
+
const remedy = supplied ? "Pass a client its class builds from the options it is given, as `Storage` does, or leave out `client`" : void 0;
|
|
1059
|
+
let twin;
|
|
1060
|
+
try {
|
|
1061
|
+
twin = twinOf(client);
|
|
1062
|
+
} catch (err) {
|
|
1063
|
+
throw refusal(supplied, `${second} could not be built`, remedy, err);
|
|
1064
|
+
}
|
|
1065
|
+
const why = unfaithful(client, twin);
|
|
1066
|
+
if (why !== void 0) throw refusal(supplied, `${second} ${why}`, remedy);
|
|
1067
|
+
return twin;
|
|
1068
|
+
}
|
|
1069
|
+
function isSdkClient(client) {
|
|
1070
|
+
const factory = client.makeAuthenticatedRequest;
|
|
1071
|
+
return client.authClient !== void 0 && typeof factory === "function" && factory.authClient === client.authClient;
|
|
1072
|
+
}
|
|
1073
|
+
function overridesBucket(client) {
|
|
1074
|
+
let defined = 0;
|
|
1075
|
+
for (let proto = Object.getPrototypeOf(client); proto !== null; ) {
|
|
1076
|
+
if (Object.hasOwn(proto, "bucket")) defined++;
|
|
1077
|
+
proto = Object.getPrototypeOf(proto);
|
|
1078
|
+
}
|
|
1079
|
+
return defined > 1;
|
|
1080
|
+
}
|
|
1081
|
+
function twinOf(client) {
|
|
1082
|
+
const Own = client.constructor;
|
|
1083
|
+
const twin = new Own({
|
|
1084
|
+
authClient: client.authClient,
|
|
1085
|
+
// The SDK treats an endpoint spelled other than its default as a custom one, whose requests go without credentials,
|
|
1086
|
+
// and then stores it with any trailing slash taken off. A client given the default in another spelling is custom
|
|
1087
|
+
// with the default's URL, which that URL with a slash reproduces.
|
|
1088
|
+
apiEndpoint: client.customEndpoint === true ? `${client.apiEndpoint}/` : client.apiEndpoint,
|
|
1089
|
+
universeDomain: client.universeDomain,
|
|
1090
|
+
projectId: client.projectId,
|
|
1091
|
+
...client.useAuthWithCustomEndpoint === void 0 ? {} : { useAuthWithCustomEndpoint: client.useAuthWithCustomEndpoint },
|
|
1092
|
+
...client.providedUserAgent === void 0 ? {} : { userAgent: client.providedUserAgent },
|
|
1093
|
+
...client.timeout === void 0 ? {} : { timeout: client.timeout },
|
|
1094
|
+
crc32cGenerator: client.crc32cGenerator,
|
|
1095
|
+
retryOptions: { ...client.retryOptions, autoRetry: false },
|
|
1096
|
+
// A view, not a copy: the client's interceptors as they are when each request is sent.
|
|
1097
|
+
interceptors_: [
|
|
1098
|
+
{
|
|
1099
|
+
request: (options) => client.getRequestInterceptors().reduce(
|
|
1100
|
+
(sent, intercept) => intercept(sent),
|
|
1101
|
+
options
|
|
1102
|
+
)
|
|
1103
|
+
}
|
|
1104
|
+
]
|
|
1105
|
+
});
|
|
1106
|
+
if (twin !== client && twin.retryOptions !== void 0 && twin.retryOptions !== client.retryOptions) {
|
|
1107
|
+
twin.retryOptions.autoRetry = false;
|
|
1108
|
+
}
|
|
1109
|
+
return twin;
|
|
1110
|
+
}
|
|
1111
|
+
function unfaithful(client, twin) {
|
|
1112
|
+
if (twin === client) return "is the same client";
|
|
1113
|
+
if (typeof twin.bucket !== "function") return "has no `bucket` method";
|
|
1114
|
+
if (twin.authClient === void 0 || twin.authClient !== client.authClient) {
|
|
1115
|
+
return "would use other credentials";
|
|
1116
|
+
}
|
|
1117
|
+
if (twin.retryOptions === client.retryOptions) return "would share its retry settings";
|
|
1118
|
+
if (twin.retryOptions?.autoRetry !== false) return "keeps the SDK's retries";
|
|
1119
|
+
if (twin.baseUrl !== client.baseUrl) {
|
|
1120
|
+
return "would send to another URL, as when `STORAGE_EMULATOR_HOST` changed after the client was built or is not a URL";
|
|
1121
|
+
}
|
|
1122
|
+
if (twin.apiEndpoint !== client.apiEndpoint) return "would name another endpoint";
|
|
1123
|
+
if (twin.customEndpoint !== client.customEndpoint) {
|
|
1124
|
+
return "would decide otherwise whether requests carry credentials";
|
|
1125
|
+
}
|
|
1126
|
+
return void 0;
|
|
1127
|
+
}
|
|
1128
|
+
function refusal(supplied, why, remedy, cause) {
|
|
1129
|
+
const err = new ValidationError5(
|
|
1130
|
+
(supplied ? `GcsStorage cannot send the supplied client's downloads without the SDK's retries: ${why}` : `GcsStorage cannot build a client that sends downloads without the SDK's retries: ${why}`) + (remedy === void 0 ? "" : `. ${remedy}`)
|
|
1131
|
+
);
|
|
1132
|
+
if (cause !== void 0) err.cause = scrubCredentials(cause);
|
|
1133
|
+
return err;
|
|
1134
|
+
}
|
|
1135
|
+
|
|
924
1136
|
// src/backend.ts
|
|
925
1137
|
var GCS_STORAGE_OPTION_KEYS = [
|
|
926
1138
|
"bucket",
|
|
@@ -937,14 +1149,14 @@ var GCS_STORAGE_OPTION_KEYS = [
|
|
|
937
1149
|
var CLIENT_SETTINGS = ["projectId", "apiEndpoint"];
|
|
938
1150
|
function refuseUnknown(name, options, keys, hint) {
|
|
939
1151
|
if (options === null || typeof options !== "object") {
|
|
940
|
-
throw new
|
|
1152
|
+
throw new ValidationError6(
|
|
941
1153
|
`${name} needs an options object \u2014 got ${options === null ? "null" : typeof options}`
|
|
942
1154
|
);
|
|
943
1155
|
}
|
|
944
1156
|
const unknown = Object.keys(options).filter((k) => !keys.includes(k));
|
|
945
1157
|
if (unknown.length > 0) {
|
|
946
1158
|
const list = (ks) => ks.map((k) => `\`${k}\``).join(", ");
|
|
947
|
-
throw new
|
|
1159
|
+
throw new ValidationError6(
|
|
948
1160
|
`${name} does not take ${list(unknown)}. It takes ${list(keys)}; ${hint}.`
|
|
949
1161
|
);
|
|
950
1162
|
}
|
|
@@ -962,22 +1174,30 @@ var GcsStorage = class {
|
|
|
962
1174
|
GCS_STORAGE_OPTION_KEYS,
|
|
963
1175
|
"a @google-cloud/storage client goes in `client`"
|
|
964
1176
|
);
|
|
1177
|
+
if (typeof options.bucket !== "string" || options.bucket.length === 0) {
|
|
1178
|
+
throw new ValidationError6("GcsStorage needs a `bucket`: the name of one, a non-empty string");
|
|
1179
|
+
}
|
|
1180
|
+
if (options.now !== void 0 && typeof options.now !== "function") {
|
|
1181
|
+
throw new ValidationError6(
|
|
1182
|
+
"GcsStorage's `now` must be a function that returns epoch milliseconds"
|
|
1183
|
+
);
|
|
1184
|
+
}
|
|
965
1185
|
if (options.client !== void 0 && options.client !== null) {
|
|
966
1186
|
const ignored = CLIENT_SETTINGS.filter((k) => options[k] !== void 0);
|
|
967
1187
|
if (ignored.length > 0) {
|
|
968
|
-
throw new
|
|
1188
|
+
throw new ValidationError6(
|
|
969
1189
|
`GcsStorage takes \`client\` OR ${CLIENT_SETTINGS.map((k) => `\`${k}\``).join(" / ")}, not both \u2014 got \`client\` with ${ignored.map((k) => `\`${k}\``).join(", ")}; the \`client\` already carries them, so configure them on the client, or drop \`client\``
|
|
970
1190
|
);
|
|
971
1191
|
}
|
|
972
1192
|
this.client = options.client;
|
|
973
|
-
readClient = options.client;
|
|
1193
|
+
readClient = downloadClient(options.client, true);
|
|
974
1194
|
} else {
|
|
975
1195
|
const settings = {
|
|
976
1196
|
...options.projectId === void 0 ? {} : { projectId: options.projectId },
|
|
977
1197
|
...options.apiEndpoint === void 0 ? {} : { apiEndpoint: options.apiEndpoint }
|
|
978
1198
|
};
|
|
979
1199
|
this.client = new GcsClient(settings);
|
|
980
|
-
readClient =
|
|
1200
|
+
readClient = downloadClient(this.client, false);
|
|
981
1201
|
}
|
|
982
1202
|
const shared = {
|
|
983
1203
|
storage: this.client,
|