@cloudbitmaps/gcs 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bucket-presence.d.ts +2 -2
- package/dist/defined.d.ts +8 -0
- package/dist/gcs-errors.d.ts +15 -1
- package/dist/index.js +117 -176
- package/dist/index.js.map +4 -4
- package/dist/registry.d.ts +2 -1
- package/dist/storage.d.ts +11 -8
- package/package.json +2 -2
- package/dist/keys.d.ts +0 -31
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* ▼
|
|
13
13
|
* list one object in the bucket
|
|
14
14
|
* ├─ answered ──────────▶ remember the bucket; the 404 is an object's
|
|
15
|
-
* ├─ 404 ───────────────▶ reject: the bucket does not exist
|
|
15
|
+
* ├─ 404 ───────────────▶ reject: ValidationError, the bucket does not exist
|
|
16
16
|
* ├─ transient ─────────▶ reject: TransientError (asked again next time)
|
|
17
17
|
* ├─ 403 ───────────────▶ remember it as unknowable; the 404 is an object's
|
|
18
18
|
* └─ anything else ─────▶ the 404 is an object's (asked again next time)
|
|
@@ -31,7 +31,7 @@ export declare class BucketPresence {
|
|
|
31
31
|
private settled;
|
|
32
32
|
private asking;
|
|
33
33
|
constructor(storage: Storage, bucket: string);
|
|
34
|
-
/** Resolve when a 404 can be read as an object's; reject when the bucket does not exist
|
|
34
|
+
/** Resolve when a 404 can be read as an object's; reject with `ValidationError` when the bucket does not exist, `TransientError` when the listing failed in transit. */
|
|
35
35
|
confirm(): Promise<void>;
|
|
36
36
|
private ask;
|
|
37
37
|
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `values` without the entries that are `undefined`, typed as optional properties, so a constructor can pass on the
|
|
3
|
+
* options it was given: `exactOptionalPropertyTypes` refuses `{ x: undefined }` for an optional `x`, and a sub-driver
|
|
4
|
+
* must see an option the caller left out as absent.
|
|
5
|
+
*/
|
|
6
|
+
export declare function defined<T extends Record<string, unknown>>(values: T): {
|
|
7
|
+
[K in keyof T]?: Exclude<T[K], undefined>;
|
|
8
|
+
};
|
package/dist/gcs-errors.d.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
* `.code` (a number); dropped/timed-out sockets surface as a Node error with a string `.code` (`ECONNRESET`,
|
|
8
8
|
* `ETIMEDOUT`, …).
|
|
9
9
|
*/
|
|
10
|
+
import { ValidationError } from '@cloudbitmaps/core/driver-kit';
|
|
10
11
|
/**
|
|
11
12
|
* A conditional `ifGenerationMatch: 0` write lost the write-once race — the object already existed, so GCS
|
|
12
13
|
* returns **412 Precondition Failed**. Maps to `WriteConflictError` (caller OCC), never a blind retry.
|
|
@@ -27,9 +28,17 @@ export declare function isNotFound(err: unknown): boolean;
|
|
|
27
28
|
* the message and each `errors[].message` are GCS's own. A wording this does not match reads as an absent object.
|
|
28
29
|
*/
|
|
29
30
|
export declare function isMissingBucket(err: unknown): boolean;
|
|
31
|
+
/**
|
|
32
|
+
* The {@link ValidationError} a missing bucket is: it names the bucket and what to check, and carries the service's
|
|
33
|
+
* error as its `cause`. A wiring fault is not an absent object, so it is not a `NotFoundError`: a read re-resolves on
|
|
34
|
+
* that, and would treat a misnamed bucket as a miss it can heal.
|
|
35
|
+
*/
|
|
36
|
+
export declare function missingBucketError(bucket: string, cause: unknown): ValidationError;
|
|
37
|
+
/** `err`, or {@link missingBucketError} when `err` says the bucket does not exist. */
|
|
38
|
+
export declare function asMissingBucket(err: unknown, bucket: string): unknown;
|
|
30
39
|
/** A range request started past EOF (HTTP 416 Requested Range Not Satisfiable). */
|
|
31
40
|
export declare function isInvalidRange(err: unknown): boolean;
|
|
32
|
-
/** A download that failed in transit rather than with an HTTP status:
|
|
41
|
+
/** A download that failed in transit rather than with an HTTP status: a connection fault ({@link isConnectionFault}). */
|
|
33
42
|
export declare function isTransportFault(err: unknown): boolean;
|
|
34
43
|
/**
|
|
35
44
|
* Whether the driver retries a failed download, which it does in the SDK's place: the statuses the SDK's predicate
|
|
@@ -44,3 +53,8 @@ export declare function isDownloadRetryable(err: unknown): boolean;
|
|
|
44
53
|
* a blind transient (a retried doomed conditional write would just fail again, and mask an OCC conflict).
|
|
45
54
|
*/
|
|
46
55
|
export declare function isTransient(err: unknown): boolean;
|
|
56
|
+
/**
|
|
57
|
+
* The message a transient GCS fault is handed on with: `transient GCS fault: <code>`, the HTTP status the SDK puts on
|
|
58
|
+
* `code` or Node's connection code, else the error's name.
|
|
59
|
+
*/
|
|
60
|
+
export declare function transientFaultMessage(err: unknown): string;
|
package/dist/index.js
CHANGED
|
@@ -1,16 +1,33 @@
|
|
|
1
1
|
// src/backend.ts
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
ValidationError as ValidationError5,
|
|
4
|
+
brandAsBackend,
|
|
5
|
+
refuseUnknownOptions
|
|
6
|
+
} from "@cloudbitmaps/core/driver-kit";
|
|
3
7
|
import { Storage as GcsClient } from "@google-cloud/storage";
|
|
4
8
|
|
|
9
|
+
// src/defined.ts
|
|
10
|
+
function defined(values) {
|
|
11
|
+
return Object.fromEntries(Object.entries(values).filter(([, v]) => v !== void 0));
|
|
12
|
+
}
|
|
13
|
+
|
|
5
14
|
// src/storage.ts
|
|
6
15
|
import {
|
|
7
16
|
NotFoundError,
|
|
8
17
|
TransientError as TransientError2,
|
|
9
18
|
ValidationError as ValidationError3,
|
|
10
19
|
WriteConflictError,
|
|
20
|
+
concatBytes,
|
|
11
21
|
isNotFoundError,
|
|
12
22
|
isValidationError,
|
|
13
|
-
isWriteConflictError
|
|
23
|
+
isWriteConflictError,
|
|
24
|
+
normalizeObjectPrefix,
|
|
25
|
+
parseGenerationFromKey,
|
|
26
|
+
requireBool,
|
|
27
|
+
requirePositiveInt,
|
|
28
|
+
segmentObjectPrefix,
|
|
29
|
+
storageObjectKey,
|
|
30
|
+
toTransient
|
|
14
31
|
} from "@cloudbitmaps/core/driver-kit";
|
|
15
32
|
import { createHash, randomBytes } from "node:crypto";
|
|
16
33
|
import { once as once2 } from "node:events";
|
|
@@ -19,6 +36,10 @@ import { once as once2 } from "node:events";
|
|
|
19
36
|
import { TransientError } from "@cloudbitmaps/core/driver-kit";
|
|
20
37
|
|
|
21
38
|
// src/gcs-errors.ts
|
|
39
|
+
import {
|
|
40
|
+
ValidationError,
|
|
41
|
+
isTransportFault as isNodeTransportFault
|
|
42
|
+
} from "@cloudbitmaps/core/driver-kit";
|
|
22
43
|
function httpStatus(err) {
|
|
23
44
|
const e = err;
|
|
24
45
|
if (typeof e?.code === "number") return e.code;
|
|
@@ -46,6 +67,17 @@ function isMissingBucket(err) {
|
|
|
46
67
|
if (says(e.message)) return true;
|
|
47
68
|
return Array.isArray(e.errors) && e.errors.some((inner) => says(inner?.message));
|
|
48
69
|
}
|
|
70
|
+
function missingBucketError(bucket, cause) {
|
|
71
|
+
return Object.assign(
|
|
72
|
+
new ValidationError(
|
|
73
|
+
`the GCS bucket does not exist: ${bucket}; check the bucket name and the project the client is built for`
|
|
74
|
+
),
|
|
75
|
+
{ cause }
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
function asMissingBucket(err, bucket) {
|
|
79
|
+
return isMissingBucket(err) ? missingBucketError(bucket, err) : err;
|
|
80
|
+
}
|
|
49
81
|
function isInvalidRange(err) {
|
|
50
82
|
return httpStatus(err) === 416;
|
|
51
83
|
}
|
|
@@ -53,22 +85,11 @@ var SDK_RETRIED_STATUSES = [408, 429, 500, 502, 503, 504];
|
|
|
53
85
|
function isSdkConnectionProblem(reason) {
|
|
54
86
|
return reason.includes("eai_again") || reason === "econnreset" || reason === "unexpected connection closure" || reason === "epipe" || reason === "socket connection timeout";
|
|
55
87
|
}
|
|
56
|
-
|
|
57
|
-
"
|
|
58
|
-
|
|
59
|
-
"ECONNABORTED",
|
|
60
|
-
"ETIMEDOUT",
|
|
61
|
-
"ENOTFOUND",
|
|
62
|
-
"EAI_AGAIN",
|
|
63
|
-
"EHOSTUNREACH",
|
|
64
|
-
"ENETUNREACH",
|
|
65
|
-
"EPIPE",
|
|
66
|
-
"ERR_STREAM_PREMATURE_CLOSE"
|
|
67
|
-
]);
|
|
88
|
+
function isConnectionFault(err) {
|
|
89
|
+
return isNodeTransportFault(err) || networkCode(err) === "ERR_STREAM_PREMATURE_CLOSE";
|
|
90
|
+
}
|
|
68
91
|
function isTransportFault(err) {
|
|
69
|
-
|
|
70
|
-
const code = networkCode(err);
|
|
71
|
-
return code !== void 0 && TRANSPORT_CODES.has(code);
|
|
92
|
+
return httpStatus(err) === void 0 && isConnectionFault(err);
|
|
72
93
|
}
|
|
73
94
|
function isDownloadRetryable(err) {
|
|
74
95
|
if (err === null || typeof err !== "object") return false;
|
|
@@ -92,8 +113,11 @@ function isTransient(err) {
|
|
|
92
113
|
const status = httpStatus(err);
|
|
93
114
|
if (status === 408 || status === 429 || status !== void 0 && status >= 500 && status < 600)
|
|
94
115
|
return true;
|
|
95
|
-
|
|
96
|
-
|
|
116
|
+
return isConnectionFault(err);
|
|
117
|
+
}
|
|
118
|
+
function transientFaultMessage(err) {
|
|
119
|
+
const e = err;
|
|
120
|
+
return `transient GCS fault: ${String(e?.code ?? e?.name ?? "unknown")}`;
|
|
97
121
|
}
|
|
98
122
|
|
|
99
123
|
// src/scrub-error.ts
|
|
@@ -186,7 +210,7 @@ var BucketPresence = class {
|
|
|
186
210
|
bucket;
|
|
187
211
|
settled = false;
|
|
188
212
|
asking;
|
|
189
|
-
/** Resolve when a 404 can be read as an object's; reject when the bucket does not exist
|
|
213
|
+
/** Resolve when a 404 can be read as an object's; reject with `ValidationError` when the bucket does not exist, `TransientError` when the listing failed in transit. */
|
|
190
214
|
async confirm() {
|
|
191
215
|
if (this.settled) return;
|
|
192
216
|
this.asking ??= this.ask().finally(() => {
|
|
@@ -205,7 +229,7 @@ var BucketPresence = class {
|
|
|
205
229
|
const err = scrubCredentials(failure.raised);
|
|
206
230
|
const status = err?.code;
|
|
207
231
|
if (status === 404) {
|
|
208
|
-
throw
|
|
232
|
+
throw missingBucketError(this.bucket, err);
|
|
209
233
|
}
|
|
210
234
|
if (isTransient(err)) {
|
|
211
235
|
throw new TransientError(
|
|
@@ -221,34 +245,6 @@ var BucketPresence = class {
|
|
|
221
245
|
}
|
|
222
246
|
};
|
|
223
247
|
|
|
224
|
-
// src/keys.ts
|
|
225
|
-
import {
|
|
226
|
-
ValidationError,
|
|
227
|
-
encodeNameForKey,
|
|
228
|
-
namespaceKeyPart,
|
|
229
|
-
prefixPart,
|
|
230
|
-
validateSegmentRef
|
|
231
|
-
} from "@cloudbitmaps/core/driver-kit";
|
|
232
|
-
import { normalizeObjectPrefix } from "@cloudbitmaps/core/driver-kit";
|
|
233
|
-
var SUFFIX = ".crbm";
|
|
234
|
-
function segmentObjectPrefix(prefix, ref) {
|
|
235
|
-
validateSegmentRef(ref);
|
|
236
|
-
return `${prefixPart(prefix)}${namespaceKeyPart(ref.namespace)}/segments/${encodeNameForKey(ref.segment)}.`;
|
|
237
|
-
}
|
|
238
|
-
function storageObjectName(prefix, key) {
|
|
239
|
-
if (!Number.isInteger(key.generation) || key.generation < 0) {
|
|
240
|
-
throw new ValidationError(`generation must be a non-negative integer; got ${key.generation}`);
|
|
241
|
-
}
|
|
242
|
-
return `${segmentObjectPrefix(prefix, key)}${key.generation}${SUFFIX}`;
|
|
243
|
-
}
|
|
244
|
-
function parseGenerationFromName(segmentPrefix, objectName) {
|
|
245
|
-
if (!objectName.startsWith(segmentPrefix) || !objectName.endsWith(SUFFIX)) return null;
|
|
246
|
-
const middle = objectName.slice(segmentPrefix.length, objectName.length - SUFFIX.length);
|
|
247
|
-
if (!/^(0|[1-9]\d*)$/.test(middle)) return null;
|
|
248
|
-
const generation = Number(middle);
|
|
249
|
-
return Number.isSafeInteger(generation) ? generation : null;
|
|
250
|
-
}
|
|
251
|
-
|
|
252
248
|
// src/read-timeout.ts
|
|
253
249
|
import { ValidationError as ValidationError2 } from "@cloudbitmaps/core/driver-kit";
|
|
254
250
|
var DEFAULT_READ_TIMEOUT_MS = 0;
|
|
@@ -411,13 +407,14 @@ function readOnce(file, options, maxBytes, oversize, deadline) {
|
|
|
411
407
|
}
|
|
412
408
|
|
|
413
409
|
// src/storage.ts
|
|
414
|
-
var
|
|
410
|
+
var GCS_MAX_OBJECT_BYTES = 5 * 1024 ** 4;
|
|
415
411
|
var WRITE_ID_KEY = "cbwid";
|
|
416
412
|
var THROTTLE_RESENDS = 3;
|
|
417
413
|
var THROTTLE_BASE_DELAY_MS = 500;
|
|
418
414
|
var REAL_TIME = {
|
|
419
415
|
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms))
|
|
420
416
|
};
|
|
417
|
+
var REAL_RNG = { next: () => Math.random() };
|
|
421
418
|
var DEFAULT_UPLOAD_THRESHOLD_BYTES = 8 * 1024 * 1024;
|
|
422
419
|
var GcsStorageDriver = class {
|
|
423
420
|
storage;
|
|
@@ -428,34 +425,25 @@ var GcsStorageDriver = class {
|
|
|
428
425
|
threshold;
|
|
429
426
|
readTimeoutMs;
|
|
430
427
|
clock;
|
|
428
|
+
rng;
|
|
431
429
|
presence;
|
|
432
430
|
conditionalDelete;
|
|
433
431
|
constructor(options) {
|
|
434
432
|
this.storage = options.storage;
|
|
435
|
-
|
|
436
|
-
throw new ValidationError3(
|
|
437
|
-
`conditionalDelete must be a boolean; got ${String(options.conditionalDelete)}`
|
|
438
|
-
);
|
|
439
|
-
}
|
|
440
|
-
this.conditionalDelete = options.conditionalDelete ?? false;
|
|
433
|
+
this.conditionalDelete = requireBool("conditionalDelete", options.conditionalDelete) ?? false;
|
|
441
434
|
this.clock = options.clock ?? REAL_TIME;
|
|
435
|
+
this.rng = options.rng ?? REAL_RNG;
|
|
442
436
|
this.readStorage = options.readStorage ?? options.storage;
|
|
443
437
|
this.bucket = options.bucket;
|
|
444
438
|
this.prefix = normalizeObjectPrefix(options.prefix);
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
]) {
|
|
449
|
-
if (value !== void 0 && (!Number.isSafeInteger(value) || value < 1)) {
|
|
450
|
-
throw new ValidationError3(`${name} must be a positive safe integer; got ${value}`);
|
|
451
|
-
}
|
|
452
|
-
}
|
|
453
|
-
if (options.maxObjectBytes !== void 0 && options.maxObjectBytes > 5 * 1024 ** 4) {
|
|
439
|
+
requirePositiveInt("maxObjectBytes", options.maxObjectBytes);
|
|
440
|
+
requirePositiveInt("simpleUploadThresholdBytes", options.simpleUploadThresholdBytes);
|
|
441
|
+
if (options.maxObjectBytes !== void 0 && options.maxObjectBytes > GCS_MAX_OBJECT_BYTES) {
|
|
454
442
|
throw new ValidationError3(
|
|
455
|
-
`maxObjectBytes must be at most GCS's 5 TiB object limit (${
|
|
443
|
+
`maxObjectBytes must be at most GCS's 5 TiB object limit (${GCS_MAX_OBJECT_BYTES}); got ${options.maxObjectBytes}`
|
|
456
444
|
);
|
|
457
445
|
}
|
|
458
|
-
this.maxObjectBytes = options.maxObjectBytes ??
|
|
446
|
+
this.maxObjectBytes = options.maxObjectBytes ?? GCS_MAX_OBJECT_BYTES;
|
|
459
447
|
this.threshold = options.simpleUploadThresholdBytes ?? DEFAULT_UPLOAD_THRESHOLD_BYTES;
|
|
460
448
|
this.readTimeoutMs = resolveReadTimeoutMs(options.readTimeoutMs);
|
|
461
449
|
this.presence = new BucketPresence(this.storage, this.bucket);
|
|
@@ -480,12 +468,13 @@ var GcsStorageDriver = class {
|
|
|
480
468
|
return startDeadline(this.readTimeoutMs, `${read} of ${key.segment}.${key.generation}`);
|
|
481
469
|
}
|
|
482
470
|
async putImmutable(key, write) {
|
|
483
|
-
const objectName =
|
|
471
|
+
const objectName = storageObjectKey(this.prefix, key);
|
|
484
472
|
const sink = new GcsUploadSink(
|
|
485
473
|
this.file(objectName),
|
|
486
474
|
this.maxObjectBytes,
|
|
487
475
|
this.threshold,
|
|
488
|
-
this.clock
|
|
476
|
+
this.clock,
|
|
477
|
+
this.rng
|
|
489
478
|
);
|
|
490
479
|
try {
|
|
491
480
|
await write(sink);
|
|
@@ -498,6 +487,7 @@ var GcsStorageDriver = class {
|
|
|
498
487
|
);
|
|
499
488
|
}
|
|
500
489
|
if (isValidationError(err) || isWriteConflictError(err) || isNotFoundError(err)) throw err;
|
|
490
|
+
if (isNotFound(err)) await this.presence.confirm();
|
|
501
491
|
throw this.mapError(err);
|
|
502
492
|
}
|
|
503
493
|
}
|
|
@@ -506,7 +496,7 @@ var GcsStorageDriver = class {
|
|
|
506
496
|
throw new ValidationError3(`invalid range offset=${offset} length=${length}`);
|
|
507
497
|
}
|
|
508
498
|
if (length === 0) return new Uint8Array(0);
|
|
509
|
-
const objectName =
|
|
499
|
+
const objectName = storageObjectKey(this.prefix, key);
|
|
510
500
|
try {
|
|
511
501
|
const deadline = this.deadline("range read", key);
|
|
512
502
|
const res = await retryDownload(
|
|
@@ -565,16 +555,16 @@ var GcsStorageDriver = class {
|
|
|
565
555
|
);
|
|
566
556
|
}
|
|
567
557
|
async getTail(key, maxBytes) {
|
|
568
|
-
const objectName =
|
|
558
|
+
const objectName = storageObjectKey(this.prefix, key);
|
|
559
|
+
if (!Number.isSafeInteger(maxBytes)) {
|
|
560
|
+
throw new ValidationError3(`invalid tail length ${String(maxBytes)}`);
|
|
561
|
+
}
|
|
569
562
|
const deadline = this.deadline("tail read", key);
|
|
570
563
|
try {
|
|
571
564
|
if (maxBytes <= 0) {
|
|
572
565
|
const meta = await this.metadataOf(objectName, deadline);
|
|
573
566
|
return withVersion({ bytes: new Uint8Array(0), size: meta.size }, meta.version);
|
|
574
567
|
}
|
|
575
|
-
if (!Number.isSafeInteger(maxBytes)) {
|
|
576
|
-
throw new ValidationError3(`invalid tail length ${maxBytes}`);
|
|
577
|
-
}
|
|
578
568
|
let res;
|
|
579
569
|
try {
|
|
580
570
|
res = await retryDownload(
|
|
@@ -662,7 +652,7 @@ var GcsStorageDriver = class {
|
|
|
662
652
|
return new ValidationError3(`GCS tail read of ${key.segment}.${key.generation} refused: ${why}`);
|
|
663
653
|
}
|
|
664
654
|
async delete(key, options) {
|
|
665
|
-
const objectName =
|
|
655
|
+
const objectName = storageObjectKey(this.prefix, key);
|
|
666
656
|
const ifVersion = options?.ifVersion;
|
|
667
657
|
const conditional = ifVersion !== void 0 && this.conditionalDelete;
|
|
668
658
|
try {
|
|
@@ -699,10 +689,11 @@ var GcsStorageDriver = class {
|
|
|
699
689
|
try {
|
|
700
690
|
[files] = await this.storage.bucket(this.bucket).getFiles({ prefix });
|
|
701
691
|
} catch (err) {
|
|
692
|
+
if (isNotFound(err)) throw missingBucketError(this.bucket, scrubCredentials(err));
|
|
702
693
|
throw this.mapError(err);
|
|
703
694
|
}
|
|
704
695
|
for (const f of files) {
|
|
705
|
-
const generation =
|
|
696
|
+
const generation = parseGenerationFromKey(prefix, f.name);
|
|
706
697
|
if (generation !== null) {
|
|
707
698
|
yield { namespace: ref.namespace, segment: ref.segment, generation };
|
|
708
699
|
}
|
|
@@ -723,31 +714,18 @@ var GcsStorageDriver = class {
|
|
|
723
714
|
return new ValidationError3(`range out of bounds for ${key.segment}.${key.generation}`);
|
|
724
715
|
}
|
|
725
716
|
if (err instanceof ReadTimedOut) return new TransientError2(err.message, { cause: err });
|
|
726
|
-
if (isTransportFault(err)) {
|
|
727
|
-
return new TransientError2(
|
|
728
|
-
`transient GCS fault: ${String(err.code)}`,
|
|
729
|
-
{
|
|
730
|
-
cause: err
|
|
731
|
-
}
|
|
732
|
-
);
|
|
733
|
-
}
|
|
734
717
|
return this.mapError(err);
|
|
735
718
|
}
|
|
736
719
|
/**
|
|
737
|
-
* Reclassify a transient GCS fault (throttle/5xx/dropped connection) as a retryable {@link TransientError},
|
|
738
|
-
*
|
|
739
|
-
* everything else propagates unchanged. The final fallback at every
|
|
740
|
-
* in transit arrive typed, and anything else (a checksum mismatch, a
|
|
720
|
+
* Reclassify a transient GCS fault (throttle/5xx/dropped connection) as a retryable {@link TransientError}, so the
|
|
721
|
+
* store's read retry can ride it out and a write's caller can tell it from a deterministic failure, and a missing
|
|
722
|
+
* bucket as a `ValidationError` that names it; everything else propagates unchanged. The final fallback at every
|
|
723
|
+
* client-call site: an HTTP status and a fault in transit arrive typed, and anything else (a checksum mismatch, a
|
|
724
|
+
* credentials failure) as the SDK raised it.
|
|
741
725
|
*/
|
|
742
726
|
mapError(err) {
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
return new TransientError2(
|
|
746
|
-
`transient GCS fault: ${err?.code ?? "unknown"}`,
|
|
747
|
-
{ cause: err }
|
|
748
|
-
);
|
|
749
|
-
}
|
|
750
|
-
return err;
|
|
727
|
+
const scrubbed = asMissingBucket(scrubCredentials(err), this.bucket);
|
|
728
|
+
return toTransient(scrubbed, isTransient, transientFaultMessage);
|
|
751
729
|
}
|
|
752
730
|
};
|
|
753
731
|
function withVersion(tail, version) {
|
|
@@ -764,26 +742,19 @@ function generationOf(version, key) {
|
|
|
764
742
|
}
|
|
765
743
|
return Number(version);
|
|
766
744
|
}
|
|
767
|
-
function concatBytes(parts, total) {
|
|
768
|
-
const out = new Uint8Array(total);
|
|
769
|
-
let offset = 0;
|
|
770
|
-
for (const p of parts) {
|
|
771
|
-
out.set(p, offset);
|
|
772
|
-
offset += p.length;
|
|
773
|
-
}
|
|
774
|
-
return out;
|
|
775
|
-
}
|
|
776
745
|
var GcsUploadSink = class _GcsUploadSink {
|
|
777
|
-
constructor(file, maxObjectBytes, threshold, clock) {
|
|
746
|
+
constructor(file, maxObjectBytes, threshold, clock, rng) {
|
|
778
747
|
this.file = file;
|
|
779
748
|
this.maxObjectBytes = maxObjectBytes;
|
|
780
749
|
this.threshold = threshold;
|
|
781
750
|
this.clock = clock;
|
|
751
|
+
this.rng = rng;
|
|
782
752
|
}
|
|
783
753
|
file;
|
|
784
754
|
maxObjectBytes;
|
|
785
755
|
threshold;
|
|
786
756
|
clock;
|
|
757
|
+
rng;
|
|
787
758
|
hash = createHash("sha256");
|
|
788
759
|
total = 0;
|
|
789
760
|
buffered = [];
|
|
@@ -837,7 +808,7 @@ var GcsUploadSink = class _GcsUploadSink {
|
|
|
837
808
|
this.failure ??= e;
|
|
838
809
|
});
|
|
839
810
|
this.done.catch(() => void 0);
|
|
840
|
-
const buf = concatBytes(this.buffered
|
|
811
|
+
const buf = concatBytes(this.buffered);
|
|
841
812
|
this.buffered.length = 0;
|
|
842
813
|
this.bufferedLen = 0;
|
|
843
814
|
stream.write(buf);
|
|
@@ -845,7 +816,7 @@ var GcsUploadSink = class _GcsUploadSink {
|
|
|
845
816
|
async finish() {
|
|
846
817
|
const sha256 = this.hash.digest("hex");
|
|
847
818
|
if (this.stream === void 0) {
|
|
848
|
-
await this.sendSimple(concatBytes(this.buffered
|
|
819
|
+
await this.sendSimple(concatBytes(this.buffered));
|
|
849
820
|
return { size: this.total, sha256 };
|
|
850
821
|
}
|
|
851
822
|
try {
|
|
@@ -881,7 +852,7 @@ var GcsUploadSink = class _GcsUploadSink {
|
|
|
881
852
|
{ cause: err }
|
|
882
853
|
);
|
|
883
854
|
}
|
|
884
|
-
await this.clock.sleep(Math.floor(
|
|
855
|
+
await this.clock.sleep(Math.floor(this.rng.next() * THROTTLE_BASE_DELAY_MS * 2 ** resent));
|
|
885
856
|
}
|
|
886
857
|
}
|
|
887
858
|
}
|
|
@@ -906,9 +877,10 @@ import {
|
|
|
906
877
|
MAX_ROW_BYTES,
|
|
907
878
|
ObjectStoreRegistry,
|
|
908
879
|
TransientError as TransientError3,
|
|
909
|
-
ValidationError as ValidationError4,
|
|
910
880
|
WriteConflictError as WriteConflictError2,
|
|
911
|
-
normalizeObjectPrefix as normalizeObjectPrefix2
|
|
881
|
+
normalizeObjectPrefix as normalizeObjectPrefix2,
|
|
882
|
+
requireBool as requireBool2,
|
|
883
|
+
toTransient as toTransient2
|
|
912
884
|
} from "@cloudbitmaps/core/driver-kit";
|
|
913
885
|
|
|
914
886
|
// src/crc32c.ts
|
|
@@ -976,7 +948,7 @@ var GcsRegistryStore = class {
|
|
|
976
948
|
await this.presence.confirm();
|
|
977
949
|
return null;
|
|
978
950
|
}
|
|
979
|
-
throw mapReadError(err);
|
|
951
|
+
throw mapReadError(err, this.bucket);
|
|
980
952
|
}
|
|
981
953
|
if (res.status !== 200) {
|
|
982
954
|
throw new IntegrityError(`registry read answered HTTP ${res.status}, not 200: ${key}`);
|
|
@@ -1004,7 +976,8 @@ var GcsRegistryStore = class {
|
|
|
1004
976
|
if (isPreconditionFailed(err)) {
|
|
1005
977
|
throw new WriteConflictError2(`registry OCC conflict for ${key}`);
|
|
1006
978
|
}
|
|
1007
|
-
|
|
979
|
+
if (isNotFound(err)) await this.presence.confirm();
|
|
980
|
+
throw mapError(err, this.bucket);
|
|
1008
981
|
}
|
|
1009
982
|
}
|
|
1010
983
|
async delete(key, expect) {
|
|
@@ -1015,7 +988,7 @@ var GcsRegistryStore = class {
|
|
|
1015
988
|
if (isNotFound(err)) await this.presence.confirm();
|
|
1016
989
|
throw new WriteConflictError2(`registry OCC conflict deleting ${key}`);
|
|
1017
990
|
}
|
|
1018
|
-
throw mapError(err);
|
|
991
|
+
throw mapError(err, this.bucket);
|
|
1019
992
|
}
|
|
1020
993
|
}
|
|
1021
994
|
async *listKeys(prefix) {
|
|
@@ -1026,7 +999,8 @@ var GcsRegistryStore = class {
|
|
|
1026
999
|
try {
|
|
1027
1000
|
[files, next] = await this.storage.bucket(this.bucket).getFiles({ prefix, autoPaginate: false, maxResults: 1e3, pageToken });
|
|
1028
1001
|
} catch (err) {
|
|
1029
|
-
throw
|
|
1002
|
+
if (isNotFound(err)) throw missingBucketError(this.bucket, scrubCredentials(err));
|
|
1003
|
+
throw mapError(err, this.bucket);
|
|
1030
1004
|
}
|
|
1031
1005
|
for (const f of files ?? []) yield f.name;
|
|
1032
1006
|
if (next?.pageToken !== void 0 && next.pageToken === pageToken) {
|
|
@@ -1043,34 +1017,17 @@ function generationFence(version, key) {
|
|
|
1043
1017
|
}
|
|
1044
1018
|
return generation;
|
|
1045
1019
|
}
|
|
1046
|
-
function mapReadError(err) {
|
|
1020
|
+
function mapReadError(err, bucket) {
|
|
1047
1021
|
err = scrubCredentials(err);
|
|
1048
1022
|
if (err instanceof ReadTimedOut) return new TransientError3(err.message, { cause: err });
|
|
1049
|
-
|
|
1050
|
-
return new TransientError3(`transient GCS fault: ${String(err.code)}`, {
|
|
1051
|
-
cause: err
|
|
1052
|
-
});
|
|
1053
|
-
}
|
|
1054
|
-
return mapError(err);
|
|
1023
|
+
return mapError(err, bucket);
|
|
1055
1024
|
}
|
|
1056
|
-
function mapError(err) {
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
);
|
|
1063
|
-
}
|
|
1064
|
-
return err;
|
|
1065
|
-
}
|
|
1066
|
-
function resolveConditionalDelete(value) {
|
|
1067
|
-
if (value === void 0) {
|
|
1068
|
-
return false;
|
|
1069
|
-
}
|
|
1070
|
-
if (typeof value !== "boolean") {
|
|
1071
|
-
throw new ValidationError4(`conditionalDelete must be a boolean; got ${String(value)}`);
|
|
1072
|
-
}
|
|
1073
|
-
return value;
|
|
1025
|
+
function mapError(err, bucket) {
|
|
1026
|
+
return toTransient2(
|
|
1027
|
+
asMissingBucket(scrubCredentials(err), bucket),
|
|
1028
|
+
isTransient,
|
|
1029
|
+
transientFaultMessage
|
|
1030
|
+
);
|
|
1074
1031
|
}
|
|
1075
1032
|
var GcsRegistryDriver = class extends ObjectStoreRegistry {
|
|
1076
1033
|
constructor(options) {
|
|
@@ -1080,7 +1037,7 @@ var GcsRegistryDriver = class extends ObjectStoreRegistry {
|
|
|
1080
1037
|
options.readStorage ?? options.storage,
|
|
1081
1038
|
options.bucket,
|
|
1082
1039
|
resolveReadTimeoutMs(options.readTimeoutMs),
|
|
1083
|
-
|
|
1040
|
+
requireBool2("conditionalDelete", options.conditionalDelete) ?? false
|
|
1084
1041
|
),
|
|
1085
1042
|
normalizeObjectPrefix2(options.prefix),
|
|
1086
1043
|
options.now ?? (() => Date.now())
|
|
@@ -1089,7 +1046,7 @@ var GcsRegistryDriver = class extends ObjectStoreRegistry {
|
|
|
1089
1046
|
};
|
|
1090
1047
|
|
|
1091
1048
|
// src/read-client.ts
|
|
1092
|
-
import { ValidationError as
|
|
1049
|
+
import { ValidationError as ValidationError4 } from "@cloudbitmaps/core/driver-kit";
|
|
1093
1050
|
function downloadClient(client, supplied) {
|
|
1094
1051
|
if (!isSdkClient(client)) {
|
|
1095
1052
|
if (client.retryOptions?.autoRetry === false) return client;
|
|
@@ -1135,12 +1092,12 @@ function isSdkClient(client) {
|
|
|
1135
1092
|
return client.authClient !== void 0 && typeof factory === "function" && factory.authClient === client.authClient;
|
|
1136
1093
|
}
|
|
1137
1094
|
function overridesBucket(client) {
|
|
1138
|
-
let
|
|
1095
|
+
let defined2 = 0;
|
|
1139
1096
|
for (let proto = Object.getPrototypeOf(client); proto !== null; ) {
|
|
1140
|
-
if (Object.hasOwn(proto, "bucket"))
|
|
1097
|
+
if (Object.hasOwn(proto, "bucket")) defined2++;
|
|
1141
1098
|
proto = Object.getPrototypeOf(proto);
|
|
1142
1099
|
}
|
|
1143
|
-
return
|
|
1100
|
+
return defined2 > 1;
|
|
1144
1101
|
}
|
|
1145
1102
|
function twinOf(client) {
|
|
1146
1103
|
const Own = client.constructor;
|
|
@@ -1190,7 +1147,7 @@ function unfaithful(client, twin) {
|
|
|
1190
1147
|
return void 0;
|
|
1191
1148
|
}
|
|
1192
1149
|
function refusal(supplied, why, remedy, cause) {
|
|
1193
|
-
const err = new
|
|
1150
|
+
const err = new ValidationError4(
|
|
1194
1151
|
(supplied ? `GcsStorage cannot send the supplied client's downloads without the SDK's retries: ${why}` : `GcsStorage cannot build a client that sends downloads without the SDK's retries: ${why}`) + (remedy === void 0 ? "" : `. ${remedy}`)
|
|
1195
1152
|
);
|
|
1196
1153
|
if (cause !== void 0) err.cause = scrubCredentials(cause);
|
|
@@ -1211,20 +1168,6 @@ var GCS_STORAGE_OPTION_KEYS = [
|
|
|
1211
1168
|
"conditionalDelete"
|
|
1212
1169
|
];
|
|
1213
1170
|
var CLIENT_SETTINGS = ["projectId", "apiEndpoint"];
|
|
1214
|
-
function refuseUnknown(name, options, keys, hint) {
|
|
1215
|
-
if (options === null || typeof options !== "object") {
|
|
1216
|
-
throw new ValidationError6(
|
|
1217
|
-
`${name} needs an options object \u2014 got ${options === null ? "null" : typeof options}`
|
|
1218
|
-
);
|
|
1219
|
-
}
|
|
1220
|
-
const unknown = Object.keys(options).filter((k) => !keys.includes(k));
|
|
1221
|
-
if (unknown.length > 0) {
|
|
1222
|
-
const list = (ks) => ks.map((k) => `\`${k}\``).join(", ");
|
|
1223
|
-
throw new ValidationError6(
|
|
1224
|
-
`${name} does not take ${list(unknown)}. It takes ${list(keys)}; ${hint}.`
|
|
1225
|
-
);
|
|
1226
|
-
}
|
|
1227
|
-
}
|
|
1228
1171
|
var GcsStorage = class {
|
|
1229
1172
|
storage;
|
|
1230
1173
|
registry;
|
|
@@ -1232,54 +1175,52 @@ var GcsStorage = class {
|
|
|
1232
1175
|
client;
|
|
1233
1176
|
constructor(options) {
|
|
1234
1177
|
let readClient;
|
|
1235
|
-
|
|
1178
|
+
refuseUnknownOptions(
|
|
1236
1179
|
"GcsStorage",
|
|
1237
1180
|
options,
|
|
1238
1181
|
GCS_STORAGE_OPTION_KEYS,
|
|
1239
1182
|
"a @google-cloud/storage client goes in `client`"
|
|
1240
1183
|
);
|
|
1241
1184
|
if (typeof options.bucket !== "string" || options.bucket.length === 0) {
|
|
1242
|
-
throw new
|
|
1185
|
+
throw new ValidationError5("GcsStorage needs a `bucket`: the name of one, a non-empty string");
|
|
1243
1186
|
}
|
|
1244
1187
|
if (options.now !== void 0 && typeof options.now !== "function") {
|
|
1245
|
-
throw new
|
|
1188
|
+
throw new ValidationError5(
|
|
1246
1189
|
"GcsStorage's `now` must be a function that returns epoch milliseconds"
|
|
1247
1190
|
);
|
|
1248
1191
|
}
|
|
1249
1192
|
if (options.client !== void 0 && options.client !== null) {
|
|
1250
1193
|
const ignored = CLIENT_SETTINGS.filter((k) => options[k] !== void 0);
|
|
1251
1194
|
if (ignored.length > 0) {
|
|
1252
|
-
throw new
|
|
1195
|
+
throw new ValidationError5(
|
|
1253
1196
|
`GcsStorage takes \`client\` OR ${CLIENT_SETTINGS.map((k) => `\`${k}\``).join(" / ")}, not both \u2014 got \`client\` with ${ignored.map((k) => `\`${k}\``).join(", ")}; the \`client\` already carries them, so configure them on the client, or drop \`client\``
|
|
1254
1197
|
);
|
|
1255
1198
|
}
|
|
1256
1199
|
this.client = options.client;
|
|
1257
1200
|
readClient = downloadClient(options.client, true);
|
|
1258
1201
|
} else {
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
};
|
|
1263
|
-
this.client = new GcsClient(settings);
|
|
1202
|
+
this.client = new GcsClient(
|
|
1203
|
+
defined({ projectId: options.projectId, apiEndpoint: options.apiEndpoint })
|
|
1204
|
+
);
|
|
1264
1205
|
readClient = downloadClient(this.client, false);
|
|
1265
1206
|
}
|
|
1266
1207
|
const shared = {
|
|
1267
1208
|
storage: this.client,
|
|
1268
1209
|
readStorage: readClient,
|
|
1269
1210
|
bucket: options.bucket,
|
|
1270
|
-
...
|
|
1271
|
-
...options.readTimeoutMs === void 0 ? {} : { readTimeoutMs: options.readTimeoutMs }
|
|
1211
|
+
...defined({ prefix: options.prefix, readTimeoutMs: options.readTimeoutMs })
|
|
1272
1212
|
};
|
|
1273
1213
|
this.storage = new GcsStorageDriver({
|
|
1274
1214
|
...shared,
|
|
1275
|
-
...
|
|
1276
|
-
|
|
1277
|
-
|
|
1215
|
+
...defined({
|
|
1216
|
+
maxObjectBytes: options.maxObjectBytes,
|
|
1217
|
+
simpleUploadThresholdBytes: options.simpleUploadThresholdBytes,
|
|
1218
|
+
conditionalDelete: options.conditionalDelete
|
|
1219
|
+
})
|
|
1278
1220
|
});
|
|
1279
1221
|
this.registry = new GcsRegistryDriver({
|
|
1280
1222
|
...shared,
|
|
1281
|
-
...
|
|
1282
|
-
...options.conditionalDelete === void 0 ? {} : { conditionalDelete: options.conditionalDelete }
|
|
1223
|
+
...defined({ now: options.now, conditionalDelete: options.conditionalDelete })
|
|
1283
1224
|
});
|
|
1284
1225
|
brandAsBackend(this);
|
|
1285
1226
|
}
|