@cloudbitmaps/gcs 0.19.1 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/backend.d.ts +7 -4
- package/dist/bucket-presence.d.ts +2 -2
- package/dist/defined.d.ts +8 -0
- package/dist/gcs-errors.d.ts +15 -1
- package/dist/index.js +187 -181
- package/dist/index.js.map +4 -4
- package/dist/registry.d.ts +3 -3
- package/dist/storage.d.ts +30 -17
- package/package.json +2 -2
- package/dist/keys.d.ts +0 -31
package/dist/index.js
CHANGED
|
@@ -1,16 +1,33 @@
|
|
|
1
1
|
// src/backend.ts
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
ValidationError as ValidationError5,
|
|
4
|
+
brandAsBackend,
|
|
5
|
+
refuseUnknownOptions
|
|
6
|
+
} from "@cloudbitmaps/core/driver-kit";
|
|
3
7
|
import { Storage as GcsClient } from "@google-cloud/storage";
|
|
4
8
|
|
|
9
|
+
// src/defined.ts
|
|
10
|
+
function defined(values) {
|
|
11
|
+
return Object.fromEntries(Object.entries(values).filter(([, v]) => v !== void 0));
|
|
12
|
+
}
|
|
13
|
+
|
|
5
14
|
// src/storage.ts
|
|
6
15
|
import {
|
|
7
16
|
NotFoundError,
|
|
8
17
|
TransientError as TransientError2,
|
|
9
18
|
ValidationError as ValidationError3,
|
|
10
19
|
WriteConflictError,
|
|
20
|
+
concatBytes,
|
|
11
21
|
isNotFoundError,
|
|
12
22
|
isValidationError,
|
|
13
|
-
isWriteConflictError
|
|
23
|
+
isWriteConflictError,
|
|
24
|
+
normalizeObjectPrefix,
|
|
25
|
+
parseGenerationFromKey,
|
|
26
|
+
requireBool,
|
|
27
|
+
requirePositiveInt,
|
|
28
|
+
segmentObjectPrefix,
|
|
29
|
+
storageObjectKey,
|
|
30
|
+
toTransient
|
|
14
31
|
} from "@cloudbitmaps/core/driver-kit";
|
|
15
32
|
import { createHash, randomBytes } from "node:crypto";
|
|
16
33
|
import { once as once2 } from "node:events";
|
|
@@ -19,6 +36,10 @@ import { once as once2 } from "node:events";
|
|
|
19
36
|
import { TransientError } from "@cloudbitmaps/core/driver-kit";
|
|
20
37
|
|
|
21
38
|
// src/gcs-errors.ts
|
|
39
|
+
import {
|
|
40
|
+
ValidationError,
|
|
41
|
+
isTransportFault as isNodeTransportFault
|
|
42
|
+
} from "@cloudbitmaps/core/driver-kit";
|
|
22
43
|
function httpStatus(err) {
|
|
23
44
|
const e = err;
|
|
24
45
|
if (typeof e?.code === "number") return e.code;
|
|
@@ -46,6 +67,17 @@ function isMissingBucket(err) {
|
|
|
46
67
|
if (says(e.message)) return true;
|
|
47
68
|
return Array.isArray(e.errors) && e.errors.some((inner) => says(inner?.message));
|
|
48
69
|
}
|
|
70
|
+
function missingBucketError(bucket, cause) {
|
|
71
|
+
return Object.assign(
|
|
72
|
+
new ValidationError(
|
|
73
|
+
`the GCS bucket does not exist: ${bucket}; check the bucket name and the project the client is built for`
|
|
74
|
+
),
|
|
75
|
+
{ cause }
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
function asMissingBucket(err, bucket) {
|
|
79
|
+
return isMissingBucket(err) ? missingBucketError(bucket, err) : err;
|
|
80
|
+
}
|
|
49
81
|
function isInvalidRange(err) {
|
|
50
82
|
return httpStatus(err) === 416;
|
|
51
83
|
}
|
|
@@ -53,22 +85,11 @@ var SDK_RETRIED_STATUSES = [408, 429, 500, 502, 503, 504];
|
|
|
53
85
|
function isSdkConnectionProblem(reason) {
|
|
54
86
|
return reason.includes("eai_again") || reason === "econnreset" || reason === "unexpected connection closure" || reason === "epipe" || reason === "socket connection timeout";
|
|
55
87
|
}
|
|
56
|
-
|
|
57
|
-
"
|
|
58
|
-
|
|
59
|
-
"ECONNABORTED",
|
|
60
|
-
"ETIMEDOUT",
|
|
61
|
-
"ENOTFOUND",
|
|
62
|
-
"EAI_AGAIN",
|
|
63
|
-
"EHOSTUNREACH",
|
|
64
|
-
"ENETUNREACH",
|
|
65
|
-
"EPIPE",
|
|
66
|
-
"ERR_STREAM_PREMATURE_CLOSE"
|
|
67
|
-
]);
|
|
88
|
+
function isConnectionFault(err) {
|
|
89
|
+
return isNodeTransportFault(err) || networkCode(err) === "ERR_STREAM_PREMATURE_CLOSE";
|
|
90
|
+
}
|
|
68
91
|
function isTransportFault(err) {
|
|
69
|
-
|
|
70
|
-
const code = networkCode(err);
|
|
71
|
-
return code !== void 0 && TRANSPORT_CODES.has(code);
|
|
92
|
+
return httpStatus(err) === void 0 && isConnectionFault(err);
|
|
72
93
|
}
|
|
73
94
|
function isDownloadRetryable(err) {
|
|
74
95
|
if (err === null || typeof err !== "object") return false;
|
|
@@ -92,8 +113,11 @@ function isTransient(err) {
|
|
|
92
113
|
const status = httpStatus(err);
|
|
93
114
|
if (status === 408 || status === 429 || status !== void 0 && status >= 500 && status < 600)
|
|
94
115
|
return true;
|
|
95
|
-
|
|
96
|
-
|
|
116
|
+
return isConnectionFault(err);
|
|
117
|
+
}
|
|
118
|
+
function transientFaultMessage(err) {
|
|
119
|
+
const e = err;
|
|
120
|
+
return `transient GCS fault: ${String(e?.code ?? e?.name ?? "unknown")}`;
|
|
97
121
|
}
|
|
98
122
|
|
|
99
123
|
// src/scrub-error.ts
|
|
@@ -186,7 +210,7 @@ var BucketPresence = class {
|
|
|
186
210
|
bucket;
|
|
187
211
|
settled = false;
|
|
188
212
|
asking;
|
|
189
|
-
/** Resolve when a 404 can be read as an object's; reject when the bucket does not exist
|
|
213
|
+
/** Resolve when a 404 can be read as an object's; reject with `ValidationError` when the bucket does not exist, `TransientError` when the listing failed in transit. */
|
|
190
214
|
async confirm() {
|
|
191
215
|
if (this.settled) return;
|
|
192
216
|
this.asking ??= this.ask().finally(() => {
|
|
@@ -205,7 +229,7 @@ var BucketPresence = class {
|
|
|
205
229
|
const err = scrubCredentials(failure.raised);
|
|
206
230
|
const status = err?.code;
|
|
207
231
|
if (status === 404) {
|
|
208
|
-
throw
|
|
232
|
+
throw missingBucketError(this.bucket, err);
|
|
209
233
|
}
|
|
210
234
|
if (isTransient(err)) {
|
|
211
235
|
throw new TransientError(
|
|
@@ -221,34 +245,6 @@ var BucketPresence = class {
|
|
|
221
245
|
}
|
|
222
246
|
};
|
|
223
247
|
|
|
224
|
-
// src/keys.ts
|
|
225
|
-
import {
|
|
226
|
-
ValidationError,
|
|
227
|
-
encodeNameForKey,
|
|
228
|
-
namespaceKeyPart,
|
|
229
|
-
prefixPart,
|
|
230
|
-
validateSegmentRef
|
|
231
|
-
} from "@cloudbitmaps/core/driver-kit";
|
|
232
|
-
import { normalizeObjectPrefix } from "@cloudbitmaps/core/driver-kit";
|
|
233
|
-
var SUFFIX = ".crbm";
|
|
234
|
-
function segmentObjectPrefix(prefix, ref) {
|
|
235
|
-
validateSegmentRef(ref);
|
|
236
|
-
return `${prefixPart(prefix)}${namespaceKeyPart(ref.namespace)}/segments/${encodeNameForKey(ref.segment)}.`;
|
|
237
|
-
}
|
|
238
|
-
function storageObjectName(prefix, key) {
|
|
239
|
-
if (!Number.isInteger(key.generation) || key.generation < 0) {
|
|
240
|
-
throw new ValidationError(`generation must be a non-negative integer; got ${key.generation}`);
|
|
241
|
-
}
|
|
242
|
-
return `${segmentObjectPrefix(prefix, key)}${key.generation}${SUFFIX}`;
|
|
243
|
-
}
|
|
244
|
-
function parseGenerationFromName(segmentPrefix, objectName) {
|
|
245
|
-
if (!objectName.startsWith(segmentPrefix) || !objectName.endsWith(SUFFIX)) return null;
|
|
246
|
-
const middle = objectName.slice(segmentPrefix.length, objectName.length - SUFFIX.length);
|
|
247
|
-
if (!/^(0|[1-9]\d*)$/.test(middle)) return null;
|
|
248
|
-
const generation = Number(middle);
|
|
249
|
-
return Number.isSafeInteger(generation) ? generation : null;
|
|
250
|
-
}
|
|
251
|
-
|
|
252
248
|
// src/read-timeout.ts
|
|
253
249
|
import { ValidationError as ValidationError2 } from "@cloudbitmaps/core/driver-kit";
|
|
254
250
|
var DEFAULT_READ_TIMEOUT_MS = 0;
|
|
@@ -411,13 +407,14 @@ function readOnce(file, options, maxBytes, oversize, deadline) {
|
|
|
411
407
|
}
|
|
412
408
|
|
|
413
409
|
// src/storage.ts
|
|
414
|
-
var
|
|
410
|
+
var GCS_MAX_OBJECT_BYTES = 5 * 1024 ** 4;
|
|
415
411
|
var WRITE_ID_KEY = "cbwid";
|
|
416
412
|
var THROTTLE_RESENDS = 3;
|
|
417
413
|
var THROTTLE_BASE_DELAY_MS = 500;
|
|
418
414
|
var REAL_TIME = {
|
|
419
415
|
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms))
|
|
420
416
|
};
|
|
417
|
+
var REAL_RNG = { next: () => Math.random() };
|
|
421
418
|
var DEFAULT_UPLOAD_THRESHOLD_BYTES = 8 * 1024 * 1024;
|
|
422
419
|
var GcsStorageDriver = class {
|
|
423
420
|
storage;
|
|
@@ -428,33 +425,36 @@ var GcsStorageDriver = class {
|
|
|
428
425
|
threshold;
|
|
429
426
|
readTimeoutMs;
|
|
430
427
|
clock;
|
|
428
|
+
rng;
|
|
431
429
|
presence;
|
|
430
|
+
conditionalDelete;
|
|
432
431
|
constructor(options) {
|
|
433
432
|
this.storage = options.storage;
|
|
433
|
+
this.conditionalDelete = requireBool("conditionalDelete", options.conditionalDelete) ?? false;
|
|
434
434
|
this.clock = options.clock ?? REAL_TIME;
|
|
435
|
+
this.rng = options.rng ?? REAL_RNG;
|
|
435
436
|
this.readStorage = options.readStorage ?? options.storage;
|
|
436
437
|
this.bucket = options.bucket;
|
|
437
438
|
this.prefix = normalizeObjectPrefix(options.prefix);
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
]) {
|
|
442
|
-
if (value !== void 0 && (!Number.isSafeInteger(value) || value < 1)) {
|
|
443
|
-
throw new ValidationError3(`${name} must be a positive safe integer; got ${value}`);
|
|
444
|
-
}
|
|
445
|
-
}
|
|
446
|
-
if (options.maxObjectBytes !== void 0 && options.maxObjectBytes > 5 * 1024 ** 4) {
|
|
439
|
+
requirePositiveInt("maxObjectBytes", options.maxObjectBytes);
|
|
440
|
+
requirePositiveInt("simpleUploadThresholdBytes", options.simpleUploadThresholdBytes);
|
|
441
|
+
if (options.maxObjectBytes !== void 0 && options.maxObjectBytes > GCS_MAX_OBJECT_BYTES) {
|
|
447
442
|
throw new ValidationError3(
|
|
448
|
-
`maxObjectBytes must be at most GCS's 5 TiB object limit (${
|
|
443
|
+
`maxObjectBytes must be at most GCS's 5 TiB object limit (${GCS_MAX_OBJECT_BYTES}); got ${options.maxObjectBytes}`
|
|
449
444
|
);
|
|
450
445
|
}
|
|
451
|
-
this.maxObjectBytes = options.maxObjectBytes ??
|
|
446
|
+
this.maxObjectBytes = options.maxObjectBytes ?? GCS_MAX_OBJECT_BYTES;
|
|
452
447
|
this.threshold = options.simpleUploadThresholdBytes ?? DEFAULT_UPLOAD_THRESHOLD_BYTES;
|
|
453
448
|
this.readTimeoutMs = resolveReadTimeoutMs(options.readTimeoutMs);
|
|
454
449
|
this.presence = new BucketPresence(this.storage, this.bucket);
|
|
455
450
|
}
|
|
456
451
|
capabilities() {
|
|
457
|
-
return {
|
|
452
|
+
return {
|
|
453
|
+
rangeRead: true,
|
|
454
|
+
maxObjectBytes: this.maxObjectBytes,
|
|
455
|
+
conditionalPut: true,
|
|
456
|
+
conditionalDelete: this.conditionalDelete
|
|
457
|
+
};
|
|
458
458
|
}
|
|
459
459
|
file(name) {
|
|
460
460
|
return this.storage.bucket(this.bucket).file(name);
|
|
@@ -468,12 +468,13 @@ var GcsStorageDriver = class {
|
|
|
468
468
|
return startDeadline(this.readTimeoutMs, `${read} of ${key.segment}.${key.generation}`);
|
|
469
469
|
}
|
|
470
470
|
async putImmutable(key, write) {
|
|
471
|
-
const objectName =
|
|
471
|
+
const objectName = storageObjectKey(this.prefix, key);
|
|
472
472
|
const sink = new GcsUploadSink(
|
|
473
473
|
this.file(objectName),
|
|
474
474
|
this.maxObjectBytes,
|
|
475
475
|
this.threshold,
|
|
476
|
-
this.clock
|
|
476
|
+
this.clock,
|
|
477
|
+
this.rng
|
|
477
478
|
);
|
|
478
479
|
try {
|
|
479
480
|
await write(sink);
|
|
@@ -486,6 +487,7 @@ var GcsStorageDriver = class {
|
|
|
486
487
|
);
|
|
487
488
|
}
|
|
488
489
|
if (isValidationError(err) || isWriteConflictError(err) || isNotFoundError(err)) throw err;
|
|
490
|
+
if (isNotFound(err)) await this.presence.confirm();
|
|
489
491
|
throw this.mapError(err);
|
|
490
492
|
}
|
|
491
493
|
}
|
|
@@ -494,7 +496,7 @@ var GcsStorageDriver = class {
|
|
|
494
496
|
throw new ValidationError3(`invalid range offset=${offset} length=${length}`);
|
|
495
497
|
}
|
|
496
498
|
if (length === 0) return new Uint8Array(0);
|
|
497
|
-
const objectName =
|
|
499
|
+
const objectName = storageObjectKey(this.prefix, key);
|
|
498
500
|
try {
|
|
499
501
|
const deadline = this.deadline("range read", key);
|
|
500
502
|
const res = await retryDownload(
|
|
@@ -553,13 +555,15 @@ var GcsStorageDriver = class {
|
|
|
553
555
|
);
|
|
554
556
|
}
|
|
555
557
|
async getTail(key, maxBytes) {
|
|
556
|
-
const objectName =
|
|
558
|
+
const objectName = storageObjectKey(this.prefix, key);
|
|
559
|
+
if (!Number.isSafeInteger(maxBytes)) {
|
|
560
|
+
throw new ValidationError3(`invalid tail length ${String(maxBytes)}`);
|
|
561
|
+
}
|
|
557
562
|
const deadline = this.deadline("tail read", key);
|
|
558
563
|
try {
|
|
559
|
-
if (maxBytes <= 0)
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
throw new ValidationError3(`invalid tail length ${maxBytes}`);
|
|
564
|
+
if (maxBytes <= 0) {
|
|
565
|
+
const meta = await this.metadataOf(objectName, deadline);
|
|
566
|
+
return withVersion({ bytes: new Uint8Array(0), size: meta.size }, meta.version);
|
|
563
567
|
}
|
|
564
568
|
let res;
|
|
565
569
|
try {
|
|
@@ -575,25 +579,32 @@ var GcsStorageDriver = class {
|
|
|
575
579
|
);
|
|
576
580
|
} catch (err) {
|
|
577
581
|
if (!isInvalidRange(err)) throw err;
|
|
578
|
-
const
|
|
579
|
-
if (size !== 0) throw err;
|
|
580
|
-
return { bytes: new Uint8Array(0), size };
|
|
582
|
+
const meta = await this.metadataOf(objectName, deadline);
|
|
583
|
+
if (meta.size !== 0) throw err;
|
|
584
|
+
return withVersion({ bytes: new Uint8Array(0), size: 0 }, meta.version);
|
|
581
585
|
}
|
|
582
586
|
if (res.bytes.length === 0) {
|
|
583
|
-
const
|
|
584
|
-
if (size !== 0)
|
|
585
|
-
|
|
587
|
+
const meta = await this.metadataOf(objectName, deadline);
|
|
588
|
+
if (meta.size !== 0) {
|
|
589
|
+
throw this.badTail(key, `no bytes returned for an object of ${meta.size}B`);
|
|
590
|
+
}
|
|
591
|
+
return withVersion({ bytes: res.bytes, size: 0 }, meta.version);
|
|
586
592
|
}
|
|
587
|
-
|
|
593
|
+
const size = this.tailSize(res, maxBytes, key);
|
|
594
|
+
return withVersion(
|
|
595
|
+
{ bytes: res.bytes, size },
|
|
596
|
+
objectGeneration(singleHeader(res.headers, "x-goog-generation"))
|
|
597
|
+
);
|
|
588
598
|
} catch (err) {
|
|
589
599
|
throw await this.readFailure(err, key);
|
|
590
600
|
}
|
|
591
601
|
}
|
|
592
602
|
/**
|
|
593
|
-
* The object's size from its metadata, validated
|
|
594
|
-
* the calling read's `deadline` ({@link withDeadline}), so
|
|
603
|
+
* The object's size from its metadata, validated, and its generation when the metadata names a usable one. Read on the
|
|
604
|
+
* download client and retried as a download is, within the calling read's `deadline` ({@link withDeadline}), so
|
|
605
|
+
* nothing but the one request in flight runs past it.
|
|
595
606
|
*/
|
|
596
|
-
async
|
|
607
|
+
async metadataOf(objectName, deadline) {
|
|
597
608
|
const [meta] = await retryDownload(
|
|
598
609
|
() => withDeadline(() => this.downloadable(objectName).getMetadata(), deadline),
|
|
599
610
|
deadline
|
|
@@ -602,7 +613,12 @@ var GcsStorageDriver = class {
|
|
|
602
613
|
if (!Number.isSafeInteger(size) || size < 0) {
|
|
603
614
|
throw new ValidationError3(`GCS returned an invalid object size: ${String(meta.size)}`);
|
|
604
615
|
}
|
|
605
|
-
return
|
|
616
|
+
return {
|
|
617
|
+
size,
|
|
618
|
+
version: objectGeneration(
|
|
619
|
+
meta.generation === void 0 ? void 0 : String(meta.generation)
|
|
620
|
+
)
|
|
621
|
+
};
|
|
606
622
|
}
|
|
607
623
|
/**
|
|
608
624
|
* The object's total size from a tail response, refused unless every header agrees with the bytes received. A 206
|
|
@@ -635,24 +651,49 @@ var GcsStorageDriver = class {
|
|
|
635
651
|
badTail(key, why) {
|
|
636
652
|
return new ValidationError3(`GCS tail read of ${key.segment}.${key.generation} refused: ${why}`);
|
|
637
653
|
}
|
|
638
|
-
async delete(key) {
|
|
654
|
+
async delete(key, options) {
|
|
655
|
+
const objectName = storageObjectKey(this.prefix, key);
|
|
656
|
+
const ifVersion = options?.ifVersion;
|
|
657
|
+
const conditional = ifVersion !== void 0 && this.conditionalDelete;
|
|
639
658
|
try {
|
|
640
|
-
await this.file(
|
|
659
|
+
await this.file(objectName).delete(
|
|
660
|
+
...conditional ? [{ ifGenerationMatch: generationOf(ifVersion, key) }] : []
|
|
661
|
+
);
|
|
641
662
|
} catch (err) {
|
|
642
663
|
if (isNotFound(err)) return this.presence.confirm();
|
|
664
|
+
if (conditional && isPreconditionFailed(err)) {
|
|
665
|
+
if (!await this.exists(objectName)) return;
|
|
666
|
+
throw new WriteConflictError(
|
|
667
|
+
`generation ${key.segment}.${key.generation} is another object than the version given; not deleted`
|
|
668
|
+
);
|
|
669
|
+
}
|
|
643
670
|
throw this.mapError(err);
|
|
644
671
|
}
|
|
645
672
|
}
|
|
673
|
+
/** Whether an object is under the name, from its metadata. A read that fails in any other way throws. */
|
|
674
|
+
async exists(objectName) {
|
|
675
|
+
try {
|
|
676
|
+
await this.downloadable(objectName).getMetadata();
|
|
677
|
+
return true;
|
|
678
|
+
} catch (err) {
|
|
679
|
+
if (isNotFound(err)) {
|
|
680
|
+
await this.presence.confirm();
|
|
681
|
+
return false;
|
|
682
|
+
}
|
|
683
|
+
throw this.mapError(scrubCredentials(err));
|
|
684
|
+
}
|
|
685
|
+
}
|
|
646
686
|
async *list(ref) {
|
|
647
687
|
const prefix = segmentObjectPrefix(this.prefix, ref);
|
|
648
688
|
let files;
|
|
649
689
|
try {
|
|
650
690
|
[files] = await this.storage.bucket(this.bucket).getFiles({ prefix });
|
|
651
691
|
} catch (err) {
|
|
692
|
+
if (isNotFound(err)) throw missingBucketError(this.bucket, scrubCredentials(err));
|
|
652
693
|
throw this.mapError(err);
|
|
653
694
|
}
|
|
654
695
|
for (const f of files) {
|
|
655
|
-
const generation =
|
|
696
|
+
const generation = parseGenerationFromKey(prefix, f.name);
|
|
656
697
|
if (generation !== null) {
|
|
657
698
|
yield { namespace: ref.namespace, segment: ref.segment, generation };
|
|
658
699
|
}
|
|
@@ -673,53 +714,47 @@ var GcsStorageDriver = class {
|
|
|
673
714
|
return new ValidationError3(`range out of bounds for ${key.segment}.${key.generation}`);
|
|
674
715
|
}
|
|
675
716
|
if (err instanceof ReadTimedOut) return new TransientError2(err.message, { cause: err });
|
|
676
|
-
if (isTransportFault(err)) {
|
|
677
|
-
return new TransientError2(
|
|
678
|
-
`transient GCS fault: ${String(err.code)}`,
|
|
679
|
-
{
|
|
680
|
-
cause: err
|
|
681
|
-
}
|
|
682
|
-
);
|
|
683
|
-
}
|
|
684
717
|
return this.mapError(err);
|
|
685
718
|
}
|
|
686
719
|
/**
|
|
687
|
-
* Reclassify a transient GCS fault (throttle/5xx/dropped connection) as a retryable {@link TransientError},
|
|
688
|
-
*
|
|
689
|
-
* everything else propagates unchanged. The final fallback at every
|
|
690
|
-
* in transit arrive typed, and anything else (a checksum mismatch, a
|
|
720
|
+
* Reclassify a transient GCS fault (throttle/5xx/dropped connection) as a retryable {@link TransientError}, so the
|
|
721
|
+
* store's read retry can ride it out and a write's caller can tell it from a deterministic failure, and a missing
|
|
722
|
+
* bucket as a `ValidationError` that names it; everything else propagates unchanged. The final fallback at every
|
|
723
|
+
* client-call site: an HTTP status and a fault in transit arrive typed, and anything else (a checksum mismatch, a
|
|
724
|
+
* credentials failure) as the SDK raised it.
|
|
691
725
|
*/
|
|
692
726
|
mapError(err) {
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
return new TransientError2(
|
|
696
|
-
`transient GCS fault: ${err?.code ?? "unknown"}`,
|
|
697
|
-
{ cause: err }
|
|
698
|
-
);
|
|
699
|
-
}
|
|
700
|
-
return err;
|
|
727
|
+
const scrubbed = asMissingBucket(scrubCredentials(err), this.bucket);
|
|
728
|
+
return toTransient(scrubbed, isTransient, transientFaultMessage);
|
|
701
729
|
}
|
|
702
730
|
};
|
|
703
|
-
function
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
731
|
+
function withVersion(tail, version) {
|
|
732
|
+
return version === void 0 ? tail : { ...tail, version };
|
|
733
|
+
}
|
|
734
|
+
function objectGeneration(value) {
|
|
735
|
+
return value !== void 0 && /^[1-9]\d*$/.test(value) && Number.isSafeInteger(Number(value)) ? value : void 0;
|
|
736
|
+
}
|
|
737
|
+
function generationOf(version, key) {
|
|
738
|
+
if (objectGeneration(version) === void 0) {
|
|
739
|
+
throw new ValidationError3(
|
|
740
|
+
`ifVersion for ${key.segment}.${key.generation} is not a GCS object generation: ${version}`
|
|
741
|
+
);
|
|
709
742
|
}
|
|
710
|
-
return
|
|
743
|
+
return Number(version);
|
|
711
744
|
}
|
|
712
745
|
var GcsUploadSink = class _GcsUploadSink {
|
|
713
|
-
constructor(file, maxObjectBytes, threshold, clock) {
|
|
746
|
+
constructor(file, maxObjectBytes, threshold, clock, rng) {
|
|
714
747
|
this.file = file;
|
|
715
748
|
this.maxObjectBytes = maxObjectBytes;
|
|
716
749
|
this.threshold = threshold;
|
|
717
750
|
this.clock = clock;
|
|
751
|
+
this.rng = rng;
|
|
718
752
|
}
|
|
719
753
|
file;
|
|
720
754
|
maxObjectBytes;
|
|
721
755
|
threshold;
|
|
722
756
|
clock;
|
|
757
|
+
rng;
|
|
723
758
|
hash = createHash("sha256");
|
|
724
759
|
total = 0;
|
|
725
760
|
buffered = [];
|
|
@@ -773,7 +808,7 @@ var GcsUploadSink = class _GcsUploadSink {
|
|
|
773
808
|
this.failure ??= e;
|
|
774
809
|
});
|
|
775
810
|
this.done.catch(() => void 0);
|
|
776
|
-
const buf = concatBytes(this.buffered
|
|
811
|
+
const buf = concatBytes(this.buffered);
|
|
777
812
|
this.buffered.length = 0;
|
|
778
813
|
this.bufferedLen = 0;
|
|
779
814
|
stream.write(buf);
|
|
@@ -781,7 +816,7 @@ var GcsUploadSink = class _GcsUploadSink {
|
|
|
781
816
|
async finish() {
|
|
782
817
|
const sha256 = this.hash.digest("hex");
|
|
783
818
|
if (this.stream === void 0) {
|
|
784
|
-
await this.sendSimple(concatBytes(this.buffered
|
|
819
|
+
await this.sendSimple(concatBytes(this.buffered));
|
|
785
820
|
return { size: this.total, sha256 };
|
|
786
821
|
}
|
|
787
822
|
try {
|
|
@@ -817,7 +852,7 @@ var GcsUploadSink = class _GcsUploadSink {
|
|
|
817
852
|
{ cause: err }
|
|
818
853
|
);
|
|
819
854
|
}
|
|
820
|
-
await this.clock.sleep(Math.floor(
|
|
855
|
+
await this.clock.sleep(Math.floor(this.rng.next() * THROTTLE_BASE_DELAY_MS * 2 ** resent));
|
|
821
856
|
}
|
|
822
857
|
}
|
|
823
858
|
}
|
|
@@ -842,9 +877,10 @@ import {
|
|
|
842
877
|
MAX_ROW_BYTES,
|
|
843
878
|
ObjectStoreRegistry,
|
|
844
879
|
TransientError as TransientError3,
|
|
845
|
-
ValidationError as ValidationError4,
|
|
846
880
|
WriteConflictError as WriteConflictError2,
|
|
847
|
-
normalizeObjectPrefix as normalizeObjectPrefix2
|
|
881
|
+
normalizeObjectPrefix as normalizeObjectPrefix2,
|
|
882
|
+
requireBool as requireBool2,
|
|
883
|
+
toTransient as toTransient2
|
|
848
884
|
} from "@cloudbitmaps/core/driver-kit";
|
|
849
885
|
|
|
850
886
|
// src/crc32c.ts
|
|
@@ -912,7 +948,7 @@ var GcsRegistryStore = class {
|
|
|
912
948
|
await this.presence.confirm();
|
|
913
949
|
return null;
|
|
914
950
|
}
|
|
915
|
-
throw mapReadError(err);
|
|
951
|
+
throw mapReadError(err, this.bucket);
|
|
916
952
|
}
|
|
917
953
|
if (res.status !== 200) {
|
|
918
954
|
throw new IntegrityError(`registry read answered HTTP ${res.status}, not 200: ${key}`);
|
|
@@ -940,7 +976,8 @@ var GcsRegistryStore = class {
|
|
|
940
976
|
if (isPreconditionFailed(err)) {
|
|
941
977
|
throw new WriteConflictError2(`registry OCC conflict for ${key}`);
|
|
942
978
|
}
|
|
943
|
-
|
|
979
|
+
if (isNotFound(err)) await this.presence.confirm();
|
|
980
|
+
throw mapError(err, this.bucket);
|
|
944
981
|
}
|
|
945
982
|
}
|
|
946
983
|
async delete(key, expect) {
|
|
@@ -951,7 +988,7 @@ var GcsRegistryStore = class {
|
|
|
951
988
|
if (isNotFound(err)) await this.presence.confirm();
|
|
952
989
|
throw new WriteConflictError2(`registry OCC conflict deleting ${key}`);
|
|
953
990
|
}
|
|
954
|
-
throw mapError(err);
|
|
991
|
+
throw mapError(err, this.bucket);
|
|
955
992
|
}
|
|
956
993
|
}
|
|
957
994
|
async *listKeys(prefix) {
|
|
@@ -962,7 +999,8 @@ var GcsRegistryStore = class {
|
|
|
962
999
|
try {
|
|
963
1000
|
[files, next] = await this.storage.bucket(this.bucket).getFiles({ prefix, autoPaginate: false, maxResults: 1e3, pageToken });
|
|
964
1001
|
} catch (err) {
|
|
965
|
-
throw
|
|
1002
|
+
if (isNotFound(err)) throw missingBucketError(this.bucket, scrubCredentials(err));
|
|
1003
|
+
throw mapError(err, this.bucket);
|
|
966
1004
|
}
|
|
967
1005
|
for (const f of files ?? []) yield f.name;
|
|
968
1006
|
if (next?.pageToken !== void 0 && next.pageToken === pageToken) {
|
|
@@ -979,34 +1017,17 @@ function generationFence(version, key) {
|
|
|
979
1017
|
}
|
|
980
1018
|
return generation;
|
|
981
1019
|
}
|
|
982
|
-
function mapReadError(err) {
|
|
1020
|
+
function mapReadError(err, bucket) {
|
|
983
1021
|
err = scrubCredentials(err);
|
|
984
1022
|
if (err instanceof ReadTimedOut) return new TransientError3(err.message, { cause: err });
|
|
985
|
-
|
|
986
|
-
return new TransientError3(`transient GCS fault: ${String(err.code)}`, {
|
|
987
|
-
cause: err
|
|
988
|
-
});
|
|
989
|
-
}
|
|
990
|
-
return mapError(err);
|
|
1023
|
+
return mapError(err, bucket);
|
|
991
1024
|
}
|
|
992
|
-
function mapError(err) {
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
);
|
|
999
|
-
}
|
|
1000
|
-
return err;
|
|
1001
|
-
}
|
|
1002
|
-
function resolveConditionalDelete(value) {
|
|
1003
|
-
if (value === void 0) {
|
|
1004
|
-
return false;
|
|
1005
|
-
}
|
|
1006
|
-
if (typeof value !== "boolean") {
|
|
1007
|
-
throw new ValidationError4(`conditionalDelete must be a boolean; got ${String(value)}`);
|
|
1008
|
-
}
|
|
1009
|
-
return value;
|
|
1025
|
+
function mapError(err, bucket) {
|
|
1026
|
+
return toTransient2(
|
|
1027
|
+
asMissingBucket(scrubCredentials(err), bucket),
|
|
1028
|
+
isTransient,
|
|
1029
|
+
transientFaultMessage
|
|
1030
|
+
);
|
|
1010
1031
|
}
|
|
1011
1032
|
var GcsRegistryDriver = class extends ObjectStoreRegistry {
|
|
1012
1033
|
constructor(options) {
|
|
@@ -1016,7 +1037,7 @@ var GcsRegistryDriver = class extends ObjectStoreRegistry {
|
|
|
1016
1037
|
options.readStorage ?? options.storage,
|
|
1017
1038
|
options.bucket,
|
|
1018
1039
|
resolveReadTimeoutMs(options.readTimeoutMs),
|
|
1019
|
-
|
|
1040
|
+
requireBool2("conditionalDelete", options.conditionalDelete) ?? false
|
|
1020
1041
|
),
|
|
1021
1042
|
normalizeObjectPrefix2(options.prefix),
|
|
1022
1043
|
options.now ?? (() => Date.now())
|
|
@@ -1025,7 +1046,7 @@ var GcsRegistryDriver = class extends ObjectStoreRegistry {
|
|
|
1025
1046
|
};
|
|
1026
1047
|
|
|
1027
1048
|
// src/read-client.ts
|
|
1028
|
-
import { ValidationError as
|
|
1049
|
+
import { ValidationError as ValidationError4 } from "@cloudbitmaps/core/driver-kit";
|
|
1029
1050
|
function downloadClient(client, supplied) {
|
|
1030
1051
|
if (!isSdkClient(client)) {
|
|
1031
1052
|
if (client.retryOptions?.autoRetry === false) return client;
|
|
@@ -1071,12 +1092,12 @@ function isSdkClient(client) {
|
|
|
1071
1092
|
return client.authClient !== void 0 && typeof factory === "function" && factory.authClient === client.authClient;
|
|
1072
1093
|
}
|
|
1073
1094
|
function overridesBucket(client) {
|
|
1074
|
-
let
|
|
1095
|
+
let defined2 = 0;
|
|
1075
1096
|
for (let proto = Object.getPrototypeOf(client); proto !== null; ) {
|
|
1076
|
-
if (Object.hasOwn(proto, "bucket"))
|
|
1097
|
+
if (Object.hasOwn(proto, "bucket")) defined2++;
|
|
1077
1098
|
proto = Object.getPrototypeOf(proto);
|
|
1078
1099
|
}
|
|
1079
|
-
return
|
|
1100
|
+
return defined2 > 1;
|
|
1080
1101
|
}
|
|
1081
1102
|
function twinOf(client) {
|
|
1082
1103
|
const Own = client.constructor;
|
|
@@ -1126,7 +1147,7 @@ function unfaithful(client, twin) {
|
|
|
1126
1147
|
return void 0;
|
|
1127
1148
|
}
|
|
1128
1149
|
function refusal(supplied, why, remedy, cause) {
|
|
1129
|
-
const err = new
|
|
1150
|
+
const err = new ValidationError4(
|
|
1130
1151
|
(supplied ? `GcsStorage cannot send the supplied client's downloads without the SDK's retries: ${why}` : `GcsStorage cannot build a client that sends downloads without the SDK's retries: ${why}`) + (remedy === void 0 ? "" : `. ${remedy}`)
|
|
1131
1152
|
);
|
|
1132
1153
|
if (cause !== void 0) err.cause = scrubCredentials(cause);
|
|
@@ -1147,20 +1168,6 @@ var GCS_STORAGE_OPTION_KEYS = [
|
|
|
1147
1168
|
"conditionalDelete"
|
|
1148
1169
|
];
|
|
1149
1170
|
var CLIENT_SETTINGS = ["projectId", "apiEndpoint"];
|
|
1150
|
-
function refuseUnknown(name, options, keys, hint) {
|
|
1151
|
-
if (options === null || typeof options !== "object") {
|
|
1152
|
-
throw new ValidationError6(
|
|
1153
|
-
`${name} needs an options object \u2014 got ${options === null ? "null" : typeof options}`
|
|
1154
|
-
);
|
|
1155
|
-
}
|
|
1156
|
-
const unknown = Object.keys(options).filter((k) => !keys.includes(k));
|
|
1157
|
-
if (unknown.length > 0) {
|
|
1158
|
-
const list = (ks) => ks.map((k) => `\`${k}\``).join(", ");
|
|
1159
|
-
throw new ValidationError6(
|
|
1160
|
-
`${name} does not take ${list(unknown)}. It takes ${list(keys)}; ${hint}.`
|
|
1161
|
-
);
|
|
1162
|
-
}
|
|
1163
|
-
}
|
|
1164
1171
|
var GcsStorage = class {
|
|
1165
1172
|
storage;
|
|
1166
1173
|
registry;
|
|
@@ -1168,53 +1175,52 @@ var GcsStorage = class {
|
|
|
1168
1175
|
client;
|
|
1169
1176
|
constructor(options) {
|
|
1170
1177
|
let readClient;
|
|
1171
|
-
|
|
1178
|
+
refuseUnknownOptions(
|
|
1172
1179
|
"GcsStorage",
|
|
1173
1180
|
options,
|
|
1174
1181
|
GCS_STORAGE_OPTION_KEYS,
|
|
1175
1182
|
"a @google-cloud/storage client goes in `client`"
|
|
1176
1183
|
);
|
|
1177
1184
|
if (typeof options.bucket !== "string" || options.bucket.length === 0) {
|
|
1178
|
-
throw new
|
|
1185
|
+
throw new ValidationError5("GcsStorage needs a `bucket`: the name of one, a non-empty string");
|
|
1179
1186
|
}
|
|
1180
1187
|
if (options.now !== void 0 && typeof options.now !== "function") {
|
|
1181
|
-
throw new
|
|
1188
|
+
throw new ValidationError5(
|
|
1182
1189
|
"GcsStorage's `now` must be a function that returns epoch milliseconds"
|
|
1183
1190
|
);
|
|
1184
1191
|
}
|
|
1185
1192
|
if (options.client !== void 0 && options.client !== null) {
|
|
1186
1193
|
const ignored = CLIENT_SETTINGS.filter((k) => options[k] !== void 0);
|
|
1187
1194
|
if (ignored.length > 0) {
|
|
1188
|
-
throw new
|
|
1195
|
+
throw new ValidationError5(
|
|
1189
1196
|
`GcsStorage takes \`client\` OR ${CLIENT_SETTINGS.map((k) => `\`${k}\``).join(" / ")}, not both \u2014 got \`client\` with ${ignored.map((k) => `\`${k}\``).join(", ")}; the \`client\` already carries them, so configure them on the client, or drop \`client\``
|
|
1190
1197
|
);
|
|
1191
1198
|
}
|
|
1192
1199
|
this.client = options.client;
|
|
1193
1200
|
readClient = downloadClient(options.client, true);
|
|
1194
1201
|
} else {
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
};
|
|
1199
|
-
this.client = new GcsClient(settings);
|
|
1202
|
+
this.client = new GcsClient(
|
|
1203
|
+
defined({ projectId: options.projectId, apiEndpoint: options.apiEndpoint })
|
|
1204
|
+
);
|
|
1200
1205
|
readClient = downloadClient(this.client, false);
|
|
1201
1206
|
}
|
|
1202
1207
|
const shared = {
|
|
1203
1208
|
storage: this.client,
|
|
1204
1209
|
readStorage: readClient,
|
|
1205
1210
|
bucket: options.bucket,
|
|
1206
|
-
...
|
|
1207
|
-
...options.readTimeoutMs === void 0 ? {} : { readTimeoutMs: options.readTimeoutMs }
|
|
1211
|
+
...defined({ prefix: options.prefix, readTimeoutMs: options.readTimeoutMs })
|
|
1208
1212
|
};
|
|
1209
1213
|
this.storage = new GcsStorageDriver({
|
|
1210
1214
|
...shared,
|
|
1211
|
-
...
|
|
1212
|
-
|
|
1215
|
+
...defined({
|
|
1216
|
+
maxObjectBytes: options.maxObjectBytes,
|
|
1217
|
+
simpleUploadThresholdBytes: options.simpleUploadThresholdBytes,
|
|
1218
|
+
conditionalDelete: options.conditionalDelete
|
|
1219
|
+
})
|
|
1213
1220
|
});
|
|
1214
1221
|
this.registry = new GcsRegistryDriver({
|
|
1215
1222
|
...shared,
|
|
1216
|
-
...
|
|
1217
|
-
...options.conditionalDelete === void 0 ? {} : { conditionalDelete: options.conditionalDelete }
|
|
1223
|
+
...defined({ now: options.now, conditionalDelete: options.conditionalDelete })
|
|
1218
1224
|
});
|
|
1219
1225
|
brandAsBackend(this);
|
|
1220
1226
|
}
|