@cloudbitmaps/gcs 0.11.2 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -1
- package/dist/backend.d.ts +29 -3
- package/dist/download-retry.d.ts +8 -2
- package/dist/gcs-errors.d.ts +6 -0
- package/dist/index.js +287 -61
- package/dist/index.js.map +4 -4
- package/dist/read-once.d.ts +16 -1
- package/dist/read-timeout.d.ts +28 -0
- package/dist/registry.d.ts +45 -5
- package/dist/storage.d.ts +32 -2
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
// src/backend.ts
|
|
2
|
-
import { ValidationError as
|
|
2
|
+
import { ValidationError as ValidationError5, brandAsBackend } from "@cloudbitmaps/core/driver-kit";
|
|
3
3
|
import { Storage as GcsClient } from "@google-cloud/storage";
|
|
4
4
|
|
|
5
5
|
// src/storage.ts
|
|
6
6
|
import {
|
|
7
7
|
NotFoundError,
|
|
8
8
|
TransientError,
|
|
9
|
-
ValidationError as
|
|
9
|
+
ValidationError as ValidationError3,
|
|
10
10
|
WriteConflictError,
|
|
11
11
|
isNotFoundError,
|
|
12
12
|
isValidationError,
|
|
@@ -72,6 +72,10 @@ function networkCode(err) {
|
|
|
72
72
|
function isPreconditionFailed(err) {
|
|
73
73
|
return httpStatus(err) === 412;
|
|
74
74
|
}
|
|
75
|
+
function isThrottle(err) {
|
|
76
|
+
const status = httpStatus(err);
|
|
77
|
+
return status === 429 || status === 503;
|
|
78
|
+
}
|
|
75
79
|
function isNotFound(err) {
|
|
76
80
|
return httpStatus(err) === 404;
|
|
77
81
|
}
|
|
@@ -125,18 +129,84 @@ function isTransient(err) {
|
|
|
125
129
|
return net === "ECONNRESET" || net === "ETIMEDOUT" || net === "ECONNREFUSED" || net === "EPIPE" || net === "EAI_AGAIN" || net === "ENOTFOUND";
|
|
126
130
|
}
|
|
127
131
|
|
|
132
|
+
// src/read-timeout.ts
|
|
133
|
+
import { ValidationError as ValidationError2 } from "@cloudbitmaps/core/driver-kit";
|
|
134
|
+
var DEFAULT_READ_TIMEOUT_MS = 0;
|
|
135
|
+
var MAX_TIMER_MS = 2147483647;
|
|
136
|
+
function resolveReadTimeoutMs(value) {
|
|
137
|
+
if (value === void 0) return DEFAULT_READ_TIMEOUT_MS;
|
|
138
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0 || value > MAX_TIMER_MS) {
|
|
139
|
+
throw new ValidationError2(
|
|
140
|
+
`readTimeoutMs must be a non-negative safe integer no larger than ${MAX_TIMER_MS}; got ${describe(value)}`
|
|
141
|
+
);
|
|
142
|
+
}
|
|
143
|
+
return value;
|
|
144
|
+
}
|
|
145
|
+
function describe(value) {
|
|
146
|
+
if (typeof value === "string") return JSON.stringify(value);
|
|
147
|
+
if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") {
|
|
148
|
+
return String(value);
|
|
149
|
+
}
|
|
150
|
+
return value === null ? "null" : typeof value;
|
|
151
|
+
}
|
|
152
|
+
var ReadTimedOut = class extends Error {
|
|
153
|
+
code = "ETIMEDOUT";
|
|
154
|
+
constructor(read, ms, cause) {
|
|
155
|
+
super(`GCS ${read} timed out after ${ms} ms`, cause === void 0 ? void 0 : { cause });
|
|
156
|
+
this.name = "ReadTimedOut";
|
|
157
|
+
}
|
|
158
|
+
};
|
|
159
|
+
var Deadline = class {
|
|
160
|
+
constructor(ms, read) {
|
|
161
|
+
this.ms = ms;
|
|
162
|
+
this.read = read;
|
|
163
|
+
this.endsAt = performance.now() + ms;
|
|
164
|
+
}
|
|
165
|
+
ms;
|
|
166
|
+
read;
|
|
167
|
+
endsAt;
|
|
168
|
+
/** What is left of it, in milliseconds: 0 once it has passed. */
|
|
169
|
+
remaining() {
|
|
170
|
+
return Math.max(0, this.endsAt - performance.now());
|
|
171
|
+
}
|
|
172
|
+
/** The error a read fails with once this has passed; `cause` is the last attempt's own error, if one failed. */
|
|
173
|
+
expired(cause) {
|
|
174
|
+
return new ReadTimedOut(this.read, this.ms, cause);
|
|
175
|
+
}
|
|
176
|
+
};
|
|
177
|
+
function startDeadline(ms, read) {
|
|
178
|
+
return ms === 0 ? void 0 : new Deadline(ms, read);
|
|
179
|
+
}
|
|
180
|
+
function withDeadline(send, deadline) {
|
|
181
|
+
if (deadline === void 0) return send();
|
|
182
|
+
if (deadline.remaining() === 0) return Promise.reject(deadline.expired());
|
|
183
|
+
const request = send();
|
|
184
|
+
let timer;
|
|
185
|
+
const timedOut = new Promise((_, reject) => {
|
|
186
|
+
timer = setTimeout(() => reject(deadline.expired()), deadline.remaining());
|
|
187
|
+
});
|
|
188
|
+
return Promise.race([request, timedOut]).finally(() => clearTimeout(timer));
|
|
189
|
+
}
|
|
190
|
+
|
|
128
191
|
// src/download-retry.ts
|
|
129
192
|
var RETRIES = 3;
|
|
130
193
|
var BASE_DELAY_MS = 100;
|
|
131
194
|
var MAX_DELAY_MS = 1e3;
|
|
132
|
-
|
|
195
|
+
var MIN_RETRY_MS = 10;
|
|
196
|
+
async function retryDownload(download, deadline) {
|
|
133
197
|
for (let attempt = 0; ; attempt++) {
|
|
134
198
|
try {
|
|
135
199
|
return await download();
|
|
136
200
|
} catch (err) {
|
|
201
|
+
if (err instanceof ReadTimedOut) throw err;
|
|
137
202
|
if (attempt >= RETRIES || !isDownloadRetryable(err)) throw err;
|
|
203
|
+
const left = deadline?.remaining() ?? Infinity;
|
|
204
|
+
if (deadline !== void 0 && left < MIN_RETRY_MS) throw deadline.expired(err);
|
|
138
205
|
const ceiling = Math.min(MAX_DELAY_MS, BASE_DELAY_MS * 2 ** attempt);
|
|
139
|
-
|
|
206
|
+
const delay = Math.random() * ceiling;
|
|
207
|
+
await new Promise((resolve) => setTimeout(resolve, Math.min(delay, left)));
|
|
208
|
+
if (deadline !== void 0 && deadline.remaining() < MIN_RETRY_MS)
|
|
209
|
+
throw deadline.expired(err);
|
|
140
210
|
}
|
|
141
211
|
}
|
|
142
212
|
}
|
|
@@ -151,25 +221,41 @@ async function saveOnce(file, body, options) {
|
|
|
151
221
|
}
|
|
152
222
|
|
|
153
223
|
// src/read-once.ts
|
|
224
|
+
var OFF_THE_SHARED_AGENT = {
|
|
225
|
+
request: (reqOpts) => ({ ...reqOpts, forever: false })
|
|
226
|
+
};
|
|
227
|
+
function downloadFile(storage, bucket, name) {
|
|
228
|
+
const file = storage.bucket(bucket).file(name);
|
|
229
|
+
file.interceptors.push(OFF_THE_SHARED_AGENT);
|
|
230
|
+
return file;
|
|
231
|
+
}
|
|
154
232
|
function singleHeader(headers, name) {
|
|
155
233
|
const value = headers[name];
|
|
156
234
|
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
157
235
|
}
|
|
158
|
-
function readOnce(file, options, maxBytes, oversize) {
|
|
236
|
+
function readOnce(file, options, maxBytes, oversize, deadline) {
|
|
237
|
+
if (deadline !== void 0 && deadline.remaining() === 0)
|
|
238
|
+
return Promise.reject(deadline.expired());
|
|
159
239
|
return new Promise((resolve, reject) => {
|
|
160
240
|
const stream = file.createReadStream(options);
|
|
161
241
|
const chunks = [];
|
|
162
242
|
let total = 0;
|
|
163
243
|
let status = 0;
|
|
164
244
|
let headers = {};
|
|
245
|
+
let responded = false;
|
|
165
246
|
let settled = false;
|
|
247
|
+
const timer = deadline === void 0 ? void 0 : setTimeout(() => fail(deadline.expired()), deadline.remaining());
|
|
248
|
+
const release = () => void setImmediate(() => stream.destroy());
|
|
166
249
|
const fail = (err) => {
|
|
167
250
|
if (settled) return;
|
|
168
251
|
settled = true;
|
|
169
|
-
|
|
252
|
+
clearTimeout(timer);
|
|
253
|
+
if (responded) release();
|
|
170
254
|
reject(err);
|
|
171
255
|
};
|
|
172
256
|
stream.on("response", (res) => {
|
|
257
|
+
responded = true;
|
|
258
|
+
if (settled) return release();
|
|
173
259
|
status = typeof res.statusCode === "number" ? res.statusCode : 0;
|
|
174
260
|
headers = res.headers ?? {};
|
|
175
261
|
if (status < 200 || status > 299) return;
|
|
@@ -188,6 +274,7 @@ function readOnce(file, options, maxBytes, oversize) {
|
|
|
188
274
|
stream.on("end", () => {
|
|
189
275
|
if (settled) return;
|
|
190
276
|
settled = true;
|
|
277
|
+
clearTimeout(timer);
|
|
191
278
|
resolve({ status, headers, bytes: new Uint8Array(Buffer.concat(chunks, total)) });
|
|
192
279
|
});
|
|
193
280
|
stream.on(
|
|
@@ -204,6 +291,11 @@ function readOnce(file, options, maxBytes, oversize) {
|
|
|
204
291
|
// src/storage.ts
|
|
205
292
|
var DEFAULT_MAX_OBJECT_BYTES = 5 * 1024 * 1024 * 1024 * 1024;
|
|
206
293
|
var WRITE_ID_KEY = "cbwid";
|
|
294
|
+
var THROTTLE_RESENDS = 3;
|
|
295
|
+
var THROTTLE_BASE_DELAY_MS = 500;
|
|
296
|
+
var REAL_TIME = {
|
|
297
|
+
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms))
|
|
298
|
+
};
|
|
207
299
|
var DEFAULT_UPLOAD_THRESHOLD_BYTES = 8 * 1024 * 1024;
|
|
208
300
|
var GcsStorageDriver = class {
|
|
209
301
|
storage;
|
|
@@ -212,8 +304,11 @@ var GcsStorageDriver = class {
|
|
|
212
304
|
prefix;
|
|
213
305
|
maxObjectBytes;
|
|
214
306
|
threshold;
|
|
307
|
+
readTimeoutMs;
|
|
308
|
+
clock;
|
|
215
309
|
constructor(options) {
|
|
216
310
|
this.storage = options.storage;
|
|
311
|
+
this.clock = options.clock ?? REAL_TIME;
|
|
217
312
|
this.readStorage = options.readStorage ?? options.storage;
|
|
218
313
|
this.bucket = options.bucket;
|
|
219
314
|
this.prefix = normalizeGcsPrefix(options.prefix);
|
|
@@ -222,11 +317,12 @@ var GcsStorageDriver = class {
|
|
|
222
317
|
["simpleUploadThresholdBytes", options.simpleUploadThresholdBytes]
|
|
223
318
|
]) {
|
|
224
319
|
if (value !== void 0 && (!Number.isSafeInteger(value) || value < 1)) {
|
|
225
|
-
throw new
|
|
320
|
+
throw new ValidationError3(`${name} must be a positive safe integer; got ${value}`);
|
|
226
321
|
}
|
|
227
322
|
}
|
|
228
323
|
this.maxObjectBytes = options.maxObjectBytes ?? DEFAULT_MAX_OBJECT_BYTES;
|
|
229
324
|
this.threshold = options.simpleUploadThresholdBytes ?? DEFAULT_UPLOAD_THRESHOLD_BYTES;
|
|
325
|
+
this.readTimeoutMs = resolveReadTimeoutMs(options.readTimeoutMs);
|
|
230
326
|
}
|
|
231
327
|
capabilities() {
|
|
232
328
|
return { rangeRead: true, maxObjectBytes: this.maxObjectBytes, conditionalPut: true };
|
|
@@ -234,13 +330,22 @@ var GcsStorageDriver = class {
|
|
|
234
330
|
file(name) {
|
|
235
331
|
return this.storage.bucket(this.bucket).file(name);
|
|
236
332
|
}
|
|
237
|
-
/** The same object on the download client. */
|
|
333
|
+
/** The same object on the download client, off the SDK's shared agent ({@link downloadFile}). */
|
|
238
334
|
downloadable(name) {
|
|
239
|
-
return this.readStorage
|
|
335
|
+
return downloadFile(this.readStorage, this.bucket, name);
|
|
336
|
+
}
|
|
337
|
+
/** The deadline a read of `key` made now runs under, every attempt included, named for its error; none when off. */
|
|
338
|
+
deadline(read, key) {
|
|
339
|
+
return startDeadline(this.readTimeoutMs, `${read} of ${key.segment}.${key.generation}`);
|
|
240
340
|
}
|
|
241
341
|
async putImmutable(key, write) {
|
|
242
342
|
const objectName = storageObjectName(this.prefix, key);
|
|
243
|
-
const sink = new GcsUploadSink(
|
|
343
|
+
const sink = new GcsUploadSink(
|
|
344
|
+
this.file(objectName),
|
|
345
|
+
this.maxObjectBytes,
|
|
346
|
+
this.threshold,
|
|
347
|
+
this.clock
|
|
348
|
+
);
|
|
244
349
|
try {
|
|
245
350
|
await write(sink);
|
|
246
351
|
return await sink.finish();
|
|
@@ -257,30 +362,75 @@ var GcsStorageDriver = class {
|
|
|
257
362
|
}
|
|
258
363
|
async getRange(key, offset, length) {
|
|
259
364
|
if (!Number.isInteger(offset) || !Number.isInteger(length) || offset < 0 || length < 0) {
|
|
260
|
-
throw new
|
|
365
|
+
throw new ValidationError3(`invalid range offset=${offset} length=${length}`);
|
|
261
366
|
}
|
|
262
367
|
if (length === 0) return new Uint8Array(0);
|
|
263
368
|
const objectName = storageObjectName(this.prefix, key);
|
|
264
369
|
try {
|
|
265
|
-
const
|
|
266
|
-
|
|
370
|
+
const deadline = this.deadline("range read", key);
|
|
371
|
+
const res = await retryDownload(
|
|
372
|
+
() => readOnce(
|
|
373
|
+
this.downloadable(objectName),
|
|
374
|
+
{ start: offset, end: offset + length - 1, decompress: false },
|
|
375
|
+
length,
|
|
376
|
+
() => this.badRange(key, `the response is longer than the ${length}B requested`),
|
|
377
|
+
deadline
|
|
378
|
+
),
|
|
379
|
+
deadline
|
|
267
380
|
);
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
`range [${offset}, ${offset + length}) out of bounds (got ${buf.length}B)`
|
|
271
|
-
);
|
|
272
|
-
}
|
|
273
|
-
return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
381
|
+
this.checkRange(res, key, offset, length);
|
|
382
|
+
return res.bytes;
|
|
274
383
|
} catch (err) {
|
|
275
384
|
throw this.mapReadError(err, key);
|
|
276
385
|
}
|
|
277
386
|
}
|
|
387
|
+
/**
|
|
388
|
+
* Refuse a range response unless it is exactly the bytes asked for. A 206 must name them in `Content-Range: bytes
|
|
389
|
+
* a-b/total`, starting at `offset`, and hold that many; a 200 (a server that ignored the range) is the whole object,
|
|
390
|
+
* which is the range only when the range starts at 0. A short read means the range ran past the end of the object:
|
|
391
|
+
* out of bounds, never a partial result.
|
|
392
|
+
*/
|
|
393
|
+
checkRange(res, key, offset, length) {
|
|
394
|
+
const received = res.bytes.length;
|
|
395
|
+
if (res.status === 200) {
|
|
396
|
+
if (offset !== 0)
|
|
397
|
+
throw this.badRange(key, `whole object returned for a range from ${offset}`);
|
|
398
|
+
} else if (res.status === 206) {
|
|
399
|
+
const header = singleHeader(res.headers, "content-range");
|
|
400
|
+
const m = header === void 0 ? null : /^bytes (\d+)-(\d+)\/(\d+)$/.exec(header);
|
|
401
|
+
if (m === null) throw this.badRange(key, "Content-Range is missing or malformed");
|
|
402
|
+
const [first, last, total] = [Number(m[1]), Number(m[2]), Number(m[3])];
|
|
403
|
+
if (![first, last, total].every(Number.isSafeInteger)) {
|
|
404
|
+
throw this.badRange(key, "Content-Range holds a number that is not a safe integer");
|
|
405
|
+
}
|
|
406
|
+
if (first !== offset || last - first + 1 !== received || last >= total) {
|
|
407
|
+
throw this.badRange(
|
|
408
|
+
key,
|
|
409
|
+
`Content-Range ${first}-${last}/${total} disagrees with ${received}B received from ${offset}`
|
|
410
|
+
);
|
|
411
|
+
}
|
|
412
|
+
} else {
|
|
413
|
+
throw this.badRange(key, `unexpected HTTP ${res.status}`);
|
|
414
|
+
}
|
|
415
|
+
if (received !== length) {
|
|
416
|
+
throw new ValidationError3(
|
|
417
|
+
`range [${offset}, ${offset + length}) out of bounds (got ${received}B)`
|
|
418
|
+
);
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
badRange(key, why) {
|
|
422
|
+
return new ValidationError3(
|
|
423
|
+
`GCS range read of ${key.segment}.${key.generation} refused: ${why}`
|
|
424
|
+
);
|
|
425
|
+
}
|
|
278
426
|
async getTail(key, maxBytes) {
|
|
279
427
|
const objectName = storageObjectName(this.prefix, key);
|
|
428
|
+
const deadline = this.deadline("tail read", key);
|
|
280
429
|
try {
|
|
281
|
-
if (maxBytes <= 0)
|
|
430
|
+
if (maxBytes <= 0)
|
|
431
|
+
return { bytes: new Uint8Array(0), size: await this.sizeOf(objectName, deadline) };
|
|
282
432
|
if (!Number.isSafeInteger(maxBytes)) {
|
|
283
|
-
throw new
|
|
433
|
+
throw new ValidationError3(`invalid tail length ${maxBytes}`);
|
|
284
434
|
}
|
|
285
435
|
let res;
|
|
286
436
|
try {
|
|
@@ -289,17 +439,19 @@ var GcsStorageDriver = class {
|
|
|
289
439
|
this.downloadable(objectName),
|
|
290
440
|
{ end: -maxBytes, decompress: false },
|
|
291
441
|
maxBytes,
|
|
292
|
-
() => this.badTail(key, `the response is longer than the ${maxBytes}B requested`)
|
|
293
|
-
|
|
442
|
+
() => this.badTail(key, `the response is longer than the ${maxBytes}B requested`),
|
|
443
|
+
deadline
|
|
444
|
+
),
|
|
445
|
+
deadline
|
|
294
446
|
);
|
|
295
447
|
} catch (err) {
|
|
296
448
|
if (!isInvalidRange(err)) throw err;
|
|
297
|
-
const size = await this.sizeOf(objectName);
|
|
449
|
+
const size = await this.sizeOf(objectName, deadline);
|
|
298
450
|
if (size !== 0) throw err;
|
|
299
451
|
return { bytes: new Uint8Array(0), size };
|
|
300
452
|
}
|
|
301
453
|
if (res.bytes.length === 0) {
|
|
302
|
-
const size = await this.sizeOf(objectName);
|
|
454
|
+
const size = await this.sizeOf(objectName, deadline);
|
|
303
455
|
if (size !== 0) throw this.badTail(key, `no bytes returned for an object of ${size}B`);
|
|
304
456
|
return { bytes: res.bytes, size };
|
|
305
457
|
}
|
|
@@ -308,12 +460,18 @@ var GcsStorageDriver = class {
|
|
|
308
460
|
throw this.mapReadError(err, key);
|
|
309
461
|
}
|
|
310
462
|
}
|
|
311
|
-
/**
|
|
312
|
-
|
|
313
|
-
|
|
463
|
+
/**
|
|
464
|
+
* The object's size from its metadata, validated. Read on the download client and retried as a download is, within
|
|
465
|
+
* the calling read's `deadline` ({@link withDeadline}), so nothing but the one request in flight runs past it.
|
|
466
|
+
*/
|
|
467
|
+
async sizeOf(objectName, deadline) {
|
|
468
|
+
const [meta] = await retryDownload(
|
|
469
|
+
() => withDeadline(() => this.downloadable(objectName).getMetadata(), deadline),
|
|
470
|
+
deadline
|
|
471
|
+
);
|
|
314
472
|
const size = Number(meta.size ?? 0);
|
|
315
473
|
if (!Number.isSafeInteger(size) || size < 0) {
|
|
316
|
-
throw new
|
|
474
|
+
throw new ValidationError3(`GCS returned an invalid object size: ${String(meta.size)}`);
|
|
317
475
|
}
|
|
318
476
|
return size;
|
|
319
477
|
}
|
|
@@ -346,7 +504,7 @@ var GcsStorageDriver = class {
|
|
|
346
504
|
return total;
|
|
347
505
|
}
|
|
348
506
|
badTail(key, why) {
|
|
349
|
-
return new
|
|
507
|
+
return new ValidationError3(`GCS tail read of ${key.segment}.${key.generation} refused: ${why}`);
|
|
350
508
|
}
|
|
351
509
|
async delete(key) {
|
|
352
510
|
try {
|
|
@@ -376,8 +534,9 @@ var GcsStorageDriver = class {
|
|
|
376
534
|
if (isNotFound(err))
|
|
377
535
|
return new NotFoundError(`no such generation: ${key.segment}.${key.generation}`);
|
|
378
536
|
if (isInvalidRange(err)) {
|
|
379
|
-
return new
|
|
537
|
+
return new ValidationError3(`range out of bounds for ${key.segment}.${key.generation}`);
|
|
380
538
|
}
|
|
539
|
+
if (err instanceof ReadTimedOut) return new TransientError(err.message, { cause: err });
|
|
381
540
|
if (isTransportFault(err)) {
|
|
382
541
|
return new TransientError(
|
|
383
542
|
`transient GCS fault: ${String(err.code)}`,
|
|
@@ -414,14 +573,16 @@ function concatBytes(parts, total) {
|
|
|
414
573
|
return out;
|
|
415
574
|
}
|
|
416
575
|
var GcsUploadSink = class _GcsUploadSink {
|
|
417
|
-
constructor(file, maxObjectBytes, threshold) {
|
|
576
|
+
constructor(file, maxObjectBytes, threshold, clock) {
|
|
418
577
|
this.file = file;
|
|
419
578
|
this.maxObjectBytes = maxObjectBytes;
|
|
420
579
|
this.threshold = threshold;
|
|
580
|
+
this.clock = clock;
|
|
421
581
|
}
|
|
422
582
|
file;
|
|
423
583
|
maxObjectBytes;
|
|
424
584
|
threshold;
|
|
585
|
+
clock;
|
|
425
586
|
hash = createHash("sha256");
|
|
426
587
|
total = 0;
|
|
427
588
|
buffered = [];
|
|
@@ -436,16 +597,26 @@ var GcsUploadSink = class _GcsUploadSink {
|
|
|
436
597
|
metadata: { contentType: "application/octet-stream" }
|
|
437
598
|
};
|
|
438
599
|
/**
|
|
439
|
-
* This write's own id,
|
|
440
|
-
*
|
|
600
|
+
* This write's own id, stored in the object's custom metadata (outside the `.crbm` bytes) so a conflict can be told
|
|
601
|
+
* from a replay of this same write: the resumable session's, or the simple upload's re-send after a throttle.
|
|
441
602
|
*/
|
|
442
603
|
writeId = randomBytes(16).toString("hex");
|
|
604
|
+
/** The upload options with this write's id in the object's custom metadata. */
|
|
605
|
+
tagged() {
|
|
606
|
+
return {
|
|
607
|
+
..._GcsUploadSink.WRITE_OPTS,
|
|
608
|
+
metadata: {
|
|
609
|
+
..._GcsUploadSink.WRITE_OPTS.metadata,
|
|
610
|
+
metadata: { [WRITE_ID_KEY]: this.writeId }
|
|
611
|
+
}
|
|
612
|
+
};
|
|
613
|
+
}
|
|
443
614
|
async write(bytes) {
|
|
444
615
|
if (this.failure !== void 0) throw this.failure;
|
|
445
616
|
if (bytes.length === 0) return;
|
|
446
617
|
this.total += bytes.length;
|
|
447
618
|
if (this.total > this.maxObjectBytes) {
|
|
448
|
-
throw new
|
|
619
|
+
throw new ValidationError3(`object exceeds maxObjectBytes ${this.maxObjectBytes}`);
|
|
449
620
|
}
|
|
450
621
|
this.hash.update(bytes);
|
|
451
622
|
if (this.stream !== void 0) {
|
|
@@ -458,14 +629,7 @@ var GcsUploadSink = class _GcsUploadSink {
|
|
|
458
629
|
}
|
|
459
630
|
/** Cross into resumable streaming: open the stream, flush the buffered bytes, keep only ~one threshold resident. */
|
|
460
631
|
startResumable() {
|
|
461
|
-
const stream = this.file.createWriteStream({
|
|
462
|
-
resumable: true,
|
|
463
|
-
..._GcsUploadSink.WRITE_OPTS,
|
|
464
|
-
metadata: {
|
|
465
|
-
..._GcsUploadSink.WRITE_OPTS.metadata,
|
|
466
|
-
metadata: { [WRITE_ID_KEY]: this.writeId }
|
|
467
|
-
}
|
|
468
|
-
});
|
|
632
|
+
const stream = this.file.createWriteStream({ resumable: true, ...this.tagged() });
|
|
469
633
|
this.stream = stream;
|
|
470
634
|
this.done = once2(stream, "finish");
|
|
471
635
|
stream.on("error", (e) => {
|
|
@@ -480,11 +644,7 @@ var GcsUploadSink = class _GcsUploadSink {
|
|
|
480
644
|
async finish() {
|
|
481
645
|
const sha256 = this.hash.digest("hex");
|
|
482
646
|
if (this.stream === void 0) {
|
|
483
|
-
await
|
|
484
|
-
this.file,
|
|
485
|
-
concatBytes(this.buffered, this.bufferedLen),
|
|
486
|
-
_GcsUploadSink.WRITE_OPTS
|
|
487
|
-
);
|
|
647
|
+
await this.sendSimple(concatBytes(this.buffered, this.bufferedLen));
|
|
488
648
|
return { size: this.total, sha256 };
|
|
489
649
|
}
|
|
490
650
|
try {
|
|
@@ -496,6 +656,34 @@ var GcsUploadSink = class _GcsUploadSink {
|
|
|
496
656
|
}
|
|
497
657
|
return { size: this.total, sha256 };
|
|
498
658
|
}
|
|
659
|
+
/**
|
|
660
|
+
* Send the simple upload once, and again after a `429` or `503`, up to {@link THROTTLE_RESENDS} times, waiting a
|
|
661
|
+
* full-jitter backoff before each. Any other failure is thrown at once. Once it has been sent again, a `412` can be
|
|
662
|
+
* an earlier send that landed after all, so the object's write id decides: this write's own is a success, and any
|
|
663
|
+
* other object, or one with no id, is the conflict. A failed read-back throws, and is neither. Exhausted, it throws
|
|
664
|
+
* {@link TransientError}; nothing is deleted.
|
|
665
|
+
*/
|
|
666
|
+
async sendSimple(body) {
|
|
667
|
+
for (let resent = 0; ; resent++) {
|
|
668
|
+
try {
|
|
669
|
+
await saveOnce(this.file, body, this.tagged());
|
|
670
|
+
return;
|
|
671
|
+
} catch (err) {
|
|
672
|
+
if (resent > 0 && isPreconditionFailed(err)) {
|
|
673
|
+
if (await this.storedWriteId() === this.writeId) return;
|
|
674
|
+
throw err;
|
|
675
|
+
}
|
|
676
|
+
if (!isThrottle(err)) throw err;
|
|
677
|
+
if (resent >= THROTTLE_RESENDS) {
|
|
678
|
+
throw new TransientError(
|
|
679
|
+
`GCS upload was throttled on each of its ${resent + 1} sends; the object may or may not exist`,
|
|
680
|
+
{ cause: err }
|
|
681
|
+
);
|
|
682
|
+
}
|
|
683
|
+
await this.clock.sleep(Math.floor(Math.random() * THROTTLE_BASE_DELAY_MS * 2 ** resent));
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
}
|
|
499
687
|
/** The id the stored object carries in its custom metadata, or `undefined` when it carries none or is gone. */
|
|
500
688
|
async storedWriteId() {
|
|
501
689
|
try {
|
|
@@ -517,30 +705,36 @@ import {
|
|
|
517
705
|
MAX_ROW_BYTES,
|
|
518
706
|
ObjectStoreRegistry,
|
|
519
707
|
TransientError as TransientError2,
|
|
708
|
+
ValidationError as ValidationError4,
|
|
520
709
|
WriteConflictError as WriteConflictError2,
|
|
521
710
|
normalizeObjectPrefix
|
|
522
711
|
} from "@cloudbitmaps/core/driver-kit";
|
|
523
|
-
var
|
|
524
|
-
constructor(storage, readStorage, bucket) {
|
|
712
|
+
var GcsRegistryStore = class {
|
|
713
|
+
constructor(storage, readStorage, bucket, readTimeoutMs, conditionalDelete) {
|
|
525
714
|
this.storage = storage;
|
|
526
715
|
this.readStorage = readStorage;
|
|
527
716
|
this.bucket = bucket;
|
|
717
|
+
this.readTimeoutMs = readTimeoutMs;
|
|
718
|
+
this.conditionalDelete = conditionalDelete;
|
|
528
719
|
}
|
|
529
720
|
storage;
|
|
530
721
|
readStorage;
|
|
531
722
|
bucket;
|
|
723
|
+
readTimeoutMs;
|
|
724
|
+
conditionalDelete;
|
|
532
725
|
label = "GCS";
|
|
533
726
|
/** A handle on one registry object. */
|
|
534
727
|
file(name) {
|
|
535
728
|
return this.storage.bucket(this.bucket).file(name);
|
|
536
729
|
}
|
|
537
|
-
/** The same file handle on the download client. */
|
|
730
|
+
/** The same file handle on the download client, off the SDK's shared agent ({@link downloadFile}). */
|
|
538
731
|
downloadable(name) {
|
|
539
|
-
return this.readStorage
|
|
732
|
+
return downloadFile(this.readStorage, this.bucket, name);
|
|
540
733
|
}
|
|
541
734
|
async read(key) {
|
|
542
735
|
let res;
|
|
543
736
|
try {
|
|
737
|
+
const deadline = startDeadline(this.readTimeoutMs, `registry read of ${key}`);
|
|
544
738
|
res = await retryDownload(
|
|
545
739
|
() => readOnce(
|
|
546
740
|
this.downloadable(key),
|
|
@@ -548,8 +742,10 @@ var GcsStore = class {
|
|
|
548
742
|
MAX_ROW_BYTES,
|
|
549
743
|
(size) => new IntegrityError(
|
|
550
744
|
size === void 0 ? `registry object exceeds cap ${MAX_ROW_BYTES}B` : `registry object ${size}B exceeds cap ${MAX_ROW_BYTES}B`
|
|
551
|
-
)
|
|
552
|
-
|
|
745
|
+
),
|
|
746
|
+
deadline
|
|
747
|
+
),
|
|
748
|
+
deadline
|
|
553
749
|
);
|
|
554
750
|
} catch (err) {
|
|
555
751
|
if (isNotFound(err)) return null;
|
|
@@ -579,6 +775,16 @@ var GcsStore = class {
|
|
|
579
775
|
throw mapError(err);
|
|
580
776
|
}
|
|
581
777
|
}
|
|
778
|
+
async delete(key, expect) {
|
|
779
|
+
try {
|
|
780
|
+
await this.file(key).delete({ ifGenerationMatch: generationFence(expect.version, key) });
|
|
781
|
+
} catch (err) {
|
|
782
|
+
if (isPreconditionFailed(err) || isNotFound(err)) {
|
|
783
|
+
throw new WriteConflictError2(`registry OCC conflict deleting ${key}`);
|
|
784
|
+
}
|
|
785
|
+
throw mapError(err);
|
|
786
|
+
}
|
|
787
|
+
}
|
|
582
788
|
async *listKeys(prefix) {
|
|
583
789
|
let pageToken;
|
|
584
790
|
do {
|
|
@@ -602,6 +808,7 @@ function generationFence(version, key) {
|
|
|
602
808
|
return generation;
|
|
603
809
|
}
|
|
604
810
|
function mapReadError(err) {
|
|
811
|
+
if (err instanceof ReadTimedOut) return new TransientError2(err.message, { cause: err });
|
|
605
812
|
if (isTransportFault(err)) {
|
|
606
813
|
return new TransientError2(`transient GCS fault: ${String(err.code)}`, {
|
|
607
814
|
cause: err
|
|
@@ -618,10 +825,25 @@ function mapError(err) {
|
|
|
618
825
|
}
|
|
619
826
|
return err;
|
|
620
827
|
}
|
|
828
|
+
function resolveConditionalDelete(value) {
|
|
829
|
+
if (value === void 0) {
|
|
830
|
+
return false;
|
|
831
|
+
}
|
|
832
|
+
if (typeof value !== "boolean") {
|
|
833
|
+
throw new ValidationError4(`conditionalDelete must be a boolean; got ${String(value)}`);
|
|
834
|
+
}
|
|
835
|
+
return value;
|
|
836
|
+
}
|
|
621
837
|
var GcsRegistryDriver = class extends ObjectStoreRegistry {
|
|
622
838
|
constructor(options) {
|
|
623
839
|
super(
|
|
624
|
-
new
|
|
840
|
+
new GcsRegistryStore(
|
|
841
|
+
options.storage,
|
|
842
|
+
options.readStorage ?? options.storage,
|
|
843
|
+
options.bucket,
|
|
844
|
+
resolveReadTimeoutMs(options.readTimeoutMs),
|
|
845
|
+
resolveConditionalDelete(options.conditionalDelete)
|
|
846
|
+
),
|
|
625
847
|
normalizeObjectPrefix(options.prefix),
|
|
626
848
|
options.now ?? (() => Date.now())
|
|
627
849
|
);
|
|
@@ -637,19 +859,21 @@ var GCS_STORAGE_OPTION_KEYS = [
|
|
|
637
859
|
"apiEndpoint",
|
|
638
860
|
"maxObjectBytes",
|
|
639
861
|
"simpleUploadThresholdBytes",
|
|
640
|
-
"now"
|
|
862
|
+
"now",
|
|
863
|
+
"readTimeoutMs",
|
|
864
|
+
"conditionalDelete"
|
|
641
865
|
];
|
|
642
866
|
var CLIENT_SETTINGS = ["projectId", "apiEndpoint"];
|
|
643
867
|
function refuseUnknown(name, options, keys, hint) {
|
|
644
868
|
if (options === null || typeof options !== "object") {
|
|
645
|
-
throw new
|
|
869
|
+
throw new ValidationError5(
|
|
646
870
|
`${name} needs an options object \u2014 got ${options === null ? "null" : typeof options}`
|
|
647
871
|
);
|
|
648
872
|
}
|
|
649
873
|
const unknown = Object.keys(options).filter((k) => !keys.includes(k));
|
|
650
874
|
if (unknown.length > 0) {
|
|
651
875
|
const list = (ks) => ks.map((k) => `\`${k}\``).join(", ");
|
|
652
|
-
throw new
|
|
876
|
+
throw new ValidationError5(
|
|
653
877
|
`${name} does not take ${list(unknown)}. It takes ${list(keys)}; ${hint}.`
|
|
654
878
|
);
|
|
655
879
|
}
|
|
@@ -670,7 +894,7 @@ var GcsStorage = class {
|
|
|
670
894
|
if (options.client !== void 0 && options.client !== null) {
|
|
671
895
|
const ignored = CLIENT_SETTINGS.filter((k) => options[k] !== void 0);
|
|
672
896
|
if (ignored.length > 0) {
|
|
673
|
-
throw new
|
|
897
|
+
throw new ValidationError5(
|
|
674
898
|
`GcsStorage takes \`client\` OR ${CLIENT_SETTINGS.map((k) => `\`${k}\``).join(" / ")}, not both \u2014 got \`client\` with ${ignored.map((k) => `\`${k}\``).join(", ")}; the \`client\` already carries them, so configure them on the client, or drop \`client\``
|
|
675
899
|
);
|
|
676
900
|
}
|
|
@@ -688,7 +912,8 @@ var GcsStorage = class {
|
|
|
688
912
|
storage: this.client,
|
|
689
913
|
readStorage: readClient,
|
|
690
914
|
bucket: options.bucket,
|
|
691
|
-
...options.prefix === void 0 ? {} : { prefix: options.prefix }
|
|
915
|
+
...options.prefix === void 0 ? {} : { prefix: options.prefix },
|
|
916
|
+
...options.readTimeoutMs === void 0 ? {} : { readTimeoutMs: options.readTimeoutMs }
|
|
692
917
|
};
|
|
693
918
|
this.storage = new GcsStorageDriver({
|
|
694
919
|
...shared,
|
|
@@ -697,7 +922,8 @@ var GcsStorage = class {
|
|
|
697
922
|
});
|
|
698
923
|
this.registry = new GcsRegistryDriver({
|
|
699
924
|
...shared,
|
|
700
|
-
...options.now === void 0 ? {} : { now: options.now }
|
|
925
|
+
...options.now === void 0 ? {} : { now: options.now },
|
|
926
|
+
...options.conditionalDelete === void 0 ? {} : { conditionalDelete: options.conditionalDelete }
|
|
701
927
|
});
|
|
702
928
|
brandAsBackend(this);
|
|
703
929
|
}
|