@cloudbitmaps/gcs 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -1
- package/dist/backend.d.ts +14 -10
- package/dist/index.js +449 -229
- package/dist/index.js.map +4 -4
- package/package.json +2 -2
- package/dist/bucket-presence.d.ts +0 -37
- package/dist/crc32c.d.ts +0 -5
- package/dist/download-retry.d.ts +0 -8
- package/dist/gcs-errors.d.ts +0 -46
- package/dist/keys.d.ts +0 -31
- package/dist/read-client.d.ts +0 -6
- package/dist/read-once.d.ts +0 -52
- package/dist/read-timeout.d.ts +0 -28
- package/dist/registry.d.ts +0 -89
- package/dist/scrub-error.d.ts +0 -18
- package/dist/send-once.d.ts +0 -5
- package/dist/storage.d.ts +0 -106
package/dist/index.js
CHANGED
|
@@ -1,16 +1,31 @@
|
|
|
1
1
|
// src/backend.ts
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
ValidationError as ValidationError5,
|
|
4
|
+
brandAsBackend,
|
|
5
|
+
refuseUnknownOptions
|
|
6
|
+
} from "@cloudbitmaps/core/driver-kit";
|
|
3
7
|
import { Storage as GcsClient } from "@google-cloud/storage";
|
|
4
8
|
|
|
9
|
+
// src/defined.ts
|
|
10
|
+
function defined(values) {
|
|
11
|
+
return Object.fromEntries(Object.entries(values).filter(([, v]) => v !== void 0));
|
|
12
|
+
}
|
|
13
|
+
|
|
5
14
|
// src/storage.ts
|
|
6
15
|
import {
|
|
16
|
+
errorKind,
|
|
7
17
|
NotFoundError,
|
|
8
18
|
TransientError as TransientError2,
|
|
9
19
|
ValidationError as ValidationError3,
|
|
10
20
|
WriteConflictError,
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
21
|
+
concatBytes,
|
|
22
|
+
normalizeObjectPrefix,
|
|
23
|
+
parseGenerationFromKey,
|
|
24
|
+
requireBool,
|
|
25
|
+
requirePositiveInt,
|
|
26
|
+
segmentObjectPrefix,
|
|
27
|
+
storageObjectKey,
|
|
28
|
+
toTransient
|
|
14
29
|
} from "@cloudbitmaps/core/driver-kit";
|
|
15
30
|
import { createHash, randomBytes } from "node:crypto";
|
|
16
31
|
import { once as once2 } from "node:events";
|
|
@@ -19,6 +34,10 @@ import { once as once2 } from "node:events";
|
|
|
19
34
|
import { TransientError } from "@cloudbitmaps/core/driver-kit";
|
|
20
35
|
|
|
21
36
|
// src/gcs-errors.ts
|
|
37
|
+
import {
|
|
38
|
+
ValidationError,
|
|
39
|
+
isTransportFault as isNodeTransportFault
|
|
40
|
+
} from "@cloudbitmaps/core/driver-kit";
|
|
22
41
|
function httpStatus(err) {
|
|
23
42
|
const e = err;
|
|
24
43
|
if (typeof e?.code === "number") return e.code;
|
|
@@ -46,6 +65,17 @@ function isMissingBucket(err) {
|
|
|
46
65
|
if (says(e.message)) return true;
|
|
47
66
|
return Array.isArray(e.errors) && e.errors.some((inner) => says(inner?.message));
|
|
48
67
|
}
|
|
68
|
+
function missingBucketError(bucket, cause) {
|
|
69
|
+
return Object.assign(
|
|
70
|
+
new ValidationError(
|
|
71
|
+
`the GCS bucket does not exist: ${bucket}; check the bucket name and the project the client is built for`
|
|
72
|
+
),
|
|
73
|
+
{ cause }
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
function asMissingBucket(err, bucket) {
|
|
77
|
+
return isMissingBucket(err) ? missingBucketError(bucket, err) : err;
|
|
78
|
+
}
|
|
49
79
|
function isInvalidRange(err) {
|
|
50
80
|
return httpStatus(err) === 416;
|
|
51
81
|
}
|
|
@@ -53,22 +83,11 @@ var SDK_RETRIED_STATUSES = [408, 429, 500, 502, 503, 504];
|
|
|
53
83
|
function isSdkConnectionProblem(reason) {
|
|
54
84
|
return reason.includes("eai_again") || reason === "econnreset" || reason === "unexpected connection closure" || reason === "epipe" || reason === "socket connection timeout";
|
|
55
85
|
}
|
|
56
|
-
|
|
57
|
-
"
|
|
58
|
-
|
|
59
|
-
"ECONNABORTED",
|
|
60
|
-
"ETIMEDOUT",
|
|
61
|
-
"ENOTFOUND",
|
|
62
|
-
"EAI_AGAIN",
|
|
63
|
-
"EHOSTUNREACH",
|
|
64
|
-
"ENETUNREACH",
|
|
65
|
-
"EPIPE",
|
|
66
|
-
"ERR_STREAM_PREMATURE_CLOSE"
|
|
67
|
-
]);
|
|
86
|
+
function isConnectionFault(err) {
|
|
87
|
+
return isNodeTransportFault(err) || networkCode(err) === "ERR_STREAM_PREMATURE_CLOSE";
|
|
88
|
+
}
|
|
68
89
|
function isTransportFault(err) {
|
|
69
|
-
|
|
70
|
-
const code = networkCode(err);
|
|
71
|
-
return code !== void 0 && TRANSPORT_CODES.has(code);
|
|
90
|
+
return httpStatus(err) === void 0 && isConnectionFault(err);
|
|
72
91
|
}
|
|
73
92
|
function isDownloadRetryable(err) {
|
|
74
93
|
if (err === null || typeof err !== "object") return false;
|
|
@@ -92,12 +111,16 @@ function isTransient(err) {
|
|
|
92
111
|
const status = httpStatus(err);
|
|
93
112
|
if (status === 408 || status === 429 || status !== void 0 && status >= 500 && status < 600)
|
|
94
113
|
return true;
|
|
95
|
-
|
|
96
|
-
|
|
114
|
+
return isConnectionFault(err);
|
|
115
|
+
}
|
|
116
|
+
function transientFaultMessage(err) {
|
|
117
|
+
const e = err;
|
|
118
|
+
return `transient GCS fault: ${String(e?.code ?? e?.name ?? "unknown")}`;
|
|
97
119
|
}
|
|
98
120
|
|
|
99
121
|
// src/scrub-error.ts
|
|
100
|
-
|
|
122
|
+
import { types } from "node:util";
|
|
123
|
+
var TRANSPORT = /* @__PURE__ */ new Set([
|
|
101
124
|
"request",
|
|
102
125
|
"agent",
|
|
103
126
|
"socket",
|
|
@@ -107,73 +130,349 @@ var DROPPED = /* @__PURE__ */ new Set([
|
|
|
107
130
|
"res",
|
|
108
131
|
"config"
|
|
109
132
|
]);
|
|
110
|
-
var
|
|
111
|
-
var
|
|
133
|
+
var RESPONSE_FIELDS = ["status", "statusText", "statusCode", "statusMessage"];
|
|
134
|
+
var CREDENTIAL_NAMES = [
|
|
135
|
+
"authorization",
|
|
136
|
+
"proxy-authorization",
|
|
137
|
+
"cookie",
|
|
138
|
+
"set-cookie",
|
|
139
|
+
"x-api-key",
|
|
140
|
+
"x-goog-api-key",
|
|
141
|
+
"x-goog-encryption-key",
|
|
142
|
+
"x-goog-copy-source-encryption-key",
|
|
143
|
+
"encryption-key",
|
|
144
|
+
"encryption-key-base64",
|
|
145
|
+
"x-goog-signature",
|
|
146
|
+
"x-goog-credential",
|
|
147
|
+
"signature",
|
|
148
|
+
"signature-provided",
|
|
149
|
+
"canonical-request",
|
|
150
|
+
"token",
|
|
151
|
+
"access-token",
|
|
152
|
+
"refresh-token",
|
|
153
|
+
"id-token",
|
|
154
|
+
"subject-token",
|
|
155
|
+
"client-secret",
|
|
156
|
+
"private-key",
|
|
157
|
+
"assertion"
|
|
158
|
+
];
|
|
159
|
+
var fold = (name) => name.trim().replace(/:$/, "").replace(/[-_]/g, "").toLowerCase();
|
|
160
|
+
var CREDENTIAL_NAME = new Set(CREDENTIAL_NAMES.map(fold));
|
|
161
|
+
var LONGEST_NAME = 64;
|
|
162
|
+
var CREDENTIAL_LINES = /^((?:proxy-)?authorization|cookie|set-cookie|x-api-key|x-goog-api-key|x-goog-(?:copy-source-)?encryption-key):[^\r\n]*/gim;
|
|
163
|
+
var AUTHORIZATION_VALUE = /((?:proxy-)?authorization\\?["']?\s*(?::|=|%3D)\s*\\?["']?)(?:[A-Za-z][\w-]*(?:\s+|%20|\+))?[^\s&"'<>\\,;]+/gi;
|
|
164
|
+
var COOKIE_VALUE = /(?<![\w-])((?:set-)?cookie\\?["']?\s*:\s*\\?["']?)[^\r\n"'<>\\]+/gi;
|
|
165
|
+
var NAMED = "x-goog-(?:copy-source-)?encryption-key|x-goog-api-key|x-goog-signature|x-goog-credential|(?:access|refresh|id|subject)[-_]token|client[-_]secret";
|
|
166
|
+
var NAMED_SECRET = new RegExp(
|
|
167
|
+
`((?:${NAMED})\\\\?["']?\\s*(?::|=|%3D)\\s*\\\\?["']?)[^\\s&"'<>\\\\,;]+`,
|
|
168
|
+
"gi"
|
|
169
|
+
);
|
|
170
|
+
var BEARER = /\b(Bearer)(\s+|%20|\+)(?=[A-Za-z0-9\-._~+/]{16})[A-Za-z0-9\-._~+/]+=*/gi;
|
|
171
|
+
var TOKEN_TEXT = /ya29\.(?=[A-Za-z0-9_.-]{16})[A-Za-z0-9_.-]+|(?<![A-Za-z0-9_-])eyJ(?=[A-Za-z0-9_-]{8})[A-Za-z0-9_-]+\.eyJ(?=[A-Za-z0-9_-]{8})[A-Za-z0-9_-]+\.(?=[A-Za-z0-9_-]{16})[A-Za-z0-9_-]+/g;
|
|
172
|
+
var SIGNATURE_NAME = /Signature(?:=|%3D)/gi;
|
|
173
|
+
var SIGNATURE_RUN = /[A-Za-z0-9+/=_.-]*/y;
|
|
174
|
+
var SIGNATURE_MIN = 16;
|
|
175
|
+
var PEM_OPEN = /-----BEGIN [A-Z0-9 ]{0,40}PRIVATE KEY-----/y;
|
|
176
|
+
var PEM_CLOSE = /-----END [A-Z0-9 ]{0,40}PRIVATE KEY-----/y;
|
|
177
|
+
var ELEMENT = /(<([A-Za-z][\w.:-]{0,63})>)([^<]*)(<\/\2>)/g;
|
|
178
|
+
var CELL = /(<([A-Za-z][\w-]{0,15})\b[^<>]{0,256}>\s{0,32}([^<>\s](?:[^<>]{0,62}[^<>\s])?)\s{0,32}<\/\2\s{0,32}>(?:\s{0,32}<\/[A-Za-z][\w-]{0,15}\s{0,32}>){0,4}(?:\s{0,32}<[A-Za-z][\w-]{0,15}\b[^<>]{0,256}>){1,4})([^<]*)/g;
|
|
112
179
|
var REDACTED = "[redacted]";
|
|
180
|
+
var MAX_DEPTH = 10;
|
|
113
181
|
var isPlain = (value) => {
|
|
114
182
|
const proto = Object.getPrototypeOf(value);
|
|
115
183
|
return proto === Object.prototype || proto === null;
|
|
116
184
|
};
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
185
|
+
var isError = (value) => value instanceof Error || types.isNativeError(value);
|
|
186
|
+
var isCredentialName = (text) => text.length <= LONGEST_NAME && CREDENTIAL_NAME.has(fold(text));
|
|
187
|
+
function redactText(text) {
|
|
188
|
+
try {
|
|
189
|
+
const named = redactPrivateKeys(text).replace(CREDENTIAL_LINES, `$1: ${REDACTED}`).replace(AUTHORIZATION_VALUE, `$1${REDACTED}`).replace(COOKIE_VALUE, `$1${REDACTED}`).replace(NAMED_SECRET, `$1${REDACTED}`).replace(BEARER, `$1$2${REDACTED}`).replace(TOKEN_TEXT, REDACTED);
|
|
190
|
+
return redactCells(
|
|
191
|
+
redactSignatures(named).replace(
|
|
192
|
+
ELEMENT,
|
|
193
|
+
(whole, open, tag, inner, close) => isCredentialName(tag) && inner.trim() !== "" ? `${open}${REDACTED}${close}` : whole
|
|
194
|
+
)
|
|
195
|
+
);
|
|
196
|
+
} catch {
|
|
197
|
+
return REDACTED;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
function redactPrivateKeys(text) {
|
|
201
|
+
let out = "";
|
|
202
|
+
let from = 0;
|
|
203
|
+
for (let at = text.indexOf("-----BEGIN "); at !== -1; at = text.indexOf("-----BEGIN ", Math.max(at + 1, from))) {
|
|
204
|
+
PEM_OPEN.lastIndex = at;
|
|
205
|
+
const open = PEM_OPEN.exec(text);
|
|
206
|
+
if (open === null) continue;
|
|
207
|
+
const body = at + open[0].length;
|
|
208
|
+
const end = text.indexOf("-----END", body);
|
|
209
|
+
let close = "";
|
|
210
|
+
if (end !== -1) {
|
|
211
|
+
PEM_CLOSE.lastIndex = end;
|
|
212
|
+
close = PEM_CLOSE.exec(text)?.[0] ?? "";
|
|
213
|
+
}
|
|
214
|
+
out += `${text.slice(from, body)}${REDACTED}${close}`;
|
|
215
|
+
from = end === -1 ? text.length : end + close.length;
|
|
216
|
+
}
|
|
217
|
+
return from === 0 ? text : out + text.slice(from);
|
|
218
|
+
}
|
|
219
|
+
function redactSignatures(text) {
|
|
220
|
+
let out = "";
|
|
221
|
+
let from = 0;
|
|
222
|
+
SIGNATURE_NAME.lastIndex = 0;
|
|
223
|
+
for (let m = SIGNATURE_NAME.exec(text); m !== null; m = SIGNATURE_NAME.exec(text)) {
|
|
224
|
+
const start = m.index + m[0].length;
|
|
225
|
+
let end = start;
|
|
226
|
+
for (; ; ) {
|
|
227
|
+
SIGNATURE_RUN.lastIndex = end;
|
|
228
|
+
SIGNATURE_RUN.exec(text);
|
|
229
|
+
end = SIGNATURE_RUN.lastIndex;
|
|
230
|
+
if (text[end] !== "%" || text.startsWith("%26", end)) break;
|
|
231
|
+
end += 1;
|
|
232
|
+
}
|
|
233
|
+
if (end - start >= SIGNATURE_MIN) {
|
|
234
|
+
out += `${text.slice(from, start)}${REDACTED}`;
|
|
235
|
+
from = end;
|
|
236
|
+
}
|
|
237
|
+
SIGNATURE_NAME.lastIndex = end;
|
|
238
|
+
}
|
|
239
|
+
return from === 0 ? text : out + text.slice(from);
|
|
240
|
+
}
|
|
241
|
+
function redactCells(text) {
|
|
242
|
+
let out = "";
|
|
243
|
+
let from = 0;
|
|
244
|
+
CELL.lastIndex = 0;
|
|
245
|
+
for (let m = CELL.exec(text); m !== null; m = CELL.exec(text)) {
|
|
246
|
+
const [whole, head, , name, value] = m;
|
|
247
|
+
if (isCredentialName(name) && value.trim() !== "") {
|
|
248
|
+
out += `${text.slice(from, m.index)}${head}${REDACTED}`;
|
|
249
|
+
from = m.index + whole.length;
|
|
250
|
+
} else {
|
|
251
|
+
CELL.lastIndex = m.index + 1;
|
|
252
|
+
}
|
|
122
253
|
}
|
|
123
|
-
return out;
|
|
254
|
+
return from === 0 ? text : out + text.slice(from);
|
|
124
255
|
}
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
256
|
+
var OMIT = /* @__PURE__ */ Symbol("omit");
|
|
257
|
+
var INDEX = /^(?:0|[1-9]\d*)$/;
|
|
258
|
+
var isIndex = (key) => typeof key === "string" && INDEX.test(key) && Number(key) < 2 ** 32 - 1;
|
|
259
|
+
function hasNamedKeys(value) {
|
|
260
|
+
if (!Array.isArray(value)) return Reflect.ownKeys(value).length > 0;
|
|
261
|
+
return Reflect.ownKeys(value).some((key) => key !== "length" && !isIndex(key));
|
|
262
|
+
}
|
|
263
|
+
function define(copy, key, value, enumerable) {
|
|
264
|
+
Object.defineProperty(copy, key, { value, writable: true, configurable: true, enumerable });
|
|
265
|
+
}
|
|
266
|
+
function copyOf(value, ctx, depth) {
|
|
267
|
+
if (typeof value === "string") {
|
|
268
|
+
const clean = redactText(value);
|
|
269
|
+
if (clean !== value) ctx.changed = true;
|
|
270
|
+
return clean;
|
|
271
|
+
}
|
|
272
|
+
if (typeof value === "function" || typeof value === "symbol") {
|
|
273
|
+
ctx.changed = true;
|
|
274
|
+
return OMIT;
|
|
275
|
+
}
|
|
276
|
+
if (typeof value !== "object" || value === null) return value;
|
|
277
|
+
if (ctx.copies.has(value)) return ctx.copies.get(value);
|
|
278
|
+
if (depth > MAX_DEPTH) {
|
|
279
|
+
ctx.changed = true;
|
|
280
|
+
return OMIT;
|
|
281
|
+
}
|
|
282
|
+
if (Array.isArray(value)) return copyList(value, ctx, depth);
|
|
283
|
+
if (value instanceof Map) return copyMap(value, ctx, depth);
|
|
284
|
+
if (value instanceof Set) return copySet(value, ctx, depth);
|
|
285
|
+
return copyRecord(value, ctx, depth);
|
|
286
|
+
}
|
|
287
|
+
function copyList(value, ctx, depth) {
|
|
288
|
+
const list = new Array(value.length);
|
|
289
|
+
ctx.copies.set(value, list);
|
|
290
|
+
if (hasNamedKeys(value)) ctx.changed = true;
|
|
291
|
+
let hidden = false;
|
|
292
|
+
let last = -1;
|
|
141
293
|
for (const key of Reflect.ownKeys(value)) {
|
|
142
|
-
if (
|
|
294
|
+
if (!isIndex(key)) continue;
|
|
295
|
+
const index = Number(key);
|
|
296
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, key);
|
|
297
|
+
if (descriptor === void 0 || !("value" in descriptor)) {
|
|
298
|
+
ctx.changed = true;
|
|
299
|
+
hidden = false;
|
|
300
|
+
continue;
|
|
301
|
+
}
|
|
302
|
+
const item = descriptor.value;
|
|
303
|
+
if (hidden && index === last + 1 && item != null) {
|
|
304
|
+
list[index] = REDACTED;
|
|
305
|
+
if (item !== REDACTED) ctx.changed = true;
|
|
306
|
+
} else {
|
|
307
|
+
const kept = copyOf(item, ctx, depth + 1);
|
|
308
|
+
if (kept === OMIT) delete list[index];
|
|
309
|
+
else list[index] = kept;
|
|
310
|
+
}
|
|
311
|
+
hidden = typeof item === "string" && isCredentialName(item);
|
|
312
|
+
last = index;
|
|
313
|
+
}
|
|
314
|
+
return list;
|
|
315
|
+
}
|
|
316
|
+
function copyMap(value, ctx, depth) {
|
|
317
|
+
const map = /* @__PURE__ */ new Map();
|
|
318
|
+
ctx.copies.set(value, map);
|
|
319
|
+
if (hasNamedKeys(value)) ctx.changed = true;
|
|
320
|
+
for (const [k, v] of value) {
|
|
321
|
+
const key = copyOf(k, ctx, depth + 1);
|
|
322
|
+
if (key === OMIT) continue;
|
|
323
|
+
if (typeof k === "string" && isCredentialName(k) && v != null) {
|
|
324
|
+
map.set(key, REDACTED);
|
|
325
|
+
if (v !== REDACTED) ctx.changed = true;
|
|
326
|
+
continue;
|
|
327
|
+
}
|
|
328
|
+
const kept = copyOf(v, ctx, depth + 1);
|
|
329
|
+
if (kept !== OMIT) map.set(key, kept);
|
|
330
|
+
}
|
|
331
|
+
return map;
|
|
332
|
+
}
|
|
333
|
+
function copySet(value, ctx, depth) {
|
|
334
|
+
const set = /* @__PURE__ */ new Set();
|
|
335
|
+
ctx.copies.set(value, set);
|
|
336
|
+
if (hasNamedKeys(value)) ctx.changed = true;
|
|
337
|
+
for (const v of value) {
|
|
338
|
+
const kept = copyOf(v, ctx, depth + 1);
|
|
339
|
+
if (kept !== OMIT) set.add(kept);
|
|
340
|
+
}
|
|
341
|
+
return set;
|
|
342
|
+
}
|
|
343
|
+
function copyRecord(value, ctx, depth) {
|
|
344
|
+
const errorLike = isError(value);
|
|
345
|
+
if (!errorLike && !isPlain(value)) {
|
|
346
|
+
ctx.changed = true;
|
|
347
|
+
return OMIT;
|
|
348
|
+
}
|
|
349
|
+
const copy = errorLike ? Object.setPrototypeOf(new Error(), Object.getPrototypeOf(value)) : Object.create(Object.getPrototypeOf(value));
|
|
350
|
+
ctx.copies.set(value, copy);
|
|
351
|
+
if (errorLike) delete copy.stack;
|
|
352
|
+
for (const key of Reflect.ownKeys(value)) {
|
|
353
|
+
if (typeof key === "string" && TRANSPORT.has(key)) {
|
|
354
|
+
ctx.changed = true;
|
|
355
|
+
} else if (errorLike && key === "stack") {
|
|
356
|
+
copyStack(value, copy, ctx, depth);
|
|
357
|
+
} else {
|
|
358
|
+
copyField(value, copy, key, ctx, depth);
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
if (errorLike) keepSlotFields(value, copy, ctx, depth);
|
|
362
|
+
redactPairs(value, copy, errorLike, ctx);
|
|
363
|
+
return copy;
|
|
364
|
+
}
|
|
365
|
+
var DESCRIBING = /* @__PURE__ */ new Set([
|
|
366
|
+
"message",
|
|
367
|
+
"domain",
|
|
368
|
+
"reason",
|
|
369
|
+
"location",
|
|
370
|
+
"locationType",
|
|
371
|
+
"extendedHelp"
|
|
372
|
+
]);
|
|
373
|
+
function redactPairs(original, copy, errorLike, ctx) {
|
|
374
|
+
const names = Reflect.ownKeys(original).filter((key) => {
|
|
375
|
+
if (typeof key !== "string" || DESCRIBING.has(key)) return false;
|
|
376
|
+
const held = Object.getOwnPropertyDescriptor(original, key)?.value;
|
|
377
|
+
return typeof held === "string" && isCredentialName(held);
|
|
378
|
+
});
|
|
379
|
+
if (names.length === 0) return;
|
|
380
|
+
const values = errorLike ? names.filter((key) => /name$/i.test(key)).map((key) => `${key.slice(0, -4)}value`.toLowerCase()) : void 0;
|
|
381
|
+
for (const key of Reflect.ownKeys(copy)) {
|
|
382
|
+
if (typeof key !== "string" || names.includes(key) || DESCRIBING.has(key)) continue;
|
|
383
|
+
if (values !== void 0 && !values.includes(key.toLowerCase())) continue;
|
|
384
|
+
const descriptor = Object.getOwnPropertyDescriptor(copy, key);
|
|
385
|
+
if (descriptor?.value == null || descriptor.value === REDACTED) continue;
|
|
386
|
+
if (typeof descriptor.value !== "string" && typeof descriptor.value !== "object") continue;
|
|
387
|
+
define(copy, key, REDACTED, descriptor.enumerable ?? false);
|
|
388
|
+
ctx.changed = true;
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
function isSummary(value) {
|
|
392
|
+
return isPlain(value) && Reflect.ownKeys(value).every((key) => {
|
|
393
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, key);
|
|
394
|
+
const held = descriptor?.value;
|
|
395
|
+
return typeof key === "string" && RESPONSE_FIELDS.includes(key) && (typeof held === "number" || typeof held === "string");
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
function copyResponse(response, ctx, depth) {
|
|
399
|
+
if (isSummary(response)) return copyOf(response, ctx, depth);
|
|
400
|
+
ctx.changed = true;
|
|
401
|
+
const summary = {};
|
|
402
|
+
for (const key of RESPONSE_FIELDS) {
|
|
143
403
|
let held;
|
|
144
404
|
try {
|
|
145
|
-
held =
|
|
405
|
+
held = response[key];
|
|
146
406
|
} catch {
|
|
147
407
|
continue;
|
|
148
408
|
}
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
else if (key === "response" && typeof held === "object" && held !== null) {
|
|
153
|
-
kept = responseSummary(held);
|
|
154
|
-
} else kept = clean(held, copies);
|
|
155
|
-
if (kept === void 0 && held !== void 0) continue;
|
|
156
|
-
Object.defineProperty(copy, key, {
|
|
157
|
-
value: kept,
|
|
158
|
-
writable: true,
|
|
159
|
-
configurable: true,
|
|
160
|
-
enumerable: descriptor?.enumerable ?? true
|
|
161
|
-
});
|
|
409
|
+
if (typeof held === "number" || typeof held === "string") {
|
|
410
|
+
summary[key] = copyOf(held, ctx, depth + 1);
|
|
411
|
+
}
|
|
162
412
|
}
|
|
163
|
-
return
|
|
413
|
+
return summary;
|
|
164
414
|
}
|
|
165
|
-
function
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
415
|
+
function copyStack(original, copy, ctx, depth) {
|
|
416
|
+
let stack;
|
|
417
|
+
try {
|
|
418
|
+
stack = original.stack;
|
|
419
|
+
} catch {
|
|
420
|
+
return;
|
|
421
|
+
}
|
|
422
|
+
const kept = copyOf(stack, ctx, depth + 1);
|
|
423
|
+
if (kept !== OMIT) define(copy, "stack", kept, false);
|
|
424
|
+
}
|
|
425
|
+
function copyField(original, copy, key, ctx, depth) {
|
|
426
|
+
const descriptor = Object.getOwnPropertyDescriptor(original, key);
|
|
427
|
+
if (descriptor === void 0) return;
|
|
428
|
+
if (!("value" in descriptor)) {
|
|
429
|
+
ctx.changed = true;
|
|
430
|
+
return;
|
|
431
|
+
}
|
|
432
|
+
const name = typeof key === "string" ? redactText(key) : key;
|
|
433
|
+
if (name !== key) ctx.changed = true;
|
|
434
|
+
const held = descriptor.value;
|
|
435
|
+
let kept;
|
|
436
|
+
if (typeof key === "string" && isCredentialName(key) && held != null) {
|
|
437
|
+
kept = REDACTED;
|
|
438
|
+
if (held !== REDACTED) ctx.changed = true;
|
|
439
|
+
} else if (key === "response" && typeof held === "object" && held !== null) {
|
|
440
|
+
kept = copyResponse(held, ctx, depth + 1);
|
|
441
|
+
} else {
|
|
442
|
+
kept = copyOf(held, ctx, depth + 1);
|
|
443
|
+
if (kept === OMIT) return;
|
|
444
|
+
}
|
|
445
|
+
Object.defineProperty(copy, name, {
|
|
446
|
+
value: kept,
|
|
447
|
+
writable: descriptor.writable ?? true,
|
|
448
|
+
configurable: true,
|
|
449
|
+
enumerable: descriptor.enumerable ?? false
|
|
450
|
+
});
|
|
451
|
+
}
|
|
452
|
+
function keepSlotFields(original, copy, ctx, depth) {
|
|
453
|
+
for (const key of ["name", "message", "code"]) {
|
|
454
|
+
if (Object.prototype.hasOwnProperty.call(original, key)) continue;
|
|
455
|
+
let held;
|
|
456
|
+
try {
|
|
457
|
+
held = original[key];
|
|
458
|
+
} catch {
|
|
459
|
+
continue;
|
|
460
|
+
}
|
|
461
|
+
let read = OMIT;
|
|
462
|
+
try {
|
|
463
|
+
read = copy[key];
|
|
464
|
+
} catch {
|
|
465
|
+
}
|
|
466
|
+
if (read === held) continue;
|
|
467
|
+
const kept = copyOf(held, ctx, depth + 1);
|
|
468
|
+
if (kept !== OMIT) define(copy, key, kept, false);
|
|
170
469
|
}
|
|
171
|
-
const cause = err.cause;
|
|
172
|
-
return typeof cause === "object" && cause !== null && holdsTransport(cause, seen);
|
|
173
470
|
}
|
|
174
471
|
function scrubCredentials(err) {
|
|
175
472
|
if (typeof err !== "object" || err === null) return err;
|
|
176
|
-
|
|
473
|
+
const ctx = { copies: /* @__PURE__ */ new WeakMap(), changed: false };
|
|
474
|
+
const copy = copyOf(err, ctx, 0);
|
|
475
|
+
return ctx.changed && copy !== OMIT ? copy : err;
|
|
177
476
|
}
|
|
178
477
|
|
|
179
478
|
// src/bucket-presence.ts
|
|
@@ -186,7 +485,7 @@ var BucketPresence = class {
|
|
|
186
485
|
bucket;
|
|
187
486
|
settled = false;
|
|
188
487
|
asking;
|
|
189
|
-
/** Resolve when a 404 can be read as an object's; reject when the bucket does not exist
|
|
488
|
+
/** Resolve when a 404 can be read as an object's; reject with `ValidationError` when the bucket does not exist, `TransientError` when the listing failed in transit. */
|
|
190
489
|
async confirm() {
|
|
191
490
|
if (this.settled) return;
|
|
192
491
|
this.asking ??= this.ask().finally(() => {
|
|
@@ -205,7 +504,7 @@ var BucketPresence = class {
|
|
|
205
504
|
const err = scrubCredentials(failure.raised);
|
|
206
505
|
const status = err?.code;
|
|
207
506
|
if (status === 404) {
|
|
208
|
-
throw
|
|
507
|
+
throw missingBucketError(this.bucket, err);
|
|
209
508
|
}
|
|
210
509
|
if (isTransient(err)) {
|
|
211
510
|
throw new TransientError(
|
|
@@ -221,34 +520,6 @@ var BucketPresence = class {
|
|
|
221
520
|
}
|
|
222
521
|
};
|
|
223
522
|
|
|
224
|
-
// src/keys.ts
|
|
225
|
-
import {
|
|
226
|
-
ValidationError,
|
|
227
|
-
encodeNameForKey,
|
|
228
|
-
namespaceKeyPart,
|
|
229
|
-
prefixPart,
|
|
230
|
-
validateSegmentRef
|
|
231
|
-
} from "@cloudbitmaps/core/driver-kit";
|
|
232
|
-
import { normalizeObjectPrefix } from "@cloudbitmaps/core/driver-kit";
|
|
233
|
-
var SUFFIX = ".crbm";
|
|
234
|
-
function segmentObjectPrefix(prefix, ref) {
|
|
235
|
-
validateSegmentRef(ref);
|
|
236
|
-
return `${prefixPart(prefix)}${namespaceKeyPart(ref.namespace)}/segments/${encodeNameForKey(ref.segment)}.`;
|
|
237
|
-
}
|
|
238
|
-
function storageObjectName(prefix, key) {
|
|
239
|
-
if (!Number.isInteger(key.generation) || key.generation < 0) {
|
|
240
|
-
throw new ValidationError(`generation must be a non-negative integer; got ${key.generation}`);
|
|
241
|
-
}
|
|
242
|
-
return `${segmentObjectPrefix(prefix, key)}${key.generation}${SUFFIX}`;
|
|
243
|
-
}
|
|
244
|
-
function parseGenerationFromName(segmentPrefix, objectName) {
|
|
245
|
-
if (!objectName.startsWith(segmentPrefix) || !objectName.endsWith(SUFFIX)) return null;
|
|
246
|
-
const middle = objectName.slice(segmentPrefix.length, objectName.length - SUFFIX.length);
|
|
247
|
-
if (!/^(0|[1-9]\d*)$/.test(middle)) return null;
|
|
248
|
-
const generation = Number(middle);
|
|
249
|
-
return Number.isSafeInteger(generation) ? generation : null;
|
|
250
|
-
}
|
|
251
|
-
|
|
252
523
|
// src/read-timeout.ts
|
|
253
524
|
import { ValidationError as ValidationError2 } from "@cloudbitmaps/core/driver-kit";
|
|
254
525
|
var DEFAULT_READ_TIMEOUT_MS = 0;
|
|
@@ -411,13 +682,18 @@ function readOnce(file, options, maxBytes, oversize, deadline) {
|
|
|
411
682
|
}
|
|
412
683
|
|
|
413
684
|
// src/storage.ts
|
|
414
|
-
var
|
|
685
|
+
var GCS_MAX_OBJECT_BYTES = 5 * 1024 ** 4;
|
|
415
686
|
var WRITE_ID_KEY = "cbwid";
|
|
416
687
|
var THROTTLE_RESENDS = 3;
|
|
417
688
|
var THROTTLE_BASE_DELAY_MS = 500;
|
|
418
689
|
var REAL_TIME = {
|
|
419
690
|
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms))
|
|
420
691
|
};
|
|
692
|
+
var REAL_RNG = { next: () => Math.random() };
|
|
693
|
+
function isTypedPassThrough(err) {
|
|
694
|
+
const kind = errorKind(err);
|
|
695
|
+
return kind === "ValidationError" || kind === "WriteConflictError" || kind === "NotFoundError";
|
|
696
|
+
}
|
|
421
697
|
var DEFAULT_UPLOAD_THRESHOLD_BYTES = 8 * 1024 * 1024;
|
|
422
698
|
var GcsStorageDriver = class {
|
|
423
699
|
storage;
|
|
@@ -428,34 +704,25 @@ var GcsStorageDriver = class {
|
|
|
428
704
|
threshold;
|
|
429
705
|
readTimeoutMs;
|
|
430
706
|
clock;
|
|
707
|
+
rng;
|
|
431
708
|
presence;
|
|
432
709
|
conditionalDelete;
|
|
433
710
|
constructor(options) {
|
|
434
711
|
this.storage = options.storage;
|
|
435
|
-
|
|
436
|
-
throw new ValidationError3(
|
|
437
|
-
`conditionalDelete must be a boolean; got ${String(options.conditionalDelete)}`
|
|
438
|
-
);
|
|
439
|
-
}
|
|
440
|
-
this.conditionalDelete = options.conditionalDelete ?? false;
|
|
712
|
+
this.conditionalDelete = requireBool("conditionalDelete", options.conditionalDelete) ?? false;
|
|
441
713
|
this.clock = options.clock ?? REAL_TIME;
|
|
714
|
+
this.rng = options.rng ?? REAL_RNG;
|
|
442
715
|
this.readStorage = options.readStorage ?? options.storage;
|
|
443
716
|
this.bucket = options.bucket;
|
|
444
717
|
this.prefix = normalizeObjectPrefix(options.prefix);
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
]) {
|
|
449
|
-
if (value !== void 0 && (!Number.isSafeInteger(value) || value < 1)) {
|
|
450
|
-
throw new ValidationError3(`${name} must be a positive safe integer; got ${value}`);
|
|
451
|
-
}
|
|
452
|
-
}
|
|
453
|
-
if (options.maxObjectBytes !== void 0 && options.maxObjectBytes > 5 * 1024 ** 4) {
|
|
718
|
+
requirePositiveInt("maxObjectBytes", options.maxObjectBytes);
|
|
719
|
+
requirePositiveInt("simpleUploadThresholdBytes", options.simpleUploadThresholdBytes);
|
|
720
|
+
if (options.maxObjectBytes !== void 0 && options.maxObjectBytes > GCS_MAX_OBJECT_BYTES) {
|
|
454
721
|
throw new ValidationError3(
|
|
455
|
-
`maxObjectBytes must be at most GCS's 5 TiB object limit (${
|
|
722
|
+
`maxObjectBytes must be at most GCS's 5 TiB object limit (${GCS_MAX_OBJECT_BYTES}); got ${options.maxObjectBytes}`
|
|
456
723
|
);
|
|
457
724
|
}
|
|
458
|
-
this.maxObjectBytes = options.maxObjectBytes ??
|
|
725
|
+
this.maxObjectBytes = options.maxObjectBytes ?? GCS_MAX_OBJECT_BYTES;
|
|
459
726
|
this.threshold = options.simpleUploadThresholdBytes ?? DEFAULT_UPLOAD_THRESHOLD_BYTES;
|
|
460
727
|
this.readTimeoutMs = resolveReadTimeoutMs(options.readTimeoutMs);
|
|
461
728
|
this.presence = new BucketPresence(this.storage, this.bucket);
|
|
@@ -480,12 +747,13 @@ var GcsStorageDriver = class {
|
|
|
480
747
|
return startDeadline(this.readTimeoutMs, `${read} of ${key.segment}.${key.generation}`);
|
|
481
748
|
}
|
|
482
749
|
async putImmutable(key, write) {
|
|
483
|
-
const objectName =
|
|
750
|
+
const objectName = storageObjectKey(this.prefix, key);
|
|
484
751
|
const sink = new GcsUploadSink(
|
|
485
752
|
this.file(objectName),
|
|
486
753
|
this.maxObjectBytes,
|
|
487
754
|
this.threshold,
|
|
488
|
-
this.clock
|
|
755
|
+
this.clock,
|
|
756
|
+
this.rng
|
|
489
757
|
);
|
|
490
758
|
try {
|
|
491
759
|
await write(sink);
|
|
@@ -497,7 +765,8 @@ var GcsStorageDriver = class {
|
|
|
497
765
|
`generation already exists (write-once): ${key.segment}.${key.generation}`
|
|
498
766
|
);
|
|
499
767
|
}
|
|
500
|
-
if (
|
|
768
|
+
if (isTypedPassThrough(err)) throw err;
|
|
769
|
+
if (isNotFound(err)) await this.presence.confirm();
|
|
501
770
|
throw this.mapError(err);
|
|
502
771
|
}
|
|
503
772
|
}
|
|
@@ -506,7 +775,7 @@ var GcsStorageDriver = class {
|
|
|
506
775
|
throw new ValidationError3(`invalid range offset=${offset} length=${length}`);
|
|
507
776
|
}
|
|
508
777
|
if (length === 0) return new Uint8Array(0);
|
|
509
|
-
const objectName =
|
|
778
|
+
const objectName = storageObjectKey(this.prefix, key);
|
|
510
779
|
try {
|
|
511
780
|
const deadline = this.deadline("range read", key);
|
|
512
781
|
const res = await retryDownload(
|
|
@@ -565,16 +834,16 @@ var GcsStorageDriver = class {
|
|
|
565
834
|
);
|
|
566
835
|
}
|
|
567
836
|
async getTail(key, maxBytes) {
|
|
568
|
-
const objectName =
|
|
837
|
+
const objectName = storageObjectKey(this.prefix, key);
|
|
838
|
+
if (!Number.isSafeInteger(maxBytes)) {
|
|
839
|
+
throw new ValidationError3(`invalid tail length ${String(maxBytes)}`);
|
|
840
|
+
}
|
|
569
841
|
const deadline = this.deadline("tail read", key);
|
|
570
842
|
try {
|
|
571
843
|
if (maxBytes <= 0) {
|
|
572
844
|
const meta = await this.metadataOf(objectName, deadline);
|
|
573
845
|
return withVersion({ bytes: new Uint8Array(0), size: meta.size }, meta.version);
|
|
574
846
|
}
|
|
575
|
-
if (!Number.isSafeInteger(maxBytes)) {
|
|
576
|
-
throw new ValidationError3(`invalid tail length ${maxBytes}`);
|
|
577
|
-
}
|
|
578
847
|
let res;
|
|
579
848
|
try {
|
|
580
849
|
res = await retryDownload(
|
|
@@ -662,7 +931,7 @@ var GcsStorageDriver = class {
|
|
|
662
931
|
return new ValidationError3(`GCS tail read of ${key.segment}.${key.generation} refused: ${why}`);
|
|
663
932
|
}
|
|
664
933
|
async delete(key, options) {
|
|
665
|
-
const objectName =
|
|
934
|
+
const objectName = storageObjectKey(this.prefix, key);
|
|
666
935
|
const ifVersion = options?.ifVersion;
|
|
667
936
|
const conditional = ifVersion !== void 0 && this.conditionalDelete;
|
|
668
937
|
try {
|
|
@@ -699,10 +968,11 @@ var GcsStorageDriver = class {
|
|
|
699
968
|
try {
|
|
700
969
|
[files] = await this.storage.bucket(this.bucket).getFiles({ prefix });
|
|
701
970
|
} catch (err) {
|
|
971
|
+
if (isNotFound(err)) throw missingBucketError(this.bucket, scrubCredentials(err));
|
|
702
972
|
throw this.mapError(err);
|
|
703
973
|
}
|
|
704
974
|
for (const f of files) {
|
|
705
|
-
const generation =
|
|
975
|
+
const generation = parseGenerationFromKey(prefix, f.name);
|
|
706
976
|
if (generation !== null) {
|
|
707
977
|
yield { namespace: ref.namespace, segment: ref.segment, generation };
|
|
708
978
|
}
|
|
@@ -716,38 +986,25 @@ var GcsStorageDriver = class {
|
|
|
716
986
|
/** Map GCS read errors to the driver vocabulary; pass everything else through {@link mapError}. */
|
|
717
987
|
mapReadError(err, key) {
|
|
718
988
|
err = scrubCredentials(err);
|
|
719
|
-
if (
|
|
989
|
+
if (errorKind(err) === "ValidationError") return err;
|
|
720
990
|
if (isNotFound(err))
|
|
721
991
|
return new NotFoundError(`no such generation: ${key.segment}.${key.generation}`);
|
|
722
992
|
if (isInvalidRange(err)) {
|
|
723
993
|
return new ValidationError3(`range out of bounds for ${key.segment}.${key.generation}`);
|
|
724
994
|
}
|
|
725
995
|
if (err instanceof ReadTimedOut) return new TransientError2(err.message, { cause: err });
|
|
726
|
-
if (isTransportFault(err)) {
|
|
727
|
-
return new TransientError2(
|
|
728
|
-
`transient GCS fault: ${String(err.code)}`,
|
|
729
|
-
{
|
|
730
|
-
cause: err
|
|
731
|
-
}
|
|
732
|
-
);
|
|
733
|
-
}
|
|
734
996
|
return this.mapError(err);
|
|
735
997
|
}
|
|
736
998
|
/**
|
|
737
|
-
* Reclassify a transient GCS fault (throttle/5xx/dropped connection) as a retryable {@link TransientError},
|
|
738
|
-
*
|
|
739
|
-
* everything else propagates unchanged. The final fallback at every
|
|
740
|
-
* in transit arrive typed, and anything else (a checksum mismatch, a
|
|
999
|
+
* Reclassify a transient GCS fault (throttle/5xx/dropped connection) as a retryable {@link TransientError}, so the
|
|
1000
|
+
* store's read retry can ride it out and a write's caller can tell it from a deterministic failure, and a missing
|
|
1001
|
+
* bucket as a `ValidationError` that names it; everything else propagates unchanged. The final fallback at every
|
|
1002
|
+
* client-call site: an HTTP status and a fault in transit arrive typed, and anything else (a checksum mismatch, a
|
|
1003
|
+
* credentials failure) as the SDK raised it.
|
|
741
1004
|
*/
|
|
742
1005
|
mapError(err) {
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
return new TransientError2(
|
|
746
|
-
`transient GCS fault: ${err?.code ?? "unknown"}`,
|
|
747
|
-
{ cause: err }
|
|
748
|
-
);
|
|
749
|
-
}
|
|
750
|
-
return err;
|
|
1006
|
+
const scrubbed = asMissingBucket(scrubCredentials(err), this.bucket);
|
|
1007
|
+
return toTransient(scrubbed, isTransient, transientFaultMessage);
|
|
751
1008
|
}
|
|
752
1009
|
};
|
|
753
1010
|
function withVersion(tail, version) {
|
|
@@ -764,26 +1021,19 @@ function generationOf(version, key) {
|
|
|
764
1021
|
}
|
|
765
1022
|
return Number(version);
|
|
766
1023
|
}
|
|
767
|
-
function concatBytes(parts, total) {
|
|
768
|
-
const out = new Uint8Array(total);
|
|
769
|
-
let offset = 0;
|
|
770
|
-
for (const p of parts) {
|
|
771
|
-
out.set(p, offset);
|
|
772
|
-
offset += p.length;
|
|
773
|
-
}
|
|
774
|
-
return out;
|
|
775
|
-
}
|
|
776
1024
|
var GcsUploadSink = class _GcsUploadSink {
|
|
777
|
-
constructor(file, maxObjectBytes, threshold, clock) {
|
|
1025
|
+
constructor(file, maxObjectBytes, threshold, clock, rng) {
|
|
778
1026
|
this.file = file;
|
|
779
1027
|
this.maxObjectBytes = maxObjectBytes;
|
|
780
1028
|
this.threshold = threshold;
|
|
781
1029
|
this.clock = clock;
|
|
1030
|
+
this.rng = rng;
|
|
782
1031
|
}
|
|
783
1032
|
file;
|
|
784
1033
|
maxObjectBytes;
|
|
785
1034
|
threshold;
|
|
786
1035
|
clock;
|
|
1036
|
+
rng;
|
|
787
1037
|
hash = createHash("sha256");
|
|
788
1038
|
total = 0;
|
|
789
1039
|
buffered = [];
|
|
@@ -837,7 +1087,7 @@ var GcsUploadSink = class _GcsUploadSink {
|
|
|
837
1087
|
this.failure ??= e;
|
|
838
1088
|
});
|
|
839
1089
|
this.done.catch(() => void 0);
|
|
840
|
-
const buf = concatBytes(this.buffered
|
|
1090
|
+
const buf = concatBytes(this.buffered);
|
|
841
1091
|
this.buffered.length = 0;
|
|
842
1092
|
this.bufferedLen = 0;
|
|
843
1093
|
stream.write(buf);
|
|
@@ -845,7 +1095,7 @@ var GcsUploadSink = class _GcsUploadSink {
|
|
|
845
1095
|
async finish() {
|
|
846
1096
|
const sha256 = this.hash.digest("hex");
|
|
847
1097
|
if (this.stream === void 0) {
|
|
848
|
-
await this.sendSimple(concatBytes(this.buffered
|
|
1098
|
+
await this.sendSimple(concatBytes(this.buffered));
|
|
849
1099
|
return { size: this.total, sha256 };
|
|
850
1100
|
}
|
|
851
1101
|
try {
|
|
@@ -881,7 +1131,7 @@ var GcsUploadSink = class _GcsUploadSink {
|
|
|
881
1131
|
{ cause: err }
|
|
882
1132
|
);
|
|
883
1133
|
}
|
|
884
|
-
await this.clock.sleep(Math.floor(
|
|
1134
|
+
await this.clock.sleep(Math.floor(this.rng.next() * THROTTLE_BASE_DELAY_MS * 2 ** resent));
|
|
885
1135
|
}
|
|
886
1136
|
}
|
|
887
1137
|
}
|
|
@@ -906,9 +1156,10 @@ import {
|
|
|
906
1156
|
MAX_ROW_BYTES,
|
|
907
1157
|
ObjectStoreRegistry,
|
|
908
1158
|
TransientError as TransientError3,
|
|
909
|
-
ValidationError as ValidationError4,
|
|
910
1159
|
WriteConflictError as WriteConflictError2,
|
|
911
|
-
normalizeObjectPrefix as normalizeObjectPrefix2
|
|
1160
|
+
normalizeObjectPrefix as normalizeObjectPrefix2,
|
|
1161
|
+
requireBool as requireBool2,
|
|
1162
|
+
toTransient as toTransient2
|
|
912
1163
|
} from "@cloudbitmaps/core/driver-kit";
|
|
913
1164
|
|
|
914
1165
|
// src/crc32c.ts
|
|
@@ -976,7 +1227,7 @@ var GcsRegistryStore = class {
|
|
|
976
1227
|
await this.presence.confirm();
|
|
977
1228
|
return null;
|
|
978
1229
|
}
|
|
979
|
-
throw mapReadError(err);
|
|
1230
|
+
throw mapReadError(err, this.bucket);
|
|
980
1231
|
}
|
|
981
1232
|
if (res.status !== 200) {
|
|
982
1233
|
throw new IntegrityError(`registry read answered HTTP ${res.status}, not 200: ${key}`);
|
|
@@ -1004,7 +1255,8 @@ var GcsRegistryStore = class {
|
|
|
1004
1255
|
if (isPreconditionFailed(err)) {
|
|
1005
1256
|
throw new WriteConflictError2(`registry OCC conflict for ${key}`);
|
|
1006
1257
|
}
|
|
1007
|
-
|
|
1258
|
+
if (isNotFound(err)) await this.presence.confirm();
|
|
1259
|
+
throw mapError(err, this.bucket);
|
|
1008
1260
|
}
|
|
1009
1261
|
}
|
|
1010
1262
|
async delete(key, expect) {
|
|
@@ -1015,7 +1267,7 @@ var GcsRegistryStore = class {
|
|
|
1015
1267
|
if (isNotFound(err)) await this.presence.confirm();
|
|
1016
1268
|
throw new WriteConflictError2(`registry OCC conflict deleting ${key}`);
|
|
1017
1269
|
}
|
|
1018
|
-
throw mapError(err);
|
|
1270
|
+
throw mapError(err, this.bucket);
|
|
1019
1271
|
}
|
|
1020
1272
|
}
|
|
1021
1273
|
async *listKeys(prefix) {
|
|
@@ -1026,7 +1278,8 @@ var GcsRegistryStore = class {
|
|
|
1026
1278
|
try {
|
|
1027
1279
|
[files, next] = await this.storage.bucket(this.bucket).getFiles({ prefix, autoPaginate: false, maxResults: 1e3, pageToken });
|
|
1028
1280
|
} catch (err) {
|
|
1029
|
-
throw
|
|
1281
|
+
if (isNotFound(err)) throw missingBucketError(this.bucket, scrubCredentials(err));
|
|
1282
|
+
throw mapError(err, this.bucket);
|
|
1030
1283
|
}
|
|
1031
1284
|
for (const f of files ?? []) yield f.name;
|
|
1032
1285
|
if (next?.pageToken !== void 0 && next.pageToken === pageToken) {
|
|
@@ -1043,34 +1296,17 @@ function generationFence(version, key) {
|
|
|
1043
1296
|
}
|
|
1044
1297
|
return generation;
|
|
1045
1298
|
}
|
|
1046
|
-
function mapReadError(err) {
|
|
1299
|
+
function mapReadError(err, bucket) {
|
|
1047
1300
|
err = scrubCredentials(err);
|
|
1048
1301
|
if (err instanceof ReadTimedOut) return new TransientError3(err.message, { cause: err });
|
|
1049
|
-
|
|
1050
|
-
return new TransientError3(`transient GCS fault: ${String(err.code)}`, {
|
|
1051
|
-
cause: err
|
|
1052
|
-
});
|
|
1053
|
-
}
|
|
1054
|
-
return mapError(err);
|
|
1055
|
-
}
|
|
1056
|
-
function mapError(err) {
|
|
1057
|
-
err = scrubCredentials(err);
|
|
1058
|
-
if (isTransient(err)) {
|
|
1059
|
-
return new TransientError3(
|
|
1060
|
-
`transient GCS fault: ${err?.code ?? err?.name ?? "unknown"}`,
|
|
1061
|
-
{ cause: err }
|
|
1062
|
-
);
|
|
1063
|
-
}
|
|
1064
|
-
return err;
|
|
1302
|
+
return mapError(err, bucket);
|
|
1065
1303
|
}
|
|
1066
|
-
function
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
}
|
|
1073
|
-
return value;
|
|
1304
|
+
function mapError(err, bucket) {
|
|
1305
|
+
return toTransient2(
|
|
1306
|
+
asMissingBucket(scrubCredentials(err), bucket),
|
|
1307
|
+
isTransient,
|
|
1308
|
+
transientFaultMessage
|
|
1309
|
+
);
|
|
1074
1310
|
}
|
|
1075
1311
|
var GcsRegistryDriver = class extends ObjectStoreRegistry {
|
|
1076
1312
|
constructor(options) {
|
|
@@ -1080,7 +1316,7 @@ var GcsRegistryDriver = class extends ObjectStoreRegistry {
|
|
|
1080
1316
|
options.readStorage ?? options.storage,
|
|
1081
1317
|
options.bucket,
|
|
1082
1318
|
resolveReadTimeoutMs(options.readTimeoutMs),
|
|
1083
|
-
|
|
1319
|
+
requireBool2("conditionalDelete", options.conditionalDelete) ?? false
|
|
1084
1320
|
),
|
|
1085
1321
|
normalizeObjectPrefix2(options.prefix),
|
|
1086
1322
|
options.now ?? (() => Date.now())
|
|
@@ -1089,7 +1325,7 @@ var GcsRegistryDriver = class extends ObjectStoreRegistry {
|
|
|
1089
1325
|
};
|
|
1090
1326
|
|
|
1091
1327
|
// src/read-client.ts
|
|
1092
|
-
import { ValidationError as
|
|
1328
|
+
import { ValidationError as ValidationError4 } from "@cloudbitmaps/core/driver-kit";
|
|
1093
1329
|
function downloadClient(client, supplied) {
|
|
1094
1330
|
if (!isSdkClient(client)) {
|
|
1095
1331
|
if (client.retryOptions?.autoRetry === false) return client;
|
|
@@ -1135,12 +1371,12 @@ function isSdkClient(client) {
|
|
|
1135
1371
|
return client.authClient !== void 0 && typeof factory === "function" && factory.authClient === client.authClient;
|
|
1136
1372
|
}
|
|
1137
1373
|
function overridesBucket(client) {
|
|
1138
|
-
let
|
|
1374
|
+
let defined2 = 0;
|
|
1139
1375
|
for (let proto = Object.getPrototypeOf(client); proto !== null; ) {
|
|
1140
|
-
if (Object.hasOwn(proto, "bucket"))
|
|
1376
|
+
if (Object.hasOwn(proto, "bucket")) defined2++;
|
|
1141
1377
|
proto = Object.getPrototypeOf(proto);
|
|
1142
1378
|
}
|
|
1143
|
-
return
|
|
1379
|
+
return defined2 > 1;
|
|
1144
1380
|
}
|
|
1145
1381
|
function twinOf(client) {
|
|
1146
1382
|
const Own = client.constructor;
|
|
@@ -1190,7 +1426,7 @@ function unfaithful(client, twin) {
|
|
|
1190
1426
|
return void 0;
|
|
1191
1427
|
}
|
|
1192
1428
|
function refusal(supplied, why, remedy, cause) {
|
|
1193
|
-
const err = new
|
|
1429
|
+
const err = new ValidationError4(
|
|
1194
1430
|
(supplied ? `GcsStorage cannot send the supplied client's downloads without the SDK's retries: ${why}` : `GcsStorage cannot build a client that sends downloads without the SDK's retries: ${why}`) + (remedy === void 0 ? "" : `. ${remedy}`)
|
|
1195
1431
|
);
|
|
1196
1432
|
if (cause !== void 0) err.cause = scrubCredentials(cause);
|
|
@@ -1211,20 +1447,6 @@ var GCS_STORAGE_OPTION_KEYS = [
|
|
|
1211
1447
|
"conditionalDelete"
|
|
1212
1448
|
];
|
|
1213
1449
|
var CLIENT_SETTINGS = ["projectId", "apiEndpoint"];
|
|
1214
|
-
function refuseUnknown(name, options, keys, hint) {
|
|
1215
|
-
if (options === null || typeof options !== "object") {
|
|
1216
|
-
throw new ValidationError6(
|
|
1217
|
-
`${name} needs an options object \u2014 got ${options === null ? "null" : typeof options}`
|
|
1218
|
-
);
|
|
1219
|
-
}
|
|
1220
|
-
const unknown = Object.keys(options).filter((k) => !keys.includes(k));
|
|
1221
|
-
if (unknown.length > 0) {
|
|
1222
|
-
const list = (ks) => ks.map((k) => `\`${k}\``).join(", ");
|
|
1223
|
-
throw new ValidationError6(
|
|
1224
|
-
`${name} does not take ${list(unknown)}. It takes ${list(keys)}; ${hint}.`
|
|
1225
|
-
);
|
|
1226
|
-
}
|
|
1227
|
-
}
|
|
1228
1450
|
var GcsStorage = class {
|
|
1229
1451
|
storage;
|
|
1230
1452
|
registry;
|
|
@@ -1232,54 +1454,52 @@ var GcsStorage = class {
|
|
|
1232
1454
|
client;
|
|
1233
1455
|
constructor(options) {
|
|
1234
1456
|
let readClient;
|
|
1235
|
-
|
|
1457
|
+
refuseUnknownOptions(
|
|
1236
1458
|
"GcsStorage",
|
|
1237
1459
|
options,
|
|
1238
1460
|
GCS_STORAGE_OPTION_KEYS,
|
|
1239
1461
|
"a @google-cloud/storage client goes in `client`"
|
|
1240
1462
|
);
|
|
1241
1463
|
if (typeof options.bucket !== "string" || options.bucket.length === 0) {
|
|
1242
|
-
throw new
|
|
1464
|
+
throw new ValidationError5("GcsStorage needs a `bucket`: the name of one, a non-empty string");
|
|
1243
1465
|
}
|
|
1244
1466
|
if (options.now !== void 0 && typeof options.now !== "function") {
|
|
1245
|
-
throw new
|
|
1467
|
+
throw new ValidationError5(
|
|
1246
1468
|
"GcsStorage's `now` must be a function that returns epoch milliseconds"
|
|
1247
1469
|
);
|
|
1248
1470
|
}
|
|
1249
1471
|
if (options.client !== void 0 && options.client !== null) {
|
|
1250
1472
|
const ignored = CLIENT_SETTINGS.filter((k) => options[k] !== void 0);
|
|
1251
1473
|
if (ignored.length > 0) {
|
|
1252
|
-
throw new
|
|
1474
|
+
throw new ValidationError5(
|
|
1253
1475
|
`GcsStorage takes \`client\` OR ${CLIENT_SETTINGS.map((k) => `\`${k}\``).join(" / ")}, not both \u2014 got \`client\` with ${ignored.map((k) => `\`${k}\``).join(", ")}; the \`client\` already carries them, so configure them on the client, or drop \`client\``
|
|
1254
1476
|
);
|
|
1255
1477
|
}
|
|
1256
1478
|
this.client = options.client;
|
|
1257
1479
|
readClient = downloadClient(options.client, true);
|
|
1258
1480
|
} else {
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
};
|
|
1263
|
-
this.client = new GcsClient(settings);
|
|
1481
|
+
this.client = new GcsClient(
|
|
1482
|
+
defined({ projectId: options.projectId, apiEndpoint: options.apiEndpoint })
|
|
1483
|
+
);
|
|
1264
1484
|
readClient = downloadClient(this.client, false);
|
|
1265
1485
|
}
|
|
1266
1486
|
const shared = {
|
|
1267
1487
|
storage: this.client,
|
|
1268
1488
|
readStorage: readClient,
|
|
1269
1489
|
bucket: options.bucket,
|
|
1270
|
-
...
|
|
1271
|
-
...options.readTimeoutMs === void 0 ? {} : { readTimeoutMs: options.readTimeoutMs }
|
|
1490
|
+
...defined({ prefix: options.prefix, readTimeoutMs: options.readTimeoutMs })
|
|
1272
1491
|
};
|
|
1273
1492
|
this.storage = new GcsStorageDriver({
|
|
1274
1493
|
...shared,
|
|
1275
|
-
...
|
|
1276
|
-
|
|
1277
|
-
|
|
1494
|
+
...defined({
|
|
1495
|
+
maxObjectBytes: options.maxObjectBytes,
|
|
1496
|
+
simpleUploadThresholdBytes: options.simpleUploadThresholdBytes,
|
|
1497
|
+
conditionalDelete: options.conditionalDelete
|
|
1498
|
+
})
|
|
1278
1499
|
});
|
|
1279
1500
|
this.registry = new GcsRegistryDriver({
|
|
1280
1501
|
...shared,
|
|
1281
|
-
...
|
|
1282
|
-
...options.conditionalDelete === void 0 ? {} : { conditionalDelete: options.conditionalDelete }
|
|
1502
|
+
...defined({ now: options.now, conditionalDelete: options.conditionalDelete })
|
|
1283
1503
|
});
|
|
1284
1504
|
brandAsBackend(this);
|
|
1285
1505
|
}
|