@cloudbitmaps/s3 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -13
- package/dist/backend.d.ts +42 -7
- package/dist/index.js +374 -153
- package/dist/index.js.map +4 -4
- package/package.json +4 -3
package/dist/index.js
CHANGED
|
@@ -2,7 +2,11 @@
|
|
|
2
2
|
import {
|
|
3
3
|
ValidationError as ValidationError5,
|
|
4
4
|
brandAsBackend,
|
|
5
|
+
normalizeObjectPrefix as normalizeObjectPrefix3,
|
|
6
|
+
prefixPart,
|
|
5
7
|
refuseUnknownOptions,
|
|
8
|
+
requireBool as requireBool3,
|
|
9
|
+
requireNonEmptyString,
|
|
6
10
|
requirePositiveInt as requirePositiveInt2
|
|
7
11
|
} from "@cloudbitmaps/core/driver-kit";
|
|
8
12
|
import { S3Client } from "@aws-sdk/client-s3";
|
|
@@ -12,38 +16,9 @@ function defined(values) {
|
|
|
12
16
|
return Object.fromEntries(Object.entries(values).filter(([, v]) => v !== void 0));
|
|
13
17
|
}
|
|
14
18
|
|
|
15
|
-
// src/
|
|
16
|
-
import {
|
|
17
|
-
|
|
18
|
-
IntegrityError,
|
|
19
|
-
NotFoundError as NotFoundError2,
|
|
20
|
-
TransientError as TransientError2,
|
|
21
|
-
ValidationError as ValidationError3,
|
|
22
|
-
WriteConflictError,
|
|
23
|
-
concatBytes,
|
|
24
|
-
normalizeObjectPrefix,
|
|
25
|
-
parseGenerationFromKey,
|
|
26
|
-
requireBool,
|
|
27
|
-
requirePositiveInt,
|
|
28
|
-
segmentObjectPrefix,
|
|
29
|
-
storageObjectKey,
|
|
30
|
-
toTransient
|
|
31
|
-
} from "@cloudbitmaps/core/driver-kit";
|
|
32
|
-
import { createHash, randomBytes } from "node:crypto";
|
|
33
|
-
import {
|
|
34
|
-
AbortMultipartUploadCommand,
|
|
35
|
-
CompleteMultipartUploadCommand,
|
|
36
|
-
CreateMultipartUploadCommand,
|
|
37
|
-
DeleteObjectCommand as DeleteObjectCommand2,
|
|
38
|
-
GetObjectCommand,
|
|
39
|
-
HeadObjectCommand,
|
|
40
|
-
ListObjectsV2Command,
|
|
41
|
-
PutObjectCommand as PutObjectCommand2,
|
|
42
|
-
UploadPartCommand
|
|
43
|
-
} from "@aws-sdk/client-s3";
|
|
44
|
-
|
|
45
|
-
// src/s3-errors.ts
|
|
46
|
-
import { ValidationError } from "@cloudbitmaps/core/driver-kit";
|
|
19
|
+
// src/lifecycle.ts
|
|
20
|
+
import { GetBucketLifecycleConfigurationCommand } from "@aws-sdk/client-s3";
|
|
21
|
+
import { TransientError as TransientError2 } from "@cloudbitmaps/core/driver-kit";
|
|
47
22
|
|
|
48
23
|
// src/aws-errors.ts
|
|
49
24
|
import { isTransportFault } from "@cloudbitmaps/core/driver-kit";
|
|
@@ -74,74 +49,14 @@ function isNetworkOrTimeout(err) {
|
|
|
74
49
|
return NETWORK_MESSAGE.test(err?.message ?? "");
|
|
75
50
|
}
|
|
76
51
|
|
|
77
|
-
// src/s3-errors.ts
|
|
78
|
-
function isPreconditionFailed(err) {
|
|
79
|
-
return errorName(err) === "PreconditionFailed" || httpStatus(err) === 412;
|
|
80
|
-
}
|
|
81
|
-
function isConditionalConflict(err) {
|
|
82
|
-
return isPreconditionFailed(err) || errorName(err) === "ConditionalRequestConflict" || httpStatus(err) === 409;
|
|
83
|
-
}
|
|
84
|
-
function isThrottle(err) {
|
|
85
|
-
return errorName(err) === "SlowDown" || httpStatus(err) === 503;
|
|
86
|
-
}
|
|
87
|
-
function isNoSuchUpload(err) {
|
|
88
|
-
return errorName(err) === "NoSuchUpload";
|
|
89
|
-
}
|
|
90
|
-
function isMissingBucket(err) {
|
|
91
|
-
return errorName(err) === "NoSuchBucket";
|
|
92
|
-
}
|
|
93
|
-
function asMissingBucket(err, bucket) {
|
|
94
|
-
if (!isMissingBucket(err)) return err;
|
|
95
|
-
return Object.assign(
|
|
96
|
-
new ValidationError(
|
|
97
|
-
`the S3 bucket does not exist: ${bucket}; check the bucket name, and the region and endpoint of the client`
|
|
98
|
-
),
|
|
99
|
-
{ cause: err }
|
|
100
|
-
);
|
|
101
|
-
}
|
|
102
|
-
function isNotFound(err) {
|
|
103
|
-
if (isMissingBucket(err)) return false;
|
|
104
|
-
const name = errorName(err);
|
|
105
|
-
return name === "NoSuchKey" || name === "NotFound" || httpStatus(err) === 404;
|
|
106
|
-
}
|
|
107
|
-
function isInvalidRange(err) {
|
|
108
|
-
return errorName(err) === "InvalidRange" || httpStatus(err) === 416;
|
|
109
|
-
}
|
|
110
|
-
function isClockSkewCorrected(err) {
|
|
111
|
-
const e = err;
|
|
112
|
-
return e?.$metadata?.clockSkewCorrected === true;
|
|
113
|
-
}
|
|
114
|
-
function isTransient(err) {
|
|
115
|
-
if (isConditionalConflict(err) || isNotFound(err) || isInvalidRange(err)) return false;
|
|
116
|
-
return errorName(err) === "SlowDown" || isServerSide(err) || isNetworkOrTimeout(err) || isSdkRetryable(err) || isClockSkewCorrected(err);
|
|
117
|
-
}
|
|
118
|
-
function transientFaultMessage(err) {
|
|
119
|
-
return `transient S3 fault: ${errorName(err) ?? "unknown"}`;
|
|
120
|
-
}
|
|
121
|
-
function spanFromContentRange(contentRange) {
|
|
122
|
-
if (contentRange === void 0) return void 0;
|
|
123
|
-
const match = /^\s*bytes\s+(\d+)-(\d+)\//.exec(contentRange);
|
|
124
|
-
if (match === null) return void 0;
|
|
125
|
-
const start = Number(match[1]);
|
|
126
|
-
const end = Number(match[2]);
|
|
127
|
-
return Number.isSafeInteger(start) && Number.isSafeInteger(end) ? { start, end } : void 0;
|
|
128
|
-
}
|
|
129
|
-
function totalFromContentRange(contentRange) {
|
|
130
|
-
if (contentRange === void 0) return void 0;
|
|
131
|
-
const match = /\/(\d+)\s*$/.exec(contentRange);
|
|
132
|
-
if (match === null) return void 0;
|
|
133
|
-
const total = Number(match[1]);
|
|
134
|
-
return Number.isSafeInteger(total) ? total : void 0;
|
|
135
|
-
}
|
|
136
|
-
|
|
137
52
|
// src/read-timeout.ts
|
|
138
|
-
import { TransientError, ValidationError
|
|
53
|
+
import { TransientError, ValidationError } from "@cloudbitmaps/core/driver-kit";
|
|
139
54
|
var DEFAULT_READ_TIMEOUT_MS = 0;
|
|
140
55
|
var MAX_TIMER_MS = 2147483647;
|
|
141
56
|
function resolveReadTimeoutMs(value) {
|
|
142
57
|
if (value === void 0) return DEFAULT_READ_TIMEOUT_MS;
|
|
143
58
|
if (!Number.isSafeInteger(value) || value < 0 || value > MAX_TIMER_MS) {
|
|
144
|
-
throw new
|
|
59
|
+
throw new ValidationError(
|
|
145
60
|
`readTimeoutMs must be a non-negative safe integer no larger than ${MAX_TIMER_MS}; got ${describe(value)}`
|
|
146
61
|
);
|
|
147
62
|
}
|
|
@@ -171,57 +86,6 @@ async function timedRead(operation, timeoutMs, read) {
|
|
|
171
86
|
}
|
|
172
87
|
}
|
|
173
88
|
|
|
174
|
-
// src/send-once.ts
|
|
175
|
-
var NO_RETRY = {
|
|
176
|
-
name: "retryMiddleware",
|
|
177
|
-
step: "finalizeRequest",
|
|
178
|
-
priority: "high",
|
|
179
|
-
override: true
|
|
180
|
-
};
|
|
181
|
-
function sendOnce(client, command) {
|
|
182
|
-
command.middlewareStack.add((next) => next, NO_RETRY);
|
|
183
|
-
return client.send(command, {});
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
// src/bounded-body.ts
|
|
187
|
-
import { NotFoundError } from "@cloudbitmaps/core/driver-kit";
|
|
188
|
-
function destroyBody(body) {
|
|
189
|
-
body?.destroy?.();
|
|
190
|
-
}
|
|
191
|
-
async function readBounded(body, maxBytes, oversize, advertised) {
|
|
192
|
-
if (body === void 0) throw new NotFoundError("S3 GetObject returned an empty body");
|
|
193
|
-
if (advertised !== void 0 && advertised > maxBytes) {
|
|
194
|
-
destroyBody(body);
|
|
195
|
-
throw oversize();
|
|
196
|
-
}
|
|
197
|
-
if (typeof body[Symbol.asyncIterator] !== "function") {
|
|
198
|
-
const bytes = await body.transformToByteArray();
|
|
199
|
-
if (bytes.length > maxBytes) throw oversize();
|
|
200
|
-
return bytes;
|
|
201
|
-
}
|
|
202
|
-
const chunks = [];
|
|
203
|
-
let total = 0;
|
|
204
|
-
let done = false;
|
|
205
|
-
try {
|
|
206
|
-
for await (const chunk of body) {
|
|
207
|
-
const piece = typeof chunk === "string" ? new TextEncoder().encode(chunk) : chunk;
|
|
208
|
-
total += piece.length;
|
|
209
|
-
if (total > maxBytes) throw oversize();
|
|
210
|
-
chunks.push(piece);
|
|
211
|
-
}
|
|
212
|
-
done = true;
|
|
213
|
-
} finally {
|
|
214
|
-
if (!done) destroyBody(body);
|
|
215
|
-
}
|
|
216
|
-
const out = new Uint8Array(total);
|
|
217
|
-
let at = 0;
|
|
218
|
-
for (const piece of chunks) {
|
|
219
|
-
out.set(piece, at);
|
|
220
|
-
at += piece.length;
|
|
221
|
-
}
|
|
222
|
-
return out;
|
|
223
|
-
}
|
|
224
|
-
|
|
225
89
|
// src/scrub-error.ts
|
|
226
90
|
import { types } from "node:util";
|
|
227
91
|
var TRANSPORT = /* @__PURE__ */ new Set([
|
|
@@ -439,6 +303,279 @@ function scrubCredentials(err) {
|
|
|
439
303
|
return ctx.changed && copy !== OMIT ? copy : err;
|
|
440
304
|
}
|
|
441
305
|
|
|
306
|
+
// src/lifecycle.ts
|
|
307
|
+
var MAX_MESSAGE = 200;
|
|
308
|
+
var MAX_FIELD = 64;
|
|
309
|
+
var MAX_NAMED_RULES = 3;
|
|
310
|
+
var READABLE_CLASSES = /* @__PURE__ */ new Set([
|
|
311
|
+
"STANDARD",
|
|
312
|
+
"STANDARD_IA",
|
|
313
|
+
"ONEZONE_IA",
|
|
314
|
+
"GLACIER_IR",
|
|
315
|
+
"INTELLIGENT_TIERING"
|
|
316
|
+
]);
|
|
317
|
+
var ARCHIVE_CLASSES = /* @__PURE__ */ new Set(["GLACIER", "DEEP_ARCHIVE"]);
|
|
318
|
+
var INFREQUENT_CLASSES = /* @__PURE__ */ new Set([
|
|
319
|
+
"STANDARD_IA",
|
|
320
|
+
"ONEZONE_IA",
|
|
321
|
+
"GLACIER_IR"
|
|
322
|
+
]);
|
|
323
|
+
var cut = (text, max) => text.length > max ? text.slice(0, max) : text;
|
|
324
|
+
var overlaps = (a, b) => a.startsWith(b) || b.startsWith(a);
|
|
325
|
+
function rulePrefix(rule) {
|
|
326
|
+
return rule.Filter?.Prefix ?? rule.Filter?.And?.Prefix ?? rule.Prefix ?? "";
|
|
327
|
+
}
|
|
328
|
+
var hasTagFilter = (rule) => rule.Filter?.Tag !== void 0 || (rule.Filter?.And?.Tags?.length ?? 0) > 0;
|
|
329
|
+
var isEnabled = (rule) => rule.Status !== "Disabled";
|
|
330
|
+
function threat(rule) {
|
|
331
|
+
if (rule.Expiration?.Days !== void 0 || rule.Expiration?.Date !== void 0) {
|
|
332
|
+
return "expires current objects";
|
|
333
|
+
}
|
|
334
|
+
const classes = (rule.Transitions ?? []).map((t) => t.StorageClass);
|
|
335
|
+
const archive = classes.find((c) => c !== void 0 && ARCHIVE_CLASSES.has(c));
|
|
336
|
+
if (archive !== void 0) return `moves current objects to ${archive}`;
|
|
337
|
+
const unknown = (rule.Transitions ?? []).find(
|
|
338
|
+
(t) => t.StorageClass === void 0 || !READABLE_CLASSES.has(t.StorageClass)
|
|
339
|
+
);
|
|
340
|
+
return unknown === void 0 ? void 0 : `moves current objects to ${cut(String(unknown.StorageClass ?? "no storage class"), MAX_FIELD)}, a storage class this check does not know`;
|
|
341
|
+
}
|
|
342
|
+
var label = (rule) => {
|
|
343
|
+
const id = rule.ID === void 0 ? "" : `${JSON.stringify(cut(rule.ID, MAX_FIELD))} `;
|
|
344
|
+
return `${id}(prefix ${JSON.stringify(cut(rulePrefix(rule), MAX_FIELD))})`;
|
|
345
|
+
};
|
|
346
|
+
var sitsInside = (rule, prefix) => prefix !== "" && rulePrefix(rule).startsWith(prefix);
|
|
347
|
+
function coversGenerations(rule, prefix) {
|
|
348
|
+
const own = rulePrefix(rule);
|
|
349
|
+
if (prefix.startsWith(own)) return true;
|
|
350
|
+
return !own.slice(prefix.length).startsWith("registry/");
|
|
351
|
+
}
|
|
352
|
+
var ABORT_ADVICE = "No rule aborts incomplete multipart uploads under this prefix; add one (AbortIncompleteMultipartUpload, a few days) so an upload a crashed writer abandoned stops billing for its parts.";
|
|
353
|
+
var INTELLIGENT_TIERING_ADVICE = "A rule moves current objects to INTELLIGENT_TIERING: if the bucket turns on its archive access tiers, an object in one needs a restore before it can be read, so leave them off for this prefix.";
|
|
354
|
+
var INFREQUENT_ADVICE = "A rule moves current objects to an infrequent-access class or GLACIER_IR: they stay readable, but each has a minimum storage duration and a per-GB retrieval fee that reads of the store will pay.";
|
|
355
|
+
function judgeRules(rules, prefix) {
|
|
356
|
+
const live = rules.filter((r) => isEnabled(r) && overlaps(rulePrefix(r), prefix));
|
|
357
|
+
const untagged = live.filter((r) => !hasTagFilter(r));
|
|
358
|
+
const threats = untagged.flatMap((r) => {
|
|
359
|
+
const how = threat(r);
|
|
360
|
+
return how === void 0 ? [] : [{ text: `rule ${label(r)} ${how}`, inside: sitsInside(r, prefix) }];
|
|
361
|
+
});
|
|
362
|
+
const problems = threats.map((t) => t.text);
|
|
363
|
+
const transitionsTo = (classes) => untagged.some(
|
|
364
|
+
(r) => (r.Transitions ?? []).some(
|
|
365
|
+
(t) => t.StorageClass !== void 0 && classes.has(t.StorageClass)
|
|
366
|
+
)
|
|
367
|
+
);
|
|
368
|
+
const advice = [
|
|
369
|
+
live.some((r) => r.AbortIncompleteMultipartUpload !== void 0 && coversGenerations(r, prefix)) ? "" : ABORT_ADVICE,
|
|
370
|
+
transitionsTo(/* @__PURE__ */ new Set(["INTELLIGENT_TIERING"])) ? INTELLIGENT_TIERING_ADVICE : "",
|
|
371
|
+
transitionsTo(INFREQUENT_CLASSES) ? INFREQUENT_ADVICE : ""
|
|
372
|
+
].filter((a) => a !== "").map((a) => ` ${a}`).join("");
|
|
373
|
+
if (problems.length > 0) {
|
|
374
|
+
const named = problems.slice(0, MAX_NAMED_RULES).join("; ");
|
|
375
|
+
const more = problems.length > MAX_NAMED_RULES ? ` and ${problems.length - MAX_NAMED_RULES} more` : "";
|
|
376
|
+
const quoted = JSON.stringify(cut(prefix, MAX_FIELD));
|
|
377
|
+
const narrow = `Narrow the rule's prefix to exclude ${quoted}, or restrict it to noncurrent versions.`;
|
|
378
|
+
const retarget = `A rule that sits inside ${quoted} cannot be narrowed to exclude it: remove it or retarget it to another prefix, or restrict it to noncurrent versions.`;
|
|
379
|
+
const where = prefix === "" ? "The store has no prefix, so every bucket-wide or prefixed rule can reach it: give the store a prefix, or scope the rule away from its paths." : [
|
|
380
|
+
threats.some((t) => !t.inside) ? narrow : "",
|
|
381
|
+
threats.some((t) => t.inside) ? retarget : ""
|
|
382
|
+
].filter((x) => x !== "").join(" ");
|
|
383
|
+
return {
|
|
384
|
+
status: "problem",
|
|
385
|
+
detail: `${named}${more}. The store keeps current objects it must be able to read back. ${where}${advice}`
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
return {
|
|
389
|
+
status: "ok",
|
|
390
|
+
detail: `no enabled lifecycle rule under ${JSON.stringify(cut(prefix, MAX_FIELD))} expires or archives current objects.${advice}`
|
|
391
|
+
};
|
|
392
|
+
}
|
|
393
|
+
function unchecked(err) {
|
|
394
|
+
if (err instanceof TransientError2) {
|
|
395
|
+
return {
|
|
396
|
+
status: "unchecked",
|
|
397
|
+
detail: "the read of the bucket lifecycle configuration got no answer in time (readTimeoutMs); run the check again"
|
|
398
|
+
};
|
|
399
|
+
}
|
|
400
|
+
const scrubbed = scrubCredentials(err);
|
|
401
|
+
if (errorName(scrubbed) === "AccessDenied") {
|
|
402
|
+
return {
|
|
403
|
+
status: "unchecked",
|
|
404
|
+
detail: "the identity may not read the bucket lifecycle configuration: grant s3:GetLifecycleConfiguration on the bucket to check it, or confirm by hand that no rule expires or archives current objects under this prefix"
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
const name = cut(String(errorName(scrubbed) ?? "Error"), MAX_FIELD);
|
|
408
|
+
const raw = scrubbed?.message;
|
|
409
|
+
const message = typeof raw === "string" ? `: ${raw.slice(0, MAX_MESSAGE)}` : "";
|
|
410
|
+
return {
|
|
411
|
+
status: "unchecked",
|
|
412
|
+
detail: `could not read the bucket lifecycle configuration (${name}${message}); check it by hand`
|
|
413
|
+
};
|
|
414
|
+
}
|
|
415
|
+
async function checkS3Lifecycle(client, bucket, prefix, readTimeoutMs = 0) {
|
|
416
|
+
try {
|
|
417
|
+
const out = await timedRead(
|
|
418
|
+
"GetBucketLifecycleConfiguration",
|
|
419
|
+
readTimeoutMs,
|
|
420
|
+
(options) => client.send(new GetBucketLifecycleConfigurationCommand({ Bucket: bucket }), options)
|
|
421
|
+
);
|
|
422
|
+
return judgeRules(out.Rules ?? [], prefix);
|
|
423
|
+
} catch (err) {
|
|
424
|
+
try {
|
|
425
|
+
if (errorName(err) === "NoSuchLifecycleConfiguration") {
|
|
426
|
+
return judgeRules([], prefix);
|
|
427
|
+
}
|
|
428
|
+
return unchecked(err);
|
|
429
|
+
} catch {
|
|
430
|
+
return { status: "unchecked", detail: "could not read the bucket lifecycle configuration" };
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
// src/storage.ts
|
|
436
|
+
import {
|
|
437
|
+
errorKind,
|
|
438
|
+
IntegrityError,
|
|
439
|
+
NotFoundError as NotFoundError2,
|
|
440
|
+
TransientError as TransientError3,
|
|
441
|
+
ValidationError as ValidationError3,
|
|
442
|
+
WriteConflictError,
|
|
443
|
+
concatBytes,
|
|
444
|
+
normalizeObjectPrefix,
|
|
445
|
+
parseGenerationFromKey,
|
|
446
|
+
checkDeleteOptions,
|
|
447
|
+
requireBool,
|
|
448
|
+
requirePositiveInt,
|
|
449
|
+
segmentObjectPrefix,
|
|
450
|
+
storageObjectKey,
|
|
451
|
+
toTransient
|
|
452
|
+
} from "@cloudbitmaps/core/driver-kit";
|
|
453
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
454
|
+
import {
|
|
455
|
+
AbortMultipartUploadCommand,
|
|
456
|
+
CompleteMultipartUploadCommand,
|
|
457
|
+
CreateMultipartUploadCommand,
|
|
458
|
+
DeleteObjectCommand as DeleteObjectCommand2,
|
|
459
|
+
GetObjectCommand,
|
|
460
|
+
HeadObjectCommand,
|
|
461
|
+
ListObjectsV2Command,
|
|
462
|
+
PutObjectCommand as PutObjectCommand2,
|
|
463
|
+
UploadPartCommand
|
|
464
|
+
} from "@aws-sdk/client-s3";
|
|
465
|
+
|
|
466
|
+
// src/s3-errors.ts
|
|
467
|
+
import { ValidationError as ValidationError2 } from "@cloudbitmaps/core/driver-kit";
|
|
468
|
+
function isPreconditionFailed(err) {
|
|
469
|
+
return errorName(err) === "PreconditionFailed" || httpStatus(err) === 412;
|
|
470
|
+
}
|
|
471
|
+
function isConditionalConflict(err) {
|
|
472
|
+
return isPreconditionFailed(err) || errorName(err) === "ConditionalRequestConflict" || httpStatus(err) === 409;
|
|
473
|
+
}
|
|
474
|
+
function isThrottle(err) {
|
|
475
|
+
const status = httpStatus(err);
|
|
476
|
+
return errorName(err) === "SlowDown" || status === 503 || status === 429;
|
|
477
|
+
}
|
|
478
|
+
function isNoSuchUpload(err) {
|
|
479
|
+
return errorName(err) === "NoSuchUpload";
|
|
480
|
+
}
|
|
481
|
+
function isMissingBucket(err) {
|
|
482
|
+
return errorName(err) === "NoSuchBucket";
|
|
483
|
+
}
|
|
484
|
+
function asMissingBucket(err, bucket) {
|
|
485
|
+
if (!isMissingBucket(err)) return err;
|
|
486
|
+
return Object.assign(
|
|
487
|
+
new ValidationError2(
|
|
488
|
+
`the S3 bucket does not exist: ${bucket}; check the bucket name, and the region and endpoint of the client`
|
|
489
|
+
),
|
|
490
|
+
{ cause: err }
|
|
491
|
+
);
|
|
492
|
+
}
|
|
493
|
+
function isNotFound(err) {
|
|
494
|
+
if (isMissingBucket(err)) return false;
|
|
495
|
+
const name = errorName(err);
|
|
496
|
+
return name === "NoSuchKey" || name === "NotFound" || httpStatus(err) === 404;
|
|
497
|
+
}
|
|
498
|
+
function isInvalidRange(err) {
|
|
499
|
+
return errorName(err) === "InvalidRange" || httpStatus(err) === 416;
|
|
500
|
+
}
|
|
501
|
+
function isClockSkewCorrected(err) {
|
|
502
|
+
const e = err;
|
|
503
|
+
return e?.$metadata?.clockSkewCorrected === true;
|
|
504
|
+
}
|
|
505
|
+
function isTransient(err) {
|
|
506
|
+
if (isConditionalConflict(err) || isNotFound(err) || isInvalidRange(err)) return false;
|
|
507
|
+
return isThrottle(err) || isServerSide(err) || isNetworkOrTimeout(err) || isSdkRetryable(err) || isClockSkewCorrected(err);
|
|
508
|
+
}
|
|
509
|
+
function transientFaultMessage(err) {
|
|
510
|
+
return `transient S3 fault: ${errorName(err) ?? "unknown"}`;
|
|
511
|
+
}
|
|
512
|
+
function spanFromContentRange(contentRange) {
|
|
513
|
+
if (contentRange === void 0) return void 0;
|
|
514
|
+
const match = /^\s*bytes\s+(\d+)-(\d+)\//.exec(contentRange);
|
|
515
|
+
if (match === null) return void 0;
|
|
516
|
+
const start = Number(match[1]);
|
|
517
|
+
const end = Number(match[2]);
|
|
518
|
+
return Number.isSafeInteger(start) && Number.isSafeInteger(end) ? { start, end } : void 0;
|
|
519
|
+
}
|
|
520
|
+
function totalFromContentRange(contentRange) {
|
|
521
|
+
if (contentRange === void 0) return void 0;
|
|
522
|
+
const match = /\/(\d+)\s*$/.exec(contentRange);
|
|
523
|
+
if (match === null) return void 0;
|
|
524
|
+
const total = Number(match[1]);
|
|
525
|
+
return Number.isSafeInteger(total) ? total : void 0;
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
// src/send-once.ts
|
|
529
|
+
var NO_RETRY = {
|
|
530
|
+
name: "retryMiddleware",
|
|
531
|
+
step: "finalizeRequest",
|
|
532
|
+
priority: "high",
|
|
533
|
+
override: true
|
|
534
|
+
};
|
|
535
|
+
function sendOnce(client, command) {
|
|
536
|
+
command.middlewareStack.add((next) => next, NO_RETRY);
|
|
537
|
+
return client.send(command, {});
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
// src/bounded-body.ts
|
|
541
|
+
import { NotFoundError } from "@cloudbitmaps/core/driver-kit";
|
|
542
|
+
function destroyBody(body) {
|
|
543
|
+
body?.destroy?.();
|
|
544
|
+
}
|
|
545
|
+
async function readBounded(body, maxBytes, oversize, advertised) {
|
|
546
|
+
if (body === void 0) throw new NotFoundError("S3 GetObject returned an empty body");
|
|
547
|
+
if (advertised !== void 0 && advertised > maxBytes) {
|
|
548
|
+
destroyBody(body);
|
|
549
|
+
throw oversize();
|
|
550
|
+
}
|
|
551
|
+
if (typeof body[Symbol.asyncIterator] !== "function") {
|
|
552
|
+
const bytes = await body.transformToByteArray();
|
|
553
|
+
if (bytes.length > maxBytes) throw oversize();
|
|
554
|
+
return bytes;
|
|
555
|
+
}
|
|
556
|
+
const chunks = [];
|
|
557
|
+
let total = 0;
|
|
558
|
+
let done = false;
|
|
559
|
+
try {
|
|
560
|
+
for await (const chunk of body) {
|
|
561
|
+
const piece = typeof chunk === "string" ? new TextEncoder().encode(chunk) : chunk;
|
|
562
|
+
total += piece.length;
|
|
563
|
+
if (total > maxBytes) throw oversize();
|
|
564
|
+
chunks.push(piece);
|
|
565
|
+
}
|
|
566
|
+
done = true;
|
|
567
|
+
} finally {
|
|
568
|
+
if (!done) destroyBody(body);
|
|
569
|
+
}
|
|
570
|
+
const out = new Uint8Array(total);
|
|
571
|
+
let at = 0;
|
|
572
|
+
for (const piece of chunks) {
|
|
573
|
+
out.set(piece, at);
|
|
574
|
+
at += piece.length;
|
|
575
|
+
}
|
|
576
|
+
return out;
|
|
577
|
+
}
|
|
578
|
+
|
|
442
579
|
// src/client-probe.ts
|
|
443
580
|
import { DeleteObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
|
|
444
581
|
var PROBE_KEY = "cloudbitmaps-probe";
|
|
@@ -542,15 +679,15 @@ var S3_LABELS = /* @__PURE__ */ new Set([
|
|
|
542
679
|
var S3_DASH_REGION_LABEL = /^s3-(fips-)?[a-z]{2}(-gov)?-[a-z]+-\d$/;
|
|
543
680
|
var S3_EXPRESS_LABEL = /^s3express-[a-z0-9-]+$/;
|
|
544
681
|
var REGION_LABEL = /^[a-z]{2}(-[a-z]+)+-\d+$/;
|
|
545
|
-
var isS3Label = (
|
|
682
|
+
var isS3Label = (label2) => S3_LABELS.has(label2) || S3_DASH_REGION_LABEL.test(label2) || S3_EXPRESS_LABEL.test(label2);
|
|
546
683
|
function isAwsS3Host(hostname) {
|
|
547
684
|
const host = hostname.toLowerCase().replace(/\.$/, "");
|
|
548
685
|
const domain = AWS_DOMAINS.find((d) => host.endsWith(d));
|
|
549
686
|
if (domain === void 0) return false;
|
|
550
687
|
const labels = host.slice(0, -domain.length).split(".");
|
|
551
688
|
let at = -1;
|
|
552
|
-
labels.forEach((
|
|
553
|
-
if (isS3Label(
|
|
689
|
+
labels.forEach((label2, i) => {
|
|
690
|
+
if (isS3Label(label2)) at = i;
|
|
554
691
|
});
|
|
555
692
|
if (at === -1) return false;
|
|
556
693
|
const rest = labels.slice(at + 1);
|
|
@@ -783,7 +920,7 @@ var S3StorageDriver = class {
|
|
|
783
920
|
}
|
|
784
921
|
async delete(key, options) {
|
|
785
922
|
const objectKey = storageObjectKey(this.prefix, key);
|
|
786
|
-
const ifVersion = options
|
|
923
|
+
const ifVersion = checkDeleteOptions(options);
|
|
787
924
|
if (ifVersion !== void 0 && this.explicitConditionalDelete !== false) await this.settle();
|
|
788
925
|
const conditional = ifVersion !== void 0 && appliesDeleteIfMatch(this.explicitConditionalDelete, this.facts);
|
|
789
926
|
try {
|
|
@@ -927,7 +1064,7 @@ var S3MultipartSink = class {
|
|
|
927
1064
|
})
|
|
928
1065
|
);
|
|
929
1066
|
if (res2.UploadId === void 0) {
|
|
930
|
-
throw new
|
|
1067
|
+
throw new TransientError3("S3 CreateMultipartUpload returned no UploadId");
|
|
931
1068
|
}
|
|
932
1069
|
this.uploadId = res2.UploadId;
|
|
933
1070
|
}
|
|
@@ -1015,7 +1152,7 @@ var S3MultipartSink = class {
|
|
|
1015
1152
|
const stored = await this.storedObject();
|
|
1016
1153
|
if (stored?.id === this.writeId) return;
|
|
1017
1154
|
if (stored === void 0 && !isPreconditionFailed(err)) {
|
|
1018
|
-
throw new
|
|
1155
|
+
throw new TransientError3(
|
|
1019
1156
|
`S3 ${operation}: ${this.objectKey} is not stored, and the first send may still be applying`,
|
|
1020
1157
|
{ cause: err }
|
|
1021
1158
|
);
|
|
@@ -1025,7 +1162,7 @@ var S3MultipartSink = class {
|
|
|
1025
1162
|
}
|
|
1026
1163
|
if (!isThrottle(err)) throw err;
|
|
1027
1164
|
if (resent >= THROTTLE_RESENDS) {
|
|
1028
|
-
throw new
|
|
1165
|
+
throw new TransientError3(
|
|
1029
1166
|
`S3 ${operation} was throttled on each of its ${resent + 1} sends; the object may or may not exist`,
|
|
1030
1167
|
{ cause: err }
|
|
1031
1168
|
);
|
|
@@ -1334,6 +1471,48 @@ function emit(sink, event) {
|
|
|
1334
1471
|
}
|
|
1335
1472
|
|
|
1336
1473
|
// src/backend.ts
|
|
1474
|
+
function checkCredentials(credentials) {
|
|
1475
|
+
if (credentials === void 0) return;
|
|
1476
|
+
if (typeof credentials === "function") {
|
|
1477
|
+
throw new ValidationError5(
|
|
1478
|
+
"S3Storage: `credentials` must be an object of `accessKeyId` and `secretAccessKey`, not a provider function; pass a provider on an S3Client given as `client`"
|
|
1479
|
+
);
|
|
1480
|
+
}
|
|
1481
|
+
const keys = [
|
|
1482
|
+
"accessKeyId",
|
|
1483
|
+
"secretAccessKey",
|
|
1484
|
+
"sessionToken",
|
|
1485
|
+
"$source",
|
|
1486
|
+
"credentialScope",
|
|
1487
|
+
"accountId",
|
|
1488
|
+
"expiration"
|
|
1489
|
+
];
|
|
1490
|
+
refuseUnknownOptions("S3Storage credentials", credentials, keys);
|
|
1491
|
+
const given = credentials;
|
|
1492
|
+
const needText = (key, value) => {
|
|
1493
|
+
if (typeof value !== "string" || value.trim().length === 0) {
|
|
1494
|
+
throw new ValidationError5(
|
|
1495
|
+
`S3Storage: \`credentials.${key}\` must be a non-empty string; got ${typeof value === "string" ? "an empty string" : value === null || value === void 0 ? String(value) : `a ${typeof value}`}`
|
|
1496
|
+
);
|
|
1497
|
+
}
|
|
1498
|
+
};
|
|
1499
|
+
needText("accessKeyId", given.accessKeyId);
|
|
1500
|
+
needText("secretAccessKey", given.secretAccessKey);
|
|
1501
|
+
if (given.sessionToken !== void 0) needText("sessionToken", given.sessionToken);
|
|
1502
|
+
}
|
|
1503
|
+
function checkEndpointUrl(endpoint) {
|
|
1504
|
+
let url;
|
|
1505
|
+
try {
|
|
1506
|
+
url = new URL(endpoint);
|
|
1507
|
+
} catch {
|
|
1508
|
+
url = void 0;
|
|
1509
|
+
}
|
|
1510
|
+
if (url === void 0 || !/^https?:$/.test(url.protocol) || url.hostname.length === 0) {
|
|
1511
|
+
throw new ValidationError5(
|
|
1512
|
+
'S3Storage: `endpoint` must be an http or https URL such as "http://localhost:9000"'
|
|
1513
|
+
);
|
|
1514
|
+
}
|
|
1515
|
+
}
|
|
1337
1516
|
var S3_STORAGE_OPTION_KEYS = [
|
|
1338
1517
|
"bucket",
|
|
1339
1518
|
"prefix",
|
|
@@ -1381,6 +1560,18 @@ var S3Storage = class {
|
|
|
1381
1560
|
/** The client both halves share — built here unless one was supplied. */
|
|
1382
1561
|
client;
|
|
1383
1562
|
sockets;
|
|
1563
|
+
bucket;
|
|
1564
|
+
/** The key prefix every object sits under: `''`, or `trimmed/`. */
|
|
1565
|
+
keyPrefix;
|
|
1566
|
+
readTimeoutMs;
|
|
1567
|
+
/**
|
|
1568
|
+
* Builds the S3 backend over one bucket, which `bucket` names. Pass `client`, or `region`, `endpoint`, `pathStyle`,
|
|
1569
|
+
* `credentials` and `maxSockets` to have one built, not both. `prefix` puts every object under a key prefix. Throws
|
|
1570
|
+
* `ValidationError` for an unknown option, no bucket, a `now` that is not a function, a client setting beside
|
|
1571
|
+
* `client`, a `maxSockets` (with no `client`) that is not a positive integer, a `partBytes` or `maxObjectBytes` that
|
|
1572
|
+
* is not a positive integer, a `conditionalDelete` that is not a boolean, a `region` or `endpoint` that is empty or not a string, a `pathStyle` that
|
|
1573
|
+
* is not a boolean, `credentials` that are not complete, or an invalid `prefix` or `readTimeoutMs`.
|
|
1574
|
+
*/
|
|
1384
1575
|
constructor(options) {
|
|
1385
1576
|
refuseUnknownOptions(
|
|
1386
1577
|
"S3Storage",
|
|
@@ -1388,7 +1579,7 @@ var S3Storage = class {
|
|
|
1388
1579
|
S3_STORAGE_OPTION_KEYS,
|
|
1389
1580
|
"an S3 client goes in `client`"
|
|
1390
1581
|
);
|
|
1391
|
-
if (typeof options.bucket !== "string" || options.bucket.length === 0) {
|
|
1582
|
+
if (typeof options.bucket !== "string" || options.bucket.trim().length === 0) {
|
|
1392
1583
|
throw new ValidationError5("S3Storage needs a `bucket`: the name of one, a non-empty string");
|
|
1393
1584
|
}
|
|
1394
1585
|
if (options.now !== void 0 && typeof options.now !== "function") {
|
|
@@ -1403,9 +1594,24 @@ var S3Storage = class {
|
|
|
1403
1594
|
`S3Storage takes \`client\` OR ${CLIENT_SETTINGS.map((k) => `\`${k}\``).join(" / ")}, not both \u2014 got \`client\` with ${ignored.map((k) => `\`${k}\``).join(", ")}; the \`client\` already carries them, so configure them on the client, or drop \`client\``
|
|
1404
1595
|
);
|
|
1405
1596
|
}
|
|
1597
|
+
if (typeof options.client.send !== "function") {
|
|
1598
|
+
throw new ValidationError5(
|
|
1599
|
+
"S3Storage: `client` must be an S3Client from @aws-sdk/client-s3 (an object with `send`)"
|
|
1600
|
+
);
|
|
1601
|
+
}
|
|
1406
1602
|
this.client = options.client;
|
|
1407
1603
|
} else {
|
|
1408
1604
|
const maxSockets = requirePositiveInt2("maxSockets", options.maxSockets) ?? DEFAULT_MAX_SOCKETS;
|
|
1605
|
+
const region = options.region;
|
|
1606
|
+
if (region !== void 0 && (typeof region !== "string" || region.trim().length === 0)) {
|
|
1607
|
+
throw new ValidationError5(
|
|
1608
|
+
"S3Storage's `region` must be a non-empty string such as 'us-east-1'"
|
|
1609
|
+
);
|
|
1610
|
+
}
|
|
1611
|
+
const endpoint = requireNonEmptyString("S3Storage: `endpoint`", options.endpoint);
|
|
1612
|
+
if (endpoint !== void 0) checkEndpointUrl(endpoint);
|
|
1613
|
+
requireBool3("S3Storage: `pathStyle`", options.pathStyle);
|
|
1614
|
+
checkCredentials(options.credentials);
|
|
1409
1615
|
this.client = new S3Client(
|
|
1410
1616
|
defined({
|
|
1411
1617
|
credentials: options.credentials,
|
|
@@ -1416,6 +1622,9 @@ var S3Storage = class {
|
|
|
1416
1622
|
);
|
|
1417
1623
|
limitSockets(this.client, maxSockets);
|
|
1418
1624
|
}
|
|
1625
|
+
this.bucket = options.bucket;
|
|
1626
|
+
this.keyPrefix = prefixPart(normalizeObjectPrefix3(options.prefix));
|
|
1627
|
+
this.readTimeoutMs = resolveReadTimeoutMs(options.readTimeoutMs);
|
|
1419
1628
|
this.sockets = new SocketAdvisory(this.client, options.bucket);
|
|
1420
1629
|
const shared = {
|
|
1421
1630
|
client: this.client,
|
|
@@ -1444,6 +1653,18 @@ var S3Storage = class {
|
|
|
1444
1653
|
attachMetrics(sink) {
|
|
1445
1654
|
this.sockets.attach(sink);
|
|
1446
1655
|
}
|
|
1656
|
+
/**
|
|
1657
|
+
* Reads the bucket's lifecycle configuration (one `GetBucketLifecycleConfiguration` request, plus the SDK's own
|
|
1658
|
+
* retries, bounded by `readTimeoutMs` when it is set) and reports a rule that would expire, or move to GLACIER,
|
|
1659
|
+
* DEEP_ARCHIVE or a class it does not know, the current objects under this store's prefix, generations and registry
|
|
1660
|
+
* rows alike. Needs `s3:GetLifecycleConfiguration`, which a load never uses (`store.preflight()`'s
|
|
1661
|
+
* lifecycle step does): without it the
|
|
1662
|
+
* answer is `unchecked`, naming the permission. A missing rule that aborts incomplete multipart uploads is advice in
|
|
1663
|
+
* `detail`, not a problem. Never rejects; a `detail` carries no credential. Rule reading: see the production guide.
|
|
1664
|
+
*/
|
|
1665
|
+
checkLifecycle() {
|
|
1666
|
+
return checkS3Lifecycle(this.client, this.bucket, this.keyPrefix, this.readTimeoutMs);
|
|
1667
|
+
}
|
|
1447
1668
|
};
|
|
1448
1669
|
export {
|
|
1449
1670
|
S3Storage
|