disk 0.8.13 → 0.8.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -0
- package/dist/cli.js +427 -25
- package/dist/index.cjs +431 -25
- package/dist/index.d.cts +318 -8
- package/dist/index.d.ts +318 -8
- package/dist/index.js +429 -25
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -128,6 +128,63 @@ if (page.isTruncated) {
|
|
|
128
128
|
}
|
|
129
129
|
```
|
|
130
130
|
|
|
131
|
+
### Large uploads and bulk delete
|
|
132
|
+
|
|
133
|
+
`putObject` handles any size with one call. Small bodies go through a single
|
|
134
|
+
request; large ones are uploaded as a multipart upload automatically — split into
|
|
135
|
+
parts, uploaded with bounded concurrency, and assembled, aborting the upload if
|
|
136
|
+
any part fails so nothing is left half-staged. You don't pick a different method
|
|
137
|
+
for big files. For very large objects the part size is grown automatically so the
|
|
138
|
+
upload never exceeds S3's 10,000-part limit.
|
|
139
|
+
|
|
140
|
+
```ts
|
|
141
|
+
// Small or multi-gigabyte — same call.
|
|
142
|
+
await d.putObject("reports/2026-01/data.json", JSON.stringify(report), "application/json");
|
|
143
|
+
|
|
144
|
+
const { etag } = await d.putObject("backups/2026-01.tar", bigBytes, {
|
|
145
|
+
contentType: "application/x-tar",
|
|
146
|
+
multipartThreshold: 5 * 1024 * 1024, // switch to multipart above 5 MiB; default = partSize
|
|
147
|
+
partSize: 32 * 1024 * 1024, // ≥ 5 MiB; default 16 MiB
|
|
148
|
+
concurrency: 8, // parts in flight at once; default 4
|
|
149
|
+
});
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
For manual control over the multipart lifecycle (e.g. uploading parts from
|
|
153
|
+
separate processes), the raw S3 primitives live in the opt-in `d.multipart`
|
|
154
|
+
namespace — `create`, `uploadPart`, `complete`, `abort`, `listParts`,
|
|
155
|
+
`listUploads`. Most code never needs these.
|
|
156
|
+
|
|
157
|
+
```ts
|
|
158
|
+
const { uploadId } = await d.multipart.create("big.bin");
|
|
159
|
+
const p1 = await d.multipart.uploadPart("big.bin", uploadId, 1, firstChunk);
|
|
160
|
+
const p2 = await d.multipart.uploadPart("big.bin", uploadId, 2, secondChunk);
|
|
161
|
+
await d.multipart.complete("big.bin", uploadId, [p1, p2]);
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
`deleteObjects` removes many keys in one round trip (auto-batched at S3's 1000-key
|
|
165
|
+
limit). Unlike `deleteObject`, per-key failures are returned rather than thrown:
|
|
166
|
+
|
|
167
|
+
```ts
|
|
168
|
+
const { deleted, errors } = await d.deleteObjects(["a.txt", "logs/b.txt", "c.txt"]);
|
|
169
|
+
for (const e of errors) console.warn(`${e.key}: ${e.code} ${e.message}`);
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
`appendObject` appends bytes to an existing object (creating it if absent) — handy
|
|
173
|
+
for log-style writes. Each call may append at most 1 MiB; append in chunks to grow
|
|
174
|
+
past that.
|
|
175
|
+
|
|
176
|
+
```ts
|
|
177
|
+
await d.appendObject("logs/app.log", "first line\n");
|
|
178
|
+
await d.appendObject("logs/app.log", "second line\n"); // concatenated
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
Transient failures (HTTP 429 and 5xx, plus network errors) are retried
|
|
182
|
+
automatically with jittered exponential backoff before surfacing; caller errors
|
|
183
|
+
(other 4xx) are not retried. The two non-idempotent operations —
|
|
184
|
+
`completeMultipartUpload` and `appendObject` — are *not* auto-retried, since a
|
|
185
|
+
retry after a succeeded-but-unacknowledged call would return a spurious
|
|
186
|
+
`NoSuchUpload` (complete) or duplicate the appended bytes (append).
|
|
187
|
+
|
|
131
188
|
Failures throw `ArchilS3Error` with `status` (HTTP status), `code` (the S3 error
|
|
132
189
|
code, e.g. `"NoSuchKey"`), `requestId`, and the raw body on `raw`. `getObject`
|
|
133
190
|
on a missing key throws a 404 — use `headObject`/`objectExists` to probe without
|
package/dist/cli.js
CHANGED
|
@@ -106,7 +106,7 @@ function deriveS3BaseUrl(controlBaseUrl) {
|
|
|
106
106
|
}
|
|
107
107
|
|
|
108
108
|
// src/version.ts
|
|
109
|
-
var VERSION = true ? "0.8.
|
|
109
|
+
var VERSION = true ? "0.8.15" : "0.0.0-dev";
|
|
110
110
|
var USER_AGENT = `archil-js/${VERSION}`;
|
|
111
111
|
|
|
112
112
|
// src/client.ts
|
|
@@ -182,6 +182,8 @@ var Disk = class {
|
|
|
182
182
|
_archilRegion;
|
|
183
183
|
/** Base URL for the S3-compatible API (port 9000 ingress). Empty if unset. */
|
|
184
184
|
_s3BaseUrl;
|
|
185
|
+
/** Lazily-constructed multipart namespace (see {@link multipart}). @internal */
|
|
186
|
+
_multipart;
|
|
185
187
|
/** @internal */
|
|
186
188
|
constructor(data, client, archilRegion, s3BaseUrl) {
|
|
187
189
|
this.id = data.id;
|
|
@@ -384,18 +386,105 @@ var Disk = class {
|
|
|
384
386
|
return await this.headObject(key) !== null;
|
|
385
387
|
}
|
|
386
388
|
/**
|
|
387
|
-
* Write an object to the disk
|
|
388
|
-
*
|
|
389
|
-
*
|
|
389
|
+
* Write an object to the disk via the S3-compatible API. Handles any size:
|
|
390
|
+
* bodies at or below `multipartThreshold` (defaults to `partSize`, i.e.
|
|
391
|
+
* 16 MiB) go through a single PutObject request; larger bodies are uploaded as
|
|
392
|
+
* a multipart upload — split into `partSize` parts, uploaded with bounded
|
|
393
|
+
* `concurrency` (default 4), and assembled. A failed part aborts the upload so
|
|
394
|
+
* nothing is left half-staged. For manual control over the multipart
|
|
395
|
+
* lifecycle, use the {@link multipart} namespace.
|
|
390
396
|
*
|
|
391
|
-
*
|
|
392
|
-
*
|
|
393
|
-
*
|
|
397
|
+
* Faster than exec for large files — no container overhead, no command-length
|
|
398
|
+
* limits. Returns the entity tag the server assigned (a multipart upload's tag
|
|
399
|
+
* is S3's `md5(concat(partMd5s))-N` form rather than a plain MD5).
|
|
400
|
+
*
|
|
401
|
+
* @param key Path on the disk (e.g. "reports/2026-01/data.json")
|
|
402
|
+
* @param body Contents as a string, Uint8Array/Buffer, or ArrayBuffer
|
|
403
|
+
* @param options Either a content-type string, or {@link PutObjectOptions}
|
|
404
|
+
* (`contentType`, `multipartThreshold`, `partSize`,
|
|
405
|
+
* `concurrency`). Content type defaults to
|
|
406
|
+
* "application/octet-stream".
|
|
407
|
+
*/
|
|
408
|
+
async putObject(key, body, options) {
|
|
409
|
+
const opts = typeof options === "string" ? { contentType: options } : options ?? {};
|
|
410
|
+
const contentType = opts.contentType ?? "application/octet-stream";
|
|
411
|
+
const partSize = Math.max(opts.partSize ?? DEFAULT_PART_SIZE, MIN_PART_SIZE);
|
|
412
|
+
const threshold = opts.multipartThreshold ?? partSize;
|
|
413
|
+
const bytes = toBytes(body);
|
|
414
|
+
if (bytes.length <= threshold) {
|
|
415
|
+
const resp = await this._s3Request("PUT", key, { body, contentType });
|
|
416
|
+
if (!resp.ok) {
|
|
417
|
+
throw parseS3Error("PutObject", resp.status, resp.statusText, decodeText(resp.body));
|
|
418
|
+
}
|
|
419
|
+
return { etag: resp.headers.get("etag") ?? void 0 };
|
|
420
|
+
}
|
|
421
|
+
return this._putMultipart(
|
|
422
|
+
key,
|
|
423
|
+
bytes,
|
|
424
|
+
contentType,
|
|
425
|
+
partSize,
|
|
426
|
+
Math.max(1, opts.concurrency ?? DEFAULT_UPLOAD_CONCURRENCY)
|
|
427
|
+
);
|
|
428
|
+
}
|
|
429
|
+
/**
|
|
430
|
+
* Upload a large body through the multipart lifecycle: split into `partSize`
|
|
431
|
+
* parts, upload them with bounded concurrency, then complete — aborting the
|
|
432
|
+
* upload if any part fails so nothing is left half-staged. @internal
|
|
433
|
+
*/
|
|
434
|
+
async _putMultipart(key, bytes, contentType, partSize, concurrency) {
|
|
435
|
+
const effectivePartSize = effectiveUploadPartSize(bytes.length, partSize);
|
|
436
|
+
const mp = this.multipart;
|
|
437
|
+
const upload = await mp.create(key, contentType);
|
|
438
|
+
try {
|
|
439
|
+
const partCount = Math.ceil(bytes.length / effectivePartSize);
|
|
440
|
+
const parts = new Array(partCount);
|
|
441
|
+
let next = 0;
|
|
442
|
+
const worker = async () => {
|
|
443
|
+
for (; ; ) {
|
|
444
|
+
const index = next++;
|
|
445
|
+
if (index >= partCount) return;
|
|
446
|
+
const start = index * effectivePartSize;
|
|
447
|
+
const slice = bytes.subarray(start, Math.min(start + effectivePartSize, bytes.length));
|
|
448
|
+
parts[index] = await mp.uploadPart(key, upload.uploadId, index + 1, slice);
|
|
449
|
+
}
|
|
450
|
+
};
|
|
451
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, partCount) }, worker));
|
|
452
|
+
const done = await mp.complete(key, upload.uploadId, parts);
|
|
453
|
+
return { etag: done.etag };
|
|
454
|
+
} catch (err) {
|
|
455
|
+
await mp.abort(key, upload.uploadId).catch(() => {
|
|
456
|
+
});
|
|
457
|
+
throw err;
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
/**
|
|
461
|
+
* Append bytes to an object via the S3-compatible PutObject append extension
|
|
462
|
+
* (`?append=true`). If the object already exists the bytes are appended to it;
|
|
463
|
+
* if it doesn't, it is created. Returns the entity tag of the full object
|
|
464
|
+
* after the append.
|
|
465
|
+
*
|
|
466
|
+
* Each call may append at most 1 MiB — the server rejects a larger body with
|
|
467
|
+
* `EntityTooLarge`. To grow an object past that, append in chunks (or use
|
|
468
|
+
* {@link putObject} for a one-shot large write).
|
|
469
|
+
*
|
|
470
|
+
* Unlike most operations this is NOT auto-retried on a transient error:
|
|
471
|
+
* append isn't idempotent, so retrying a succeeded-but-unacknowledged append
|
|
472
|
+
* would duplicate the bytes. On a transient failure, re-append yourself only
|
|
473
|
+
* after confirming the object's size.
|
|
474
|
+
*
|
|
475
|
+
* @param key Path on the disk (e.g. "logs/app.log")
|
|
476
|
+
* @param body Bytes to append (string, Uint8Array/Buffer, or ArrayBuffer)
|
|
477
|
+
* @param contentType MIME type, applied only when the object is newly created.
|
|
394
478
|
*/
|
|
395
|
-
async
|
|
396
|
-
const resp = await this._s3Request("PUT", key, {
|
|
479
|
+
async appendObject(key, body, contentType = "application/octet-stream") {
|
|
480
|
+
const resp = await this._s3Request("PUT", key, {
|
|
481
|
+
body,
|
|
482
|
+
contentType,
|
|
483
|
+
query: { append: "true" },
|
|
484
|
+
retry: false
|
|
485
|
+
});
|
|
397
486
|
if (!resp.ok) {
|
|
398
|
-
throw parseS3Error("
|
|
487
|
+
throw parseS3Error("AppendObject", resp.status, resp.statusText, decodeText(resp.body));
|
|
399
488
|
}
|
|
400
489
|
return { etag: resp.headers.get("etag") ?? void 0 };
|
|
401
490
|
}
|
|
@@ -488,6 +577,50 @@ var Disk = class {
|
|
|
488
577
|
}
|
|
489
578
|
return parseListObjectsResult(decodeText(resp.body));
|
|
490
579
|
}
|
|
580
|
+
/**
|
|
581
|
+
* Delete up to many objects in a single S3-compatible DeleteObjects request.
|
|
582
|
+
* Unlike {@link deleteObject}, failures are reported per key rather than
|
|
583
|
+
* thrown: the result's `deleted` lists the keys that were removed and
|
|
584
|
+
* `errors` lists the ones that weren't (with the server's code/message). A
|
|
585
|
+
* key that didn't exist still counts as deleted, per S3 semantics.
|
|
586
|
+
*
|
|
587
|
+
* The server caps a single request at 1000 keys; this method transparently
|
|
588
|
+
* splits larger inputs into 1000-key batches and merges the results.
|
|
589
|
+
*
|
|
590
|
+
* @param keys Object keys to delete.
|
|
591
|
+
* @param opts `quiet` suppresses the per-key success list server-side.
|
|
592
|
+
*/
|
|
593
|
+
async deleteObjects(keys2, opts = {}) {
|
|
594
|
+
const deleted = [];
|
|
595
|
+
const errors = [];
|
|
596
|
+
for (let i = 0; i < keys2.length; i += MAX_DELETE_OBJECTS_PER_REQUEST) {
|
|
597
|
+
const batch = keys2.slice(i, i + MAX_DELETE_OBJECTS_PER_REQUEST);
|
|
598
|
+
const xml = buildDeleteObjectsXml(batch, opts.quiet ?? false);
|
|
599
|
+
const resp = await this._s3Request("POST", "", {
|
|
600
|
+
query: { delete: "" },
|
|
601
|
+
body: xml,
|
|
602
|
+
contentType: "application/xml"
|
|
603
|
+
});
|
|
604
|
+
if (!resp.ok) {
|
|
605
|
+
throw parseS3Error("DeleteObjects", resp.status, resp.statusText, decodeText(resp.body));
|
|
606
|
+
}
|
|
607
|
+
const parsed = parseDeleteObjectsResult(decodeText(resp.body));
|
|
608
|
+
deleted.push(...parsed.deleted);
|
|
609
|
+
errors.push(...parsed.errors);
|
|
610
|
+
}
|
|
611
|
+
return { deleted, errors };
|
|
612
|
+
}
|
|
613
|
+
/**
|
|
614
|
+
* The advanced, opt-in multipart-upload API. Drive the raw lifecycle
|
|
615
|
+
* yourself — `create` → `uploadPart` → `complete` (or `abort`), plus
|
|
616
|
+
* `listParts` / `listUploads`. Most callers don't need this: {@link putObject}
|
|
617
|
+
* runs the whole lifecycle automatically for large bodies. Reach for it only
|
|
618
|
+
* when you need manual control (e.g. uploading parts from separate processes),
|
|
619
|
+
* and note you then own part-size, memory, and concurrency management.
|
|
620
|
+
*/
|
|
621
|
+
get multipart() {
|
|
622
|
+
return this._multipart ??= new DiskMultipart(this.id, this._s3Request.bind(this));
|
|
623
|
+
}
|
|
491
624
|
/**
|
|
492
625
|
* Send a single request to the disk's S3-compatible endpoint. This reuses the
|
|
493
626
|
* control-plane client purely for its credential and transport — the same
|
|
@@ -512,33 +645,55 @@ var Disk = class {
|
|
|
512
645
|
const trimmedKey = key.replace(/^\//, "");
|
|
513
646
|
const encodedKey = trimmedKey.split("/").map(encodeURIComponent).join("/");
|
|
514
647
|
const path = encodedKey ? `/${this.id}/${encodedKey}` : `/${this.id}`;
|
|
515
|
-
const
|
|
648
|
+
const init = {
|
|
516
649
|
baseUrl: this._s3BaseUrl,
|
|
517
650
|
parseAs: "stream",
|
|
518
651
|
...opts.query ? { params: { query: opts.query } } : {},
|
|
519
652
|
// fetch accepts string / Uint8Array / ArrayBuffer bodies directly; pass it
|
|
520
|
-
// through unchanged (no Node Buffer in the data path).
|
|
653
|
+
// through unchanged (no Node Buffer in the data path). Bodies are fully
|
|
654
|
+
// buffered (never streams), so re-sending one on a retry is safe.
|
|
521
655
|
...opts.body !== void 0 ? { body: opts.body, bodySerializer: (b) => b } : {},
|
|
522
656
|
...opts.contentType ? { headers: { "Content-Type": opts.contentType } } : {}
|
|
523
|
-
}
|
|
524
|
-
|
|
525
|
-
|
|
657
|
+
};
|
|
658
|
+
const hasBody = method === "GET" || method === "POST";
|
|
659
|
+
const maxRetries = opts.retry === false ? 0 : MAX_S3_RETRIES;
|
|
660
|
+
for (let attempt = 0; ; attempt++) {
|
|
661
|
+
let error;
|
|
662
|
+
let response;
|
|
663
|
+
try {
|
|
664
|
+
({ error, response } = await call(path, init));
|
|
665
|
+
} catch (transportError) {
|
|
666
|
+
if (attempt < maxRetries) {
|
|
667
|
+
await sleep(s3RetryDelayMs(attempt));
|
|
668
|
+
continue;
|
|
669
|
+
}
|
|
670
|
+
throw transportError;
|
|
671
|
+
}
|
|
672
|
+
if (!response.ok) {
|
|
673
|
+
if (isTransientS3Status(response.status) && attempt < maxRetries) {
|
|
674
|
+
await response.body?.cancel().catch(() => {
|
|
675
|
+
});
|
|
676
|
+
await sleep(s3RetryDelayMs(attempt));
|
|
677
|
+
continue;
|
|
678
|
+
}
|
|
679
|
+
const message = typeof error === "string" ? error : error ? JSON.stringify(error) : "";
|
|
680
|
+
return {
|
|
681
|
+
ok: false,
|
|
682
|
+
status: response.status,
|
|
683
|
+
statusText: response.statusText,
|
|
684
|
+
headers: response.headers,
|
|
685
|
+
body: new TextEncoder().encode(message)
|
|
686
|
+
};
|
|
687
|
+
}
|
|
688
|
+
const bytes = hasBody ? new Uint8Array(await response.arrayBuffer()) : new Uint8Array(0);
|
|
526
689
|
return {
|
|
527
|
-
ok:
|
|
690
|
+
ok: true,
|
|
528
691
|
status: response.status,
|
|
529
692
|
statusText: response.statusText,
|
|
530
693
|
headers: response.headers,
|
|
531
|
-
body:
|
|
694
|
+
body: bytes
|
|
532
695
|
};
|
|
533
696
|
}
|
|
534
|
-
const bytes = method === "GET" ? new Uint8Array(await response.arrayBuffer()) : new Uint8Array(0);
|
|
535
|
-
return {
|
|
536
|
-
ok: true,
|
|
537
|
-
status: response.status,
|
|
538
|
-
statusText: response.statusText,
|
|
539
|
-
headers: response.headers,
|
|
540
|
-
body: bytes
|
|
541
|
-
};
|
|
542
697
|
}
|
|
543
698
|
/**
|
|
544
699
|
* Connect to this disk's data plane via the native ArchilClient.
|
|
@@ -569,6 +724,199 @@ var Disk = class {
|
|
|
569
724
|
});
|
|
570
725
|
}
|
|
571
726
|
};
|
|
727
|
+
var DiskMultipart = class {
|
|
728
|
+
/** @internal */
|
|
729
|
+
constructor(diskId, s3Request) {
|
|
730
|
+
this.diskId = diskId;
|
|
731
|
+
this.s3Request = s3Request;
|
|
732
|
+
}
|
|
733
|
+
diskId;
|
|
734
|
+
s3Request;
|
|
735
|
+
/**
|
|
736
|
+
* Start a multipart upload (CreateMultipartUpload) and return its `uploadId`.
|
|
737
|
+
*
|
|
738
|
+
* @param key Path on the disk the finished object will live at.
|
|
739
|
+
* @param contentType MIME type to store the object with.
|
|
740
|
+
*/
|
|
741
|
+
async create(key, contentType) {
|
|
742
|
+
const resp = await this.s3Request("POST", key, { query: { uploads: "" }, contentType });
|
|
743
|
+
if (!resp.ok) {
|
|
744
|
+
throw parseS3Error("CreateMultipartUpload", resp.status, resp.statusText, decodeText(resp.body));
|
|
745
|
+
}
|
|
746
|
+
const root = parseXml(decodeText(resp.body)).InitiateMultipartUploadResult ?? {};
|
|
747
|
+
const uploadId = optionalString(root.UploadId);
|
|
748
|
+
if (!uploadId) {
|
|
749
|
+
throw new ArchilS3Error({
|
|
750
|
+
operation: "CreateMultipartUpload",
|
|
751
|
+
statusCode: resp.status,
|
|
752
|
+
message: "response did not contain an UploadId",
|
|
753
|
+
raw: decodeText(resp.body)
|
|
754
|
+
});
|
|
755
|
+
}
|
|
756
|
+
return {
|
|
757
|
+
uploadId,
|
|
758
|
+
key: optionalString(root.Key) ?? key,
|
|
759
|
+
bucket: optionalString(root.Bucket) ?? this.diskId
|
|
760
|
+
};
|
|
761
|
+
}
|
|
762
|
+
/**
|
|
763
|
+
* Upload one part (UploadPart) and return its entity tag, which you must
|
|
764
|
+
* collect (with its part number) and pass to {@link complete}. Every part
|
|
765
|
+
* except the last must be at least 5 MiB.
|
|
766
|
+
*
|
|
767
|
+
* @param key The upload's object key.
|
|
768
|
+
* @param uploadId The id from {@link create}.
|
|
769
|
+
* @param partNumber 1-based part number (1..=10000).
|
|
770
|
+
* @param body Part contents as a string, Uint8Array/Buffer, or ArrayBuffer.
|
|
771
|
+
*/
|
|
772
|
+
async uploadPart(key, uploadId, partNumber, body) {
|
|
773
|
+
const resp = await this.s3Request("PUT", key, { query: { uploadId, partNumber }, body });
|
|
774
|
+
if (!resp.ok) {
|
|
775
|
+
throw parseS3Error("UploadPart", resp.status, resp.statusText, decodeText(resp.body));
|
|
776
|
+
}
|
|
777
|
+
return { partNumber, etag: resp.headers.get("etag") ?? "" };
|
|
778
|
+
}
|
|
779
|
+
/**
|
|
780
|
+
* Finish a multipart upload (CompleteMultipartUpload), assembling the listed
|
|
781
|
+
* parts into one object. Parts are sorted by part number before submission
|
|
782
|
+
* (the server requires strictly-increasing order).
|
|
783
|
+
*
|
|
784
|
+
* Unlike the other operations this is NOT auto-retried on a transient error:
|
|
785
|
+
* the gateway isn't idempotent for completion, so a retry after a
|
|
786
|
+
* successful-but-unacknowledged complete would return a spurious NoSuchUpload.
|
|
787
|
+
* On a transient failure, re-drive completion yourself only after confirming
|
|
788
|
+
* the object isn't already present.
|
|
789
|
+
*
|
|
790
|
+
* @param key The upload's object key.
|
|
791
|
+
* @param uploadId The id from {@link create}.
|
|
792
|
+
* @param parts The `{ partNumber, etag }` pairs from {@link uploadPart}.
|
|
793
|
+
*/
|
|
794
|
+
async complete(key, uploadId, parts) {
|
|
795
|
+
const sorted = [...parts].sort((a, b) => a.partNumber - b.partNumber);
|
|
796
|
+
const xml = buildCompleteMultipartUploadXml(sorted);
|
|
797
|
+
const resp = await this.s3Request("POST", key, {
|
|
798
|
+
query: { uploadId },
|
|
799
|
+
body: xml,
|
|
800
|
+
contentType: "application/xml",
|
|
801
|
+
retry: false
|
|
802
|
+
});
|
|
803
|
+
if (!resp.ok) {
|
|
804
|
+
throw parseS3Error("CompleteMultipartUpload", resp.status, resp.statusText, decodeText(resp.body));
|
|
805
|
+
}
|
|
806
|
+
const root = parseXml(decodeText(resp.body)).CompleteMultipartUploadResult ?? {};
|
|
807
|
+
return {
|
|
808
|
+
etag: optionalString(root.ETag),
|
|
809
|
+
location: optionalString(root.Location),
|
|
810
|
+
bucket: optionalString(root.Bucket),
|
|
811
|
+
key: optionalString(root.Key)
|
|
812
|
+
};
|
|
813
|
+
}
|
|
814
|
+
/**
|
|
815
|
+
* Abort a multipart upload (AbortMultipartUpload), discarding every staged
|
|
816
|
+
* part. Idempotent against an upload that's already gone (404 / NoSuchUpload
|
|
817
|
+
* resolves successfully).
|
|
818
|
+
*
|
|
819
|
+
* @param key The upload's object key.
|
|
820
|
+
* @param uploadId The id from {@link create}.
|
|
821
|
+
*/
|
|
822
|
+
async abort(key, uploadId) {
|
|
823
|
+
const resp = await this.s3Request("DELETE", key, { query: { uploadId } });
|
|
824
|
+
if (!resp.ok && resp.status !== 404) {
|
|
825
|
+
throw parseS3Error("AbortMultipartUpload", resp.status, resp.statusText, decodeText(resp.body));
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
/**
|
|
829
|
+
* List the parts already uploaded for an in-progress upload (ListParts).
|
|
830
|
+
* Returns a single page; follow `nextPartNumberMarker` (when `isTruncated`)
|
|
831
|
+
* to page through the rest.
|
|
832
|
+
*
|
|
833
|
+
* @param key The upload's object key.
|
|
834
|
+
* @param uploadId The id from {@link create}.
|
|
835
|
+
* @param opts `maxParts` / `partNumberMarker` pagination controls.
|
|
836
|
+
*/
|
|
837
|
+
async listParts(key, uploadId, opts = {}) {
|
|
838
|
+
const query = { uploadId };
|
|
839
|
+
if (opts.maxParts !== void 0) query["max-parts"] = opts.maxParts;
|
|
840
|
+
if (opts.partNumberMarker !== void 0) query["part-number-marker"] = opts.partNumberMarker;
|
|
841
|
+
const resp = await this.s3Request("GET", key, { query });
|
|
842
|
+
if (!resp.ok) {
|
|
843
|
+
throw parseS3Error("ListParts", resp.status, resp.statusText, decodeText(resp.body));
|
|
844
|
+
}
|
|
845
|
+
return parseListPartsResult(decodeText(resp.body));
|
|
846
|
+
}
|
|
847
|
+
/**
|
|
848
|
+
* List in-progress multipart uploads on the disk (ListMultipartUploads).
|
|
849
|
+
* Returns a single page; follow `nextKeyMarker` / `nextUploadIdMarker` (when
|
|
850
|
+
* `isTruncated`) for the rest.
|
|
851
|
+
*
|
|
852
|
+
* @param opts Prefix/delimiter filter and pagination markers.
|
|
853
|
+
*/
|
|
854
|
+
async listUploads(opts = {}) {
|
|
855
|
+
const query = { uploads: "" };
|
|
856
|
+
if (opts.prefix !== void 0) query.prefix = opts.prefix;
|
|
857
|
+
if (opts.delimiter !== void 0) query.delimiter = opts.delimiter;
|
|
858
|
+
if (opts.keyMarker !== void 0) query["key-marker"] = opts.keyMarker;
|
|
859
|
+
if (opts.uploadIdMarker !== void 0) query["upload-id-marker"] = opts.uploadIdMarker;
|
|
860
|
+
if (opts.maxUploads !== void 0) query["max-uploads"] = opts.maxUploads;
|
|
861
|
+
const resp = await this.s3Request("GET", "", { query });
|
|
862
|
+
if (!resp.ok) {
|
|
863
|
+
throw parseS3Error("ListMultipartUploads", resp.status, resp.statusText, decodeText(resp.body));
|
|
864
|
+
}
|
|
865
|
+
return parseListMultipartUploadsResult(decodeText(resp.body));
|
|
866
|
+
}
|
|
867
|
+
};
|
|
868
|
+
var MAX_DELETE_OBJECTS_PER_REQUEST = 1e3;
|
|
869
|
+
var MAX_S3_RETRIES = 3;
|
|
870
|
+
var S3_RETRY_BASE_MS = 100;
|
|
871
|
+
var S3_RETRY_CAP_MS = 2e3;
|
|
872
|
+
function isTransientS3Status(status) {
|
|
873
|
+
return status === 429 || status === 500 || status === 502 || status === 503 || status === 504;
|
|
874
|
+
}
|
|
875
|
+
function s3RetryDelayMs(attempt) {
|
|
876
|
+
const ceiling = Math.min(S3_RETRY_CAP_MS, S3_RETRY_BASE_MS * 2 ** attempt);
|
|
877
|
+
return Math.random() * ceiling;
|
|
878
|
+
}
|
|
879
|
+
function sleep(ms) {
|
|
880
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
881
|
+
}
|
|
882
|
+
var MIN_PART_SIZE = 5 * 1024 * 1024;
|
|
883
|
+
var DEFAULT_PART_SIZE = 16 * 1024 * 1024;
|
|
884
|
+
var DEFAULT_UPLOAD_CONCURRENCY = 4;
|
|
885
|
+
var MAX_PARTS_PER_UPLOAD = 1e4;
|
|
886
|
+
function effectiveUploadPartSize(totalBytes, requestedPartSize) {
|
|
887
|
+
if (Math.ceil(totalBytes / requestedPartSize) <= MAX_PARTS_PER_UPLOAD) {
|
|
888
|
+
return requestedPartSize;
|
|
889
|
+
}
|
|
890
|
+
const MiB = 1024 * 1024;
|
|
891
|
+
const needed = Math.ceil(totalBytes / MAX_PARTS_PER_UPLOAD);
|
|
892
|
+
return Math.ceil(needed / MiB) * MiB;
|
|
893
|
+
}
|
|
894
|
+
function toBytes(body) {
|
|
895
|
+
if (typeof body === "string") return new TextEncoder().encode(body);
|
|
896
|
+
if (body instanceof Uint8Array) return body;
|
|
897
|
+
return new Uint8Array(body);
|
|
898
|
+
}
|
|
899
|
+
function escapeXml(value) {
|
|
900
|
+
return value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
|
901
|
+
}
|
|
902
|
+
function buildDeleteObjectsXml(keys2, quiet) {
|
|
903
|
+
const objects = keys2.map((k) => `<Object><Key>${escapeXml(k)}</Key></Object>`).join("");
|
|
904
|
+
const quietTag = quiet ? "<Quiet>true</Quiet>" : "";
|
|
905
|
+
return `<?xml version="1.0" encoding="UTF-8"?><Delete>${objects}${quietTag}</Delete>`;
|
|
906
|
+
}
|
|
907
|
+
function buildCompleteMultipartUploadXml(parts) {
|
|
908
|
+
const body = parts.map(
|
|
909
|
+
(p) => `<Part><PartNumber>${p.partNumber}</PartNumber><ETag>${escapeXml(p.etag)}</ETag></Part>`
|
|
910
|
+
).join("");
|
|
911
|
+
return `<?xml version="1.0" encoding="UTF-8"?><CompleteMultipartUpload>${body}</CompleteMultipartUpload>`;
|
|
912
|
+
}
|
|
913
|
+
function asArray(value) {
|
|
914
|
+
if (value === void 0 || value === null) return [];
|
|
915
|
+
return Array.isArray(value) ? value : [value];
|
|
916
|
+
}
|
|
917
|
+
function optionalNumber(value) {
|
|
918
|
+
return value === void 0 || value === null ? void 0 : Number(value);
|
|
919
|
+
}
|
|
572
920
|
function optionalString(value) {
|
|
573
921
|
return value === void 0 || value === null ? void 0 : String(value);
|
|
574
922
|
}
|
|
@@ -597,6 +945,60 @@ function parseListObjectsResult(xml) {
|
|
|
597
945
|
prefix: optionalString(root.Prefix)
|
|
598
946
|
};
|
|
599
947
|
}
|
|
948
|
+
function xmlIsTruncated(value) {
|
|
949
|
+
return value === "true" || value === true;
|
|
950
|
+
}
|
|
951
|
+
function parseDeleteObjectsResult(xml) {
|
|
952
|
+
const root = parseXml(xml).DeleteResult ?? {};
|
|
953
|
+
const deleted = asArray(root.Deleted).map((d) => optionalString(d.Key)).filter((k) => k !== void 0);
|
|
954
|
+
const errors = asArray(root.Error).map((e) => ({
|
|
955
|
+
key: String(e.Key ?? ""),
|
|
956
|
+
code: optionalString(e.Code),
|
|
957
|
+
message: optionalString(e.Message)
|
|
958
|
+
}));
|
|
959
|
+
return { deleted, errors };
|
|
960
|
+
}
|
|
961
|
+
function parseListPartsResult(xml) {
|
|
962
|
+
const root = parseXml(xml).ListPartsResult ?? {};
|
|
963
|
+
const parts = asArray(root.Part).map((p) => ({
|
|
964
|
+
partNumber: Number(p.PartNumber ?? 0),
|
|
965
|
+
etag: optionalString(p.ETag),
|
|
966
|
+
size: Number(p.Size ?? 0),
|
|
967
|
+
lastModified: optionalDate(p.LastModified)
|
|
968
|
+
}));
|
|
969
|
+
return {
|
|
970
|
+
bucket: optionalString(root.Bucket),
|
|
971
|
+
key: optionalString(root.Key),
|
|
972
|
+
uploadId: optionalString(root.UploadId),
|
|
973
|
+
parts,
|
|
974
|
+
isTruncated: xmlIsTruncated(root.IsTruncated),
|
|
975
|
+
partNumberMarker: Number(root.PartNumberMarker ?? 0),
|
|
976
|
+
nextPartNumberMarker: optionalNumber(root.NextPartNumberMarker),
|
|
977
|
+
maxParts: Number(root.MaxParts ?? parts.length)
|
|
978
|
+
};
|
|
979
|
+
}
|
|
980
|
+
function parseListMultipartUploadsResult(xml) {
|
|
981
|
+
const root = parseXml(xml).ListMultipartUploadsResult ?? {};
|
|
982
|
+
const uploads = asArray(root.Upload).map((u) => ({
|
|
983
|
+
key: String(u.Key ?? ""),
|
|
984
|
+
uploadId: String(u.UploadId ?? ""),
|
|
985
|
+
initiated: optionalDate(u.Initiated)
|
|
986
|
+
}));
|
|
987
|
+
const commonPrefixes = asArray(root.CommonPrefixes).map((cp) => optionalString(cp.Prefix)).filter((p) => p !== void 0);
|
|
988
|
+
return {
|
|
989
|
+
bucket: optionalString(root.Bucket),
|
|
990
|
+
uploads,
|
|
991
|
+
commonPrefixes,
|
|
992
|
+
isTruncated: xmlIsTruncated(root.IsTruncated),
|
|
993
|
+
keyMarker: optionalString(root.KeyMarker),
|
|
994
|
+
uploadIdMarker: optionalString(root.UploadIdMarker),
|
|
995
|
+
nextKeyMarker: optionalString(root.NextKeyMarker),
|
|
996
|
+
nextUploadIdMarker: optionalString(root.NextUploadIdMarker),
|
|
997
|
+
prefix: optionalString(root.Prefix),
|
|
998
|
+
delimiter: optionalString(root.Delimiter),
|
|
999
|
+
maxUploads: optionalNumber(root.MaxUploads)
|
|
1000
|
+
};
|
|
1001
|
+
}
|
|
600
1002
|
|
|
601
1003
|
// src/disks.ts
|
|
602
1004
|
var Disks = class {
|