disk 0.8.14 → 0.8.16

package/dist/index.js

@@ -100,7 +100,7 @@ function deriveS3BaseUrl(controlBaseUrl) {
 }
 
 // src/version.ts
-var VERSION = true ? "0.8.14" : "0.0.0-dev";
+var VERSION = true ? "0.8.16" : "0.0.0-dev";
 var USER_AGENT = `archil-js/${VERSION}`;
 
 // src/client.ts
@@ -176,6 +176,8 @@ var Disk = class {
   _archilRegion;
   /** Base URL for the S3-compatible API (port 9000 ingress). Empty if unset. */
   _s3BaseUrl;
+  /** Lazily-constructed multipart namespace (see {@link multipart}). @internal */
+  _multipart;
   /** @internal */
   constructor(data, client, archilRegion, s3BaseUrl) {
     this.id = data.id;
@@ -378,18 +380,105 @@ var Disk = class {
     return await this.headObject(key) !== null;
   }
   /**
-   * Write an object to the disk using the S3-compatible PutObject API. Faster
-   * than exec for large files — no container overhead, no command-length limits.
-   * Returns the entity tag the server assigned.
+   * Write an object to the disk via the S3-compatible API. Handles any size:
+   * bodies at or below `multipartThreshold` (defaults to `partSize`, i.e.
+   * 16 MiB) go through a single PutObject request; larger bodies are uploaded as
+   * a multipart upload — split into `partSize` parts, uploaded with bounded
+   * `concurrency` (default 4), and assembled. A failed part aborts the upload so
+   * nothing is left half-staged. For manual control over the multipart
+   * lifecycle, use the {@link multipart} namespace.
    *
-   * @param key          Path on the disk (e.g. "reports/2026-01/data.json")
-   * @param body         Contents as a string, Uint8Array/Buffer, or ArrayBuffer
-   * @param contentType  MIME type (default: "application/octet-stream")
+   * Faster than exec for large files — no container overhead, no command-length
+   * limits. Returns the entity tag the server assigned (a multipart upload's tag
+   * is S3's `md5(concat(partMd5s))-N` form rather than a plain MD5).
+   *
+   * @param key      Path on the disk (e.g. "reports/2026-01/data.json")
+   * @param body     Contents as a string, Uint8Array/Buffer, or ArrayBuffer
+   * @param options  Either a content-type string, or {@link PutObjectOptions}
+   *                 (`contentType`, `multipartThreshold`, `partSize`,
+   *                 `concurrency`). Content type defaults to
+   *                 "application/octet-stream".
+   */
+  async putObject(key, body, options) {
+    const opts = typeof options === "string" ? { contentType: options } : options ?? {};
+    const contentType = opts.contentType ?? "application/octet-stream";
+    const partSize = Math.max(opts.partSize ?? DEFAULT_PART_SIZE, MIN_PART_SIZE);
+    const threshold = opts.multipartThreshold ?? partSize;
+    const bytes = toBytes(body);
+    if (bytes.length <= threshold) {
+      const resp = await this._s3Request("PUT", key, { body, contentType });
+      if (!resp.ok) {
+        throw parseS3Error("PutObject", resp.status, resp.statusText, decodeText(resp.body));
+      }
+      return { etag: resp.headers.get("etag") ?? void 0 };
+    }
+    return this._putMultipart(
+      key,
+      bytes,
+      contentType,
+      partSize,
+      Math.max(1, opts.concurrency ?? DEFAULT_UPLOAD_CONCURRENCY)
+    );
+  }
+  /**
+   * Upload a large body through the multipart lifecycle: split into `partSize`
+   * parts, upload them with bounded concurrency, then complete — aborting the
+   * upload if any part fails so nothing is left half-staged. @internal
    */
-  async putObject(key, body, contentType = "application/octet-stream") {
-    const resp = await this._s3Request("PUT", key, { body, contentType });
+  async _putMultipart(key, bytes, contentType, partSize, concurrency) {
+    const effectivePartSize = effectiveUploadPartSize(bytes.length, partSize);
+    const mp = this.multipart;
+    const upload = await mp.create(key, contentType);
+    try {
+      const partCount = Math.ceil(bytes.length / effectivePartSize);
+      const parts = new Array(partCount);
+      let next = 0;
+      const worker = async () => {
+        for (; ; ) {
+          const index = next++;
+          if (index >= partCount) return;
+          const start = index * effectivePartSize;
+          const slice = bytes.subarray(start, Math.min(start + effectivePartSize, bytes.length));
+          parts[index] = await mp.uploadPart(key, upload.uploadId, index + 1, slice);
+        }
+      };
+      await Promise.all(Array.from({ length: Math.min(concurrency, partCount) }, worker));
+      const done = await mp.complete(key, upload.uploadId, parts);
+      return { etag: done.etag };
+    } catch (err) {
+      await mp.abort(key, upload.uploadId).catch(() => {
+      });
+      throw err;
+    }
+  }
+  /**
+   * Append bytes to an object via the S3-compatible PutObject append extension
+   * (`?append=true`). If the object already exists the bytes are appended to it;
+   * if it doesn't, it is created. Returns the entity tag of the full object
+   * after the append.
+   *
+   * Each call may append at most 1 MiB — the server rejects a larger body with
+   * `EntityTooLarge`. To grow an object past that, append in chunks (or use
+   * {@link putObject} for a one-shot large write).
+   *
+   * Unlike most operations this is NOT auto-retried on a transient error:
+   * append isn't idempotent, so retrying a succeeded-but-unacknowledged append
+   * would duplicate the bytes. On a transient failure, re-append yourself only
+   * after confirming the object's size.
+   *
+   * @param key          Path on the disk (e.g. "logs/app.log")
+   * @param body         Bytes to append (string, Uint8Array/Buffer, or ArrayBuffer)
+   * @param contentType  MIME type, applied only when the object is newly created.
+   */
+  async appendObject(key, body, contentType = "application/octet-stream") {
+    const resp = await this._s3Request("PUT", key, {
+      body,
+      contentType,
+      query: { append: "true" },
+      retry: false
+    });
     if (!resp.ok) {
-      throw parseS3Error("PutObject", resp.status, resp.statusText, decodeText(resp.body));
+      throw parseS3Error("AppendObject", resp.status, resp.statusText, decodeText(resp.body));
     }
     return { etag: resp.headers.get("etag") ?? void 0 };
   }
@@ -482,6 +571,50 @@ var Disk = class {
     }
     return parseListObjectsResult(decodeText(resp.body));
   }
+  /**
+   * Delete up to many objects in a single S3-compatible DeleteObjects request.
+   * Unlike {@link deleteObject}, failures are reported per key rather than
+   * thrown: the result's `deleted` lists the keys that were removed and
+   * `errors` lists the ones that weren't (with the server's code/message). A
+   * key that didn't exist still counts as deleted, per S3 semantics.
+   *
+   * The server caps a single request at 1000 keys; this method transparently
+   * splits larger inputs into 1000-key batches and merges the results.
+   *
+   * @param keys  Object keys to delete.
+   * @param opts  `quiet` suppresses the per-key success list server-side.
+   */
+  async deleteObjects(keys, opts = {}) {
+    const deleted = [];
+    const errors = [];
+    for (let i = 0; i < keys.length; i += MAX_DELETE_OBJECTS_PER_REQUEST) {
+      const batch = keys.slice(i, i + MAX_DELETE_OBJECTS_PER_REQUEST);
+      const xml = buildDeleteObjectsXml(batch, opts.quiet ?? false);
+      const resp = await this._s3Request("POST", "", {
+        query: { delete: "" },
+        body: xml,
+        contentType: "application/xml"
+      });
+      if (!resp.ok) {
+        throw parseS3Error("DeleteObjects", resp.status, resp.statusText, decodeText(resp.body));
+      }
+      const parsed = parseDeleteObjectsResult(decodeText(resp.body));
+      deleted.push(...parsed.deleted);
+      errors.push(...parsed.errors);
+    }
+    return { deleted, errors };
+  }
+  /**
+   * The advanced, opt-in multipart-upload API. Drive the raw lifecycle
+   * yourself — `create` → `uploadPart` → `complete` (or `abort`), plus
+   * `listParts` / `listUploads`. Most callers don't need this: {@link putObject}
+   * runs the whole lifecycle automatically for large bodies. Reach for it only
+   * when you need manual control (e.g. uploading parts from separate processes),
+   * and note you then own part-size, memory, and concurrency management.
+   */
+  get multipart() {
+    return this._multipart ??= new DiskMultipart(this.id, this._s3Request.bind(this));
+  }
   /**
    * Send a single request to the disk's S3-compatible endpoint. This reuses the
    * control-plane client purely for its credential and transport — the same
@@ -506,33 +639,55 @@ var Disk = class {
     const trimmedKey = key.replace(/^\//, "");
     const encodedKey = trimmedKey.split("/").map(encodeURIComponent).join("/");
     const path = encodedKey ? `/${this.id}/${encodedKey}` : `/${this.id}`;
-    const { error, response } = await call(path, {
+    const init = {
       baseUrl: this._s3BaseUrl,
       parseAs: "stream",
       ...opts.query ? { params: { query: opts.query } } : {},
       // fetch accepts string / Uint8Array / ArrayBuffer bodies directly; pass it
-      // through unchanged (no Node Buffer in the data path).
+      // through unchanged (no Node Buffer in the data path). Bodies are fully
+      // buffered (never streams), so re-sending one on a retry is safe.
       ...opts.body !== void 0 ? { body: opts.body, bodySerializer: (b) => b } : {},
       ...opts.contentType ? { headers: { "Content-Type": opts.contentType } } : {}
-    });
-    if (!response.ok) {
-      const message = typeof error === "string" ? error : error ? JSON.stringify(error) : "";
+    };
+    const hasBody = method === "GET" || method === "POST";
+    const maxRetries = opts.retry === false ? 0 : MAX_S3_RETRIES;
+    for (let attempt = 0; ; attempt++) {
+      let error;
+      let response;
+      try {
+        ({ error, response } = await call(path, init));
+      } catch (transportError) {
+        if (attempt < maxRetries) {
+          await sleep(s3RetryDelayMs(attempt));
+          continue;
+        }
+        throw transportError;
+      }
+      if (!response.ok) {
+        if (isTransientS3Status(response.status) && attempt < maxRetries) {
+          await response.body?.cancel().catch(() => {
+          });
+          await sleep(s3RetryDelayMs(attempt));
+          continue;
+        }
+        const message = typeof error === "string" ? error : error ? JSON.stringify(error) : "";
+        return {
+          ok: false,
+          status: response.status,
+          statusText: response.statusText,
+          headers: response.headers,
+          body: new TextEncoder().encode(message)
+        };
+      }
+      const bytes = hasBody ? new Uint8Array(await response.arrayBuffer()) : new Uint8Array(0);
       return {
-        ok: false,
+        ok: true,
         status: response.status,
         statusText: response.statusText,
         headers: response.headers,
-        body: new TextEncoder().encode(message)
+        body: bytes
       };
     }
-    const bytes = method === "GET" ? new Uint8Array(await response.arrayBuffer()) : new Uint8Array(0);
-    return {
-      ok: true,
-      status: response.status,
-      statusText: response.statusText,
-      headers: response.headers,
-      body: bytes
-    };
   }
   /**
    * Connect to this disk's data plane via the native ArchilClient.
@@ -563,6 +718,199 @@ var Disk = class {
     });
   }
 };
+var DiskMultipart = class {
+  /** @internal */
+  constructor(diskId, s3Request) {
+    this.diskId = diskId;
+    this.s3Request = s3Request;
+  }
+  diskId;
+  s3Request;
+  /**
+   * Start a multipart upload (CreateMultipartUpload) and return its `uploadId`.
+   *
+   * @param key          Path on the disk the finished object will live at.
+   * @param contentType  MIME type to store the object with.
+   */
+  async create(key, contentType) {
+    const resp = await this.s3Request("POST", key, { query: { uploads: "" }, contentType });
+    if (!resp.ok) {
+      throw parseS3Error("CreateMultipartUpload", resp.status, resp.statusText, decodeText(resp.body));
+    }
+    const root = parseXml(decodeText(resp.body)).InitiateMultipartUploadResult ?? {};
+    const uploadId = optionalString(root.UploadId);
+    if (!uploadId) {
+      throw new ArchilS3Error({
+        operation: "CreateMultipartUpload",
+        statusCode: resp.status,
+        message: "response did not contain an UploadId",
+        raw: decodeText(resp.body)
+      });
+    }
+    return {
+      uploadId,
+      key: optionalString(root.Key) ?? key,
+      bucket: optionalString(root.Bucket) ?? this.diskId
+    };
+  }
+  /**
+   * Upload one part (UploadPart) and return its entity tag, which you must
+   * collect (with its part number) and pass to {@link complete}. Every part
+   * except the last must be at least 5 MiB.
+   *
+   * @param key         The upload's object key.
+   * @param uploadId    The id from {@link create}.
+   * @param partNumber  1-based part number (1..=10000).
+   * @param body        Part contents as a string, Uint8Array/Buffer, or ArrayBuffer.
+   */
+  async uploadPart(key, uploadId, partNumber, body) {
+    const resp = await this.s3Request("PUT", key, { query: { uploadId, partNumber }, body });
+    if (!resp.ok) {
+      throw parseS3Error("UploadPart", resp.status, resp.statusText, decodeText(resp.body));
+    }
+    return { partNumber, etag: resp.headers.get("etag") ?? "" };
+  }
+  /**
+   * Finish a multipart upload (CompleteMultipartUpload), assembling the listed
+   * parts into one object. Parts are sorted by part number before submission
+   * (the server requires strictly-increasing order).
+   *
+   * Unlike the other operations this is NOT auto-retried on a transient error:
+   * the gateway isn't idempotent for completion, so a retry after a
+   * successful-but-unacknowledged complete would return a spurious NoSuchUpload.
+   * On a transient failure, re-drive completion yourself only after confirming
+   * the object isn't already present.
+   *
+   * @param key       The upload's object key.
+   * @param uploadId  The id from {@link create}.
+   * @param parts     The `{ partNumber, etag }` pairs from {@link uploadPart}.
+   */
+  async complete(key, uploadId, parts) {
+    const sorted = [...parts].sort((a, b) => a.partNumber - b.partNumber);
+    const xml = buildCompleteMultipartUploadXml(sorted);
+    const resp = await this.s3Request("POST", key, {
+      query: { uploadId },
+      body: xml,
+      contentType: "application/xml",
+      retry: false
+    });
+    if (!resp.ok) {
+      throw parseS3Error("CompleteMultipartUpload", resp.status, resp.statusText, decodeText(resp.body));
+    }
+    const root = parseXml(decodeText(resp.body)).CompleteMultipartUploadResult ?? {};
+    return {
+      etag: optionalString(root.ETag),
+      location: optionalString(root.Location),
+      bucket: optionalString(root.Bucket),
+      key: optionalString(root.Key)
+    };
+  }
+  /**
+   * Abort a multipart upload (AbortMultipartUpload), discarding every staged
+   * part. Idempotent against an upload that's already gone (404 / NoSuchUpload
+   * resolves successfully).
+   *
+   * @param key       The upload's object key.
+   * @param uploadId  The id from {@link create}.
+   */
+  async abort(key, uploadId) {
+    const resp = await this.s3Request("DELETE", key, { query: { uploadId } });
+    if (!resp.ok && resp.status !== 404) {
+      throw parseS3Error("AbortMultipartUpload", resp.status, resp.statusText, decodeText(resp.body));
+    }
+  }
+  /**
+   * List the parts already uploaded for an in-progress upload (ListParts).
+   * Returns a single page; follow `nextPartNumberMarker` (when `isTruncated`)
+   * to page through the rest.
+   *
+   * @param key       The upload's object key.
+   * @param uploadId  The id from {@link create}.
+   * @param opts      `maxParts` / `partNumberMarker` pagination controls.
+   */
+  async listParts(key, uploadId, opts = {}) {
+    const query = { uploadId };
+    if (opts.maxParts !== void 0) query["max-parts"] = opts.maxParts;
+    if (opts.partNumberMarker !== void 0) query["part-number-marker"] = opts.partNumberMarker;
+    const resp = await this.s3Request("GET", key, { query });
+    if (!resp.ok) {
+      throw parseS3Error("ListParts", resp.status, resp.statusText, decodeText(resp.body));
+    }
+    return parseListPartsResult(decodeText(resp.body));
+  }
+  /**
+   * List in-progress multipart uploads on the disk (ListMultipartUploads).
+   * Returns a single page; follow `nextKeyMarker` / `nextUploadIdMarker` (when
+   * `isTruncated`) for the rest.
+   *
+   * @param opts  Prefix/delimiter filter and pagination markers.
+   */
+  async listUploads(opts = {}) {
+    const query = { uploads: "" };
+    if (opts.prefix !== void 0) query.prefix = opts.prefix;
+    if (opts.delimiter !== void 0) query.delimiter = opts.delimiter;
+    if (opts.keyMarker !== void 0) query["key-marker"] = opts.keyMarker;
+    if (opts.uploadIdMarker !== void 0) query["upload-id-marker"] = opts.uploadIdMarker;
+    if (opts.maxUploads !== void 0) query["max-uploads"] = opts.maxUploads;
+    const resp = await this.s3Request("GET", "", { query });
+    if (!resp.ok) {
+      throw parseS3Error("ListMultipartUploads", resp.status, resp.statusText, decodeText(resp.body));
+    }
+    return parseListMultipartUploadsResult(decodeText(resp.body));
+  }
+};
+var MAX_DELETE_OBJECTS_PER_REQUEST = 1e3;
+var MAX_S3_RETRIES = 3;
+var S3_RETRY_BASE_MS = 100;
+var S3_RETRY_CAP_MS = 2e3;
+function isTransientS3Status(status) {
+  return status === 429 || status === 500 || status === 502 || status === 503 || status === 504;
+}
+function s3RetryDelayMs(attempt) {
+  const ceiling = Math.min(S3_RETRY_CAP_MS, S3_RETRY_BASE_MS * 2 ** attempt);
+  return Math.random() * ceiling;
+}
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+var MIN_PART_SIZE = 5 * 1024 * 1024;
+var DEFAULT_PART_SIZE = 16 * 1024 * 1024;
+var DEFAULT_UPLOAD_CONCURRENCY = 4;
+var MAX_PARTS_PER_UPLOAD = 1e4;
+function effectiveUploadPartSize(totalBytes, requestedPartSize) {
+  if (Math.ceil(totalBytes / requestedPartSize) <= MAX_PARTS_PER_UPLOAD) {
+    return requestedPartSize;
+  }
+  const MiB = 1024 * 1024;
+  const needed = Math.ceil(totalBytes / MAX_PARTS_PER_UPLOAD);
+  return Math.ceil(needed / MiB) * MiB;
+}
+function toBytes(body) {
+  if (typeof body === "string") return new TextEncoder().encode(body);
+  if (body instanceof Uint8Array) return body;
+  return new Uint8Array(body);
+}
+function escapeXml(value) {
+  return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+}
+function buildDeleteObjectsXml(keys, quiet) {
+  const objects = keys.map((k) => `<Object><Key>${escapeXml(k)}</Key></Object>`).join("");
+  const quietTag = quiet ? "<Quiet>true</Quiet>" : "";
+  return `<?xml version="1.0" encoding="UTF-8"?><Delete>${objects}${quietTag}</Delete>`;
+}
+function buildCompleteMultipartUploadXml(parts) {
+  const body = parts.map(
+    (p) => `<Part><PartNumber>${p.partNumber}</PartNumber><ETag>${escapeXml(p.etag)}</ETag></Part>`
+  ).join("");
+  return `<?xml version="1.0" encoding="UTF-8"?><CompleteMultipartUpload>${body}</CompleteMultipartUpload>`;
+}
+function asArray(value) {
+  if (value === void 0 || value === null) return [];
+  return Array.isArray(value) ? value : [value];
+}
+function optionalNumber(value) {
+  return value === void 0 || value === null ? void 0 : Number(value);
+}
 function optionalString(value) {
   return value === void 0 || value === null ? void 0 : String(value);
 }
@@ -591,6 +939,60 @@ function parseListObjectsResult(xml) {
     prefix: optionalString(root.Prefix)
   };
 }
+function xmlIsTruncated(value) {
+  return value === "true" || value === true;
+}
+function parseDeleteObjectsResult(xml) {
+  const root = parseXml(xml).DeleteResult ?? {};
+  const deleted = asArray(root.Deleted).map((d) => optionalString(d.Key)).filter((k) => k !== void 0);
+  const errors = asArray(root.Error).map((e) => ({
+    key: String(e.Key ?? ""),
+    code: optionalString(e.Code),
+    message: optionalString(e.Message)
+  }));
+  return { deleted, errors };
+}
+function parseListPartsResult(xml) {
+  const root = parseXml(xml).ListPartsResult ?? {};
+  const parts = asArray(root.Part).map((p) => ({
+    partNumber: Number(p.PartNumber ?? 0),
+    etag: optionalString(p.ETag),
+    size: Number(p.Size ?? 0),
+    lastModified: optionalDate(p.LastModified)
+  }));
+  return {
+    bucket: optionalString(root.Bucket),
+    key: optionalString(root.Key),
+    uploadId: optionalString(root.UploadId),
+    parts,
+    isTruncated: xmlIsTruncated(root.IsTruncated),
+    partNumberMarker: Number(root.PartNumberMarker ?? 0),
+    nextPartNumberMarker: optionalNumber(root.NextPartNumberMarker),
+    maxParts: Number(root.MaxParts ?? parts.length)
+  };
+}
+function parseListMultipartUploadsResult(xml) {
+  const root = parseXml(xml).ListMultipartUploadsResult ?? {};
+  const uploads = asArray(root.Upload).map((u) => ({
+    key: String(u.Key ?? ""),
+    uploadId: String(u.UploadId ?? ""),
+    initiated: optionalDate(u.Initiated)
+  }));
+  const commonPrefixes = asArray(root.CommonPrefixes).map((cp) => optionalString(cp.Prefix)).filter((p) => p !== void 0);
+  return {
+    bucket: optionalString(root.Bucket),
+    uploads,
+    commonPrefixes,
+    isTruncated: xmlIsTruncated(root.IsTruncated),
+    keyMarker: optionalString(root.KeyMarker),
+    uploadIdMarker: optionalString(root.UploadIdMarker),
+    nextKeyMarker: optionalString(root.NextKeyMarker),
+    nextUploadIdMarker: optionalString(root.NextUploadIdMarker),
+    prefix: optionalString(root.Prefix),
+    delimiter: optionalString(root.Delimiter),
+    maxUploads: optionalNumber(root.MaxUploads)
+  };
+}
 
 // src/disks.ts
 var Disks = class {
@@ -783,6 +1185,7 @@ export {
   ArchilError,
   ArchilS3Error,
   Disk,
+  DiskMultipart,
   Disks,
   Tokens,
   USER_AGENT,
@@ -791,6 +1194,7 @@ export {
   createApiKey,
   createDisk,
   deleteApiKey,
+  effectiveUploadPartSize,
   exec,
   getDisk,
   listApiKeys,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "disk",
-  "version": "0.8.14",
+  "version": "0.8.16",
   "description": "Pure-JS client and CLI for Archil disks",
   "type": "module",
   "main": "dist/index.cjs",