@procedures/s3 0.0.0-stage → 0.1.0

package/src/module.ts

@@ -0,0 +1,211 @@
+/**
+ * Module (no side effects): the procedures of this package as data, for `createRegistry()`.
+ *
+ * Procedure Registration for S3 operations
+ *
+ * Provides s3.upload, s3.download, s3.list, s3.delete, and s3.multipart.* procedures.
+ * Credentials are loaded from environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY).
+ */
+
+import { createProcedure, zodAdapter, defineModule, type ProcedureModule } from "@procedures/client";
+import {
+  s3Upload,
+  s3Download,
+  s3List,
+  s3ListAll,
+  s3Delete,
+  s3MultipartInit,
+  s3MultipartUpload,
+  s3MultipartComplete,
+  s3MultipartAbort,
+} from "./procedures/s3/index.js";
+import {
+  S3UploadInputSchema,
+  S3DownloadInputSchema,
+  S3ListInputSchema,
+  S3ListAllInputSchema,
+  S3DeleteInputSchema,
+  S3MultipartInitInputSchema,
+  S3MultipartUploadInputSchema,
+  S3MultipartCompleteInputSchema,
+  S3MultipartAbortInputSchema,
+  type S3UploadInput,
+  type S3DownloadInput,
+  type S3ListInput,
+  type S3ListAllInput,
+  type S3DeleteInput,
+  type S3MultipartInitInput,
+  type S3MultipartUploadInput,
+  type S3MultipartCompleteInput,
+  type S3MultipartAbortInput,
+  type S3UploadOutput,
+  type S3DownloadOutput,
+  type S3ListOutput,
+  type S3ListAllOutput,
+  type S3DeleteOutput,
+  type S3MultipartInitOutput,
+  type S3MultipartUploadOutput,
+  type S3MultipartCompleteOutput,
+  type S3MultipartAbortOutput,
+  S3UploadOutputSchema,
+  S3DownloadOutputSchema,
+  S3ListOutputSchema,
+  S3ListAllOutputSchema,
+  S3DeleteOutputSchema,
+  S3MultipartInitOutputSchema,
+  S3MultipartUploadOutputSchema,
+  S3MultipartCompleteOutputSchema,
+  S3MultipartAbortOutputSchema,
+} from "./types.js";
+
+// =============================================================================
+// Procedures
+// =============================================================================
+
+const s3UploadProcedure = createProcedure()
+  .path(["s3", "upload"])
+  .input(zodAdapter<S3UploadInput>(S3UploadInputSchema))
+  .output(zodAdapter<S3UploadOutput>(S3UploadOutputSchema))
+  .meta({
+    description: "Upload content to S3 bucket",
+    shorts: { bucket: "b", key: "k" },
+    output: "json",
+  })
+  .handler(async (input: S3UploadInput): Promise<S3UploadOutput> => {
+    return s3Upload(input);
+  })
+  .build();
+
+const s3DownloadProcedure = createProcedure()
+  .path(["s3", "download"])
+  .input(zodAdapter<S3DownloadInput>(S3DownloadInputSchema))
+  .output(zodAdapter<S3DownloadOutput>(S3DownloadOutputSchema))
+  .meta({
+    description: "Download content from S3 bucket",
+    shorts: { bucket: "b", key: "k", encoding: "e" },
+    output: "json",
+  })
+  .handler(async (input: S3DownloadInput): Promise<S3DownloadOutput> => {
+    return s3Download(input);
+  })
+  .build();
+
+const s3ListProcedure = createProcedure()
+  .path(["s3", "list"])
+  .input(zodAdapter<S3ListInput>(S3ListInputSchema))
+  .output(zodAdapter<S3ListOutput>(S3ListOutputSchema))
+  .meta({
+    description: "List objects in S3 bucket",
+    shorts: { bucket: "b", prefix: "p", maxKeys: "n" },
+    output: "json",
+  })
+  .handler(async (input: S3ListInput): Promise<S3ListOutput> => {
+    return s3List(input);
+  })
+  .build();
+
+const s3ListAllProcedure = createProcedure()
+  .path(["s3", "listAll"])
+  .input(zodAdapter<S3ListAllInput>(S3ListAllInputSchema))
+  .output(zodAdapter<S3ListAllOutput>(S3ListAllOutputSchema))
+  .meta({
+    description: "List ALL objects in S3 bucket, following continuation tokens (unbounded by 1000)",
+    shorts: { bucket: "b", prefix: "p", maxKeys: "n" },
+    output: "json",
+  })
+  .handler(async (input: S3ListAllInput): Promise<S3ListAllOutput> => {
+    return s3ListAll(input);
+  })
+  .build();
+
+const s3DeleteProcedure = createProcedure()
+  .path(["s3", "delete"])
+  .input(zodAdapter<S3DeleteInput>(S3DeleteInputSchema))
+  .output(zodAdapter<S3DeleteOutput>(S3DeleteOutputSchema))
+  .meta({
+    description: "Delete object from S3 bucket",
+    shorts: { bucket: "b", key: "k" },
+    output: "json",
+  })
+  .handler(async (input: S3DeleteInput): Promise<S3DeleteOutput> => {
+    return s3Delete(input);
+  })
+  .build();
+
+const s3MultipartInitProcedure = createProcedure()
+  .path(["s3", "multipart", "init"])
+  .input(zodAdapter<S3MultipartInitInput>(S3MultipartInitInputSchema))
+  .output(zodAdapter<S3MultipartInitOutput>(S3MultipartInitOutputSchema))
+  .meta({
+    description: "Initialize multipart upload for large files",
+    shorts: { bucket: "b", key: "k" },
+    output: "json",
+  })
+  .handler(async (input: S3MultipartInitInput): Promise<S3MultipartInitOutput> => {
+    return s3MultipartInit(input);
+  })
+  .build();
+
+const s3MultipartUploadProcedure = createProcedure()
+  .path(["s3", "multipart", "upload"])
+  .input(zodAdapter<S3MultipartUploadInput>(S3MultipartUploadInputSchema))
+  .output(zodAdapter<S3MultipartUploadOutput>(S3MultipartUploadOutputSchema))
+  .meta({
+    description: "Upload a part in multipart upload",
+    shorts: { bucket: "b", key: "k", partNumber: "n" },
+    output: "json",
+  })
+  .handler(async (input: S3MultipartUploadInput): Promise<S3MultipartUploadOutput> => {
+    return s3MultipartUpload(input);
+  })
+  .build();
+
+const s3MultipartCompleteProcedure = createProcedure()
+  .path(["s3", "multipart", "complete"])
+  .input(zodAdapter<S3MultipartCompleteInput>(S3MultipartCompleteInputSchema))
+  .output(zodAdapter<S3MultipartCompleteOutput>(S3MultipartCompleteOutputSchema))
+  .meta({
+    description: "Complete multipart upload by assembling parts",
+    shorts: { bucket: "b", key: "k" },
+    output: "json",
+  })
+  .handler(async (input: S3MultipartCompleteInput): Promise<S3MultipartCompleteOutput> => {
+    return s3MultipartComplete(input);
+  })
+  .build();
+
+const s3MultipartAbortProcedure = createProcedure()
+  .path(["s3", "multipart", "abort"])
+  .input(zodAdapter<S3MultipartAbortInput>(S3MultipartAbortInputSchema))
+  .output(zodAdapter<S3MultipartAbortOutput>(S3MultipartAbortOutputSchema))
+  .meta({
+    description: "Abort multipart upload and clean up parts",
+    shorts: { bucket: "b", key: "k" },
+    output: "json",
+  })
+  .handler(async (input: S3MultipartAbortInput): Promise<S3MultipartAbortOutput> => {
+    return s3MultipartAbort(input);
+  })
+  .build();
+
+// =============================================================================
+// Registration
+// =============================================================================
+
+/**
+ * The module of this package, for `createRegistry()`: its procedures as data. Importing this
+ * file registers nothing. (`register.ts` registers the same procedures in the global registry.)
+ */
+export const s3Module: ProcedureModule = defineModule("@procedures/s3", [
+    s3UploadProcedure,
+    s3DownloadProcedure,
+    s3ListProcedure,
+    s3ListAllProcedure,
+    s3DeleteProcedure,
+    s3MultipartInitProcedure,
+    s3MultipartUploadProcedure,
+    s3MultipartCompleteProcedure,
+    s3MultipartAbortProcedure,
+  ]);
+
+export default s3Module;

package/src/procedures/s3/delete.ts

@@ -0,0 +1,45 @@
+/**
+ * s3.delete procedure
+ *
+ * Delete object from S3 bucket.
+ */
+
+import { DeleteObjectCommand } from "@aws-sdk/client-s3";
+import { getS3Client } from "../../s3-client.js";
+import type { S3DeleteInput, S3DeleteOutput } from "../../types.js";
+
+/**
+ * Delete object from S3
+ *
+ * @example
+ * // Delete an object
+ * await client.call(["s3", "delete"], {
+ *   bucket: "my-bucket",
+ *   key: "path/to/file.json",
+ * });
+ *
+ * @example
+ * // Delete specific version
+ * await client.call(["s3", "delete"], {
+ *   bucket: "my-bucket",
+ *   key: "path/to/file.json",
+ *   versionId: "abc123",
+ * });
+ */
+export async function s3Delete(input: S3DeleteInput): Promise<S3DeleteOutput> {
+  const client = getS3Client();
+
+  const command = new DeleteObjectCommand({
+    Bucket: input.bucket,
+    Key: input.key,
+    VersionId: input.versionId,
+  });
+
+  const response = await client.send(command);
+
+  return {
+    deleted: true,
+    deleteMarker: response.DeleteMarker,
+    versionId: response.VersionId,
+  };
+}

package/src/procedures/s3/download.ts

@@ -0,0 +1,155 @@
+/**
+ * s3.download procedure
+ *
+ * Download content from S3 bucket.
+ */
+
+import { randomUUID } from "node:crypto";
+import { createWriteStream } from "node:fs";
+import { rename, rm } from "node:fs/promises";
+import { Transform, type Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { GetObjectCommand } from "@aws-sdk/client-s3";
+import { getS3Client } from "../../s3-client.js";
+import type { S3DownloadInput, S3DownloadOutput } from "../../types.js";
+
+/** Largest object loaded into memory by default (the base64 body is a third larger again) */
+const DEFAULT_MAX_BYTES = 64 * 1024 * 1024;
+
+function tooLarge(bytes: number | string, maxBytes: number): Error {
+  return new Error(
+    `Object is ${bytes} bytes, more than maxBytes (${maxBytes}). Use destPath to download it to a file.`
+  );
+}
+
+/** A stream step that counts the bytes and fails past `maxBytes`. */
+function byteLimit(maxBytes: number): Transform {
+  let total = 0;
+  return new Transform({
+    transform(chunk: Buffer, _encoding, callback) {
+      total += chunk.length;
+      if (total > maxBytes) {
+        callback(tooLarge(`more than ${maxBytes}`, maxBytes));
+        return;
+      }
+      callback(null, chunk);
+    },
+  });
+}
+
+/**
+ * Read a body into memory, counting the bytes. `Content-Length` can be missing or wrong, so the
+ * count is the limit (deep dive DATA-14).
+ */
+async function readLimited(body: Readable, maxBytes: number): Promise<Buffer> {
+  const chunks: Buffer[] = [];
+  let total = 0;
+  for await (const chunk of body) {
+    const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array);
+    total += buffer.length;
+    if (total > maxBytes) {
+      body.destroy();
+      throw tooLarge(`more than ${maxBytes}`, maxBytes);
+    }
+    chunks.push(buffer);
+  }
+  return Buffer.concat(chunks);
+}
+
+/**
+ * Download content from S3
+ *
+ * @example
+ * // Download text file
+ * const result = await client.call(["s3", "download"], {
+ *   bucket: "my-bucket",
+ *   key: "path/to/file.json",
+ *   encoding: "utf8",
+ * });
+ * console.log(result.body); // JSON string
+ *
+ * @example
+ * // Download binary as base64
+ * const result = await client.call(["s3", "download"], {
+ *   bucket: "my-bucket",
+ *   key: "path/to/image.png",
+ * });
+ * console.log(result.body); // base64 encoded
+ *
+ * @example
+ * // Download a large object straight to a file
+ * await client.call(["s3", "download"], {
+ *   bucket: "my-bucket",
+ *   key: "backups/archive.tar.gz",
+ *   destPath: "/tmp/archive.tar.gz",
+ * });
+ */
+export async function s3Download(input: S3DownloadInput): Promise<S3DownloadOutput> {
+  const client = getS3Client();
+
+  const command = new GetObjectCommand({
+    Bucket: input.bucket,
+    Key: input.key,
+    VersionId: input.versionId,
+  });
+
+  const response = await client.send(command);
+
+  // To a file: stream the body, nothing is held in memory (BUGS-2026-07 M22)
+  if (input.destPath) {
+    if (!response.Body) {
+      throw new Error("Empty response body");
+    }
+    // Write a part file, then rename it. A failed transfer leaves the previous file
+    // (createWriteStream on destPath truncated it first: deep dive DATA-14).
+    const partPath = `${input.destPath}.${randomUUID().slice(0, 8)}.part`;
+    const source = response.Body as Readable;
+    try {
+      if (input.maxBytes !== undefined) {
+        await pipeline(source, byteLimit(input.maxBytes), createWriteStream(partPath));
+      } else {
+        await pipeline(source, createWriteStream(partPath));
+      }
+      await rename(partPath, input.destPath);
+    } catch (error) {
+      await rm(partPath, { force: true });
+      throw error;
+    }
+    return {
+      body: "",
+      path: input.destPath,
+      contentLength: response.ContentLength ?? 0,
+      contentType: response.ContentType,
+      etag: response.ETag?.replace(/"/g, ""),
+      lastModified: response.LastModified?.toISOString(),
+    };
+  }
+
+  // Into memory: refuse an object that is too large before reading it (BUGS-2026-07 M22)
+  const maxBytes = input.maxBytes ?? DEFAULT_MAX_BYTES;
+  if (response.ContentLength !== undefined && response.ContentLength > maxBytes) {
+    (response.Body as Readable | undefined)?.destroy?.();
+    throw tooLarge(response.ContentLength, maxBytes);
+  }
+
+  if (!response.Body) {
+    throw new Error("Empty response body");
+  }
+  const bodyBytes = await readLimited(response.Body as Readable, maxBytes);
+
+  // Convert to string based on encoding
+  let body: string;
+  if (input.encoding) {
+    body = bodyBytes.toString(input.encoding as BufferEncoding);
+  } else {
+    body = bodyBytes.toString("base64");
+  }
+
+  return {
+    body,
+    contentLength: bodyBytes.length,
+    contentType: response.ContentType,
+    etag: response.ETag?.replace(/"/g, ""),
+    lastModified: response.LastModified?.toISOString(),
+  };
+}

package/src/procedures/s3/index.ts

@@ -0,0 +1,13 @@
+/**
+ * S3 Procedures Index
+ */
+
+export { s3Upload } from "./upload.js";
+export { s3Download } from "./download.js";
+export { s3List } from "./list.js";
+export { s3ListAll } from "./list-all.js";
+export { s3Delete } from "./delete.js";
+export { s3MultipartInit } from "./multipart-init.js";
+export { s3MultipartUpload } from "./multipart-upload.js";
+export { s3MultipartComplete } from "./multipart-complete.js";
+export { s3MultipartAbort } from "./multipart-abort.js";

package/src/procedures/s3/list-all.ts

@@ -0,0 +1,63 @@
+/**
+ * s3.listAll procedure
+ *
+ * List ALL objects in an S3 bucket, transparently following the
+ * ListObjectsV2 continuation token so that buckets/prefixes with more than
+ * 1000 keys are fully reachable in a single call.
+ */
+
+import { s3List } from "./list.js";
+import type { S3ListAllInput, S3ListAllOutput, S3ListObject } from "../../types.js";
+
+/**
+ * List every object under a bucket/prefix, following pagination automatically.
+ *
+ * Unlike `s3.list` (which returns at most one page — up to 1000 keys — and
+ * exposes `nextContinuationToken` for manual paging), `s3.listAll` loops until
+ * the result is no longer truncated and returns the aggregated contents.
+ *
+ * @example
+ * const { contents, keyCount } = await client.call(["s3", "listAll"], {
+ *   bucket: "my-bucket",
+ *   prefix: "snapshots/",
+ * });
+ * console.log(keyCount); // e.g. 4213, not clamped to 1000
+ */
+export async function s3ListAll(input: S3ListAllInput): Promise<S3ListAllOutput> {
+  const contents: S3ListObject[] = [];
+  const commonPrefixes: string[] = [];
+  const seenPrefixes = new Set<string>();
+  let continuationToken: string | undefined = undefined;
+  let pageCount = 0;
+
+  do {
+    const page = await s3List({
+      bucket: input.bucket,
+      prefix: input.prefix,
+      delimiter: input.delimiter,
+      maxKeys: input.maxKeys,
+      continuationToken,
+    });
+    pageCount += 1;
+
+    for (const object of page.contents) {
+      contents.push(object);
+    }
+    for (const prefix of page.commonPrefixes) {
+      if (!seenPrefixes.has(prefix)) {
+        seenPrefixes.add(prefix);
+        commonPrefixes.push(prefix);
+      }
+    }
+
+    // Only continue while S3 reports more results AND hands back a token.
+    continuationToken = page.isTruncated ? page.nextContinuationToken : undefined;
+  } while (continuationToken);
+
+  return {
+    contents,
+    commonPrefixes,
+    keyCount: contents.length,
+    pageCount,
+  };
+}

package/src/procedures/s3/list.ts

@@ -0,0 +1,63 @@
+/**
+ * s3.list procedure
+ *
+ * List objects in S3 bucket.
+ */
+
+import { ListObjectsV2Command } from "@aws-sdk/client-s3";
+import { getS3Client } from "../../s3-client.js";
+import type { S3ListInput, S3ListOutput, S3ListObject } from "../../types.js";
+
+/**
+ * List objects in S3 bucket
+ *
+ * @example
+ * // List all objects with prefix
+ * const result = await client.call(["s3", "list"], {
+ *   bucket: "my-bucket",
+ *   prefix: "snapshots/",
+ *   maxKeys: 100,
+ * });
+ *
+ * @example
+ * // Hierarchical listing with delimiter
+ * const result = await client.call(["s3", "list"], {
+ *   bucket: "my-bucket",
+ *   prefix: "data/",
+ *   delimiter: "/",
+ * });
+ * console.log(result.commonPrefixes); // ["data/2024/", "data/2025/"]
+ */
+export async function s3List(input: S3ListInput): Promise<S3ListOutput> {
+  const client = getS3Client();
+
+  const command = new ListObjectsV2Command({
+    Bucket: input.bucket,
+    Prefix: input.prefix,
+    MaxKeys: input.maxKeys || 1000,
+    ContinuationToken: input.continuationToken,
+    Delimiter: input.delimiter,
+  });
+
+  const response = await client.send(command);
+
+  const contents: S3ListObject[] = (response.Contents || []).map((obj) => ({
+    key: obj.Key || "",
+    size: obj.Size || 0,
+    lastModified: obj.LastModified?.toISOString() || "",
+    etag: obj.ETag?.replace(/"/g, "") || "",
+    storageClass: obj.StorageClass,
+  }));
+
+  const commonPrefixes = (response.CommonPrefixes || [])
+    .map((cp) => cp.Prefix || "")
+    .filter(Boolean);
+
+  return {
+    contents,
+    commonPrefixes,
+    isTruncated: response.IsTruncated || false,
+    nextContinuationToken: response.NextContinuationToken,
+    keyCount: response.KeyCount || contents.length,
+  };
+}

package/src/procedures/s3/multipart-abort.ts

@@ -0,0 +1,40 @@
+/**
+ * s3.multipart.abort procedure
+ *
+ * Abort a multipart upload and clean up parts.
+ */
+
+import { AbortMultipartUploadCommand } from "@aws-sdk/client-s3";
+import { getS3Client } from "../../s3-client.js";
+import type { S3MultipartAbortInput, S3MultipartAbortOutput } from "../../types.js";
+
+/**
+ * Abort multipart upload
+ *
+ * Use to cancel an in-progress multipart upload.
+ * This cleans up any parts that were already uploaded.
+ *
+ * @example
+ * await client.call(["s3", "multipart", "abort"], {
+ *   bucket: "my-bucket",
+ *   key: "large-file.tar.gz",
+ *   uploadId: "abc123",
+ * });
+ */
+export async function s3MultipartAbort(
+  input: S3MultipartAbortInput
+): Promise<S3MultipartAbortOutput> {
+  const client = getS3Client();
+
+  const command = new AbortMultipartUploadCommand({
+    Bucket: input.bucket,
+    Key: input.key,
+    UploadId: input.uploadId,
+  });
+
+  await client.send(command);
+
+  return {
+    aborted: true,
+  };
+}

package/src/procedures/s3/multipart-complete.ts

@@ -0,0 +1,58 @@
+/**
+ * s3.multipart.complete procedure
+ *
+ * Complete a multipart upload by assembling parts.
+ */
+
+import { CompleteMultipartUploadCommand } from "@aws-sdk/client-s3";
+import { getS3Client } from "../../s3-client.js";
+import type { S3MultipartCompleteInput, S3MultipartCompleteOutput } from "../../types.js";
+
+/**
+ * Complete multipart upload
+ *
+ * Call after all parts have been uploaded. Parts must be provided
+ * in order with their ETags from the upload responses.
+ *
+ * @example
+ * const result = await client.call(["s3", "multipart", "complete"], {
+ *   bucket: "my-bucket",
+ *   key: "large-file.tar.gz",
+ *   uploadId: "abc123",
+ *   parts: [
+ *     { partNumber: 1, etag: "etag1" },
+ *     { partNumber: 2, etag: "etag2" },
+ *     { partNumber: 3, etag: "etag3" },
+ *   ],
+ * });
+ */
+export async function s3MultipartComplete(
+  input: S3MultipartCompleteInput
+): Promise<S3MultipartCompleteOutput> {
+  const client = getS3Client();
+
+  const command = new CompleteMultipartUploadCommand({
+    Bucket: input.bucket,
+    Key: input.key,
+    UploadId: input.uploadId,
+    MultipartUpload: {
+      // S3 refuses parts out of order. Parts that were uploaded in parallel can arrive in any
+      // order, so the procedure sorts them by number.
+      Parts: [...input.parts]
+        .sort((a, b) => a.partNumber - b.partNumber)
+        .map((part) => ({
+          ETag: `"${part.etag}"`,
+          PartNumber: part.partNumber,
+        })),
+    },
+  });
+
+  const response = await client.send(command);
+
+  return {
+    location: response.Location || `s3://${input.bucket}/${input.key}`,
+    bucket: response.Bucket || input.bucket,
+    key: response.Key || input.key,
+    etag: response.ETag?.replace(/"/g, "") || "",
+  };
+}

package/src/procedures/s3/multipart-init.ts

@@ -0,0 +1,50 @@
+/**
+ * s3.multipart.init procedure
+ *
+ * Initialize a multipart upload for large files.
+ */
+
+import { CreateMultipartUploadCommand } from "@aws-sdk/client-s3";
+import { getS3Client } from "../../s3-client.js";
+import type { S3MultipartInitInput, S3MultipartInitOutput } from "../../types.js";
+
+/**
+ * Initialize multipart upload
+ *
+ * Use this for files larger than 5MB. Workflow:
+ * 1. s3.multipart.init - get uploadId
+ * 2. s3.multipart.upload - upload each part (5MB min, except last)
+ * 3. s3.multipart.complete - finalize with part ETags
+ *
+ * @example
+ * const init = await client.call(["s3", "multipart", "init"], {
+ *   bucket: "my-bucket",
+ *   key: "large-file.tar.gz",
+ *   contentType: "application/gzip",
+ * });
+ * console.log(init.uploadId); // Use for subsequent part uploads
+ */
+export async function s3MultipartInit(
+  input: S3MultipartInitInput
+): Promise<S3MultipartInitOutput> {
+  const client = getS3Client();
+
+  const command = new CreateMultipartUploadCommand({
+    Bucket: input.bucket,
+    Key: input.key,
+    ContentType: input.contentType,
+    Metadata: input.metadata,
+  });
+
+  const response = await client.send(command);
+
+  if (!response.UploadId) {
+    throw new Error("Failed to initialize multipart upload: no UploadId returned");
+  }
+
+  return {
+    uploadId: response.UploadId,
+    bucket: input.bucket,
+    key: input.key,
+  };
+}