@statewalker/webrun-files-s3 0.5.0 → 0.7.0

package/src/s3-files-api.ts

@@ -1,33 +1,40 @@
 /**
- * S3 implementation of IFilesApi
+ * S3 implementation of FilesApi
  *
  * Provides a filesystem-like interface over S3 object storage.
- * Supports all IFilesApi methods including the optional move, copy, and mkdir.
  */
 
 import type { S3Client } from "@aws-sdk/client-s3";
 import {
+  AbortMultipartUploadCommand,
+  CompleteMultipartUploadCommand,
   CopyObjectCommand,
+  CreateMultipartUploadCommand,
   DeleteObjectCommand,
+  GetObjectCommand,
   HeadObjectCommand,
   ListObjectsV2Command,
   PutObjectCommand,
+  UploadPartCommand,
 } from "@aws-sdk/client-s3";
 import type {
-  CopyOptions,
-  FileHandle,
   FileInfo,
-  FileRef,
-  IFilesApi,
+  FileStats,
+  FilesApi,
   ListOptions,
+  ReadOptions,
 } from "@statewalker/webrun-files";
-import { basename, resolveFileRef } from "@statewalker/webrun-files";
-import { S3FileHandle } from "./s3-file-handle.js";
+import { basename, normalizePath } from "@statewalker/webrun-files";
+
+/** Default part size for multipart uploads: 5MB (S3 minimum) */
+const DEFAULT_PART_SIZE = 5 * 1024 * 1024;
+
+/** Threshold for using multipart upload: 5MB */
+const MULTIPART_THRESHOLD = 5 * 1024 * 1024;
 
 export interface S3FilesApiOptions {
   /**
    * Pre-configured S3Client instance.
-   * Allows full control over credentials, region, endpoint.
    */
   client: S3Client;
 
@@ -38,42 +45,40 @@ export interface S3FilesApiOptions {
 
   /**
    * Optional key prefix (acts as root directory).
-   * All paths will be relative to this prefix.
    * @example "projects/my-app/data"
    */
   prefix?: string;
 
   /**
-   * Minimum part size for multipart uploads (5MB minimum for S3).
-   * @default 5 * 1024 * 1024 (5MB)
+   * Part size for multipart uploads in bytes.
+   * S3 requires minimum 5MB for all parts except the last.
+   * @default 5242880 (5MB)
    */
   multipartPartSize?: number;
 }
 
-export class S3FilesApi implements IFilesApi {
+/**
+ * S3 filesystem implementation of FilesApi.
+ *
+ * Provides a filesystem-like interface over S3 object storage.
+ * Works with AWS S3 and S3-compatible services.
+ */
+export class S3FilesApi implements FilesApi {
   private client: S3Client;
   private bucket: string;
   private prefix: string;
-  private multipartPartSize: number;
+  private partSize: number;
 
   constructor(options: S3FilesApiOptions) {
     this.client = options.client;
     this.bucket = options.bucket;
-    // Normalize prefix: remove leading/trailing slashes
     this.prefix = (options.prefix ?? "").replace(/^\/+|\/+$/g, "");
-    this.multipartPartSize = options.multipartPartSize ?? 5 * 1024 * 1024;
+    this.partSize = options.multipartPartSize ?? DEFAULT_PART_SIZE;
   }
 
-  /**
-   * Converts a virtual path to an S3 key.
-   * @example resolveKey("/docs/file.txt") => "prefix/docs/file.txt"
-   */
-  private resolveKey(file: FileRef): string {
-    const normalized = resolveFileRef(file);
-    // Remove leading slash and combine with prefix
-    const relativePath = normalized.startsWith("/")
-      ? normalized.substring(1)
-      : normalized;
+  private resolveKey(path: string): string {
+    const normalized = normalizePath(path);
+    const relativePath = normalized.startsWith("/") ? normalized.substring(1) : normalized;
 
     if (this.prefix) {
       return relativePath ? `${this.prefix}/${relativePath}` : this.prefix;
@@ -81,9 +86,6 @@ export class S3FilesApi implements IFilesApi {
     return relativePath;
   }
 
-  /**
-   * Converts an S3 key back to a virtual path.
-   */
   private keyToPath(key: string): string {
     let relativePath = key;
     if (this.prefix && key.startsWith(this.prefix)) {
@@ -92,21 +94,193 @@ export class S3FilesApi implements IFilesApi {
         relativePath = relativePath.substring(1);
       }
     }
-    return "/" + relativePath;
+    return `/${relativePath}`;
   }
 
-  /**
-   * Lists objects in a virtual directory.
-   *
-   * Uses S3's delimiter feature to simulate directory listing.
-   * CommonPrefixes represent subdirectories.
-   */
-  async *list(file: FileRef, options: ListOptions = {}): AsyncGenerator<FileInfo> {
-    const { recursive = false } = options;
-    const prefix = this.resolveKey(file);
-    // For root path ("/"), resolveKey returns empty string or prefix
-    // We need to ensure proper prefix handling
-    const normalizedPrefix = prefix ? prefix + "/" : (this.prefix ? this.prefix + "/" : "");
+  async *read(path: string, options?: ReadOptions): AsyncIterable<Uint8Array> {
+    const key = this.resolveKey(path);
+
+    try {
+      const start = options?.start ?? 0;
+      const length = options?.length;
+
+      // Return empty if length is explicitly 0
+      if (length === 0) {
+        return;
+      }
+
+      let range: string | undefined;
+      if (start > 0 || length !== undefined) {
+        const end = length !== undefined ? start + length - 1 : "";
+        range = `bytes=${start}-${end}`;
+      }
+
+      const command = new GetObjectCommand({
+        Bucket: this.bucket,
+        Key: key,
+        Range: range,
+      });
+
+      const response = await this.client.send(command);
+
+      if (response.Body) {
+        const stream = response.Body as AsyncIterable<Uint8Array>;
+        for await (const chunk of stream) {
+          yield chunk;
+        }
+      }
+    } catch (error: unknown) {
+      const err = error as { name?: string; $metadata?: { httpStatusCode?: number } };
+      // Handle NotFound (404) and InvalidRange (416) - return empty
+      if (
+        err.name === "NotFound" ||
+        err.name === "InvalidRange" ||
+        err.$metadata?.httpStatusCode === 404 ||
+        err.$metadata?.httpStatusCode === 416
+      ) {
+        return;
+      }
+      throw error;
+    }
+  }
+
+  async write(
+    path: string,
+    content: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,
+  ): Promise<void> {
+    const key = this.resolveKey(path);
+
+    // Buffer for accumulating small writes
+    let buffer: Uint8Array[] = [];
+    let bufferSize = 0;
+    let uploadId: string | undefined;
+    const parts: { ETag: string; PartNumber: number }[] = [];
+    let partNumber = 1;
+
+    const flushPart = async (isLast: boolean) => {
+      if (buffer.length === 0) return;
+
+      const partLength = buffer.reduce((sum, c) => sum + c.length, 0);
+      const partBuffer = new Uint8Array(partLength);
+      let offset = 0;
+      for (const chunk of buffer) {
+        partBuffer.set(chunk, offset);
+        offset += chunk.length;
+      }
+      buffer = [];
+      bufferSize = 0;
+
+      // For small files (under threshold) and this is the only/last part, use simple PutObject
+      if (!uploadId && isLast && partLength < MULTIPART_THRESHOLD) {
+        const command = new PutObjectCommand({
+          Bucket: this.bucket,
+          Key: key,
+          Body: partBuffer,
+          ContentLength: partLength,
+        });
+        await this.client.send(command);
+        return;
+      }
+
+      // Start multipart upload if not already started
+      if (!uploadId) {
+        const createCommand = new CreateMultipartUploadCommand({
+          Bucket: this.bucket,
+          Key: key,
+        });
+        const { UploadId } = await this.client.send(createCommand);
+        if (!UploadId) {
+          throw new Error("Failed to create multipart upload");
+        }
+        uploadId = UploadId;
+      }
+
+      // Upload the part
+      const uploadCommand = new UploadPartCommand({
+        Bucket: this.bucket,
+        Key: key,
+        UploadId: uploadId,
+        PartNumber: partNumber,
+        Body: partBuffer,
+      });
+
+      const response = await this.client.send(uploadCommand);
+      if (!response.ETag) {
+        throw new Error(`Failed to upload part ${partNumber}`);
+      }
+
+      parts.push({
+        ETag: response.ETag,
+        PartNumber: partNumber,
+      });
+      partNumber++;
+    };
+
+    try {
+      // Process chunks as they arrive
+      for await (const chunk of content) {
+        buffer.push(chunk);
+        bufferSize += chunk.length;
+
+        // Flush when buffer reaches part size
+        if (bufferSize >= this.partSize) {
+          await flushPart(false);
+        }
+      }
+
+      // Flush remaining data
+      await flushPart(true);
+
+      // Complete multipart upload if one was started
+      if (uploadId) {
+        const completeCommand = new CompleteMultipartUploadCommand({
+          Bucket: this.bucket,
+          Key: key,
+          UploadId: uploadId,
+          MultipartUpload: {
+            Parts: parts.sort((a, b) => a.PartNumber - b.PartNumber),
+          },
+        });
+        await this.client.send(completeCommand);
+      }
+    } catch (error) {
+      // Abort multipart upload on error
+      if (uploadId) {
+        const abortCommand = new AbortMultipartUploadCommand({
+          Bucket: this.bucket,
+          Key: key,
+          UploadId: uploadId,
+        });
+
+        try {
+          await this.client.send(abortCommand);
+        } catch {
+          // Ignore abort errors
+        }
+      }
+
+      throw error;
+    }
+  }
+
+  async mkdir(path: string): Promise<void> {
+    const key = this.resolveKey(path);
+    const dirKey = key.endsWith("/") ? key : `${key}/`;
+
+    const command = new PutObjectCommand({
+      Bucket: this.bucket,
+      Key: dirKey,
+      Body: new Uint8Array(0),
+      ContentLength: 0,
+    });
+
+    await this.client.send(command);
+  }
+
+  async *list(path: string, options?: ListOptions): AsyncIterable<FileInfo> {
+    const recursive = options?.recursive ?? false;
+    const prefix = this.resolveKey(path);
+    const normalizedPrefix = prefix ? `${prefix}/` : this.prefix ? `${this.prefix}/` : "";
 
     let continuationToken: string | undefined;
 
@@ -121,45 +295,38 @@ export class S3FilesApi implements IFilesApi {
 
       const response = await this.client.send(command);
 
-      // Yield directories (CommonPrefixes)
       if (!recursive && response.CommonPrefixes) {
         for (const cpPrefix of response.CommonPrefixes) {
           if (!cpPrefix.Prefix) continue;
 
-          // Remove trailing slash
           const dirKey = cpPrefix.Prefix.endsWith("/")
             ? cpPrefix.Prefix.slice(0, -1)
             : cpPrefix.Prefix;
 
-          const path = this.keyToPath(dirKey);
+          const dirPath = this.keyToPath(dirKey);
 
           yield {
             kind: "directory",
-            name: basename(path),
-            path,
+            name: basename(dirPath),
+            path: dirPath,
             lastModified: 0,
           };
         }
       }
 
-      // Yield files (Contents)
       if (response.Contents) {
         for (const obj of response.Contents) {
           if (!obj.Key) continue;
-
-          // Skip if it's just the prefix itself (directory marker)
           if (obj.Key === normalizedPrefix) continue;
-          if (obj.Key + "/" === normalizedPrefix) continue;
-
-          // Skip directory markers (keys ending with /)
+          if (`${obj.Key}/` === normalizedPrefix) continue;
           if (obj.Key.endsWith("/")) continue;
 
-          const path = this.keyToPath(obj.Key);
+          const filePath = this.keyToPath(obj.Key);
 
           yield {
             kind: "file",
-            name: basename(path),
-            path,
+            name: basename(filePath),
+            path: filePath,
             size: obj.Size ?? 0,
             lastModified: obj.LastModified?.getTime() ?? 0,
           };
@@ -170,19 +337,13 @@ export class S3FilesApi implements IFilesApi {
     } while (continuationToken);
   }
 
-  /**
-   * Gets object metadata using HEAD request.
-   */
-  async stats(file: FileRef): Promise<FileInfo | undefined> {
-    const normalized = resolveFileRef(file);
-    const key = this.resolveKey(file);
+  async stats(path: string): Promise<FileStats | undefined> {
+    const normalized = normalizePath(path);
+    const key = this.resolveKey(path);
 
-    // Handle root directory
     if (normalized === "/") {
       return {
         kind: "directory",
-        name: "",
-        path: "/",
         lastModified: 0,
       };
     }
@@ -198,11 +359,8 @@ export class S3FilesApi implements IFilesApi {
 
       return {
         kind: "file",
-        name: basename(normalized),
-        path: normalized,
         size: response.ContentLength ?? 0,
         lastModified: response.LastModified?.getTime() ?? 0,
-        type: response.ContentType,
       };
     } catch (error: unknown) {
       const err = error as { name?: string; $metadata?: { httpStatusCode?: number } };
@@ -211,8 +369,8 @@ export class S3FilesApi implements IFilesApi {
       }
     }
 
-    // Check if it's a virtual directory by listing with prefix
-    const prefix = key ? key + "/" : "";
+    // Check if it's a virtual directory
+    const prefix = key ? `${key}/` : "";
     const listCommand = new ListObjectsV2Command({
       Bucket: this.bucket,
       Prefix: prefix,
@@ -227,8 +385,6 @@ export class S3FilesApi implements IFilesApi {
     ) {
       return {
         kind: "directory",
-        name: basename(normalized),
-        path: normalized,
         lastModified: 0,
       };
     }
@@ -236,15 +392,16 @@ export class S3FilesApi implements IFilesApi {
     return undefined;
   }
 
-  /**
-   * Deletes an object or all objects under a prefix (directory).
-   */
-  async remove(file: FileRef): Promise<boolean> {
-    const key = this.resolveKey(file);
+  async exists(path: string): Promise<boolean> {
+    const stats = await this.stats(path);
+    return stats !== undefined;
+  }
+
+  async remove(path: string): Promise<boolean> {
+    const key = this.resolveKey(path);
 
     // First try to delete as a single object
     try {
-      // Check if object exists
       const headCommand = new HeadObjectCommand({
         Bucket: this.bucket,
         Key: key,
@@ -252,7 +409,6 @@ export class S3FilesApi implements IFilesApi {
 
       await this.client.send(headCommand);
 
-      // Object exists, delete it
       const deleteCommand = new DeleteObjectCommand({
         Bucket: this.bucket,
         Key: key,
@@ -265,11 +421,10 @@ export class S3FilesApi implements IFilesApi {
       if (err.name !== "NotFound" && err.$metadata?.httpStatusCode !== 404) {
         throw error;
       }
-      // Object doesn't exist, continue to check if it's a directory
     }
 
     // Delete all objects under this prefix (directory)
-    const prefix = key ? key + "/" : "";
+    const prefix = key ? `${key}/` : "";
     const keysToDelete: string[] = [];
 
     let continuationToken: string | undefined;
@@ -298,7 +453,6 @@ export class S3FilesApi implements IFilesApi {
       return false;
     }
 
-    // Delete objects one by one (more compatible with MinIO and other S3-compatible stores)
     for (const keyToDelete of keysToDelete) {
       const deleteCommand = new DeleteObjectCommand({
         Bucket: this.bucket,
@@ -311,69 +465,20 @@ export class S3FilesApi implements IFilesApi {
     return true;
   }
 
-  /**
-   * Opens an S3 object for random access.
-   * Creates the object if it doesn't exist.
-   */
-  async open(file: FileRef): Promise<FileHandle> {
-    const key = this.resolveKey(file);
-
-    let size = 0;
-
-    // Get current size if object exists
-    try {
-      const command = new HeadObjectCommand({
-        Bucket: this.bucket,
-        Key: key,
-      });
-
-      const response = await this.client.send(command);
-      size = response.ContentLength ?? 0;
-    } catch (error: unknown) {
-      const err = error as { name?: string; $metadata?: { httpStatusCode?: number } };
-      if (err.name !== "NotFound" && err.$metadata?.httpStatusCode !== 404) {
-        throw error;
-      }
-      // Object doesn't exist, will be created on write
-    }
-
-    return new S3FileHandle({
-      client: this.client,
-      bucket: this.bucket,
-      key,
-      size,
-      multipartPartSize: this.multipartPartSize,
-    });
-  }
-
-  /**
-   * Moves an object using Copy + Delete.
-   */
-  async move(source: FileRef, target: FileRef): Promise<boolean> {
+  async move(source: string, target: string): Promise<boolean> {
     const copied = await this.copy(source, target);
     if (!copied) return false;
-
     return this.remove(source);
   }
 
-  /**
-   * Copies an object or directory using S3 CopyObject.
-   */
-  async copy(
-    source: FileRef,
-    target: FileRef,
-    options: CopyOptions = {},
-  ): Promise<boolean> {
+  async copy(source: string, target: string): Promise<boolean> {
     const sourceKey = this.resolveKey(source);
     const targetKey = this.resolveKey(target);
-    const { recursive = true } = options;
 
-    // Check if source is a file
-    const sourceInfo = await this.stats(source);
-    if (!sourceInfo) return false;
+    const sourceStats = await this.stats(source);
+    if (!sourceStats) return false;
 
-    if (sourceInfo.kind === "file") {
-      // Copy single object
+    if (sourceStats.kind === "file") {
       const command = new CopyObjectCommand({
         Bucket: this.bucket,
         Key: targetKey,
@@ -385,12 +490,8 @@ export class S3FilesApi implements IFilesApi {
     }
 
     // Copy directory (all objects under prefix)
-    if (!recursive) {
-      return false;
-    }
-
-    const sourcePrefix = sourceKey + "/";
-    const targetPrefix = targetKey + "/";
+    const sourcePrefix = `${sourceKey}/`;
+    const targetPrefix = `${targetKey}/`;
 
     let continuationToken: string | undefined;
     let copied = false;
@@ -427,29 +528,4 @@ export class S3FilesApi implements IFilesApi {
 
     return copied;
   }
-
-  /**
-   * Creates a directory marker in S3.
-   *
-   * S3 doesn't have real directories, but some tools expect
-   * a zero-byte object with trailing slash as a directory marker.
-   * However, for our purposes we don't actually need to create markers
-   * since directories are implied by files within them.
-   */
-  async mkdir(file: FileRef): Promise<void> {
-    // For S3, directories don't really need to be created explicitly
-    // They exist implicitly when files exist inside them
-    // But we create an empty object as a marker for compatibility
-    const key = this.resolveKey(file);
-    const dirKey = key.endsWith("/") ? key : key + "/";
-
-    const command = new PutObjectCommand({
-      Bucket: this.bucket,
-      Key: dirKey,
-      Body: new Uint8Array(0),
-      ContentLength: 0,
-    });
-
-    await this.client.send(command);
-  }
 }

package/dist/s3-file-handle.d.ts

@@ -1,75 +0,0 @@
-/**
- * S3 FileHandle implementation
- *
- * Provides random access read/write operations for S3 objects.
- * Uses HTTP Range headers for efficient partial reads.
- * Since S3 objects are immutable, writes require re-uploading the entire object.
- */
-import type { S3Client } from "@aws-sdk/client-s3";
-import type { AppendOptions, BinaryStream, FileHandle, ReadStreamOptions, WriteStreamOptions } from "@statewalker/webrun-files";
-export interface S3FileHandleOptions {
-    client: S3Client;
-    bucket: string;
-    key: string;
-    size: number;
-    multipartPartSize: number;
-}
-export declare class S3FileHandle implements FileHandle {
-    private client;
-    private bucket;
-    private key;
-    private _size;
-    private multipartPartSize;
-    private closed;
-    constructor(options: S3FileHandleOptions);
-    get size(): number;
-    close(): Promise<void>;
-    /**
-     * Reads a range of bytes from the S3 object.
-     * Uses HTTP Range header for efficient partial reads.
-     * Streams data directly from S3 without buffering the entire response.
-     */
-    createReadStream(options?: ReadStreamOptions): AsyncGenerator<Uint8Array>;
-    /**
-     * Writes content to the S3 object using streaming.
-     *
-     * S3 objects are immutable, so this operation replaces the entire object.
-     * Uses streaming multipart upload to avoid buffering the entire file in memory.
-     * Only buffers one part at a time (5MB by default).
-     *
-     * Note: start position > 0 uses UploadPartCopy to preserve existing content
-     * without downloading it.
-     */
-    createWriteStream(data: BinaryStream, options?: WriteStreamOptions): Promise<number>;
-    /**
-     * Appends content to the end of the S3 object using streaming.
-     *
-     * Uses UploadPartCopy to preserve existing content without downloading it,
-     * then streams new data as additional parts.
-     */
-    appendFile(data: BinaryStream, options?: AppendOptions): Promise<number>;
-    /**
-     * Streaming multipart upload that buffers only one part at a time.
-     * Uses UploadPartCopy to preserve existing content without downloading it
-     * for large files. For small existing content (< part size), downloads and
-     * combines with new data to meet S3's minimum part size requirement.
-     */
-    private streamingMultipartUpload;
-    /**
-     * Download a range of bytes from the S3 object.
-     */
-    private downloadRange;
-    /**
-     * Upload a single part and record it in the parts array.
-     */
-    private uploadPart;
-    /**
-     * Abort a multipart upload, ignoring errors.
-     */
-    private abortMultipartUpload;
-    /**
-     * Random access read from S3 object using HTTP Range header.
-     */
-    read(buffer: Uint8Array, offset: number, length: number, position: number): Promise<number>;
-}
-//# sourceMappingURL=s3-file-handle.d.ts.map

package/dist/s3-file-handle.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"s3-file-handle.d.ts","sourceRoot":"","sources":["../src/s3-file-handle.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AASnD,OAAO,KAAK,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,iBAAiB,EACjB,kBAAkB,EACnB,MAAM,2BAA2B,CAAC;AAGnC,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,QAAQ,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AA+BD,qBAAa,YAAa,YAAW,UAAU;IAC7C,OAAO,CAAC,MAAM,CAAW;IACzB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,MAAM,CAAS;gBAEX,OAAO,EAAE,mBAAmB;IAQxC,IAAI,IAAI,IAAI,MAAM,CAEjB;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;;;OAIG;IACI,gBAAgB,CACrB,OAAO,GAAE,iBAAsB,GAC9B,cAAc,CAAC,UAAU,CAAC;IA6B7B;;;;;;;;;OASG;IACG,iBAAiB,CACrB,IAAI,EAAE,YAAY,EAClB,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,MAAM,CAAC;IAWlB;;;;;OAKG;IACG,UAAU,CAAC,IAAI,EAAE,YAAY,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,MAAM,CAAC;IAgBlF;;;;;OAKG;YACW,wBAAwB;IAgLtC;;OAEG;YACW,aAAa;IAwC3B;;OAEG;YACW,UAAU;IAuBxB;;OAEG;YACW,oBAAoB;IAYlC;;OAEG;IACG,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAoDlG"}