@statewalker/webrun-files-s3 0.5.0

package/dist/cjs/index.cjs

@@ -0,0 +1,579 @@
+let __aws_sdk_client_s3 = require("@aws-sdk/client-s3");
+let __statewalker_webrun_files = require("@statewalker/webrun-files");
+
+//#region src/s3-file-handle.ts
+/**
+* Helper to stream S3 response body as async iterable of Uint8Array chunks.
+* Handles both Node.js Readable streams and browser ReadableStreams.
+* Yields chunks as they arrive without buffering the entire response.
+*/
+async function* streamBody(body) {
+	if (!body) return;
+	if (Symbol.asyncIterator in body) {
+		for await (const chunk of body) yield chunk instanceof Uint8Array ? chunk : new Uint8Array(chunk);
+		return;
+	}
+	if (body instanceof Uint8Array) {
+		yield body;
+		return;
+	}
+	if (body instanceof ArrayBuffer) {
+		yield new Uint8Array(body);
+		return;
+	}
+	throw new Error("Unsupported body type");
+}
+var S3FileHandle = class {
+	client;
+	bucket;
+	key;
+	_size;
+	multipartPartSize;
+	closed = false;
+	constructor(options) {
+		this.client = options.client;
+		this.bucket = options.bucket;
+		this.key = options.key;
+		this._size = options.size;
+		this.multipartPartSize = options.multipartPartSize;
+	}
+	get size() {
+		return this._size;
+	}
+	async close() {
+		this.closed = true;
+	}
+	/**
+	* Reads a range of bytes from the S3 object.
+	* Uses HTTP Range header for efficient partial reads.
+	* Streams data directly from S3 without buffering the entire response.
+	*/
+	async *createReadStream(options = {}) {
+		if (this.closed) throw new Error("FileHandle is closed");
+		const { start = 0, end = Infinity, signal } = options;
+		const actualEnd = Math.min(end, this._size);
+		if (start >= actualEnd || this._size === 0) return;
+		const command = new __aws_sdk_client_s3.GetObjectCommand({
+			Bucket: this.bucket,
+			Key: this.key,
+			Range: `bytes=${start}-${actualEnd - 1}`
+		});
+		const response = await this.client.send(command, { abortSignal: signal });
+		if (!response.Body) return;
+		for await (const chunk of streamBody(response.Body)) {
+			if (signal?.aborted) throw new Error("Operation aborted");
+			yield chunk;
+		}
+	}
+	/**
+	* Writes content to the S3 object using streaming.
+	*
+	* S3 objects are immutable, so this operation replaces the entire object.
+	* Uses streaming multipart upload to avoid buffering the entire file in memory.
+	* Only buffers one part at a time (5MB by default).
+	*
+	* Note: start position > 0 uses UploadPartCopy to preserve existing content
+	* without downloading it.
+	*/
+	async createWriteStream(data, options = {}) {
+		if (this.closed) throw new Error("FileHandle is closed");
+		const { start = 0, signal } = options;
+		return this.streamingMultipartUpload(data, start, signal);
+	}
+	/**
+	* Appends content to the end of the S3 object using streaming.
+	*
+	* Uses UploadPartCopy to preserve existing content without downloading it,
+	* then streams new data as additional parts.
+	*/
+	async appendFile(data, options = {}) {
+		if (this.closed) throw new Error("FileHandle is closed");
+		const { signal } = options;
+		return this.streamingMultipartUpload(data, this._size, signal);
+	}
+	/**
+	* Streaming multipart upload that buffers only one part at a time.
+	* Uses UploadPartCopy to preserve existing content without downloading it
+	* for large files. For small existing content (< part size), downloads and
+	* combines with new data to meet S3's minimum part size requirement.
+	*/
+	async streamingMultipartUpload(data, start, signal) {
+		const createCommand = new __aws_sdk_client_s3.CreateMultipartUploadCommand({
+			Bucket: this.bucket,
+			Key: this.key
+		});
+		const { UploadId } = await this.client.send(createCommand, { abortSignal: signal });
+		if (!UploadId) throw new Error("Failed to initiate multipart upload");
+		const parts = [];
+		let partNumber = 1;
+		let bytesWritten = 0;
+		let totalSize = 0;
+		let buffer = new Uint8Array(this.multipartPartSize);
+		let bufferOffset = 0;
+		try {
+			if (start > 0 && this._size > 0) {
+				const preserveEnd = Math.min(start, this._size);
+				const fullPartsEnd = Math.floor(preserveEnd / this.multipartPartSize) * this.multipartPartSize;
+				let copyOffset = 0;
+				while (copyOffset < fullPartsEnd) {
+					if (signal?.aborted) throw new Error("Operation aborted");
+					const copyEnd = copyOffset + this.multipartPartSize;
+					const copyCommand = new __aws_sdk_client_s3.UploadPartCopyCommand({
+						Bucket: this.bucket,
+						Key: this.key,
+						UploadId,
+						PartNumber: partNumber,
+						CopySource: `${this.bucket}/${this.key}`,
+						CopySourceRange: `bytes=${copyOffset}-${copyEnd - 1}`
+					});
+					const { CopyPartResult } = await this.client.send(copyCommand, { abortSignal: signal });
+					if (CopyPartResult?.ETag) {
+						parts.push({
+							ETag: CopyPartResult.ETag,
+							PartNumber: partNumber
+						});
+						partNumber++;
+						totalSize += this.multipartPartSize;
+					}
+					copyOffset = copyEnd;
+				}
+				if (fullPartsEnd < preserveEnd) {
+					const remainingBytes = await this.downloadRange(fullPartsEnd, preserveEnd, signal);
+					buffer.set(remainingBytes, bufferOffset);
+					bufferOffset += remainingBytes.length;
+					totalSize += remainingBytes.length;
+				}
+				if (start > this._size) {
+					const paddingSize = start - this._size;
+					let paddingOffset = 0;
+					while (paddingOffset < paddingSize) {
+						const spaceInBuffer = this.multipartPartSize - bufferOffset;
+						const bytesToAdd = Math.min(spaceInBuffer, paddingSize - paddingOffset);
+						bufferOffset += bytesToAdd;
+						paddingOffset += bytesToAdd;
+						totalSize += bytesToAdd;
+						if (bufferOffset >= this.multipartPartSize) {
+							await this.uploadPart(UploadId, partNumber, buffer, parts, signal);
+							partNumber++;
+							buffer = new Uint8Array(this.multipartPartSize);
+							bufferOffset = 0;
+						}
+					}
+				}
+			} else if (start > 0) {
+				let paddingOffset = 0;
+				while (paddingOffset < start) {
+					const spaceInBuffer = this.multipartPartSize - bufferOffset;
+					const bytesToAdd = Math.min(spaceInBuffer, start - paddingOffset);
+					bufferOffset += bytesToAdd;
+					paddingOffset += bytesToAdd;
+					totalSize += bytesToAdd;
+					if (bufferOffset >= this.multipartPartSize) {
+						await this.uploadPart(UploadId, partNumber, buffer, parts, signal);
+						partNumber++;
+						buffer = new Uint8Array(this.multipartPartSize);
+						bufferOffset = 0;
+					}
+				}
+			}
+			for await (const chunk of (0, __statewalker_webrun_files.toBinaryAsyncIterable)(data)) {
+				if (signal?.aborted) throw new Error("Operation aborted");
+				let chunkOffset = 0;
+				while (chunkOffset < chunk.length) {
+					const spaceInBuffer = this.multipartPartSize - bufferOffset;
+					const bytesToCopy = Math.min(spaceInBuffer, chunk.length - chunkOffset);
+					buffer.set(chunk.subarray(chunkOffset, chunkOffset + bytesToCopy), bufferOffset);
+					bufferOffset += bytesToCopy;
+					chunkOffset += bytesToCopy;
+					bytesWritten += bytesToCopy;
+					totalSize += bytesToCopy;
+					if (bufferOffset >= this.multipartPartSize) {
+						await this.uploadPart(UploadId, partNumber, buffer, parts, signal);
+						partNumber++;
+						buffer = new Uint8Array(this.multipartPartSize);
+						bufferOffset = 0;
+					}
+				}
+			}
+			if (bufferOffset > 0) {
+				const lastPart = buffer.subarray(0, bufferOffset);
+				await this.uploadPart(UploadId, partNumber, lastPart, parts, signal);
+			}
+			if (parts.length === 0) await this.uploadPart(UploadId, 1, new Uint8Array(0), parts, signal);
+			const completeCommand = new __aws_sdk_client_s3.CompleteMultipartUploadCommand({
+				Bucket: this.bucket,
+				Key: this.key,
+				UploadId,
+				MultipartUpload: { Parts: parts }
+			});
+			await this.client.send(completeCommand, { abortSignal: signal });
+			this._size = totalSize;
+			return bytesWritten;
+		} catch (error) {
+			await this.abortMultipartUpload(UploadId);
+			throw error;
+		}
+	}
+	/**
+	* Download a range of bytes from the S3 object.
+	*/
+	async downloadRange(start, end, signal) {
+		if (end <= start) return new Uint8Array(0);
+		const command = new __aws_sdk_client_s3.GetObjectCommand({
+			Bucket: this.bucket,
+			Key: this.key,
+			Range: `bytes=${start}-${end - 1}`
+		});
+		const response = await this.client.send(command, { abortSignal: signal });
+		if (!response.Body) return new Uint8Array(0);
+		const chunks = [];
+		for await (const chunk of streamBody(response.Body)) chunks.push(chunk);
+		if (chunks.length === 0) return new Uint8Array(0);
+		if (chunks.length === 1) return chunks[0];
+		const totalLength = chunks.reduce((sum, c) => sum + c.length, 0);
+		const result = new Uint8Array(totalLength);
+		let offset = 0;
+		for (const chunk of chunks) {
+			result.set(chunk, offset);
+			offset += chunk.length;
+		}
+		return result;
+	}
+	/**
+	* Upload a single part and record it in the parts array.
+	*/
+	async uploadPart(uploadId, partNumber, data, parts, signal) {
+		const command = new __aws_sdk_client_s3.UploadPartCommand({
+			Bucket: this.bucket,
+			Key: this.key,
+			UploadId: uploadId,
+			PartNumber: partNumber,
+			Body: data,
+			ContentLength: data.length
+		});
+		const { ETag } = await this.client.send(command, { abortSignal: signal });
+		if (ETag) parts.push({
+			ETag,
+			PartNumber: partNumber
+		});
+	}
+	/**
+	* Abort a multipart upload, ignoring errors.
+	*/
+	async abortMultipartUpload(uploadId) {
+		const command = new __aws_sdk_client_s3.AbortMultipartUploadCommand({
+			Bucket: this.bucket,
+			Key: this.key,
+			UploadId: uploadId
+		});
+		await this.client.send(command).catch(() => {});
+	}
+	/**
+	* Random access read from S3 object using HTTP Range header.
+	*/
+	async read(buffer, offset, length, position) {
+		if (this.closed) throw new Error("FileHandle is closed");
+		let len = Math.min(length, buffer.length - offset);
+		len = Math.min(len, this._size - position);
+		len = Math.max(0, len);
+		if (len === 0) return 0;
+		const command = new __aws_sdk_client_s3.GetObjectCommand({
+			Bucket: this.bucket,
+			Key: this.key,
+			Range: `bytes=${position}-${position + len - 1}`
+		});
+		const response = await this.client.send(command);
+		if (!response.Body) return 0;
+		const chunks = [];
+		for await (const chunk of streamBody(response.Body)) chunks.push(chunk);
+		if (chunks.length === 0) return 0;
+		let data;
+		if (chunks.length === 1) data = chunks[0];
+		else {
+			const totalLength = chunks.reduce((sum, c) => sum + c.length, 0);
+			data = new Uint8Array(totalLength);
+			let pos = 0;
+			for (const chunk of chunks) {
+				data.set(chunk, pos);
+				pos += chunk.length;
+			}
+		}
+		const bytesToCopy = Math.min(data.length, len);
+		buffer.set(data.subarray(0, bytesToCopy), offset);
+		return bytesToCopy;
+	}
+};
+
+//#endregion
+//#region src/s3-files-api.ts
+var S3FilesApi = class {
+	client;
+	bucket;
+	prefix;
+	multipartPartSize;
+	constructor(options) {
+		this.client = options.client;
+		this.bucket = options.bucket;
+		this.prefix = (options.prefix ?? "").replace(/^\/+|\/+$/g, "");
+		this.multipartPartSize = options.multipartPartSize ?? 5 * 1024 * 1024;
+	}
+	/**
+	* Converts a virtual path to an S3 key.
+	* @example resolveKey("/docs/file.txt") => "prefix/docs/file.txt"
+	*/
+	resolveKey(file) {
+		const normalized = (0, __statewalker_webrun_files.resolveFileRef)(file);
+		const relativePath = normalized.startsWith("/") ? normalized.substring(1) : normalized;
+		if (this.prefix) return relativePath ? `${this.prefix}/${relativePath}` : this.prefix;
+		return relativePath;
+	}
+	/**
+	* Converts an S3 key back to a virtual path.
+	*/
+	keyToPath(key) {
+		let relativePath = key;
+		if (this.prefix && key.startsWith(this.prefix)) {
+			relativePath = key.substring(this.prefix.length);
+			if (relativePath.startsWith("/")) relativePath = relativePath.substring(1);
+		}
+		return "/" + relativePath;
+	}
+	/**
+	* Lists objects in a virtual directory.
+	*
+	* Uses S3's delimiter feature to simulate directory listing.
+	* CommonPrefixes represent subdirectories.
+	*/
+	async *list(file, options = {}) {
+		const { recursive = false } = options;
+		const prefix = this.resolveKey(file);
+		const normalizedPrefix = prefix ? prefix + "/" : this.prefix ? this.prefix + "/" : "";
+		let continuationToken;
+		do {
+			const command = new __aws_sdk_client_s3.ListObjectsV2Command({
+				Bucket: this.bucket,
+				Prefix: normalizedPrefix,
+				Delimiter: recursive ? void 0 : "/",
+				ContinuationToken: continuationToken,
+				MaxKeys: 1e3
+			});
+			const response = await this.client.send(command);
+			if (!recursive && response.CommonPrefixes) for (const cpPrefix of response.CommonPrefixes) {
+				if (!cpPrefix.Prefix) continue;
+				const dirKey = cpPrefix.Prefix.endsWith("/") ? cpPrefix.Prefix.slice(0, -1) : cpPrefix.Prefix;
+				const path = this.keyToPath(dirKey);
+				yield {
+					kind: "directory",
+					name: (0, __statewalker_webrun_files.basename)(path),
+					path,
+					lastModified: 0
+				};
+			}
+			if (response.Contents) for (const obj of response.Contents) {
+				if (!obj.Key) continue;
+				if (obj.Key === normalizedPrefix) continue;
+				if (obj.Key + "/" === normalizedPrefix) continue;
+				if (obj.Key.endsWith("/")) continue;
+				const path = this.keyToPath(obj.Key);
+				yield {
+					kind: "file",
+					name: (0, __statewalker_webrun_files.basename)(path),
+					path,
+					size: obj.Size ?? 0,
+					lastModified: obj.LastModified?.getTime() ?? 0
+				};
+			}
+			continuationToken = response.NextContinuationToken;
+		} while (continuationToken);
+	}
+	/**
+	* Gets object metadata using HEAD request.
+	*/
+	async stats(file) {
+		const normalized = (0, __statewalker_webrun_files.resolveFileRef)(file);
+		const key = this.resolveKey(file);
+		if (normalized === "/") return {
+			kind: "directory",
+			name: "",
+			path: "/",
+			lastModified: 0
+		};
+		try {
+			const command = new __aws_sdk_client_s3.HeadObjectCommand({
+				Bucket: this.bucket,
+				Key: key
+			});
+			const response = await this.client.send(command);
+			return {
+				kind: "file",
+				name: (0, __statewalker_webrun_files.basename)(normalized),
+				path: normalized,
+				size: response.ContentLength ?? 0,
+				lastModified: response.LastModified?.getTime() ?? 0,
+				type: response.ContentType
+			};
+		} catch (error) {
+			const err = error;
+			if (err.name !== "NotFound" && err.$metadata?.httpStatusCode !== 404) throw error;
+		}
+		const prefix = key ? key + "/" : "";
+		const listCommand = new __aws_sdk_client_s3.ListObjectsV2Command({
+			Bucket: this.bucket,
+			Prefix: prefix,
+			MaxKeys: 1
+		});
+		const listResponse = await this.client.send(listCommand);
+		if (listResponse.Contents && listResponse.Contents.length > 0 || listResponse.CommonPrefixes && listResponse.CommonPrefixes.length > 0) return {
+			kind: "directory",
+			name: (0, __statewalker_webrun_files.basename)(normalized),
+			path: normalized,
+			lastModified: 0
+		};
+	}
+	/**
+	* Deletes an object or all objects under a prefix (directory).
+	*/
+	async remove(file) {
+		const key = this.resolveKey(file);
+		try {
+			const headCommand = new __aws_sdk_client_s3.HeadObjectCommand({
+				Bucket: this.bucket,
+				Key: key
+			});
+			await this.client.send(headCommand);
+			const deleteCommand = new __aws_sdk_client_s3.DeleteObjectCommand({
+				Bucket: this.bucket,
+				Key: key
+			});
+			await this.client.send(deleteCommand);
+			return true;
+		} catch (error) {
+			const err = error;
+			if (err.name !== "NotFound" && err.$metadata?.httpStatusCode !== 404) throw error;
+		}
+		const prefix = key ? key + "/" : "";
+		const keysToDelete = [];
+		let continuationToken;
+		do {
+			const listCommand = new __aws_sdk_client_s3.ListObjectsV2Command({
+				Bucket: this.bucket,
+				Prefix: prefix,
+				ContinuationToken: continuationToken
+			});
+			const response = await this.client.send(listCommand);
+			if (response.Contents) {
+				for (const obj of response.Contents) if (obj.Key) keysToDelete.push(obj.Key);
+			}
+			continuationToken = response.NextContinuationToken;
+		} while (continuationToken);
+		if (keysToDelete.length === 0) return false;
+		for (const keyToDelete of keysToDelete) {
+			const deleteCommand = new __aws_sdk_client_s3.DeleteObjectCommand({
+				Bucket: this.bucket,
+				Key: keyToDelete
+			});
+			await this.client.send(deleteCommand);
+		}
+		return true;
+	}
+	/**
+	* Opens an S3 object for random access.
+	* Creates the object if it doesn't exist.
+	*/
+	async open(file) {
+		const key = this.resolveKey(file);
+		let size = 0;
+		try {
+			const command = new __aws_sdk_client_s3.HeadObjectCommand({
+				Bucket: this.bucket,
+				Key: key
+			});
+			size = (await this.client.send(command)).ContentLength ?? 0;
+		} catch (error) {
+			const err = error;
+			if (err.name !== "NotFound" && err.$metadata?.httpStatusCode !== 404) throw error;
+		}
+		return new S3FileHandle({
+			client: this.client,
+			bucket: this.bucket,
+			key,
+			size,
+			multipartPartSize: this.multipartPartSize
+		});
+	}
+	/**
+	* Moves an object using Copy + Delete.
+	*/
+	async move(source, target) {
+		if (!await this.copy(source, target)) return false;
+		return this.remove(source);
+	}
+	/**
+	* Copies an object or directory using S3 CopyObject.
+	*/
+	async copy(source, target, options = {}) {
+		const sourceKey = this.resolveKey(source);
+		const targetKey = this.resolveKey(target);
+		const { recursive = true } = options;
+		const sourceInfo = await this.stats(source);
+		if (!sourceInfo) return false;
+		if (sourceInfo.kind === "file") {
+			const command = new __aws_sdk_client_s3.CopyObjectCommand({
+				Bucket: this.bucket,
+				Key: targetKey,
+				CopySource: encodeURIComponent(`${this.bucket}/${sourceKey}`)
+			});
+			await this.client.send(command);
+			return true;
+		}
+		if (!recursive) return false;
+		const sourcePrefix = sourceKey + "/";
+		const targetPrefix = targetKey + "/";
+		let continuationToken;
+		let copied = false;
+		do {
+			const listCommand = new __aws_sdk_client_s3.ListObjectsV2Command({
+				Bucket: this.bucket,
+				Prefix: sourcePrefix,
+				ContinuationToken: continuationToken
+			});
+			const response = await this.client.send(listCommand);
+			if (response.Contents) for (const obj of response.Contents) {
+				if (!obj.Key) continue;
+				const newKey = targetPrefix + obj.Key.substring(sourcePrefix.length);
+				const copyCommand = new __aws_sdk_client_s3.CopyObjectCommand({
+					Bucket: this.bucket,
+					Key: newKey,
+					CopySource: encodeURIComponent(`${this.bucket}/${obj.Key}`)
+				});
+				await this.client.send(copyCommand);
+				copied = true;
+			}
+			continuationToken = response.NextContinuationToken;
+		} while (continuationToken);
+		return copied;
+	}
+	/**
+	* Creates a directory marker in S3.
+	*
+	* S3 doesn't have real directories, but some tools expect
+	* a zero-byte object with trailing slash as a directory marker.
+	* However, for our purposes we don't actually need to create markers
+	* since directories are implied by files within them.
+	*/
+	async mkdir(file) {
+		const key = this.resolveKey(file);
+		const dirKey = key.endsWith("/") ? key : key + "/";
+		const command = new __aws_sdk_client_s3.PutObjectCommand({
+			Bucket: this.bucket,
+			Key: dirKey,
+			Body: new Uint8Array(0),
+			ContentLength: 0
+		});
+		await this.client.send(command);
+	}
+};
+
+//#endregion
+exports.S3FileHandle = S3FileHandle;
+exports.S3FilesApi = S3FilesApi;