@statewalker/webrun-files-s3 0.10.4 → 0.11.0

package/README.md

@@ -83,6 +83,20 @@ for await (const chunk of files.read('/data/large.bin', { start: 1000, length: 5
 }
 ```
 
+### A lock object, shared by every client of the bucket
+
+```typescript
+const locked = await files.createExclusive('/jobs/rebuild.lock', [new TextEncoder().encode(host)]);
+if (!locked) return; // another client created it first
+```
+
+`createExclusive` sends the request that makes the object visible, `PutObject` or, above
+5 MiB, `CompleteMultipartUpload`, with `If-None-Match: *`. The service creates the key only if no
+object has it and answers `412 Precondition Failed` otherwise, which the method returns as
+`false`. Every client of the bucket is excluded. AWS S3 has supported conditional writes since
+August 2024, and RustFS, which the integration tests use, honours them. A service that ignores
+the header gives no exclusion at all: check yours before relying on it.
+
 ## Internals
 
 ### How calls map to S3
@@ -95,6 +109,7 @@ path "/docs/file.txt" + prefix "my-app/data"  ->  key "my-app/data/docs/file.txt
 | --- | --- |
 | `read` | `GetObject`, with `Range: bytes=start-end` when `start`/`length` are given; streams the body |
 | `write` | under 5 MiB: one `PutObject`; otherwise `CreateMultipartUpload` + one `UploadPart` per `multipartPartSize` + `CompleteMultipartUpload`; `AbortMultipartUpload` on any error |
+| `createExclusive` | a `stats` check (any object or prefix at the path returns `false`), then as `write`, with `If-None-Match: *` on the final `PutObject` or `CompleteMultipartUpload`; `412` returns `false`, a `409 ConditionalRequestConflict` (another conditional write in flight) is retried up to four times |
 | `mkdir` | `PutObject` of an empty `dir/` marker key |
 | `list` | `ListObjectsV2` with `Delimiter: "/"` (or none when recursive), `StartAfter` for `after` |
 | `copy` | `CopyObject` per object, server-side |
@@ -124,10 +139,17 @@ under `a/` both exist, `/a` is listed once, as the file, the same answer `stats(
 
 ### What breaks, and how it looks
 
-- **`move` and directory `copy` are not atomic.** They are many `CopyObject` and `DeleteObject`
-  calls; a failure midway leaves some objects copied and the source partly removed.
+- **`move` and directory `copy` are not atomic.** S3 has no rename: they are many `CopyObject`
+  and `DeleteObject` calls; a failure midway leaves some objects copied and the source partly
+  removed. When the copy succeeded but the source cannot be deleted, `move` retries the delete
+  once, then throws a `MoveIncompleteError` (`move /a -> /b: the target was replaced, but the
+  source could not be removed`) instead of returning `false`.
 - **Writing zero chunks creates nothing.** `write(path, [])` sends no request, so the path does not
   exist afterwards. Pass `[new Uint8Array(0)]` to create an empty file.
+- **Only the key is exclusive.** A directory is a key prefix, so an existing one makes
+  `createExclusive` return `false` through the check before the upload, but a directory created
+  concurrently with the call is not excluded. Unlike `write`, `createExclusive(path, [])`
+  creates an empty object.
 - **Single-request copies.** Each object is copied with one `CopyObject`, which S3 limits to 5 GB
   per object.
 - **Errors other than not-found propagate.** `read` treats `404` and `416` as an empty read; any
@@ -135,11 +157,13 @@ under `a/` both exist, `/a` is listed once, as the file, the same answer `stats(
 
 ### Testing
 
-`pnpm test` runs only unit tests (a fake client exercises listing order across pages).
+`pnpm test` runs only unit tests, against fake clients: listing order across pages, and how
+`createExclusive` handles `412` (returns `false`, also from `CompleteMultipartUpload`) and `409`
+(retried, then thrown).
 `pnpm test:integration` needs Docker: it starts
 [RustFS](https://github.com/rustfs/rustfs) through testcontainers (`rustfs/rustfs:latest`, override
 with `RUSTFS_IMAGE`) and runs the shared `createFilesApiTests` and `createBigFilesApiTests` suites
-against it.
+against it, the `createExclusive` cases included.
 
 ### Dependencies
 

package/dist/cjs/index.cjs

@@ -1,7 +1,7 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
 let _aws_sdk_client_s3 = require("@aws-sdk/client-s3");
 let _statewalker_webrun_files = require("@statewalker/webrun-files");
-//#region src/s3-files-api.ts
+// src/s3-files-api.ts
 /** Default part size for multipart uploads: 5MB (S3 minimum) */
 const DEFAULT_PART_SIZE = 5242880;
 /** Threshold for using multipart upload: 5MB */
@@ -62,14 +62,40 @@ var S3FilesApi = class {
 		}
 	}
 	async write(path, content) {
-		const key = this.resolveKey(path);
+		await this.upload(this.resolveKey(path), content, false);
+	}
+	/**
+	* A conditional write, `If-None-Match: *`, on the `PutObject` (or, above the multipart
+	* threshold, on the `CompleteMultipartUpload`) that makes the object visible: the
+	* service creates the key only if no object has it, and answers 412 otherwise. That
+	* excludes every client of the bucket. AWS S3 supports it since August 2024; an
+	* S3-compatible service that ignores the header gives no exclusion at all, so check
+	* yours before relying on this.
+	*
+	* Only the key itself is exclusive. A directory is a key prefix, so an existing one is
+	* turned away by a check before the upload, but one created concurrently with this
+	* call is not excluded. Nothing is visible until the content is complete; a failing
+	* source aborts the multipart upload and leaves no object.
+	*/
+	async createExclusive(path, content) {
+		if ((0, _statewalker_webrun_files.normalizePath)(path) === "/" || await this.stats(path)) return false;
+		return this.upload(this.resolveKey(path), content, true);
+	}
+	/**
+	* Upload `content` to `key`: one PutObject below the multipart threshold, else a
+	* multipart upload. With `exclusive`, the request that creates the object carries
+	* `If-None-Match: *`; returns false when the service refuses it because the key exists.
+	*/
+	async upload(key, content, exclusive) {
+		const ifNoneMatch = exclusive ? "*" : void 0;
+		const sendLast = (send) => exclusive ? sendConditional(send) : send();
 		let buffer = [];
 		let bufferSize = 0;
 		let uploadId;
 		const parts = [];
 		let partNumber = 1;
 		const flushPart = async (isLast) => {
-			if (buffer.length === 0) return;
+			if (buffer.length === 0 && !(exclusive && isLast && !uploadId)) return;
 			const partLength = buffer.reduce((sum, c) => sum + c.length, 0);
 			const partBuffer = new Uint8Array(partLength);
 			let offset = 0;
@@ -84,9 +110,10 @@ var S3FilesApi = class {
 					Bucket: this.bucket,
 					Key: key,
 					Body: partBuffer,
-					ContentLength: partLength
+					ContentLength: partLength,
+					IfNoneMatch: ifNoneMatch
 				});
-				await this.client.send(command);
+				await sendLast(() => this.client.send(command));
 				return;
 			}
 			if (!uploadId) {
@@ -125,10 +152,12 @@ var S3FilesApi = class {
 					Bucket: this.bucket,
 					Key: key,
 					UploadId: uploadId,
-					MultipartUpload: { Parts: parts.sort((a, b) => a.PartNumber - b.PartNumber) }
+					MultipartUpload: { Parts: parts.sort((a, b) => a.PartNumber - b.PartNumber) },
+					IfNoneMatch: ifNoneMatch
 				});
-				await this.client.send(completeCommand);
+				await sendLast(() => this.client.send(completeCommand));
 			}
+			return true;
 		} catch (error) {
 			if (uploadId) {
 				const abortCommand = new _aws_sdk_client_s3.AbortMultipartUploadCommand({
@@ -140,6 +169,7 @@ var S3FilesApi = class {
 					await this.client.send(abortCommand);
 				} catch {}
 			}
+			if (exclusive && isPreconditionFailed(error)) return false;
 			throw error;
 		}
 	}
@@ -316,9 +346,18 @@ var S3FilesApi = class {
 		}
 		return true;
 	}
+	/**
+	* S3 has no rename, so a move is a copy, then a delete, and is NOT atomic: while it
+	* runs, the object exists under both keys, and a directory (a key prefix) moves object
+	* by object. A failed copy throws and keeps the source. If the copy succeeds but the
+	* source cannot be deleted, the delete is retried once, then a `MoveIncompleteError`
+	* is thrown: the target holds the new content and the source still exists. False is
+	* never returned for a half-done move.
+	*/
 	async move(source, target) {
 		if (!await this.copy(source, target)) return false;
-		return this.remove(source);
+		await (0, _statewalker_webrun_files.removeMovedSource)(this, (0, _statewalker_webrun_files.normalizePath)(source), (0, _statewalker_webrun_files.normalizePath)(target));
+		return true;
 	}
 	async copy(source, target) {
 		const sourceKey = this.resolveKey(source);
@@ -377,5 +416,23 @@ function insertInOrder(entries, entry) {
 	while (i > 0 && (0, _statewalker_webrun_files.comparePaths)(entries[i - 1].path, entry.path) > 0) i--;
 	entries.splice(i, 0, entry);
 }
-//#endregion
+/** The service refused a conditional write because the key exists (HTTP 412). */
+function isPreconditionFailed(error) {
+	const err = error;
+	return err?.name === "PreconditionFailed" || err?.$metadata?.httpStatusCode === 412;
+}
+/**
+* Send a request that may carry `If-None-Match`, retrying a 409
+* `ConditionalRequestConflict`: S3 answers that while another conditional write to the
+* key is in flight, and asks the client to retry, after which one of them gets 412.
+*/
+async function sendConditional(send) {
+	for (let attempt = 0;; attempt++) try {
+		return await send();
+	} catch (error) {
+		const err = error;
+		if (!(err?.name === "ConditionalRequestConflict" || err?.$metadata?.httpStatusCode === 409) || attempt >= 4) throw error;
+		await new Promise((resolve) => setTimeout(resolve, 50 * 2 ** attempt));
+	}
+}
 exports.S3FilesApi = S3FilesApi;

package/dist/esm/index.js

@@ -1,6 +1,6 @@
 import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CopyObjectCommand, CreateMultipartUploadCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, UploadPartCommand } from "@aws-sdk/client-s3";
-import { basename, comparePaths, normalizePath } from "@statewalker/webrun-files";
-//#region src/s3-files-api.ts
+import { basename, comparePaths, normalizePath, removeMovedSource } from "@statewalker/webrun-files";
+// src/s3-files-api.ts
 /** Default part size for multipart uploads: 5MB (S3 minimum) */
 const DEFAULT_PART_SIZE = 5242880;
 /** Threshold for using multipart upload: 5MB */
@@ -61,14 +61,40 @@ var S3FilesApi = class {
 		}
 	}
 	async write(path, content) {
-		const key = this.resolveKey(path);
+		await this.upload(this.resolveKey(path), content, false);
+	}
+	/**
+	* A conditional write, `If-None-Match: *`, on the `PutObject` (or, above the multipart
+	* threshold, on the `CompleteMultipartUpload`) that makes the object visible: the
+	* service creates the key only if no object has it, and answers 412 otherwise. That
+	* excludes every client of the bucket. AWS S3 supports it since August 2024; an
+	* S3-compatible service that ignores the header gives no exclusion at all, so check
+	* yours before relying on this.
+	*
+	* Only the key itself is exclusive. A directory is a key prefix, so an existing one is
+	* turned away by a check before the upload, but one created concurrently with this
+	* call is not excluded. Nothing is visible until the content is complete; a failing
+	* source aborts the multipart upload and leaves no object.
+	*/
+	async createExclusive(path, content) {
+		if (normalizePath(path) === "/" || await this.stats(path)) return false;
+		return this.upload(this.resolveKey(path), content, true);
+	}
+	/**
+	* Upload `content` to `key`: one PutObject below the multipart threshold, else a
+	* multipart upload. With `exclusive`, the request that creates the object carries
+	* `If-None-Match: *`; returns false when the service refuses it because the key exists.
+	*/
+	async upload(key, content, exclusive) {
+		const ifNoneMatch = exclusive ? "*" : void 0;
+		const sendLast = (send) => exclusive ? sendConditional(send) : send();
 		let buffer = [];
 		let bufferSize = 0;
 		let uploadId;
 		const parts = [];
 		let partNumber = 1;
 		const flushPart = async (isLast) => {
-			if (buffer.length === 0) return;
+			if (buffer.length === 0 && !(exclusive && isLast && !uploadId)) return;
 			const partLength = buffer.reduce((sum, c) => sum + c.length, 0);
 			const partBuffer = new Uint8Array(partLength);
 			let offset = 0;
@@ -83,9 +109,10 @@ var S3FilesApi = class {
 					Bucket: this.bucket,
 					Key: key,
 					Body: partBuffer,
-					ContentLength: partLength
+					ContentLength: partLength,
+					IfNoneMatch: ifNoneMatch
 				});
-				await this.client.send(command);
+				await sendLast(() => this.client.send(command));
 				return;
 			}
 			if (!uploadId) {
@@ -124,10 +151,12 @@ var S3FilesApi = class {
 					Bucket: this.bucket,
 					Key: key,
 					UploadId: uploadId,
-					MultipartUpload: { Parts: parts.sort((a, b) => a.PartNumber - b.PartNumber) }
+					MultipartUpload: { Parts: parts.sort((a, b) => a.PartNumber - b.PartNumber) },
+					IfNoneMatch: ifNoneMatch
 				});
-				await this.client.send(completeCommand);
+				await sendLast(() => this.client.send(completeCommand));
 			}
+			return true;
 		} catch (error) {
 			if (uploadId) {
 				const abortCommand = new AbortMultipartUploadCommand({
@@ -139,6 +168,7 @@ var S3FilesApi = class {
 					await this.client.send(abortCommand);
 				} catch {}
 			}
+			if (exclusive && isPreconditionFailed(error)) return false;
 			throw error;
 		}
 	}
@@ -315,9 +345,18 @@ var S3FilesApi = class {
 		}
 		return true;
 	}
+	/**
+	* S3 has no rename, so a move is a copy, then a delete, and is NOT atomic: while it
+	* runs, the object exists under both keys, and a directory (a key prefix) moves object
+	* by object. A failed copy throws and keeps the source. If the copy succeeds but the
+	* source cannot be deleted, the delete is retried once, then a `MoveIncompleteError`
+	* is thrown: the target holds the new content and the source still exists. False is
+	* never returned for a half-done move.
+	*/
 	async move(source, target) {
 		if (!await this.copy(source, target)) return false;
-		return this.remove(source);
+		await removeMovedSource(this, normalizePath(source), normalizePath(target));
+		return true;
 	}
 	async copy(source, target) {
 		const sourceKey = this.resolveKey(source);
@@ -376,5 +415,23 @@ function insertInOrder(entries, entry) {
 	while (i > 0 && comparePaths(entries[i - 1].path, entry.path) > 0) i--;
 	entries.splice(i, 0, entry);
 }
-//#endregion
+/** The service refused a conditional write because the key exists (HTTP 412). */
+function isPreconditionFailed(error) {
+	const err = error;
+	return err?.name === "PreconditionFailed" || err?.$metadata?.httpStatusCode === 412;
+}
+/**
+* Send a request that may carry `If-None-Match`, retrying a 409
+* `ConditionalRequestConflict`: S3 answers that while another conditional write to the
+* key is in flight, and asks the client to retry, after which one of them gets 412.
+*/
+async function sendConditional(send) {
+	for (let attempt = 0;; attempt++) try {
+		return await send();
+	} catch (error) {
+		const err = error;
+		if (!(err?.name === "ConditionalRequestConflict" || err?.$metadata?.httpStatusCode === 409) || attempt >= 4) throw error;
+		await new Promise((resolve) => setTimeout(resolve, 50 * 2 ** attempt));
+	}
+}
 export { S3FilesApi };

package/dist/s3-files-api.d.ts

@@ -42,6 +42,26 @@ export declare class S3FilesApi implements FilesApi {
     private keyToPath;
     read(path: string, options?: ReadOptions): AsyncIterable<Uint8Array>;
     write(path: string, content: Iterable<Uint8Array> | AsyncIterable<Uint8Array>): Promise<void>;
+    /**
+     * A conditional write, `If-None-Match: *`, on the `PutObject` (or, above the multipart
+     * threshold, on the `CompleteMultipartUpload`) that makes the object visible: the
+     * service creates the key only if no object has it, and answers 412 otherwise. That
+     * excludes every client of the bucket. AWS S3 supports it since August 2024; an
+     * S3-compatible service that ignores the header gives no exclusion at all, so check
+     * yours before relying on this.
+     *
+     * Only the key itself is exclusive. A directory is a key prefix, so an existing one is
+     * turned away by a check before the upload, but one created concurrently with this
+     * call is not excluded. Nothing is visible until the content is complete; a failing
+     * source aborts the multipart upload and leaves no object.
+     */
+    createExclusive(path: string, content: Iterable<Uint8Array> | AsyncIterable<Uint8Array>): Promise<boolean>;
+    /**
+     * Upload `content` to `key`: one PutObject below the multipart threshold, else a
+     * multipart upload. With `exclusive`, the request that creates the object carries
+     * `If-None-Match: *`; returns false when the service refuses it because the key exists.
+     */
+    private upload;
     mkdir(path: string): Promise<void>;
     /**
      * Keys list in UTF-8 byte order, which is `comparePaths` order for files.
@@ -54,6 +74,14 @@ export declare class S3FilesApi implements FilesApi {
     stats(path: string): Promise<FileStats | undefined>;
     exists(path: string): Promise<boolean>;
     remove(path: string): Promise<boolean>;
+    /**
+     * S3 has no rename, so a move is a copy, then a delete, and is NOT atomic: while it
+     * runs, the object exists under both keys, and a directory (a key prefix) moves object
+     * by object. A failed copy throws and keeps the source. If the copy succeeds but the
+     * source cannot be deleted, the delete is retried once, then a `MoveIncompleteError`
+     * is thrown: the target holds the new content and the source still exists. False is
+     * never returned for a half-done move.
+     */
     move(source: string, target: string): Promise<boolean>;
     copy(source: string, target: string): Promise<boolean>;
 }

package/dist/s3-files-api.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"s3-files-api.d.ts","sourceRoot":"","sources":["../src/s3-files-api.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAanD,OAAO,KAAK,EACV,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,WAAW,EACX,WAAW,EACZ,MAAM,2BAA2B,CAAC;AASnC,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,MAAM,EAAE,QAAQ,CAAC;IAEjB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;;GAKG;AACH,qBAAa,UAAW,YAAW,QAAQ;IACzC,OAAO,CAAC,MAAM,CAAW;IACzB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,QAAQ,CAAS;IAEzB,YAAY,OAAO,EAAE,iBAAiB,EAKrC;IAED,OAAO,CAAC,UAAU;IAUlB,OAAO,CAAC,SAAS;IAWV,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,aAAa,CAAC,UAAU,CAAC,CA6C1E;IAEK,KAAK,CACT,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,GACxD,OAAO,CAAC,IAAI,CAAC,CAkHf;IAEK,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAYvC;IAED;;;;;;OAMG;IACI,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,aAAa,CAAC,QAAQ,CAAC,CA0GxE;IAEK,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC,CA+CxD;IAEK,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAG3C;IAEK,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAkE3C;IAEK,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAI3D;IAEK,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAwD3D;CACF"}
+{"version":3,"file":"s3-files-api.d.ts","sourceRoot":"","sources":["../src/s3-files-api.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAanD,OAAO,KAAK,EACV,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,WAAW,EACX,WAAW,EACZ,MAAM,2BAA2B,CAAC;AAcnC,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,MAAM,EAAE,QAAQ,CAAC;IAEjB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;;GAKG;AACH,qBAAa,UAAW,YAAW,QAAQ;IACzC,OAAO,CAAC,MAAM,CAAW;IACzB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,QAAQ,CAAS;IAEzB,YAAY,OAAO,EAAE,iBAAiB,EAKrC;IAED,OAAO,CAAC,UAAU;IAUlB,OAAO,CAAC,SAAS;IAWV,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,aAAa,CAAC,UAAU,CAAC,CA6C1E;IAEK,KAAK,CACT,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,GACxD,OAAO,CAAC,IAAI,CAAC,CAEf;IAED;;;;;;;;;;;;OAYG;IACG,eAAe,CACnB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,GACxD,OAAO,CAAC,OAAO,CAAC,CAGlB;IAED;;;;OAIG;YACW,MAAM;IA8Hd,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAYvC;IAED;;;;;;OAMG;IACI,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,aAAa,CAAC,QAAQ,CAAC,CA0GxE;IAEK,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC,CA+CxD;IAEK,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAG3C;IAEK,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAkE3C;IAED;;;;;;;OAOG;IACG,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAK3D;IAEK,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAwD3D;CACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@statewalker/webrun-files-s3",
-  "version": "0.10.4",
+  "version": "0.11.0",
   "private": false,
   "type": "module",
   "description": "S3-backed FilesApi implementation for webrun-files",
@@ -34,7 +34,7 @@
     "src"
   ],
   "dependencies": {
-    "@statewalker/webrun-files": "^0.10.4"
+    "@statewalker/webrun-files": "^0.11.0"
   },
   "peerDependencies": {
     "@aws-sdk/client-s3": "^3.1032.0"

package/src/s3-files-api.ts

@@ -24,7 +24,12 @@ import type {
   ListOptions,
   ReadOptions,
 } from "@statewalker/webrun-files";
-import { basename, comparePaths, normalizePath } from "@statewalker/webrun-files";
+import {
+  basename,
+  comparePaths,
+  normalizePath,
+  removeMovedSource,
+} from "@statewalker/webrun-files";
 
 /** Default part size for multipart uploads: 5MB (S3 minimum) */
 const DEFAULT_PART_SIZE = 5 * 1024 * 1024;
@@ -148,7 +153,42 @@ export class S3FilesApi implements FilesApi {
     path: string,
     content: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,
   ): Promise<void> {
-    const key = this.resolveKey(path);
+    await this.upload(this.resolveKey(path), content, false);
+  }
+
+  /**
+   * A conditional write, `If-None-Match: *`, on the `PutObject` (or, above the multipart
+   * threshold, on the `CompleteMultipartUpload`) that makes the object visible: the
+   * service creates the key only if no object has it, and answers 412 otherwise. That
+   * excludes every client of the bucket. AWS S3 supports it since August 2024; an
+   * S3-compatible service that ignores the header gives no exclusion at all, so check
+   * yours before relying on this.
+   *
+   * Only the key itself is exclusive. A directory is a key prefix, so an existing one is
+   * turned away by a check before the upload, but one created concurrently with this
+   * call is not excluded. Nothing is visible until the content is complete; a failing
+   * source aborts the multipart upload and leaves no object.
+   */
+  async createExclusive(
+    path: string,
+    content: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,
+  ): Promise<boolean> {
+    if (normalizePath(path) === "/" || (await this.stats(path))) return false;
+    return this.upload(this.resolveKey(path), content, true);
+  }
+
+  /**
+   * Upload `content` to `key`: one PutObject below the multipart threshold, else a
+   * multipart upload. With `exclusive`, the request that creates the object carries
+   * `If-None-Match: *`; returns false when the service refuses it because the key exists.
+   */
+  private async upload(
+    key: string,
+    content: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,
+    exclusive: boolean,
+  ): Promise<boolean> {
+    const ifNoneMatch = exclusive ? "*" : undefined;
+    const sendLast = <T>(send: () => Promise<T>) => (exclusive ? sendConditional(send) : send());
 
     // Buffer for accumulating small writes
     let buffer: Uint8Array[] = [];
@@ -158,7 +198,8 @@ export class S3FilesApi implements FilesApi {
     let partNumber = 1;
 
     const flushPart = async (isLast: boolean) => {
-      if (buffer.length === 0) return;
+      // An exclusive create of empty content still has to create the object.
+      if (buffer.length === 0 && !(exclusive && isLast && !uploadId)) return;
 
       const partLength = buffer.reduce((sum, c) => sum + c.length, 0);
       const partBuffer = new Uint8Array(partLength);
@@ -177,8 +218,9 @@ export class S3FilesApi implements FilesApi {
           Key: key,
           Body: partBuffer,
           ContentLength: partLength,
+          IfNoneMatch: ifNoneMatch,
         });
-        await this.client.send(command);
+        await sendLast(() => this.client.send(command));
         return;
       }
 
@@ -240,9 +282,11 @@ export class S3FilesApi implements FilesApi {
           MultipartUpload: {
             Parts: parts.sort((a, b) => a.PartNumber - b.PartNumber),
           },
+          IfNoneMatch: ifNoneMatch,
         });
-        await this.client.send(completeCommand);
+        await sendLast(() => this.client.send(completeCommand));
       }
+      return true;
     } catch (error) {
       // Abort multipart upload on error
       if (uploadId) {
@@ -259,6 +303,7 @@ export class S3FilesApi implements FilesApi {
         }
       }
 
+      if (exclusive && isPreconditionFailed(error)) return false;
       throw error;
     }
   }
@@ -514,10 +559,19 @@ export class S3FilesApi implements FilesApi {
     return true;
   }
 
+  /**
+   * S3 has no rename, so a move is a copy, then a delete, and is NOT atomic: while it
+   * runs, the object exists under both keys, and a directory (a key prefix) moves object
+   * by object. A failed copy throws and keeps the source. If the copy succeeds but the
+   * source cannot be deleted, the delete is retried once, then a `MoveIncompleteError`
+   * is thrown: the target holds the new content and the source still exists. False is
+   * never returned for a half-done move.
+   */
   async move(source: string, target: string): Promise<boolean> {
     const copied = await this.copy(source, target);
     if (!copied) return false;
-    return this.remove(source);
+    await removeMovedSource(this, normalizePath(source), normalizePath(target));
+    return true;
   }
 
   async copy(source: string, target: string): Promise<boolean> {
@@ -599,3 +653,28 @@ function insertInOrder(entries: FileInfo[], entry: FileInfo): void {
   while (i > 0 && comparePaths(entries[i - 1].path, entry.path) > 0) i--;
   entries.splice(i, 0, entry);
 }
+
+/** The service refused a conditional write because the key exists (HTTP 412). */
+function isPreconditionFailed(error: unknown): boolean {
+  const err = error as { name?: string; $metadata?: { httpStatusCode?: number } };
+  return err?.name === "PreconditionFailed" || err?.$metadata?.httpStatusCode === 412;
+}
+
+/**
+ * Send a request that may carry `If-None-Match`, retrying a 409
+ * `ConditionalRequestConflict`: S3 answers that while another conditional write to the
+ * key is in flight, and asks the client to retry, after which one of them gets 412.
+ */
+async function sendConditional<T>(send: () => Promise<T>): Promise<T> {
+  for (let attempt = 0; ; attempt++) {
+    try {
+      return await send();
+    } catch (error) {
+      const err = error as { name?: string; $metadata?: { httpStatusCode?: number } };
+      const conflict =
+        err?.name === "ConditionalRequestConflict" || err?.$metadata?.httpStatusCode === 409;
+      if (!conflict || attempt >= 4) throw error;
+      await new Promise((resolve) => setTimeout(resolve, 50 * 2 ** attempt));
+    }
+  }
+}