@stim-cli/cache-s3 1.21.0 → 1.22.0

package/README.md

@@ -33,6 +33,12 @@ SDK's standard credential chain. For a named profile in `~/.aws/credentials`,
 set `cache.options.profile` to `stim-r2`, or run commands with
 `AWS_PROFILE=stim-r2`, for example `AWS_PROFILE=stim-r2 stim ios`. An explicit
 cache profile overrides the ambient AWS profile. Credentials never belong in `.stim.json`.
+Programmatic Stim clients use their captured environment, including profile files
+and `credential_process`. The optional provider resolves credentials and environment-backed request settings
+in a cancellable helper process, keeping the calling process environment unchanged.
+Captured endpoint settings are pinned for the request, with no fallback to the
+calling process's endpoint. Credentials and settings return over private IPC and
+are reused until the credentials need refreshing.
 
 Create a private R2 bucket and an R2 API token with Object Read & Write access
 scoped to that bucket. Copy the S3 endpoint shown by Cloudflare; jurisdiction
@@ -47,6 +53,11 @@ existing build entries unless overwrite was requested. Use matching native
 toolchains for the initial test; the provider does not add toolchain information
 to Stim's existing build key.
 
+Archive extraction runs in an owned helper. Cancellation waits for that process
+to exit before removing download staging, so extraction cannot keep writing
+after the operation settles. A cleanup failure reports the retained directory
+and filesystem cause; bounded plans refuse instead of counting it as a miss.
+
 The file transport is also exported as `createS3FileCache()` from
 `@stim-cli/cache-s3`, with the `FileCacheCapability` interface exported from
 `@stim-cli/cache`. A cache adapter supplies a compatible key and a completed
@@ -61,5 +72,7 @@ Objects use `<prefix>/builds/<platform>/<key>` and
 `<prefix>/native-android/v1/<key>`. A prefix defaults to `stim/v1`. Keep
 incompatible snapshot formats in separate key namespaces. Retention belongs to
 the bucket lifecycle configuration; local GC never removes shared objects.
-Uploads use a single S3 PUT, so entries larger than S3's 5 GiB limit are not
-supported by this initial provider.
+Uploads use a single-object S3 PUT, so entries larger than S3's 5 GiB limit are
+not supported. A connection reset retries the completed file with a fresh stream,
+up to three PUT attempts with short, cancellable delays. Conditional writes still
+preserve an existing entry if the first upload committed but its reply was lost.

package/dist/credentials-worker.d.mts

@@ -0,0 +1 @@
+export {}

package/dist/credentials-worker.mjs

@@ -0,0 +1,46 @@
+import { S3Client } from "@aws-sdk/client-s3";
+//#region credentials-worker.ts
+process.once("message", async (input) => {
+	const client = new S3Client({
+		...input,
+		logger: {
+			debug() {},
+			info() {},
+			warn() {},
+			error() {}
+		}
+	});
+	try {
+		const config = client.config;
+		const credentials = await config.credentials();
+		const endpoint = input.endpoint ?? await config.serviceConfiguredEndpoint?.();
+		const message = {
+			credentials: {
+				...credentials,
+				expiration: credentials.expiration?.toISOString()
+			},
+			requestConfig: {
+				...endpoint === void 0 ? {} : { endpoint },
+				disableClockSkewCorrection: await config.disableClockSkewCorrection(),
+				useFipsEndpoint: await config.useFipsEndpoint(),
+				useDualstackEndpoint: await config.useDualstackEndpoint(),
+				useArnRegion: typeof config.useArnRegion === "function" ? await config.useArnRegion() : config.useArnRegion,
+				disableS3ExpressSessionAuth: typeof config.disableS3ExpressSessionAuth === "function" ? await config.disableS3ExpressSessionAuth() : config.disableS3ExpressSessionAuth,
+				authSchemePreference: typeof config.authSchemePreference === "function" ? await config.authSchemePreference() : config.authSchemePreference,
+				sigv4aSigningRegionSet: typeof config.sigv4aSigningRegionSet === "function" ? await config.sigv4aSigningRegionSet() : config.sigv4aSigningRegionSet,
+				maxAttempts: typeof config.maxAttempts === "function" ? await config.maxAttempts() : config.maxAttempts,
+				retryMode: typeof config.retryMode === "function" ? await config.retryMode() : config.retryMode,
+				defaultsMode: typeof config.defaultsMode === "function" ? await config.defaultsMode() : config.defaultsMode,
+				userAgentAppId: typeof config.userAgentAppId === "function" ? await config.userAgentAppId() : config.userAgentAppId
+			}
+		};
+		process.send?.({ value: message }, () => process.exit(0));
+	} catch {
+		process.exitCode = 1;
+		process.disconnect?.();
+	} finally {
+		client.destroy();
+	}
+});
+//#endregion
+export {};

package/dist/extract-worker.d.mts

@@ -0,0 +1 @@
+export {}

package/dist/extract-worker.mjs

@@ -0,0 +1,36 @@
+import { createReadStream } from "node:fs";
+import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
+import { pipeline } from "node:stream/promises";
+import { extract } from "tar";
+//#region extract-worker.ts
+process.once("message", async ({ archive, staging }) => {
+	try {
+		let invalidLink = false;
+		await pipeline(createReadStream(archive), extract({
+			cwd: staging,
+			strict: true,
+			filter(path, entry) {
+				if (!("type" in entry)) return true;
+				if (entry.type !== "SymbolicLink" && entry.type !== "Link") return true;
+				const link = entry.linkpath ?? "";
+				const target = resolve(staging, entry.type === "SymbolicLink" ? dirname(path) : "", link);
+				const root = relative(staging, resolve(staging, path)).split(sep)[0];
+				const outside = relative(resolve(staging, root ?? ""), target);
+				if (isAbsolute(link) || outside === ".." || outside.startsWith(`..${sep}`)) {
+					invalidLink = true;
+					return false;
+				}
+				return true;
+			}
+		}));
+		if (invalidLink) throw new Error("The S3 build entry contains a link outside the artifact.");
+		process.send?.({ value: true }, () => process.exit(0));
+	} catch (error) {
+		process.send?.({ error: {
+			message: error instanceof Error ? error.message : String(error),
+			...error instanceof Error && "code" in error && typeof error.code === "string" ? { code: error.code } : {}
+		} }, () => process.exit(1));
+	}
+});
+//#endregion
+export {};

package/dist/index.d.mts

@@ -8,12 +8,15 @@ interface S3FileCacheOptions {
   region?: string | undefined;
   profile?: string | undefined;
   forcePathStyle?: boolean | undefined;
+  env?: NodeJS.ProcessEnv | undefined;
+  projectRoot?: string | undefined;
 }
 /** Stores opaque files; each cache adapter owns its key and serialization. Credentials use the AWS SDK chain. */
 declare function createS3FileCache(options: S3FileCacheOptions): FileCacheCapability & Required<Pick<FileCacheCapability, "stat">>;
-declare function createCacheProvider({ options }: {
+declare function createCacheProvider({ options, projectRoot, env }: {
   projectRoot: string;
   options: Record<string, unknown>;
+  env?: NodeJS.ProcessEnv;
 }): CacheProvider;
 //#endregion
 export { S3FileCacheOptions, apiVersion, createCacheProvider, createS3FileCache };

package/dist/index.mjs

@@ -3,12 +3,118 @@ import { createHash, randomUUID } from "node:crypto";
 import { createReadStream, createWriteStream } from "node:fs";
 import { lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { basename, dirname, join } from "node:path";
 import { Readable, Transform } from "node:stream";
 import { pipeline } from "node:stream/promises";
+import { setTimeout as setTimeout$1 } from "node:timers/promises";
 import { deserialize, serialize } from "node:v8";
-import { create, extract } from "tar";
-import { CACHE_PROVIDER_API_VERSION } from "@stim-cli/cache";
+import { create } from "tar";
+import { CACHE_PROVIDER_API_VERSION, buildResolveTimeoutMs } from "@stim-cli/cache";
+import { spawn } from "node:child_process";
+import { fileURLToPath } from "node:url";
+//#region worker.ts
+function scopedWorker(worker, input, { env, cwd, signal, timeoutMs, name }) {
+	signal.throwIfAborted();
+	return new Promise((resolve, reject) => {
+		const child = spawn(process.execPath, [fileURLToPath(worker)], {
+			cwd,
+			env,
+			detached: process.platform !== "win32",
+			stdio: [
+				"ignore",
+				"ignore",
+				"ignore",
+				"ipc"
+			]
+		});
+		let reply;
+		let failure;
+		let closed = false;
+		let stopping;
+		const stop = () => {
+			if (closed || !child.pid) return;
+			if (process.platform === "win32") stopping ??= new Promise((done) => {
+				const killer = spawn(join(process.env.SystemRoot || "C:\\Windows", "System32", "taskkill.exe"), [
+					"/PID",
+					String(child.pid),
+					"/T",
+					"/F"
+				], {
+					stdio: "ignore",
+					windowsHide: true
+				});
+				killer.once("error", (error) => {
+					failure ??= error;
+				});
+				killer.once("close", (code) => {
+					if (code !== 0) failure ??= /* @__PURE__ */ new Error(`${name} process cleanup failed.`);
+					done();
+				});
+			});
+			else try {
+				process.kill(-child.pid, "SIGKILL");
+			} catch (error) {
+				if (!(error instanceof Error && "code" in error && error.code === "ESRCH")) failure ??= error;
+			}
+		};
+		const timer = setTimeout(() => {
+			failure = /* @__PURE__ */ new Error(`${name} did not finish within ${timeoutMs / 1e3} seconds.`);
+			stop();
+		}, timeoutMs);
+		signal.addEventListener("abort", stop, { once: true });
+		if (signal.aborted) stop();
+		child.once("spawn", () => {
+			if (signal.aborted) stop();
+			else child.send(input, (error) => {
+				if (error) {
+					failure = error;
+					stop();
+				}
+			});
+		});
+		child.once("message", (message) => {
+			reply = message;
+		});
+		child.once("error", (error) => {
+			failure = error;
+		});
+		child.once("close", async (code) => {
+			closed = true;
+			clearTimeout(timer);
+			signal.removeEventListener("abort", stop);
+			await stopping;
+			if (signal.aborted) reject(signal.reason);
+			else if (failure) reject(failure);
+			else if (reply?.error) reject(Object.assign(new Error(reply.error.message), { code: reply.error.code }));
+			else if (code !== 0 || reply?.value === void 0) reject(/* @__PURE__ */ new Error(`Could not complete ${name}.`));
+			else resolve(reply.value);
+		});
+	});
+}
+//#endregion
+//#region credentials.ts
+function scopedClientContext(env, cwd, options, signal) {
+	return scopedWorker(new URL(import.meta.url.endsWith(".ts") ? "./credentials-worker.ts" : "./credentials-worker.mjs", import.meta.url), options, {
+		env: {
+			...env,
+			...options.profile ? { AWS_PROFILE: options.profile } : {}
+		},
+		cwd,
+		signal,
+		timeoutMs: 1e4,
+		name: "S3 credential resolution"
+	}).then((context) => {
+		const { expiration, ...credentials } = context.credentials;
+		return {
+			credentials: {
+				...credentials,
+				...expiration ? { expiration: new Date(expiration) } : {}
+			},
+			requestConfig: context.requestConfig
+		};
+	});
+}
+//#endregion
 //#region index.ts
 const apiVersion = CACHE_PROVIDER_API_VERSION;
 async function fileDigests(path, signal) {
@@ -46,93 +152,147 @@ function createS3FileCache(options) {
 			error() {}
 		}
 	};
-	const client = new S3Client(config);
+	const environment = options.env === void 0 ? void 0 : { ...options.env };
+	const defaultClient = environment ? void 0 : new S3Client(config);
+	let context;
+	async function withClient(signal, run) {
+		signal.throwIfAborted();
+		if (defaultClient) return run(defaultClient);
+		if (!context || context.credentials.expiration && context.credentials.expiration.getTime() - Date.now() < 3e5) context = await scopedClientContext(environment ?? {}, options.projectRoot ?? process.cwd(), {
+			profile: options.profile,
+			region: options.region ?? "us-east-1",
+			endpoint: options.endpoint
+		}, signal);
+		signal.throwIfAborted();
+		const captured = context;
+		const scoped = new S3Client({
+			...config,
+			...captured.requestConfig,
+			credentials: captured.credentials,
+			ignoreConfiguredEndpointUrls: true,
+			useArnRegion: async () => captured.requestConfig.useArnRegion,
+			disableS3ExpressSessionAuth: async () => captured.requestConfig.disableS3ExpressSessionAuth,
+			sigv4aSigningRegionSet: async () => captured.requestConfig.sigv4aSigningRegionSet,
+			userAgentAppId: async () => captured.requestConfig.userAgentAppId
+		});
+		try {
+			return await run(scoped);
+		} finally {
+			scoped.destroy();
+		}
+	}
 	const prefix = (options.prefix ?? "stim/v1").replace(/^\/+|\/+$/g, "");
 	const objectKey = (key) => `${prefix}/${key}`;
 	return {
 		async stat({ key, signal }) {
-			signal.throwIfAborted();
-			try {
-				const object = await client.send(new HeadObjectCommand({
-					Bucket: options.bucket,
-					Key: objectKey(key)
-				}), { abortSignal: signal });
-				return typeof object.ContentLength === "number" ? { size: object.ContentLength } : null;
-			} catch (error) {
-				if (status(error) === 404) return null;
-				throw error;
-			}
+			return withClient(signal, async (client) => {
+				signal.throwIfAborted();
+				try {
+					const object = await client.send(new HeadObjectCommand({
+						Bucket: options.bucket,
+						Key: objectKey(key)
+					}), { abortSignal: signal });
+					return typeof object.ContentLength === "number" ? { size: object.ContentLength } : null;
+				} catch (error) {
+					if (status(error) === 404) return null;
+					throw error;
+				}
+			});
 		},
 		async resolve({ key, destinationDir, signal }) {
-			signal.throwIfAborted();
-			let object;
-			try {
-				object = await client.send(new GetObjectCommand({
-					Bucket: options.bucket,
-					Key: objectKey(key)
-				}), { abortSignal: signal });
-			} catch (error) {
-				if (status(error) === 404) return null;
-				throw error;
-			}
-			const expected = object.Metadata?.sha256;
-			if (!(object.Body instanceof Readable) || !expected || !/^[a-f0-9]{64}$/.test(expected)) {
-				if (object.Body instanceof Readable) object.Body.destroy();
-				throw new Error("The S3 cache entry has no SHA-256 metadata.");
-			}
-			const temporary = join(destinationDir, `${randomUUID()}.partial`);
-			const complete = join(destinationDir, `${randomUUID()}.cache`);
-			const digest = createHash("sha256");
-			try {
-				await mkdir(destinationDir, { recursive: true });
-				await pipeline(object.Body, new Transform({ transform(chunk, _encoding, callback) {
-					digest.update(chunk);
-					callback(null, chunk);
-				} }), createWriteStream(temporary, { flags: "wx" }), { signal });
-				if (digest.digest("hex") !== expected) throw new Error("The S3 cache entry failed SHA-256 verification.");
+			return withClient(signal, async (client) => {
 				signal.throwIfAborted();
-				await rename(temporary, complete);
-				return complete;
-			} finally {
-				object.Body.destroy();
-				await rm(temporary, { force: true });
-			}
+				let object;
+				try {
+					object = await client.send(new GetObjectCommand({
+						Bucket: options.bucket,
+						Key: objectKey(key)
+					}), { abortSignal: signal });
+				} catch (error) {
+					if (status(error) === 404) return null;
+					throw error;
+				}
+				const expected = object.Metadata?.sha256;
+				if (!(object.Body instanceof Readable) || !expected || !/^[a-f0-9]{64}$/.test(expected)) {
+					if (object.Body instanceof Readable) object.Body.destroy();
+					throw new Error("The S3 cache entry has no SHA-256 metadata.");
+				}
+				const temporary = join(destinationDir, `${randomUUID()}.partial`);
+				const complete = join(destinationDir, `${randomUUID()}.cache`);
+				const digest = createHash("sha256");
+				try {
+					await mkdir(destinationDir, { recursive: true });
+					await pipeline(object.Body, new Transform({ transform(chunk, _encoding, callback) {
+						digest.update(chunk);
+						callback(null, chunk);
+					} }), createWriteStream(temporary, { flags: "wx" }), { signal });
+					if (digest.digest("hex") !== expected) throw new Error("The S3 cache entry failed SHA-256 verification.");
+					signal.throwIfAborted();
+					await rename(temporary, complete);
+					return complete;
+				} finally {
+					object.Body.destroy();
+					await rm(temporary, { force: true });
+				}
+			});
 		},
 		async store({ key, sourcePath, overwrite, signal }) {
-			signal.throwIfAborted();
-			const digests = await fileDigests(sourcePath, signal);
-			const body = createReadStream(sourcePath, { signal });
-			try {
-				await client.send(new PutObjectCommand({
-					Bucket: options.bucket,
-					Key: objectKey(key),
-					Body: body,
-					ContentLength: (await stat(sourcePath)).size,
-					ContentMD5: digests.md5,
-					Metadata: { sha256: digests.sha256 },
-					IfNoneMatch: overwrite ? void 0 : "*"
-				}), { abortSignal: signal });
-			} catch (error) {
-				if (overwrite || status(error) !== 412) throw error;
-			} finally {
-				body.destroy();
-			}
+			return withClient(signal, async (client) => {
+				signal.throwIfAborted();
+				const digests = await fileDigests(sourcePath, signal);
+				const size = (await stat(sourcePath)).size;
+				for (let attempt = 0;; attempt++) {
+					signal.throwIfAborted();
+					const body = createReadStream(sourcePath, { signal });
+					try {
+						await client.send(new PutObjectCommand({
+							Bucket: options.bucket,
+							Key: objectKey(key),
+							Body: body,
+							ContentLength: size,
+							ContentMD5: digests.md5,
+							Metadata: { sha256: digests.sha256 },
+							IfNoneMatch: overwrite ? void 0 : "*"
+						}), { abortSignal: signal });
+						return;
+					} catch (error) {
+						if (!overwrite && status(error) === 412) return;
+						if (attempt === 2 || signal.aborted || !(error instanceof Error) || !("code" in error) || error.code !== "ECONNRESET") throw error;
+					} finally {
+						body.destroy();
+					}
+					await setTimeout$1(100 * 2 ** attempt, void 0, { signal });
+				}
+			});
 		}
 	};
 }
-async function scratch(run) {
-	const dir = await mkdtemp(join(tmpdir(), "stim-s3-cache-"));
-	try {
-		return await run(dir);
-	} finally {
-		await rm(dir, {
-			recursive: true,
-			force: true
-		});
-	}
+function cleanupFailure(path, cause) {
+	return Object.assign(new Error(`S3 cache cleanup failed. Retained ${path}.`, { cause }), {
+		code: "STIM_CLAIM_UNAVAILABLE",
+		reportCleanup: true,
+		remedy: "Check the retained directory permissions and remove it before retrying."
+	});
 }
-function createCacheProvider({ options }) {
+async function scratch(run, parent) {
+	if (parent) await mkdir(parent, { recursive: true });
+	const dir = await mkdtemp(join(parent ?? tmpdir(), "stim-s3-cache-"));
+	return Promise.resolve().then(() => run(dir)).finally(async () => {
+		try {
+			await rm(dir, {
+				recursive: true,
+				force: true
+			});
+		} catch (cause) {
+			throw cleanupFailure(parent ?? dir, cause);
+		}
+	});
+}
+function createCacheProvider({ options, projectRoot, env }) {
+	const environment = env === void 0 ? void 0 : { ...env };
 	const files = createS3FileCache({
+		env: environment,
+		projectRoot,
 		bucket: typeof options.bucket === "string" ? options.bucket : "",
 		prefix: typeof options.prefix === "string" ? options.prefix : void 0,
 		endpoint: typeof options.endpoint === "string" ? options.endpoint : void 0,
@@ -153,25 +313,16 @@ function createCacheProvider({ options }) {
 					if (!archive) return null;
 					const staging = await mkdtemp(join(destinationDir, "s3-"));
 					try {
-						let invalidLink = false;
-						await pipeline(createReadStream(archive), extract({
-							cwd: staging,
-							strict: true,
-							filter(path, entry) {
-								if (!("type" in entry)) return true;
-								if (entry.type !== "SymbolicLink" && entry.type !== "Link") return true;
-								const link = entry.linkpath ?? "";
-								const target = resolve(staging, entry.type === "SymbolicLink" ? dirname(path) : "", link);
-								const root = relative(staging, resolve(staging, path)).split(sep)[0];
-								const outside = relative(resolve(staging, root ?? ""), target);
-								if (isAbsolute(link) || outside === ".." || outside.startsWith(`..${sep}`)) {
-									invalidLink = true;
-									return false;
-								}
-								return true;
-							}
-						}), { signal });
-						if (invalidLink) throw new Error("The S3 build entry contains a link outside the artifact.");
+						await scopedWorker(new URL(import.meta.url.endsWith(".ts") ? "./extract-worker.ts" : "./extract-worker.mjs", import.meta.url), {
+							archive,
+							staging
+						}, {
+							env: { ...environment ?? process.env },
+							cwd: projectRoot,
+							signal,
+							timeoutMs: buildResolveTimeoutMs(environment),
+							name: "S3 archive extraction"
+						});
 						signal.throwIfAborted();
 						const entries = await readdir(staging);
 						const [entry] = entries;
@@ -181,13 +332,17 @@ function createCacheProvider({ options }) {
 						if (platform === "ios" ? !info.isDirectory() || !path.endsWith(".app") : !info.isFile() || !path.endsWith(".apk")) throw new Error("The S3 build entry has the wrong artifact type.");
 						return path;
 					} catch (error) {
-						await rm(staging, {
-							recursive: true,
-							force: true
-						});
+						try {
+							await rm(staging, {
+								recursive: true,
+								force: true
+							});
+						} catch (cause) {
+							throw cleanupFailure(destinationDir, cause);
+						}
 						throw error;
 					}
-				});
+				}, destinationDir);
 			},
 			async store({ platform, key, sourcePath, overwrite, signal }) {
 				await scratch(async (dir) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stim-cli/cache-s3",
-  "version": "1.21.0",
+  "version": "1.22.0",
   "description": "Optional S3-compatible file, native artifact and Metro cache provider for Stim.",
   "license": "MIT",
   "repository": {
@@ -25,7 +25,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.1149.0",
-    "@stim-cli/cache": "^1.21.0",
+    "@stim-cli/cache": "^1.22.0",
     "tar": "^7.5.22"
   },
   "engines": {