@aws-blocks/core 0.3.0 → 0.4.0

package/src/lambda-handler.test.ts

@@ -3,10 +3,11 @@
 
 import { describe, it, beforeEach } from 'node:test';
 import assert from 'node:assert';
-import { createLambdaHandler, _resetCorsPatterns, requestCookies, isApiGatewayHttpEvent, computeHttpDeadlineMs, classifyEvent, buildEventUrl, isLoopbackForwardedHost } from './lambda-handler.js';
+import { createLambdaHandler, _resetCorsPatterns, requestCookies, isApiGatewayHttpEvent, computeHttpDeadlineMs, classifyEvent, buildEventUrl, isLoopbackForwardedHost, TransientConfigError } from './lambda-handler.js';
 import type { LambdaContext } from './lambda-handler.js';
 import { registerRoute, clearRouteRegistry } from './raw-route.js';
 import { decodeRpcResponse } from './rpc.js';
+import { _resetConfigCache, _setS3Fetcher } from './common/config.js';
 import type { BlocksContext } from './api.js';
 
 beforeEach(() => {
@@ -29,6 +30,145 @@ async function invoke(backend: any, event: any): Promise<any> {
   return handler(event) as any;
 }
 
+// ── init self-heal (retry on failed initialization) ─────────────────────────
+
+describe('createLambdaHandler — init self-heal', () => {
+  it('retries initialize() on a later request instead of caching a failed init', async () => {
+    let initCalls = 0;
+    const backend = {
+      api: (_ctx: BlocksContext) => ({
+        async echo(msg: string) {
+          return { msg };
+        },
+      }),
+    };
+    // Fail the first initialization (e.g. config not readable yet in the brief post-deploy
+    // window), then succeed. The SAME handler instance must recover — a cached rejected
+    // initPromise would poison the container and fail every subsequent request.
+    const backendFactory = async () => {
+      initCalls++;
+      if (initCalls === 1) throw new Error('transient init failure');
+      return backend;
+    };
+    const handler = createLambdaHandler(backendFactory);
+
+    // 1st request: init fails, so the handler rejects.
+    await assert.rejects(() => handler(makeEvent()) as any, /transient init failure/);
+
+    // 2nd request: init is retried and succeeds → 200 (not a re-thrown cached rejection).
+    const res = (await handler(makeEvent())) as any;
+    assert.strictEqual(res.statusCode, 200);
+    assert.strictEqual(initCalls, 2, 'initialize() must be retried on the next request, not cached');
+  });
+
+  it('recovers from a transient-empty (post-deploy 404) config load on the next request', async () => {
+    // Reproduces the poisoned-container bug: the first S3 load hits the transient
+    // post-deploy window (NoSuchKey), so config resolves empty; the handler must
+    // NOT lock in that empty config, and the NEXT request must re-fetch and pick
+    // up the now-present config.
+    _resetConfigCache();
+    process.env.BLOCKS_CONFIG_BUCKET = 'test-bucket';
+    process.env.BLOCKS_CONFIG_KEY = 'blocks-config.json';
+    delete process.env.SELFHEAL_KEY;
+
+    let fetchCall = 0;
+    const notFound = new Error('The specified key does not exist.');
+    (notFound as any).name = 'NoSuchKey';
+    _setS3Fetcher(async () => {
+      fetchCall++;
+      if (fetchCall === 1) throw notFound; // config not readable yet
+      return JSON.stringify({ SELFHEAL_KEY: 'ready' });
+    });
+
+    let backendImports = 0;
+    const handler = createLambdaHandler(async () => {
+      backendImports++;
+      return {
+        api: (_ctx: BlocksContext) => ({
+          async echo(msg: string) { return { msg, cfg: process.env.SELFHEAL_KEY }; },
+        }),
+      };
+    });
+
+    try {
+      // 1st request: transient-empty load → initialize() throws TransientConfigError
+      // BEFORE importing the backend (so we never lock in empty config), and the
+      // request rejects.
+      await assert.rejects(() => handler(makeEvent()) as any, TransientConfigError);
+      assert.strictEqual(backendImports, 0, 'backend must NOT be imported while config is unresolved');
+
+      // 2nd request: config now present → initialize() re-runs, re-fetches, and
+      // injects the config into process.env before importing the backend.
+      const res = (await handler(makeEvent())) as any;
+      assert.strictEqual(res.statusCode, 200);
+      const body = JSON.parse(res.body);
+      assert.strictEqual(body.result.cfg, 'ready', 'now-present config was injected on the retry');
+      assert.strictEqual(backendImports, 1, 'backend imported exactly once, on the successful retry');
+      assert.strictEqual(fetchCall, 2, 'S3 was re-fetched on the retry (transient miss is not cached)');
+    } finally {
+      _resetConfigCache();
+      delete process.env.BLOCKS_CONFIG_BUCKET;
+      delete process.env.BLOCKS_CONFIG_KEY;
+      delete process.env.SELFHEAL_KEY;
+    }
+  });
+
+  it('does NOT re-init for a config-less app (no bucket, local dev) — initialize runs once', async () => {
+    _resetConfigCache();
+    delete process.env.BLOCKS_CONFIG_BUCKET;
+    delete process.env.BLOCKS_CONFIG_KEY;
+
+    let backendImports = 0;
+    const handler = createLambdaHandler(async () => {
+      backendImports++;
+      return { api: (_ctx: BlocksContext) => ({ async echo(msg: string) { return { msg }; } }) };
+    });
+
+    try {
+      const r1 = (await handler(makeEvent())) as any;
+      const r2 = (await handler(makeEvent())) as any;
+      assert.strictEqual(r1.statusCode, 200);
+      assert.strictEqual(r2.statusCode, 200);
+      assert.strictEqual(backendImports, 1, 'no re-init for a genuinely config-less (local dev) app');
+    } finally {
+      _resetConfigCache();
+    }
+  });
+
+  it('does NOT re-init or spin for a genuinely-empty ({}) S3 config', async () => {
+    // A real, readable empty config must be treated as resolved — distinct from
+    // the transient 404 miss — so the container does not throw/retry forever nor
+    // re-fetch S3 on every request.
+    _resetConfigCache();
+    process.env.BLOCKS_CONFIG_BUCKET = 'test-bucket';
+    process.env.BLOCKS_CONFIG_KEY = 'blocks-config.json';
+
+    let fetchCall = 0;
+    _setS3Fetcher(async () => { fetchCall++; return JSON.stringify({}); });
+
+    let backendImports = 0;
+    const handler = createLambdaHandler(async () => {
+      backendImports++;
+      return { api: (_ctx: BlocksContext) => ({ async echo(msg: string) { return { msg }; } }) };
+    });
+
+    try {
+      const r1 = (await handler(makeEvent())) as any;
+      const r2 = (await handler(makeEvent())) as any;
+      const r3 = (await handler(makeEvent())) as any;
+      assert.strictEqual(r1.statusCode, 200);
+      assert.strictEqual(r2.statusCode, 200);
+      assert.strictEqual(r3.statusCode, 200);
+      assert.strictEqual(backendImports, 1, 'genuinely-empty config resolves; no re-init');
+      assert.strictEqual(fetchCall, 1, 'S3 fetched once and cached — no per-request re-fetch spin');
+    } finally {
+      _resetConfigCache();
+      delete process.env.BLOCKS_CONFIG_BUCKET;
+      delete process.env.BLOCKS_CONFIG_KEY;
+    }
+  });
+});
+
 // ── RPC body tests ──────────────────────────────────────────────────────────
 
 describe('createLambdaHandler — RPC body handling', () => {

package/src/lambda-handler.ts

@@ -7,7 +7,7 @@ import { ApiError } from './errors.js';
 import { BLOCKS_RPC_PREFIX } from './constants.js';
 import { matchRoute, lockRouteRegistry } from './raw-route.js';
 import { registerBuiltinRoutes } from './builtin-routes.js';
-import { loadConfigToProcessEnv } from './common/config.js';
+import { loadConfigToProcessEnv, isConfigResolved } from './common/config.js';
 import {
   parseRpcRequest,
   successResponse,
@@ -297,6 +297,26 @@ export function createLambdaHandler(backendFactory: () => Promise<any>) {
   async function initialize() {
     await loadConfigToProcessEnv();
 
+    // If the app has config coordinates (BLOCKS_CONFIG_BUCKET/KEY set) but the
+    // load didn't actually resolve the config, we're in the transient post-deploy
+    // S3 window where blocks-config.json isn't readable yet. loadConfigFromS3()
+    // deliberately does NOT cache that empty result, so throw a typed transient
+    // error here — BEFORE importing the backend — so the createLambdaHandler()
+    // catch resets initPromise and the next request re-runs initialize(), which
+    // re-fetches from S3 and picks up the now-present config. Without this throw,
+    // initialize() would succeed with empty config, `handler` would be assigned,
+    // and the `if (!handler)` guard below would never fire again, poisoning the
+    // container for its whole life (it would import the backend against empty
+    // process.env and serve "not configured" 500s forever).
+    //
+    // A genuinely config-less app does NOT throw: the no-bucket local-dev path
+    // and a real empty `{}` config both cache their result, so isConfigResolved()
+    // is true and the retry never spins. Once the blob is readable the successful
+    // load caches, so there is no unbounded per-request S3 re-fetch either.
+    if (process.env.BLOCKS_CONFIG_BUCKET && process.env.BLOCKS_CONFIG_KEY && !isConfigResolved()) {
+      throw new TransientConfigError();
+    }
+
     // Merge hosting-provided CORS origins into the main env var so the lazy
     // getCorsPatterns() sees a combined value on first access.
     // loadConfigToProcessEnv() won't override CORS_ALLOWED_ORIGINS if it's
@@ -318,7 +338,18 @@ export function createLambdaHandler(backendFactory: () => Promise<any>) {
 
   return async (event: any, context?: LambdaContext) => {
     if (!handler) {
-      if (!initPromise) initPromise = initialize();
+      // Retry init on failure instead of caching the rejection. If initialize() throws (most often
+      // because loadConfigToProcessEnv() couldn't read blocks-config.json during the brief
+      // post-deploy window before it's readable), a memoized rejected promise would poison this
+      // container for its whole lifetime — every later request re-awaits the same rejection and 500s.
+      // Resetting initPromise lets the next invocation re-run initialize() (config.ts re-fetches,
+      // since it doesn't cache failures), so the handler self-heals once config is available.
+      if (!initPromise) {
+        initPromise = initialize().catch((err) => {
+          initPromise = null;
+          throw err;
+        });
+      }
       await initPromise;
     }
 
@@ -390,6 +421,27 @@ class HandlerTimeoutError extends Error {
   }
 }
 
+/**
+ * Thrown by `initialize()` when the app has config coordinates
+ * (BLOCKS_CONFIG_BUCKET/KEY) but the S3 config load resolved empty because
+ * blocks-config.json wasn't readable yet — the transient window right after a
+ * deploy, before the BucketDeployment settles.
+ *
+ * It flows through the `createLambdaHandler()` init catch, which resets
+ * `initPromise` so the NEXT request re-runs `initialize()` and picks up the
+ * now-present config (config.ts does not cache a not-found result, so the retry
+ * re-fetches). A distinct type keeps the recovery path greppable and testable
+ * and separates it from real init failures, which surface with their own error.
+ *
+ * @internal Exported for testing only.
+ */
+export class TransientConfigError extends Error {
+  constructor() {
+    super('[Blocks] Config not readable yet (transient post-deploy S3 window); will retry on next request');
+    this.name = 'TransientConfigError';
+  }
+}
+
 /**
  * Extract the request path from a Lambda event, normalizing between
  * API Gateway v1 (REST) and v2 (HTTP API) event shapes.

package/src/scripts/sandbox-empty-buckets.test.ts

@@ -0,0 +1,191 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, it } from 'node:test';
+import assert from 'node:assert';
+import type { CloudFormationClient } from '@aws-sdk/client-cloudformation';
+import type { S3Client } from '@aws-sdk/client-s3';
+
+import { emptyBucket, listStackBucketNames, runDestroyWithRetries, toDeleteObjects } from './sandbox.js';
+
+// The teardown bucket-emptying must delete BOTH live versions and delete
+// markers — a versioned bucket left with either still blocks `cdk destroy`.
+// This is the exact combination that, done wrong, silently no-ops.
+describe('toDeleteObjects', () => {
+	it('combines versions and delete markers', () => {
+		const out = toDeleteObjects({
+			Versions: [
+				{ Key: 'a', VersionId: 'v1' },
+				{ Key: 'b', VersionId: 'v2' },
+			],
+			DeleteMarkers: [{ Key: 'a', VersionId: 'dm1' }],
+		});
+		assert.deepStrictEqual(out, [
+			{ Key: 'a', VersionId: 'v1' },
+			{ Key: 'b', VersionId: 'v2' },
+			{ Key: 'a', VersionId: 'dm1' },
+		]);
+	});
+
+	it('returns [] for an empty bucket (no Versions/DeleteMarkers keys)', () => {
+		assert.deepStrictEqual(toDeleteObjects({}), []);
+	});
+
+	it('tolerates only versions or only delete markers', () => {
+		assert.deepStrictEqual(toDeleteObjects({ Versions: [{ Key: 'x', VersionId: 'v' }] }), [
+			{ Key: 'x', VersionId: 'v' },
+		]);
+		assert.deepStrictEqual(toDeleteObjects({ DeleteMarkers: [{ Key: 'y', VersionId: 'd' }] }), [
+			{ Key: 'y', VersionId: 'd' },
+		]);
+	});
+
+	it('drops entries without a Key rather than emitting a bad DeleteObjects payload', () => {
+		assert.deepStrictEqual(toDeleteObjects({ Versions: [{ VersionId: 'orphan' }] }), []);
+	});
+});
+
+// A fake AWS client: routes on the command class name and returns queued/canned
+// responses. Cast to the real client type — this is test plumbing, not customer code.
+type Send = (cmd: { constructor: { name: string }; input: any }) => Promise<any>;
+
+describe('listStackBucketNames', () => {
+	it('follows NextToken across pages and returns only S3 buckets', async () => {
+		const seen: Array<string | undefined> = [];
+		const send: Send = async (cmd) => {
+			seen.push(cmd.input.NextToken);
+			if (cmd.input.NextToken === undefined) {
+				return {
+					StackResourceSummaries: [
+						{ ResourceType: 'AWS::S3::Bucket', PhysicalResourceId: 'b1' },
+						{ ResourceType: 'AWS::Lambda::Function', PhysicalResourceId: 'fn' },
+					],
+					NextToken: 't2',
+				};
+			}
+			return { StackResourceSummaries: [{ ResourceType: 'AWS::S3::Bucket', PhysicalResourceId: 'b2' }] };
+		};
+		const cfn = { send } as unknown as CloudFormationClient;
+		const buckets = await listStackBucketNames(cfn, 'stack');
+		assert.deepStrictEqual(buckets, ['b1', 'b2']);
+		assert.deepStrictEqual(seen, [undefined, 't2']); // paged exactly twice
+	});
+});
+
+describe('emptyBucket', () => {
+	it('deletes versions + markers, re-lists, and stops when empty', async () => {
+		let lists = 0;
+		const deletePayloads: unknown[] = [];
+		const send: Send = async (cmd) => {
+			if (cmd.constructor.name === 'ListObjectVersionsCommand') {
+				lists++;
+				return lists === 1
+					? { Versions: [{ Key: 'a', VersionId: 'v1' }], DeleteMarkers: [{ Key: 'a', VersionId: 'd1' }] }
+					: {};
+			}
+			if (cmd.constructor.name === 'DeleteObjectsCommand') {
+				deletePayloads.push(cmd.input.Delete.Objects);
+				return {};
+			}
+			throw new Error(`unexpected ${cmd.constructor.name}`);
+		};
+		await emptyBucket({ send } as unknown as S3Client, 'bucket');
+		assert.strictEqual(lists, 2); // listed, deleted, re-listed → empty
+		assert.deepStrictEqual(deletePayloads, [
+			[
+				{ Key: 'a', VersionId: 'v1' },
+				{ Key: 'a', VersionId: 'd1' },
+			],
+		]);
+	});
+
+	it('stops (no infinite loop) when DeleteObjects reports Errors', async () => {
+		let lists = 0;
+		let deletes = 0;
+		const send: Send = async (cmd) => {
+			if (cmd.constructor.name === 'ListObjectVersionsCommand') {
+				lists++;
+				return { Versions: [{ Key: 'locked', VersionId: 'v1' }] };
+			}
+			if (cmd.constructor.name === 'DeleteObjectsCommand') {
+				deletes++;
+				return { Errors: [{ Key: 'locked', Code: 'AccessDenied' }] };
+			}
+			throw new Error(`unexpected ${cmd.constructor.name}`);
+		};
+		await emptyBucket({ send } as unknown as S3Client, 'bucket');
+		assert.strictEqual(lists, 1); // did NOT re-list after Errors
+		assert.strictEqual(deletes, 1);
+	});
+});
+
+// The sev2 fix is the wiring: on a destroy failure, buckets get emptied BEFORE
+// the retry. runDestroyWithRetries takes its side effects as deps so we can
+// exercise that ordering without spawning cdk.
+describe('runDestroyWithRetries', () => {
+	it('empties buckets before retrying a failed destroy', async () => {
+		const events: string[] = [];
+		let attempts = 0;
+		await runDestroyWithRetries({
+			runDestroy: () => {
+				attempts++;
+				events.push(`destroy#${attempts}`);
+				if (attempts === 1) throw new Error('bucket not empty');
+			},
+			listStackNames: () => {
+				events.push('list');
+				return ['stackA', 'stackB'];
+			},
+			emptyBuckets: async (names) => {
+				events.push(`empty:${names.join(',')}`);
+			},
+			sleep: async () => {
+				events.push('sleep');
+			},
+			retryDelays: [1],
+		});
+		assert.strictEqual(attempts, 2);
+		// buckets emptied (with the resolved stack names) before the retry destroy
+		assert.deepStrictEqual(events, ['destroy#1', 'list', 'empty:stackA,stackB', 'sleep', 'destroy#2']);
+	});
+
+	it('does not empty or sleep when the first destroy succeeds', async () => {
+		let emptied = false;
+		let slept = false;
+		let listed = false;
+		await runDestroyWithRetries({
+			runDestroy: () => {},
+			listStackNames: () => {
+				listed = true;
+				return [];
+			},
+			emptyBuckets: async () => {
+				emptied = true;
+			},
+			sleep: async () => {
+				slept = true;
+			},
+		});
+		assert.strictEqual(emptied, false);
+		assert.strictEqual(slept, false);
+		assert.strictEqual(listed, false);
+	});
+
+	it('throws after exhausting retries', async () => {
+		let attempts = 0;
+		await assert.rejects(
+			runDestroyWithRetries({
+				runDestroy: () => {
+					attempts++;
+					throw new Error('still failing');
+				},
+				listStackNames: () => [],
+				emptyBuckets: async () => {},
+				sleep: async () => {},
+				retryDelays: [1, 1],
+			}),
+			/still failing/,
+		);
+		assert.strictEqual(attempts, 3); // initial attempt + 2 retries
+	});
+});

package/src/scripts/sandbox.ts

@@ -14,6 +14,8 @@ import { classifyError } from '../telemetry/trackCommand.js';
 import { getCdkTelemetryEnv } from './cdk-telemetry-env.js';
 import { runSync, spawnCommand } from './run-command.js';
 import { terminateProcessTree } from './process-tree.js';
+import type { CloudFormationClient } from '@aws-sdk/client-cloudformation';
+import type { S3Client } from '@aws-sdk/client-s3';
 
 /**
  * Import the backend definition to populate the Scope BB registry.
@@ -321,6 +323,180 @@ export async function startSandbox(options: SandboxOptions) {
   await new Promise(() => {});
 }
 
+/**
+ * Flatten a ListObjectVersions response into the `{ Key, VersionId }[]` shape
+ * DeleteObjects expects, combining live versions AND delete markers. Exported
+ * for unit testing — getting this combination wrong (e.g. missing the delete
+ * markers) leaves a "versioned" bucket that still can't be deleted.
+ */
+export function toDeleteObjects(listed: {
+	Versions?: Array<{ Key?: string; VersionId?: string }>;
+	DeleteMarkers?: Array<{ Key?: string; VersionId?: string }>;
+}): Array<{ Key: string; VersionId?: string }> {
+	return [...(listed.Versions ?? []), ...(listed.DeleteMarkers ?? [])]
+		.filter((v): v is { Key: string; VersionId?: string } => v.Key !== undefined)
+		.map((v) => ({ Key: v.Key, VersionId: v.VersionId }));
+}
+
+/** Collect a stack's S3 bucket physical names (paginated — a large stack has
+ * >100 resources, so a single page can miss the hosting bucket that blocks the
+ * delete). Returns [] if the stack is already gone / not accessible. */
+export async function listStackBucketNames(cfn: CloudFormationClient, stackName: string): Promise<string[]> {
+	const { ListStackResourcesCommand } = await import('@aws-sdk/client-cloudformation');
+	const buckets: string[] = [];
+	let token: string | undefined;
+	do {
+		const res = await cfn.send(new ListStackResourcesCommand({ StackName: stackName, NextToken: token }));
+		for (const r of res.StackResourceSummaries ?? []) {
+			if (r.ResourceType === 'AWS::S3::Bucket' && r.PhysicalResourceId) buckets.push(r.PhysicalResourceId);
+		}
+		token = res.NextToken;
+	} while (token);
+	return buckets;
+}
+
+/** Empty a versioned bucket: delete every object version + delete marker in
+ * pages of 1000 (the DeleteObjects limit). Stops if a page reports per-key
+ * errors (object-lock / retention) so it can't loop forever. */
+export async function emptyBucket(s3: S3Client, bucket: string): Promise<void> {
+	const { ListObjectVersionsCommand, DeleteObjectsCommand } = await import('@aws-sdk/client-s3');
+	while (true) {
+		const listed = await s3.send(new ListObjectVersionsCommand({ Bucket: bucket, MaxKeys: 1000 }));
+		const objects = toDeleteObjects(listed);
+		if (objects.length === 0) return;
+		const res = await s3.send(new DeleteObjectsCommand({ Bucket: bucket, Delete: { Objects: objects, Quiet: true } }));
+		if (res.Errors && res.Errors.length > 0) {
+			// A bucket we can't fully empty (object-lock / retention / AccessDenied)
+			// will keep blocking `cdk destroy`, so the stack won't tear down until it's
+			// resolved by hand. Log loudly with the first error code — this is the
+			// kind of silent skip that leaks a stack, so make it visible in CI output.
+			const first = res.Errors[0];
+			console.warn(
+				`  ⚠️  ${bucket}: ${res.Errors.length} object(s) could NOT be deleted ` +
+					`(e.g. ${first.Key}: ${first.Code}). This bucket will block stack teardown ` +
+					`until cleared manually.`,
+			);
+			return;
+		}
+	}
+}
+
+/**
+ * Empty every versioned S3 bucket owned by the given stacks. `cdk destroy`
+ * relies on each bucket's `autoDeleteObjects` custom-resource Lambda to empty
+ * it on teardown — but that Lambda never provisions if the CREATE failed (e.g.
+ * a partial/aborted deploy), so the versioned bucket blocks the delete and the
+ * stack (and its IAM roles) leak. Emptying out-of-band before the retry lets
+ * the delete complete. Best-effort: any per-stack/per-bucket error is logged
+ * and skipped so teardown still proceeds.
+ */
+async function emptySandboxBuckets(stackNames: string[]): Promise<void> {
+	try {
+		const { CloudFormationClient } = await import('@aws-sdk/client-cloudformation');
+		const { S3Client } = await import('@aws-sdk/client-s3');
+		const cfn = new CloudFormationClient({});
+		// `followRegionRedirects` makes the client transparently retry against a
+		// bucket's real region on a PermanentRedirect, so a bucket that lives
+		// outside AWS_REGION (e.g. a us-east-1 Lambda@Edge stack's bucket) is still
+		// emptied instead of failing silently — closing the exact no-op this fix
+		// targets. It needs no GetBucketLocation permission.
+		const s3 = new S3Client({ followRegionRedirects: true });
+		for (const stackName of stackNames) {
+			let buckets: string[] = [];
+			try {
+				buckets = await listStackBucketNames(cfn, stackName);
+			} catch {
+				continue; // stack already gone / not accessible
+			}
+			for (const b of buckets) {
+				try {
+					await emptyBucket(s3, b);
+				} catch (e) {
+					console.warn(`  ⚠️  could not empty ${b}: ${(e as Error).message}`);
+				}
+			}
+		}
+	} catch (e) {
+		console.warn(`  ⚠️  bucket-emptying step skipped: ${(e as Error).message}`);
+	}
+}
+
+/** Resolve the sandbox app's stack names via `cdk ls` (best-effort; [] on error). */
+function listSandboxStackNames(backendPath: string, cdkEnv: NodeJS.ProcessEnv): string[] {
+	try {
+		const out = execFileSync(
+			'npm',
+			// Single-quote backendPath: CDK re-runs the --app string through a shell,
+			// so an unquoted path containing a space would split.
+			['exec', 'cdk', '--', 'ls', '--context', 'sandboxMode=true', '--app', `npm exec tsx -- -C cdk '${backendPath}'`],
+			// Capture stderr so a genuine synth/`cdk ls` failure can be logged below
+			// rather than being indistinguishable from "app has zero stacks".
+			{ env: cdkEnv, encoding: 'utf-8', stdio: ['ignore', 'pipe', 'pipe'] },
+		);
+		return out
+			.split('\n')
+			.map((s) => s.trim())
+			.filter(Boolean);
+	} catch (e) {
+		// Warn (don't stay silent): if this returns [] on a real failure, the retry
+		// skips bucket-emptying — the exact silent-no-op this change exists to avoid.
+		const detail = (e as { stderr?: string; message?: string }).stderr || (e as Error).message;
+		console.warn(`  ⚠️  could not list sandbox stacks via 'cdk ls'; skipping bucket-emptying: ${detail}`);
+		return [];
+	}
+}
+
+/** Injectable dependencies for {@link runDestroyWithRetries} — real ones in
+ * `destroySandbox`, fakes in tests so the retry/empty-before-retry wiring (the
+ * sev2 fix) is exercised without spawning cdk. */
+export interface DestroyRetryDeps {
+	/** Run one `cdk destroy` attempt; throws on failure. */
+	runDestroy: () => void;
+	/** Resolve the app's stack names (for bucket enumeration). */
+	listStackNames: () => string[];
+	/** Empty the given stacks' versioned S3 buckets. */
+	emptyBuckets: (stackNames: string[]) => Promise<void>;
+	/** Sleep (VPC-ENI detach window). */
+	sleep: (ms: number) => Promise<void>;
+	/** Backoff between retries. Default: 1min, then 2min. */
+	retryDelays?: number[];
+}
+
+/**
+ * Retry `cdk destroy`, clearing the two known teardown blockers before each
+ * retry: (1) non-empty versioned S3 buckets — `cdk destroy` relies on each
+ * bucket's autoDeleteObjects Lambda, which never provisioned if the CREATE
+ * failed, so the bucket blocks the delete and the stack (+ its IAM roles) leaks;
+ * we empty them out-of-band — and (2) VPC ENIs that take 60-120s to detach (the
+ * backoff). Stack names are resolved once and reused (topology is stable across
+ * attempts). Factored out with injectable deps so this behavior is unit-testable.
+ */
+export async function runDestroyWithRetries(deps: DestroyRetryDeps): Promise<void> {
+	const retryDelays = deps.retryDelays ?? [60_000, 120_000];
+	let stackNames: string[] | undefined;
+	for (let attempt = 0; ; attempt++) {
+		try {
+			deps.runDestroy();
+			console.log(attempt === 0 ? '\n✅ Sandbox destroyed!' : '\n✅ Sandbox destroyed on retry!');
+			return;
+		} catch (error) {
+			if (attempt < retryDelays.length) {
+				if (stackNames === undefined) stackNames = deps.listStackNames();
+				if (stackNames.length > 0) {
+					console.log('\n🧹 Emptying versioned S3 buckets before retry...');
+					await deps.emptyBuckets(stackNames);
+				}
+				const delaySec = retryDelays[attempt] / 1000;
+				console.log(`\n⏳ Stack deletion failed. Retrying in ${delaySec}s (waiting for resource cleanup)...`);
+				await deps.sleep(retryDelays[attempt]);
+			} else {
+				console.error('\n❌ Destroy failed after retries.');
+				throw error;
+			}
+		}
+	}
+}
+
 export async function destroySandbox(backendPath: string) {
   return trackCommand('sandbox:destroy', async () => {
     console.log("🗑️  Destroying sandbox...");
@@ -334,31 +510,16 @@ export async function destroySandbox(backendPath: string) {
       "exec", "cdk", "--", "destroy",
       "--force",
       "--context", "sandboxMode=true",
-      "--app", `npm exec tsx -- -C cdk ${backendPath}`,
+      // Single-quote backendPath: CDK re-runs the --app string through a shell,
+      // so an unquoted path containing a space would split.
+      "--app", `npm exec tsx -- -C cdk '${backendPath}'`,
     ];
     const cdkEnv = { ...process.env, NODE_OPTIONS: "--conditions=cdk", ...getCdkTelemetryEnv('sandbox') };
-    // Retry with backoff for VPC-dependent resources (e.g. Aurora clusters).
-    // CloudFormation deletes the cluster first, but its ENIs take 60-120s to
-    // detach from the VPC subnets asynchronously. The initial destroy fails
-    // because the subnets still have attached ENIs; retrying after the cleanup
-    // window lets the VPC delete succeed.
-    const retryDelays = [60_000, 120_000]; // 1min, then 2min
-
-    for (let attempt = 0; ; attempt++) {
-      try {
-        runSync("npm", cdkArgs, { stdio: "inherit", env: cdkEnv });
-        console.log(attempt === 0 ? "\n✅ Sandbox destroyed!" : "\n✅ Sandbox destroyed on retry!");
-        return;
-      } catch (error) {
-        if (attempt < retryDelays.length) {
-          const delaySec = retryDelays[attempt] / 1000;
-          console.log(`\n⏳ Stack deletion failed. Retrying in ${delaySec}s (waiting for resource cleanup)...`);
-          await new Promise(r => setTimeout(r, retryDelays[attempt]));
-        } else {
-          console.error("\n❌ Destroy failed after retries.");
-          throw error;
-        }
-      }
-    }
+    await runDestroyWithRetries({
+      runDestroy: () => runSync('npm', cdkArgs, { stdio: 'inherit', env: cdkEnv }),
+      listStackNames: () => listSandboxStackNames(backendPath, cdkEnv),
+      emptyBuckets: emptySandboxBuckets,
+      sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
+    });
   });
 }

package/src/version.ts

@@ -1,2 +1,2 @@
 // Auto-generated by scripts/generate-version.mjs — do not edit manually
-export const CORE_VERSION = '0.3.0';
+export const CORE_VERSION = '0.4.0';