@aws-blocks/core 0.2.0 → 0.4.0

package/src/scripts/sandbox-args.test.ts

@@ -0,0 +1,92 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, it } from 'node:test';
+import assert from 'node:assert';
+
+import { buildSandboxDeployArgs, sandboxFailureRecoveryHint } from './sandbox.js';
+import { buildCdkDeployArgs } from './deploy-stream.js';
+
+/**
+ * Return the value that follows `flag` in `args`, failing with a clear message
+ * if the flag is absent. Reading `args[indexOf(flag) + 1]` directly silently
+ * reads `args[0]` when the flag is missing (indexOf returns -1), which turns a
+ * dropped flag into a confusing value mismatch instead of a "flag missing".
+ *
+ * NOTE: resolves only the FIRST occurrence of `flag`. `--context` legitimately
+ * appears twice in this argv (projectRoot, then sandboxMode=true), so do not
+ * use `valueAfter(args, '--context')` expecting the second value — it silently
+ * returns the first. Assert repeated flags with `args.includes(...)` instead.
+ */
+function valueAfter(args: string[], flag: string): string {
+  const i = args.indexOf(flag);
+  assert.ok(i !== -1, `expected ${flag} in: ${args.join(' ')}`);
+  return args[i + 1];
+}
+
+describe('buildSandboxDeployArgs — express mode (sandbox only)', () => {
+  const args = buildSandboxDeployArgs({
+    outDir: '.blocks-sandbox',
+    projectRoot: '/app',
+    backendPath: 'aws-blocks/index.ts',
+  });
+
+  it('enables CloudFormation Express Mode via --express', () => {
+    assert.ok(args.includes('--express'), `expected --express in: ${args.join(' ')}`);
+  });
+
+  it('pairs express mode with the direct deployment method', () => {
+    assert.strictEqual(valueAfter(args, '--method'), 'direct');
+  });
+
+  it('leaves express-mode rollback at its default (off) — no --rollback', () => {
+    // Express Mode disables automatic rollback by default; we keep that default
+    // for the throwaway sandbox loop. Passing --rollback here would silently
+    // give up the speedup.
+    assert.ok(!args.includes('--rollback'), `sandbox must not force --rollback: ${args.join(' ')}`);
+  });
+
+  it('deploys every stack so Lambda@Edge apps synth cleanly', () => {
+    assert.ok(args.includes('--all'), `expected --all in: ${args.join(' ')}`);
+  });
+
+  it('keeps the non-interactive deploy contract and sandbox context', () => {
+    assert.deepStrictEqual(args.slice(0, 4), ['exec', 'cdk', '--', 'deploy']);
+    assert.strictEqual(valueAfter(args, '--require-approval'), 'never');
+    assert.strictEqual(valueAfter(args, '--outputs-file'), '.blocks-sandbox/outputs.json');
+    assert.ok(args.includes('sandboxMode=true'), `expected sandboxMode=true in: ${args.join(' ')}`);
+    assert.strictEqual(valueAfter(args, '--app'), 'npm exec tsx -- -C cdk aws-blocks/index.ts');
+  });
+
+  it('interpolates the provided outDir and projectRoot', () => {
+    const custom = buildSandboxDeployArgs({ outDir: 'out', projectRoot: '/other', backendPath: 'b.ts' });
+    assert.strictEqual(valueAfter(custom, '--outputs-file'), 'out/outputs.json');
+    assert.strictEqual(valueAfter(custom, '--context'), 'projectRoot=/other');
+  });
+
+  it('does NOT leak express mode into the production deploy path', () => {
+    // A production deploy must keep a reviewable change set and full
+    // stabilization with rollback: neither `--express` nor `--method direct`
+    // (both sandbox-only speedups) may appear in buildCdkDeployArgs.
+    const prod = buildCdkDeployArgs({ projectRoot: '/app', outputsFile: '.blocks-sandbox/outputs.json' });
+    assert.ok(!prod.includes('--express'), `production deploy must not use --express: ${prod.join(' ')}`);
+    assert.ok(!prod.includes('--method'), `production deploy must not use --method: ${prod.join(' ')}`);
+  });
+});
+
+describe('sandboxFailureRecoveryHint — no silent dead-end after a failed express deploy', () => {
+  const hint = sandboxFailureRecoveryHint();
+
+  it('points to the repo-native destroy + redeploy recovery', () => {
+    assert.match(hint, /npm run sandbox:destroy/);
+    assert.match(hint, /npm run sandbox\b/);
+  });
+
+  it('covers the manual UPDATE_ROLLBACK_FAILED escape hatch', () => {
+    assert.match(hint, /continue-update-rollback/);
+  });
+
+  it('explains why (rollback is off under express mode)', () => {
+    assert.match(hint, /rollback is off|automatic rollback/i);
+  });
+});

package/src/scripts/sandbox-empty-buckets.test.ts

@@ -0,0 +1,191 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, it } from 'node:test';
+import assert from 'node:assert';
+import type { CloudFormationClient } from '@aws-sdk/client-cloudformation';
+import type { S3Client } from '@aws-sdk/client-s3';
+
+import { emptyBucket, listStackBucketNames, runDestroyWithRetries, toDeleteObjects } from './sandbox.js';
+
+// The teardown bucket-emptying must delete BOTH live versions and delete
+// markers — a versioned bucket left with either still blocks `cdk destroy`.
+// This is the exact combination that, done wrong, silently no-ops.
+describe('toDeleteObjects', () => {
+	it('combines versions and delete markers', () => {
+		const out = toDeleteObjects({
+			Versions: [
+				{ Key: 'a', VersionId: 'v1' },
+				{ Key: 'b', VersionId: 'v2' },
+			],
+			DeleteMarkers: [{ Key: 'a', VersionId: 'dm1' }],
+		});
+		assert.deepStrictEqual(out, [
+			{ Key: 'a', VersionId: 'v1' },
+			{ Key: 'b', VersionId: 'v2' },
+			{ Key: 'a', VersionId: 'dm1' },
+		]);
+	});
+
+	it('returns [] for an empty bucket (no Versions/DeleteMarkers keys)', () => {
+		assert.deepStrictEqual(toDeleteObjects({}), []);
+	});
+
+	it('tolerates only versions or only delete markers', () => {
+		assert.deepStrictEqual(toDeleteObjects({ Versions: [{ Key: 'x', VersionId: 'v' }] }), [
+			{ Key: 'x', VersionId: 'v' },
+		]);
+		assert.deepStrictEqual(toDeleteObjects({ DeleteMarkers: [{ Key: 'y', VersionId: 'd' }] }), [
+			{ Key: 'y', VersionId: 'd' },
+		]);
+	});
+
+	it('drops entries without a Key rather than emitting a bad DeleteObjects payload', () => {
+		assert.deepStrictEqual(toDeleteObjects({ Versions: [{ VersionId: 'orphan' }] }), []);
+	});
+});
+
+// A fake AWS client: routes on the command class name and returns queued/canned
+// responses. Cast to the real client type — this is test plumbing, not customer code.
+type Send = (cmd: { constructor: { name: string }; input: any }) => Promise<any>;
+
+describe('listStackBucketNames', () => {
+	it('follows NextToken across pages and returns only S3 buckets', async () => {
+		const seen: Array<string | undefined> = [];
+		const send: Send = async (cmd) => {
+			seen.push(cmd.input.NextToken);
+			if (cmd.input.NextToken === undefined) {
+				return {
+					StackResourceSummaries: [
+						{ ResourceType: 'AWS::S3::Bucket', PhysicalResourceId: 'b1' },
+						{ ResourceType: 'AWS::Lambda::Function', PhysicalResourceId: 'fn' },
+					],
+					NextToken: 't2',
+				};
+			}
+			return { StackResourceSummaries: [{ ResourceType: 'AWS::S3::Bucket', PhysicalResourceId: 'b2' }] };
+		};
+		const cfn = { send } as unknown as CloudFormationClient;
+		const buckets = await listStackBucketNames(cfn, 'stack');
+		assert.deepStrictEqual(buckets, ['b1', 'b2']);
+		assert.deepStrictEqual(seen, [undefined, 't2']); // paged exactly twice
+	});
+});
+
+describe('emptyBucket', () => {
+	it('deletes versions + markers, re-lists, and stops when empty', async () => {
+		let lists = 0;
+		const deletePayloads: unknown[] = [];
+		const send: Send = async (cmd) => {
+			if (cmd.constructor.name === 'ListObjectVersionsCommand') {
+				lists++;
+				return lists === 1
+					? { Versions: [{ Key: 'a', VersionId: 'v1' }], DeleteMarkers: [{ Key: 'a', VersionId: 'd1' }] }
+					: {};
+			}
+			if (cmd.constructor.name === 'DeleteObjectsCommand') {
+				deletePayloads.push(cmd.input.Delete.Objects);
+				return {};
+			}
+			throw new Error(`unexpected ${cmd.constructor.name}`);
+		};
+		await emptyBucket({ send } as unknown as S3Client, 'bucket');
+		assert.strictEqual(lists, 2); // listed, deleted, re-listed → empty
+		assert.deepStrictEqual(deletePayloads, [
+			[
+				{ Key: 'a', VersionId: 'v1' },
+				{ Key: 'a', VersionId: 'd1' },
+			],
+		]);
+	});
+
+	it('stops (no infinite loop) when DeleteObjects reports Errors', async () => {
+		let lists = 0;
+		let deletes = 0;
+		const send: Send = async (cmd) => {
+			if (cmd.constructor.name === 'ListObjectVersionsCommand') {
+				lists++;
+				return { Versions: [{ Key: 'locked', VersionId: 'v1' }] };
+			}
+			if (cmd.constructor.name === 'DeleteObjectsCommand') {
+				deletes++;
+				return { Errors: [{ Key: 'locked', Code: 'AccessDenied' }] };
+			}
+			throw new Error(`unexpected ${cmd.constructor.name}`);
+		};
+		await emptyBucket({ send } as unknown as S3Client, 'bucket');
+		assert.strictEqual(lists, 1); // did NOT re-list after Errors
+		assert.strictEqual(deletes, 1);
+	});
+});
+
+// The sev2 fix is the wiring: on a destroy failure, buckets get emptied BEFORE
+// the retry. runDestroyWithRetries takes its side effects as deps so we can
+// exercise that ordering without spawning cdk.
+describe('runDestroyWithRetries', () => {
+	it('empties buckets before retrying a failed destroy', async () => {
+		const events: string[] = [];
+		let attempts = 0;
+		await runDestroyWithRetries({
+			runDestroy: () => {
+				attempts++;
+				events.push(`destroy#${attempts}`);
+				if (attempts === 1) throw new Error('bucket not empty');
+			},
+			listStackNames: () => {
+				events.push('list');
+				return ['stackA', 'stackB'];
+			},
+			emptyBuckets: async (names) => {
+				events.push(`empty:${names.join(',')}`);
+			},
+			sleep: async () => {
+				events.push('sleep');
+			},
+			retryDelays: [1],
+		});
+		assert.strictEqual(attempts, 2);
+		// buckets emptied (with the resolved stack names) before the retry destroy
+		assert.deepStrictEqual(events, ['destroy#1', 'list', 'empty:stackA,stackB', 'sleep', 'destroy#2']);
+	});
+
+	it('does not empty or sleep when the first destroy succeeds', async () => {
+		let emptied = false;
+		let slept = false;
+		let listed = false;
+		await runDestroyWithRetries({
+			runDestroy: () => {},
+			listStackNames: () => {
+				listed = true;
+				return [];
+			},
+			emptyBuckets: async () => {
+				emptied = true;
+			},
+			sleep: async () => {
+				slept = true;
+			},
+		});
+		assert.strictEqual(emptied, false);
+		assert.strictEqual(slept, false);
+		assert.strictEqual(listed, false);
+	});
+
+	it('throws after exhausting retries', async () => {
+		let attempts = 0;
+		await assert.rejects(
+			runDestroyWithRetries({
+				runDestroy: () => {
+					attempts++;
+					throw new Error('still failing');
+				},
+				listStackNames: () => [],
+				emptyBuckets: async () => {},
+				sleep: async () => {},
+				retryDelays: [1, 1],
+			}),
+			/still failing/,
+		);
+		assert.strictEqual(attempts, 3); // initial attempt + 2 retries
+	});
+});

package/src/scripts/sandbox.ts

@@ -6,12 +6,16 @@ import { writeFileSync, mkdirSync, readFileSync } from "node:fs";
 import { join, resolve, dirname } from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import { ensureSecrets, loadEnvFile } from './ensure-secrets.js';
+import { assertAwsCredentials } from './preflight-credentials.js';
 import { applyExternalMigrations } from './external-migrations-step.js';
 import { trackCommand } from '../telemetry/trackCommand.js';
 import { buildAndSendEvent } from '../telemetry/client.js';
+import { classifyError } from '../telemetry/trackCommand.js';
 import { getCdkTelemetryEnv } from './cdk-telemetry-env.js';
 import { runSync, spawnCommand } from './run-command.js';
 import { terminateProcessTree } from './process-tree.js';
+import type { CloudFormationClient } from '@aws-sdk/client-cloudformation';
+import type { S3Client } from '@aws-sdk/client-s3';
 
 /**
  * Import the backend definition to populate the Scope BB registry.
@@ -38,6 +42,85 @@ export interface SandboxOptions {
   devCommand?: string;
 }
 
+export interface SandboxDeployArgsOptions {
+  /** Directory CDK writes stack outputs to (relative to the project root). */
+  outDir: string;
+  /** Project root passed to synth as `--context projectRoot=…` (usually `process.cwd()`). */
+  projectRoot: string;
+  /** Backend entry passed to `--app` so CDK synthesizes from the app definition. */
+  backendPath: string;
+}
+
+/**
+ * Build the `npm exec cdk -- deploy …` argv used by the sandbox deploy.
+ *
+ * Kept pure (no I/O) so the argv contract — in particular the flags below — can
+ * be asserted directly, the same way {@link buildCdkDeployArgs} is for the
+ * production path.
+ *
+ * - `--all`: an app that uses Lambda@Edge (e.g. a Next.js route with
+ *   `export const runtime = 'edge'`) synthesizes a SECOND stack
+ *   (`edge-lambda-stack-*`, region us-east-1) in addition to the main hosting
+ *   stack. Without `--all`, CDK refuses with "specify which stacks to use".
+ *   Deploying every stack in a sandbox app is the intended behavior.
+ * - `--method direct`: skip CloudFormation change-set creation and call
+ *   UpdateStack directly. This is the baseline sandbox deployment method that
+ *   `--express` builds on.
+ * - `--express` (**CloudFormation Express Mode — sandbox only**): the real
+ *   speedup. CloudFormation reports each stack operation complete as soon as
+ *   the resource's configuration is applied, WITHOUT waiting for full
+ *   stabilization. Because it skips stabilization it also disables automatic
+ *   rollback by default, so a failed deploy can leave the stack in a failed
+ *   state — acceptable for the throwaway sandbox iteration loop, never for
+ *   production. We leave rollback at Express Mode's default (off) and do NOT
+ *   pass `--rollback`. The production deploy path (`deploy.ts`) deliberately
+ *   uses neither `--express` nor `--method direct`: it keeps a reviewable
+ *   change set and full stabilization with rollback, which are exactly the
+ *   safety signals a production deploy should keep. Requires an aws-cdk CLI
+ *   new enough to expose `--express`; our pinned `^2.1138.0` has it.
+ */
+export function buildSandboxDeployArgs({ outDir, projectRoot, backendPath }: SandboxDeployArgsOptions): string[] {
+  return [
+    "exec", "cdk", "--", "deploy",
+    "--all",
+    "--method", "direct",
+    "--express",
+    "--require-approval", "never",
+    "--outputs-file", `${outDir}/outputs.json`,
+    "--context", `projectRoot=${projectRoot}`,
+    "--context", "sandboxMode=true",
+    "--app", `npm exec tsx -- -C cdk ${backendPath}`,
+  ];
+}
+
+/**
+ * Operator-facing recovery guidance printed when a sandbox deploy fails.
+ *
+ * Express Mode (see {@link buildSandboxDeployArgs}) disables CloudFormation's
+ * automatic rollback, so a failed deploy does NOT clean up after itself: a
+ * failed *first* deploy leaves the stack in `CREATE_FAILED`/`ROLLBACK_COMPLETE`
+ * and a failed update can land in `UPDATE_ROLLBACK_FAILED`. In those states
+ * CloudFormation refuses the next `UpdateStack`, so a plain re-run of
+ * `npm run sandbox` would hit the same wall — a silent dead-end for the fast
+ * loop. This spells out the exact way out instead of leaving the operator to
+ * discover it via a cryptic CloudFormation error.
+ *
+ * Kept pure so it can be asserted in a unit test.
+ */
+export function sandboxFailureRecoveryHint(): string {
+  return [
+    "",
+    "ℹ️  Express Mode leaves failed sandbox stacks in place (automatic rollback is off).",
+    "   If the next `npm run sandbox` reports the stack cannot be updated, recover with:",
+    "",
+    "     npm run sandbox:destroy   # tears the failed stack down (handles ROLLBACK_COMPLETE)",
+    "     npm run sandbox           # redeploy from a clean slate",
+    "",
+    "   If destroy reports UPDATE_ROLLBACK_FAILED, first run:",
+    "     aws cloudformation continue-update-rollback --stack-name <your-sandbox-stack>",
+  ].join("\n");
+}
+
 export async function startSandbox(options: SandboxOptions) {
   const { backendPath, outDir = ".blocks-sandbox", clientPort = 3000, deployOnly = false, devCommand } = options;
   const sandboxStartTime = Date.now();
@@ -49,6 +132,23 @@ export async function startSandbox(options: SandboxOptions) {
 
   process.env.BLOCKS_STAGE = 'sandbox';
 
+  // Fail fast if AWS credentials are missing/expired, before spending ~10s in
+  // synth only to hit an opaque CDK credential error. Unlike deploy(), startSandbox
+  // isn't wrapped in trackCommand, so emit the FAIL event manually (mirroring the
+  // CDK-deploy failure path below) before rethrowing — otherwise a no-creds abort
+  // sends no telemetry.
+  try {
+    await assertAwsCredentials('sandbox');
+  } catch (error) {
+    buildAndSendEvent({
+      command: 'sandbox',
+      state: 'FAIL',
+      duration: Date.now() - sandboxStartTime,
+      error: classifyError(error),
+    });
+    throw error;
+  }
+
   // Provision connection string to SSM SecureString.
   // On first deploy, creates the parameter. On subsequent deploys, updates if changed.
   // projectRoot is process.cwd() — the same value passed to cdk as --context
@@ -75,21 +175,9 @@ export async function startSandbox(options: SandboxOptions) {
   try {
     runSync(
       "npm",
-      [
-        "exec", "cdk", "--", "deploy",
-        // `--all`: an app that uses Lambda@Edge (e.g. a Next.js route with
-        // `export const runtime = 'edge'`) synthesizes a SECOND stack
-        // (`edge-lambda-stack-*`, region us-east-1) in addition to the main
-        // hosting stack. Without `--all`, CDK refuses with "specify which
-        // stacks to use". Deploying every stack in a sandbox app is the
-        // intended behavior, so select them all.
-        "--all",
-        "--require-approval", "never",
-        "--outputs-file", `${outDir}/outputs.json`,
-        "--context", `projectRoot=${process.cwd()}`,
-        "--context", "sandboxMode=true",
-        "--app", `npm exec tsx -- -C cdk ${backendPath}`,
-      ],
+      // Argv (including sandbox-only CloudFormation Express Mode, `--express`)
+      // is built by buildSandboxDeployArgs — see its doc for why each flag exists.
+      buildSandboxDeployArgs({ outDir, projectRoot: process.cwd(), backendPath }),
       {
         stdio: "inherit",
         env: { ...process.env, NODE_OPTIONS: "--conditions=cdk", ...getCdkTelemetryEnv('sandbox') },
@@ -103,6 +191,7 @@ export async function startSandbox(options: SandboxOptions) {
       error: { code: 'CDK_DEPLOY_FAILED', phase: 'deploy' },
     });
     console.error("\n❌ Deployment failed.");
+    console.error(sandboxFailureRecoveryHint());
     throw error;
   }
 
@@ -234,6 +323,180 @@ export async function startSandbox(options: SandboxOptions) {
   await new Promise(() => {});
 }
 
+/**
+ * Flatten a ListObjectVersions response into the `{ Key, VersionId }[]` shape
+ * DeleteObjects expects, combining live versions AND delete markers. Exported
+ * for unit testing — getting this combination wrong (e.g. missing the delete
+ * markers) leaves a "versioned" bucket that still can't be deleted.
+ */
+export function toDeleteObjects(listed: {
+	Versions?: Array<{ Key?: string; VersionId?: string }>;
+	DeleteMarkers?: Array<{ Key?: string; VersionId?: string }>;
+}): Array<{ Key: string; VersionId?: string }> {
+	return [...(listed.Versions ?? []), ...(listed.DeleteMarkers ?? [])]
+		.filter((v): v is { Key: string; VersionId?: string } => v.Key !== undefined)
+		.map((v) => ({ Key: v.Key, VersionId: v.VersionId }));
+}
+
+/** Collect a stack's S3 bucket physical names (paginated — a large stack has
+ * >100 resources, so a single page can miss the hosting bucket that blocks the
+ * delete). Returns [] if the stack is already gone / not accessible. */
+export async function listStackBucketNames(cfn: CloudFormationClient, stackName: string): Promise<string[]> {
+	const { ListStackResourcesCommand } = await import('@aws-sdk/client-cloudformation');
+	const buckets: string[] = [];
+	let token: string | undefined;
+	do {
+		const res = await cfn.send(new ListStackResourcesCommand({ StackName: stackName, NextToken: token }));
+		for (const r of res.StackResourceSummaries ?? []) {
+			if (r.ResourceType === 'AWS::S3::Bucket' && r.PhysicalResourceId) buckets.push(r.PhysicalResourceId);
+		}
+		token = res.NextToken;
+	} while (token);
+	return buckets;
+}
+
+/** Empty a versioned bucket: delete every object version + delete marker in
+ * pages of 1000 (the DeleteObjects limit). Stops if a page reports per-key
+ * errors (object-lock / retention) so it can't loop forever. */
+export async function emptyBucket(s3: S3Client, bucket: string): Promise<void> {
+	const { ListObjectVersionsCommand, DeleteObjectsCommand } = await import('@aws-sdk/client-s3');
+	while (true) {
+		const listed = await s3.send(new ListObjectVersionsCommand({ Bucket: bucket, MaxKeys: 1000 }));
+		const objects = toDeleteObjects(listed);
+		if (objects.length === 0) return;
+		const res = await s3.send(new DeleteObjectsCommand({ Bucket: bucket, Delete: { Objects: objects, Quiet: true } }));
+		if (res.Errors && res.Errors.length > 0) {
+			// A bucket we can't fully empty (object-lock / retention / AccessDenied)
+			// will keep blocking `cdk destroy`, so the stack won't tear down until it's
+			// resolved by hand. Log loudly with the first error code — this is the
+			// kind of silent skip that leaks a stack, so make it visible in CI output.
+			const first = res.Errors[0];
+			console.warn(
+				`  ⚠️  ${bucket}: ${res.Errors.length} object(s) could NOT be deleted ` +
+					`(e.g. ${first.Key}: ${first.Code}). This bucket will block stack teardown ` +
+					`until cleared manually.`,
+			);
+			return;
+		}
+	}
+}
+
+/**
+ * Empty every versioned S3 bucket owned by the given stacks. `cdk destroy`
+ * relies on each bucket's `autoDeleteObjects` custom-resource Lambda to empty
+ * it on teardown — but that Lambda never provisions if the CREATE failed (e.g.
+ * a partial/aborted deploy), so the versioned bucket blocks the delete and the
+ * stack (and its IAM roles) leak. Emptying out-of-band before the retry lets
+ * the delete complete. Best-effort: any per-stack/per-bucket error is logged
+ * and skipped so teardown still proceeds.
+ */
+async function emptySandboxBuckets(stackNames: string[]): Promise<void> {
+	try {
+		const { CloudFormationClient } = await import('@aws-sdk/client-cloudformation');
+		const { S3Client } = await import('@aws-sdk/client-s3');
+		const cfn = new CloudFormationClient({});
+		// `followRegionRedirects` makes the client transparently retry against a
+		// bucket's real region on a PermanentRedirect, so a bucket that lives
+		// outside AWS_REGION (e.g. a us-east-1 Lambda@Edge stack's bucket) is still
+		// emptied instead of failing silently — closing the exact no-op this fix
+		// targets. It needs no GetBucketLocation permission.
+		const s3 = new S3Client({ followRegionRedirects: true });
+		for (const stackName of stackNames) {
+			let buckets: string[] = [];
+			try {
+				buckets = await listStackBucketNames(cfn, stackName);
+			} catch {
+				continue; // stack already gone / not accessible
+			}
+			for (const b of buckets) {
+				try {
+					await emptyBucket(s3, b);
+				} catch (e) {
+					console.warn(`  ⚠️  could not empty ${b}: ${(e as Error).message}`);
+				}
+			}
+		}
+	} catch (e) {
+		console.warn(`  ⚠️  bucket-emptying step skipped: ${(e as Error).message}`);
+	}
+}
+
+/** Resolve the sandbox app's stack names via `cdk ls` (best-effort; [] on error). */
+function listSandboxStackNames(backendPath: string, cdkEnv: NodeJS.ProcessEnv): string[] {
+	try {
+		const out = execFileSync(
+			'npm',
+			// Single-quote backendPath: CDK re-runs the --app string through a shell,
+			// so an unquoted path containing a space would split.
+			['exec', 'cdk', '--', 'ls', '--context', 'sandboxMode=true', '--app', `npm exec tsx -- -C cdk '${backendPath}'`],
+			// Capture stderr so a genuine synth/`cdk ls` failure can be logged below
+			// rather than being indistinguishable from "app has zero stacks".
+			{ env: cdkEnv, encoding: 'utf-8', stdio: ['ignore', 'pipe', 'pipe'] },
+		);
+		return out
+			.split('\n')
+			.map((s) => s.trim())
+			.filter(Boolean);
+	} catch (e) {
+		// Warn (don't stay silent): if this returns [] on a real failure, the retry
+		// skips bucket-emptying — the exact silent-no-op this change exists to avoid.
+		const detail = (e as { stderr?: string; message?: string }).stderr || (e as Error).message;
+		console.warn(`  ⚠️  could not list sandbox stacks via 'cdk ls'; skipping bucket-emptying: ${detail}`);
+		return [];
+	}
+}
+
+/** Injectable dependencies for {@link runDestroyWithRetries} — real ones in
+ * `destroySandbox`, fakes in tests so the retry/empty-before-retry wiring (the
+ * sev2 fix) is exercised without spawning cdk. */
+export interface DestroyRetryDeps {
+	/** Run one `cdk destroy` attempt; throws on failure. */
+	runDestroy: () => void;
+	/** Resolve the app's stack names (for bucket enumeration). */
+	listStackNames: () => string[];
+	/** Empty the given stacks' versioned S3 buckets. */
+	emptyBuckets: (stackNames: string[]) => Promise<void>;
+	/** Sleep (VPC-ENI detach window). */
+	sleep: (ms: number) => Promise<void>;
+	/** Backoff between retries. Default: 1min, then 2min. */
+	retryDelays?: number[];
+}
+
+/**
+ * Retry `cdk destroy`, clearing the two known teardown blockers before each
+ * retry: (1) non-empty versioned S3 buckets — `cdk destroy` relies on each
+ * bucket's autoDeleteObjects Lambda, which never provisioned if the CREATE
+ * failed, so the bucket blocks the delete and the stack (+ its IAM roles) leaks;
+ * we empty them out-of-band — and (2) VPC ENIs that take 60-120s to detach (the
+ * backoff). Stack names are resolved once and reused (topology is stable across
+ * attempts). Factored out with injectable deps so this behavior is unit-testable.
+ */
+export async function runDestroyWithRetries(deps: DestroyRetryDeps): Promise<void> {
+	const retryDelays = deps.retryDelays ?? [60_000, 120_000];
+	let stackNames: string[] | undefined;
+	for (let attempt = 0; ; attempt++) {
+		try {
+			deps.runDestroy();
+			console.log(attempt === 0 ? '\n✅ Sandbox destroyed!' : '\n✅ Sandbox destroyed on retry!');
+			return;
+		} catch (error) {
+			if (attempt < retryDelays.length) {
+				if (stackNames === undefined) stackNames = deps.listStackNames();
+				if (stackNames.length > 0) {
+					console.log('\n🧹 Emptying versioned S3 buckets before retry...');
+					await deps.emptyBuckets(stackNames);
+				}
+				const delaySec = retryDelays[attempt] / 1000;
+				console.log(`\n⏳ Stack deletion failed. Retrying in ${delaySec}s (waiting for resource cleanup)...`);
+				await deps.sleep(retryDelays[attempt]);
+			} else {
+				console.error('\n❌ Destroy failed after retries.');
+				throw error;
+			}
+		}
+	}
+}
+
 export async function destroySandbox(backendPath: string) {
   return trackCommand('sandbox:destroy', async () => {
     console.log("🗑️  Destroying sandbox...");
@@ -247,31 +510,16 @@ export async function destroySandbox(backendPath: string) {
       "exec", "cdk", "--", "destroy",
       "--force",
       "--context", "sandboxMode=true",
-      "--app", `npm exec tsx -- -C cdk ${backendPath}`,
+      // Single-quote backendPath: CDK re-runs the --app string through a shell,
+      // so an unquoted path containing a space would split.
+      "--app", `npm exec tsx -- -C cdk '${backendPath}'`,
     ];
     const cdkEnv = { ...process.env, NODE_OPTIONS: "--conditions=cdk", ...getCdkTelemetryEnv('sandbox') };
-    // Retry with backoff for VPC-dependent resources (e.g. Aurora clusters).
-    // CloudFormation deletes the cluster first, but its ENIs take 60-120s to
-    // detach from the VPC subnets asynchronously. The initial destroy fails
-    // because the subnets still have attached ENIs; retrying after the cleanup
-    // window lets the VPC delete succeed.
-    const retryDelays = [60_000, 120_000]; // 1min, then 2min
-
-    for (let attempt = 0; ; attempt++) {
-      try {
-        runSync("npm", cdkArgs, { stdio: "inherit", env: cdkEnv });
-        console.log(attempt === 0 ? "\n✅ Sandbox destroyed!" : "\n✅ Sandbox destroyed on retry!");
-        return;
-      } catch (error) {
-        if (attempt < retryDelays.length) {
-          const delaySec = retryDelays[attempt] / 1000;
-          console.log(`\n⏳ Stack deletion failed. Retrying in ${delaySec}s (waiting for resource cleanup)...`);
-          await new Promise(r => setTimeout(r, retryDelays[attempt]));
-        } else {
-          console.error("\n❌ Destroy failed after retries.");
-          throw error;
-        }
-      }
-    }
+    await runDestroyWithRetries({
+      runDestroy: () => runSync('npm', cdkArgs, { stdio: 'inherit', env: cdkEnv }),
+      listStackNames: () => listSandboxStackNames(backendPath, cdkEnv),
+      emptyBuckets: emptySandboxBuckets,
+      sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
+    });
   });
 }