@aws-blocks/core 0.2.0 → 0.4.0

package/dist/scripts/sandbox-empty-buckets.test.js

@@ -0,0 +1,171 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+import { describe, it } from 'node:test';
+import assert from 'node:assert';
+import { emptyBucket, listStackBucketNames, runDestroyWithRetries, toDeleteObjects } from './sandbox.js';
+// The teardown bucket-emptying must delete BOTH live versions and delete
+// markers — a versioned bucket left with either still blocks `cdk destroy`.
+// This is the exact combination that, done wrong, silently no-ops.
+describe('toDeleteObjects', () => {
+    it('combines versions and delete markers', () => {
+        const out = toDeleteObjects({
+            Versions: [
+                { Key: 'a', VersionId: 'v1' },
+                { Key: 'b', VersionId: 'v2' },
+            ],
+            DeleteMarkers: [{ Key: 'a', VersionId: 'dm1' }],
+        });
+        assert.deepStrictEqual(out, [
+            { Key: 'a', VersionId: 'v1' },
+            { Key: 'b', VersionId: 'v2' },
+            { Key: 'a', VersionId: 'dm1' },
+        ]);
+    });
+    it('returns [] for an empty bucket (no Versions/DeleteMarkers keys)', () => {
+        assert.deepStrictEqual(toDeleteObjects({}), []);
+    });
+    it('tolerates only versions or only delete markers', () => {
+        assert.deepStrictEqual(toDeleteObjects({ Versions: [{ Key: 'x', VersionId: 'v' }] }), [
+            { Key: 'x', VersionId: 'v' },
+        ]);
+        assert.deepStrictEqual(toDeleteObjects({ DeleteMarkers: [{ Key: 'y', VersionId: 'd' }] }), [
+            { Key: 'y', VersionId: 'd' },
+        ]);
+    });
+    it('drops entries without a Key rather than emitting a bad DeleteObjects payload', () => {
+        assert.deepStrictEqual(toDeleteObjects({ Versions: [{ VersionId: 'orphan' }] }), []);
+    });
+});
+describe('listStackBucketNames', () => {
+    it('follows NextToken across pages and returns only S3 buckets', async () => {
+        const seen = [];
+        const send = async (cmd) => {
+            seen.push(cmd.input.NextToken);
+            if (cmd.input.NextToken === undefined) {
+                return {
+                    StackResourceSummaries: [
+                        { ResourceType: 'AWS::S3::Bucket', PhysicalResourceId: 'b1' },
+                        { ResourceType: 'AWS::Lambda::Function', PhysicalResourceId: 'fn' },
+                    ],
+                    NextToken: 't2',
+                };
+            }
+            return { StackResourceSummaries: [{ ResourceType: 'AWS::S3::Bucket', PhysicalResourceId: 'b2' }] };
+        };
+        const cfn = { send };
+        const buckets = await listStackBucketNames(cfn, 'stack');
+        assert.deepStrictEqual(buckets, ['b1', 'b2']);
+        assert.deepStrictEqual(seen, [undefined, 't2']); // paged exactly twice
+    });
+});
+describe('emptyBucket', () => {
+    it('deletes versions + markers, re-lists, and stops when empty', async () => {
+        let lists = 0;
+        const deletePayloads = [];
+        const send = async (cmd) => {
+            if (cmd.constructor.name === 'ListObjectVersionsCommand') {
+                lists++;
+                return lists === 1
+                    ? { Versions: [{ Key: 'a', VersionId: 'v1' }], DeleteMarkers: [{ Key: 'a', VersionId: 'd1' }] }
+                    : {};
+            }
+            if (cmd.constructor.name === 'DeleteObjectsCommand') {
+                deletePayloads.push(cmd.input.Delete.Objects);
+                return {};
+            }
+            throw new Error(`unexpected ${cmd.constructor.name}`);
+        };
+        await emptyBucket({ send }, 'bucket');
+        assert.strictEqual(lists, 2); // listed, deleted, re-listed → empty
+        assert.deepStrictEqual(deletePayloads, [
+            [
+                { Key: 'a', VersionId: 'v1' },
+                { Key: 'a', VersionId: 'd1' },
+            ],
+        ]);
+    });
+    it('stops (no infinite loop) when DeleteObjects reports Errors', async () => {
+        let lists = 0;
+        let deletes = 0;
+        const send = async (cmd) => {
+            if (cmd.constructor.name === 'ListObjectVersionsCommand') {
+                lists++;
+                return { Versions: [{ Key: 'locked', VersionId: 'v1' }] };
+            }
+            if (cmd.constructor.name === 'DeleteObjectsCommand') {
+                deletes++;
+                return { Errors: [{ Key: 'locked', Code: 'AccessDenied' }] };
+            }
+            throw new Error(`unexpected ${cmd.constructor.name}`);
+        };
+        await emptyBucket({ send }, 'bucket');
+        assert.strictEqual(lists, 1); // did NOT re-list after Errors
+        assert.strictEqual(deletes, 1);
+    });
+});
+// The sev2 fix is the wiring: on a destroy failure, buckets get emptied BEFORE
+// the retry. runDestroyWithRetries takes its side effects as deps so we can
+// exercise that ordering without spawning cdk.
+describe('runDestroyWithRetries', () => {
+    it('empties buckets before retrying a failed destroy', async () => {
+        const events = [];
+        let attempts = 0;
+        await runDestroyWithRetries({
+            runDestroy: () => {
+                attempts++;
+                events.push(`destroy#${attempts}`);
+                if (attempts === 1)
+                    throw new Error('bucket not empty');
+            },
+            listStackNames: () => {
+                events.push('list');
+                return ['stackA', 'stackB'];
+            },
+            emptyBuckets: async (names) => {
+                events.push(`empty:${names.join(',')}`);
+            },
+            sleep: async () => {
+                events.push('sleep');
+            },
+            retryDelays: [1],
+        });
+        assert.strictEqual(attempts, 2);
+        // buckets emptied (with the resolved stack names) before the retry destroy
+        assert.deepStrictEqual(events, ['destroy#1', 'list', 'empty:stackA,stackB', 'sleep', 'destroy#2']);
+    });
+    it('does not empty or sleep when the first destroy succeeds', async () => {
+        let emptied = false;
+        let slept = false;
+        let listed = false;
+        await runDestroyWithRetries({
+            runDestroy: () => { },
+            listStackNames: () => {
+                listed = true;
+                return [];
+            },
+            emptyBuckets: async () => {
+                emptied = true;
+            },
+            sleep: async () => {
+                slept = true;
+            },
+        });
+        assert.strictEqual(emptied, false);
+        assert.strictEqual(slept, false);
+        assert.strictEqual(listed, false);
+    });
+    it('throws after exhausting retries', async () => {
+        let attempts = 0;
+        await assert.rejects(runDestroyWithRetries({
+            runDestroy: () => {
+                attempts++;
+                throw new Error('still failing');
+            },
+            listStackNames: () => [],
+            emptyBuckets: async () => { },
+            sleep: async () => { },
+            retryDelays: [1, 1],
+        }), /still failing/);
+        assert.strictEqual(attempts, 3); // initial attempt + 2 retries
+    });
+});

package/dist/scripts/sandbox.d.ts

@@ -1,3 +1,5 @@
+import type { CloudFormationClient } from '@aws-sdk/client-cloudformation';
+import type { S3Client } from '@aws-sdk/client-s3';
 export interface SandboxOptions {
     backendPath: string;
     outDir?: string;
@@ -6,6 +8,110 @@ export interface SandboxOptions {
     /** Custom dev server command. Defaults to 'npx vite'. For Next.js use 'npx next dev'. */
     devCommand?: string;
 }
+export interface SandboxDeployArgsOptions {
+    /** Directory CDK writes stack outputs to (relative to the project root). */
+    outDir: string;
+    /** Project root passed to synth as `--context projectRoot=…` (usually `process.cwd()`). */
+    projectRoot: string;
+    /** Backend entry passed to `--app` so CDK synthesizes from the app definition. */
+    backendPath: string;
+}
+/**
+ * Build the `npm exec cdk -- deploy …` argv used by the sandbox deploy.
+ *
+ * Kept pure (no I/O) so the argv contract — in particular the flags below — can
+ * be asserted directly, the same way {@link buildCdkDeployArgs} is for the
+ * production path.
+ *
+ * - `--all`: an app that uses Lambda@Edge (e.g. a Next.js route with
+ *   `export const runtime = 'edge'`) synthesizes a SECOND stack
+ *   (`edge-lambda-stack-*`, region us-east-1) in addition to the main hosting
+ *   stack. Without `--all`, CDK refuses with "specify which stacks to use".
+ *   Deploying every stack in a sandbox app is the intended behavior.
+ * - `--method direct`: skip CloudFormation change-set creation and call
+ *   UpdateStack directly. This is the baseline sandbox deployment method that
+ *   `--express` builds on.
+ * - `--express` (**CloudFormation Express Mode — sandbox only**): the real
+ *   speedup. CloudFormation reports each stack operation complete as soon as
+ *   the resource's configuration is applied, WITHOUT waiting for full
+ *   stabilization. Because it skips stabilization it also disables automatic
+ *   rollback by default, so a failed deploy can leave the stack in a failed
+ *   state — acceptable for the throwaway sandbox iteration loop, never for
+ *   production. We leave rollback at Express Mode's default (off) and do NOT
+ *   pass `--rollback`. The production deploy path (`deploy.ts`) deliberately
+ *   uses neither `--express` nor `--method direct`: it keeps a reviewable
+ *   change set and full stabilization with rollback, which are exactly the
+ *   safety signals a production deploy should keep. Requires an aws-cdk CLI
+ *   new enough to expose `--express`; our pinned `^2.1138.0` has it.
+ */
+export declare function buildSandboxDeployArgs({ outDir, projectRoot, backendPath }: SandboxDeployArgsOptions): string[];
+/**
+ * Operator-facing recovery guidance printed when a sandbox deploy fails.
+ *
+ * Express Mode (see {@link buildSandboxDeployArgs}) disables CloudFormation's
+ * automatic rollback, so a failed deploy does NOT clean up after itself: a
+ * failed *first* deploy leaves the stack in `CREATE_FAILED`/`ROLLBACK_COMPLETE`
+ * and a failed update can land in `UPDATE_ROLLBACK_FAILED`. In those states
+ * CloudFormation refuses the next `UpdateStack`, so a plain re-run of
+ * `npm run sandbox` would hit the same wall — a silent dead-end for the fast
+ * loop. This spells out the exact way out instead of leaving the operator to
+ * discover it via a cryptic CloudFormation error.
+ *
+ * Kept pure so it can be asserted in a unit test.
+ */
+export declare function sandboxFailureRecoveryHint(): string;
 export declare function startSandbox(options: SandboxOptions): Promise<string | undefined>;
+/**
+ * Flatten a ListObjectVersions response into the `{ Key, VersionId }[]` shape
+ * DeleteObjects expects, combining live versions AND delete markers. Exported
+ * for unit testing — getting this combination wrong (e.g. missing the delete
+ * markers) leaves a "versioned" bucket that still can't be deleted.
+ */
+export declare function toDeleteObjects(listed: {
+    Versions?: Array<{
+        Key?: string;
+        VersionId?: string;
+    }>;
+    DeleteMarkers?: Array<{
+        Key?: string;
+        VersionId?: string;
+    }>;
+}): Array<{
+    Key: string;
+    VersionId?: string;
+}>;
+/** Collect a stack's S3 bucket physical names (paginated — a large stack has
+ * >100 resources, so a single page can miss the hosting bucket that blocks the
+ * delete). Returns [] if the stack is already gone / not accessible. */
+export declare function listStackBucketNames(cfn: CloudFormationClient, stackName: string): Promise<string[]>;
+/** Empty a versioned bucket: delete every object version + delete marker in
+ * pages of 1000 (the DeleteObjects limit). Stops if a page reports per-key
+ * errors (object-lock / retention) so it can't loop forever. */
+export declare function emptyBucket(s3: S3Client, bucket: string): Promise<void>;
+/** Injectable dependencies for {@link runDestroyWithRetries} — real ones in
+ * `destroySandbox`, fakes in tests so the retry/empty-before-retry wiring (the
+ * sev2 fix) is exercised without spawning cdk. */
+export interface DestroyRetryDeps {
+    /** Run one `cdk destroy` attempt; throws on failure. */
+    runDestroy: () => void;
+    /** Resolve the app's stack names (for bucket enumeration). */
+    listStackNames: () => string[];
+    /** Empty the given stacks' versioned S3 buckets. */
+    emptyBuckets: (stackNames: string[]) => Promise<void>;
+    /** Sleep (VPC-ENI detach window). */
+    sleep: (ms: number) => Promise<void>;
+    /** Backoff between retries. Default: 1min, then 2min. */
+    retryDelays?: number[];
+}
+/**
+ * Retry `cdk destroy`, clearing the two known teardown blockers before each
+ * retry: (1) non-empty versioned S3 buckets — `cdk destroy` relies on each
+ * bucket's autoDeleteObjects Lambda, which never provisioned if the CREATE
+ * failed, so the bucket blocks the delete and the stack (+ its IAM roles) leaks;
+ * we empty them out-of-band — and (2) VPC ENIs that take 60-120s to detach (the
+ * backoff). Stack names are resolved once and reused (topology is stable across
+ * attempts). Factored out with injectable deps so this behavior is unit-testable.
+ */
+export declare function runDestroyWithRetries(deps: DestroyRetryDeps): Promise<void>;
 export declare function destroySandbox(backendPath: string): Promise<void>;
 //# sourceMappingURL=sandbox.d.ts.map

package/dist/scripts/sandbox.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sandbox.d.ts","sourceRoot":"","sources":["../../src/scripts/sandbox.ts"],"names":[],"mappings":"AA+BA,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,yFAAyF;IACzF,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,wBAAsB,YAAY,CAAC,OAAO,EAAE,cAAc,+BAkMzD;AAED,wBAAsB,cAAc,CAAC,WAAW,EAAE,MAAM,iBAwCvD"}
+{"version":3,"file":"sandbox.d.ts","sourceRoot":"","sources":["../../src/scripts/sandbox.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAC;AAC3E,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAkBnD,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,yFAAyF;IACzF,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,wBAAwB;IACvC,4EAA4E;IAC5E,MAAM,EAAE,MAAM,CAAC;IACf,2FAA2F;IAC3F,WAAW,EAAE,MAAM,CAAC;IACpB,kFAAkF;IAClF,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,sBAAsB,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,EAAE,wBAAwB,GAAG,MAAM,EAAE,CAY/G;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,0BAA0B,IAAI,MAAM,CAYnD;AAED,wBAAsB,YAAY,CAAC,OAAO,EAAE,cAAc,+BAwMzD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE;IACvC,QAAQ,CAAC,EAAE,KAAK,CAAC;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACvD,aAAa,CAAC,EAAE,KAAK,CAAC;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC5D,GAAG,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAI7C;AAED;;wEAEwE;AACxE,wBAAsB,oBAAoB,CAAC,GAAG,EAAE,oBAAoB,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAY1G;AAED;;gEAEgE;AAChE,wBAAsB,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAqB7E;AAmED;;kDAEkD;AAClD,MAAM,WAAW,gBAAgB;IAChC,wDAAwD;IACxD,UAAU,EAAE,MAAM,IAAI,CAAC;IACvB,8DAA8D;IAC9D,cAAc,EAAE,MAAM,MAAM,EAAE,CAAC;IAC/B,oDAAoD;IACpD,YAAY,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,qCAAqC;IACrC,KAAK,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACrC,yDAAyD;IACzD,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED;;;;;;;;GAQG;AACH,wBAAsB,qBAAqB,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAwBjF;AAED,wBAAsB,cAAc,CAAC,WAAW,EAAE,MAAM,iBAyBvD"}

package/dist/scripts/sandbox.js

@@ -5,9 +5,11 @@ import { writeFileSync, mkdirSync, readFileSync } from "node:fs";
 import { join, resolve, dirname } from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import { ensureSecrets, loadEnvFile } from './ensure-secrets.js';
+import { assertAwsCredentials } from './preflight-credentials.js';
 import { applyExternalMigrations } from './external-migrations-step.js';
 import { trackCommand } from '../telemetry/trackCommand.js';
 import { buildAndSendEvent } from '../telemetry/client.js';
+import { classifyError } from '../telemetry/trackCommand.js';
 import { getCdkTelemetryEnv } from './cdk-telemetry-env.js';
 import { runSync, spawnCommand } from './run-command.js';
 import { terminateProcessTree } from './process-tree.js';
@@ -27,6 +29,74 @@ async function importBackendForRegistry(backendPath) {
         // best-effort — telemetry never affects the command
     }
 }
+/**
+ * Build the `npm exec cdk -- deploy …` argv used by the sandbox deploy.
+ *
+ * Kept pure (no I/O) so the argv contract — in particular the flags below — can
+ * be asserted directly, the same way {@link buildCdkDeployArgs} is for the
+ * production path.
+ *
+ * - `--all`: an app that uses Lambda@Edge (e.g. a Next.js route with
+ *   `export const runtime = 'edge'`) synthesizes a SECOND stack
+ *   (`edge-lambda-stack-*`, region us-east-1) in addition to the main hosting
+ *   stack. Without `--all`, CDK refuses with "specify which stacks to use".
+ *   Deploying every stack in a sandbox app is the intended behavior.
+ * - `--method direct`: skip CloudFormation change-set creation and call
+ *   UpdateStack directly. This is the baseline sandbox deployment method that
+ *   `--express` builds on.
+ * - `--express` (**CloudFormation Express Mode — sandbox only**): the real
+ *   speedup. CloudFormation reports each stack operation complete as soon as
+ *   the resource's configuration is applied, WITHOUT waiting for full
+ *   stabilization. Because it skips stabilization it also disables automatic
+ *   rollback by default, so a failed deploy can leave the stack in a failed
+ *   state — acceptable for the throwaway sandbox iteration loop, never for
+ *   production. We leave rollback at Express Mode's default (off) and do NOT
+ *   pass `--rollback`. The production deploy path (`deploy.ts`) deliberately
+ *   uses neither `--express` nor `--method direct`: it keeps a reviewable
+ *   change set and full stabilization with rollback, which are exactly the
+ *   safety signals a production deploy should keep. Requires an aws-cdk CLI
+ *   new enough to expose `--express`; our pinned `^2.1138.0` has it.
+ */
+export function buildSandboxDeployArgs({ outDir, projectRoot, backendPath }) {
+    return [
+        "exec", "cdk", "--", "deploy",
+        "--all",
+        "--method", "direct",
+        "--express",
+        "--require-approval", "never",
+        "--outputs-file", `${outDir}/outputs.json`,
+        "--context", `projectRoot=${projectRoot}`,
+        "--context", "sandboxMode=true",
+        "--app", `npm exec tsx -- -C cdk ${backendPath}`,
+    ];
+}
+/**
+ * Operator-facing recovery guidance printed when a sandbox deploy fails.
+ *
+ * Express Mode (see {@link buildSandboxDeployArgs}) disables CloudFormation's
+ * automatic rollback, so a failed deploy does NOT clean up after itself: a
+ * failed *first* deploy leaves the stack in `CREATE_FAILED`/`ROLLBACK_COMPLETE`
+ * and a failed update can land in `UPDATE_ROLLBACK_FAILED`. In those states
+ * CloudFormation refuses the next `UpdateStack`, so a plain re-run of
+ * `npm run sandbox` would hit the same wall — a silent dead-end for the fast
+ * loop. This spells out the exact way out instead of leaving the operator to
+ * discover it via a cryptic CloudFormation error.
+ *
+ * Kept pure so it can be asserted in a unit test.
+ */
+export function sandboxFailureRecoveryHint() {
+    return [
+        "",
+        "ℹ️  Express Mode leaves failed sandbox stacks in place (automatic rollback is off).",
+        "   If the next `npm run sandbox` reports the stack cannot be updated, recover with:",
+        "",
+        "     npm run sandbox:destroy   # tears the failed stack down (handles ROLLBACK_COMPLETE)",
+        "     npm run sandbox           # redeploy from a clean slate",
+        "",
+        "   If destroy reports UPDATE_ROLLBACK_FAILED, first run:",
+        "     aws cloudformation continue-update-rollback --stack-name <your-sandbox-stack>",
+    ].join("\n");
+}
 export async function startSandbox(options) {
     const { backendPath, outDir = ".blocks-sandbox", clientPort = 3000, deployOnly = false, devCommand } = options;
     const sandboxStartTime = Date.now();
@@ -39,6 +109,23 @@ export async function startSandbox(options) {
             throw e;
     }
     process.env.BLOCKS_STAGE = 'sandbox';
+    // Fail fast if AWS credentials are missing/expired, before spending ~10s in
+    // synth only to hit an opaque CDK credential error. Unlike deploy(), startSandbox
+    // isn't wrapped in trackCommand, so emit the FAIL event manually (mirroring the
+    // CDK-deploy failure path below) before rethrowing — otherwise a no-creds abort
+    // sends no telemetry.
+    try {
+        await assertAwsCredentials('sandbox');
+    }
+    catch (error) {
+        buildAndSendEvent({
+            command: 'sandbox',
+            state: 'FAIL',
+            duration: Date.now() - sandboxStartTime,
+            error: classifyError(error),
+        });
+        throw error;
+    }
     // Provision connection string to SSM SecureString.
     // On first deploy, creates the parameter. On subsequent deploys, updates if changed.
     // projectRoot is process.cwd() — the same value passed to cdk as --context
@@ -59,21 +146,10 @@ export async function startSandbox(options) {
     console.log("🚀 Deploying to AWS...");
     console.log("   (This may take a few minutes on first deploy)");
     try {
-        runSync("npm", [
-            "exec", "cdk", "--", "deploy",
-            // `--all`: an app that uses Lambda@Edge (e.g. a Next.js route with
-            // `export const runtime = 'edge'`) synthesizes a SECOND stack
-            // (`edge-lambda-stack-*`, region us-east-1) in addition to the main
-            // hosting stack. Without `--all`, CDK refuses with "specify which
-            // stacks to use". Deploying every stack in a sandbox app is the
-            // intended behavior, so select them all.
-            "--all",
-            "--require-approval", "never",
-            "--outputs-file", `${outDir}/outputs.json`,
-            "--context", `projectRoot=${process.cwd()}`,
-            "--context", "sandboxMode=true",
-            "--app", `npm exec tsx -- -C cdk ${backendPath}`,
-        ], {
+        runSync("npm", 
+        // Argv (including sandbox-only CloudFormation Express Mode, `--express`)
+        // is built by buildSandboxDeployArgs — see its doc for why each flag exists.
+        buildSandboxDeployArgs({ outDir, projectRoot: process.cwd(), backendPath }), {
             stdio: "inherit",
             env: { ...process.env, NODE_OPTIONS: "--conditions=cdk", ...getCdkTelemetryEnv('sandbox') },
         });
@@ -86,6 +162,7 @@ export async function startSandbox(options) {
             error: { code: 'CDK_DEPLOY_FAILED', phase: 'deploy' },
         });
         console.error("\n❌ Deployment failed.");
+        console.error(sandboxFailureRecoveryHint());
         throw error;
     }
     const outputs = JSON.parse(readFileSync(`${outDir}/outputs.json`, "utf-8"));
@@ -202,6 +279,160 @@ export async function startSandbox(options) {
     process.on("SIGTERM", cleanup);
     await new Promise(() => { });
 }
+/**
+ * Flatten a ListObjectVersions response into the `{ Key, VersionId }[]` shape
+ * DeleteObjects expects, combining live versions AND delete markers. Exported
+ * for unit testing — getting this combination wrong (e.g. missing the delete
+ * markers) leaves a "versioned" bucket that still can't be deleted.
+ */
+export function toDeleteObjects(listed) {
+    return [...(listed.Versions ?? []), ...(listed.DeleteMarkers ?? [])]
+        .filter((v) => v.Key !== undefined)
+        .map((v) => ({ Key: v.Key, VersionId: v.VersionId }));
+}
+/** Collect a stack's S3 bucket physical names (paginated — a large stack has
+ * >100 resources, so a single page can miss the hosting bucket that blocks the
+ * delete). Returns [] if the stack is already gone / not accessible. */
+export async function listStackBucketNames(cfn, stackName) {
+    const { ListStackResourcesCommand } = await import('@aws-sdk/client-cloudformation');
+    const buckets = [];
+    let token;
+    do {
+        const res = await cfn.send(new ListStackResourcesCommand({ StackName: stackName, NextToken: token }));
+        for (const r of res.StackResourceSummaries ?? []) {
+            if (r.ResourceType === 'AWS::S3::Bucket' && r.PhysicalResourceId)
+                buckets.push(r.PhysicalResourceId);
+        }
+        token = res.NextToken;
+    } while (token);
+    return buckets;
+}
+/** Empty a versioned bucket: delete every object version + delete marker in
+ * pages of 1000 (the DeleteObjects limit). Stops if a page reports per-key
+ * errors (object-lock / retention) so it can't loop forever. */
+export async function emptyBucket(s3, bucket) {
+    const { ListObjectVersionsCommand, DeleteObjectsCommand } = await import('@aws-sdk/client-s3');
+    while (true) {
+        const listed = await s3.send(new ListObjectVersionsCommand({ Bucket: bucket, MaxKeys: 1000 }));
+        const objects = toDeleteObjects(listed);
+        if (objects.length === 0)
+            return;
+        const res = await s3.send(new DeleteObjectsCommand({ Bucket: bucket, Delete: { Objects: objects, Quiet: true } }));
+        if (res.Errors && res.Errors.length > 0) {
+            // A bucket we can't fully empty (object-lock / retention / AccessDenied)
+            // will keep blocking `cdk destroy`, so the stack won't tear down until it's
+            // resolved by hand. Log loudly with the first error code — this is the
+            // kind of silent skip that leaks a stack, so make it visible in CI output.
+            const first = res.Errors[0];
+            console.warn(`  ⚠️  ${bucket}: ${res.Errors.length} object(s) could NOT be deleted ` +
+                `(e.g. ${first.Key}: ${first.Code}). This bucket will block stack teardown ` +
+                `until cleared manually.`);
+            return;
+        }
+    }
+}
+/**
+ * Empty every versioned S3 bucket owned by the given stacks. `cdk destroy`
+ * relies on each bucket's `autoDeleteObjects` custom-resource Lambda to empty
+ * it on teardown — but that Lambda never provisions if the CREATE failed (e.g.
+ * a partial/aborted deploy), so the versioned bucket blocks the delete and the
+ * stack (and its IAM roles) leak. Emptying out-of-band before the retry lets
+ * the delete complete. Best-effort: any per-stack/per-bucket error is logged
+ * and skipped so teardown still proceeds.
+ */
+async function emptySandboxBuckets(stackNames) {
+    try {
+        const { CloudFormationClient } = await import('@aws-sdk/client-cloudformation');
+        const { S3Client } = await import('@aws-sdk/client-s3');
+        const cfn = new CloudFormationClient({});
+        // `followRegionRedirects` makes the client transparently retry against a
+        // bucket's real region on a PermanentRedirect, so a bucket that lives
+        // outside AWS_REGION (e.g. a us-east-1 Lambda@Edge stack's bucket) is still
+        // emptied instead of failing silently — closing the exact no-op this fix
+        // targets. It needs no GetBucketLocation permission.
+        const s3 = new S3Client({ followRegionRedirects: true });
+        for (const stackName of stackNames) {
+            let buckets = [];
+            try {
+                buckets = await listStackBucketNames(cfn, stackName);
+            }
+            catch {
+                continue; // stack already gone / not accessible
+            }
+            for (const b of buckets) {
+                try {
+                    await emptyBucket(s3, b);
+                }
+                catch (e) {
+                    console.warn(`  ⚠️  could not empty ${b}: ${e.message}`);
+                }
+            }
+        }
+    }
+    catch (e) {
+        console.warn(`  ⚠️  bucket-emptying step skipped: ${e.message}`);
+    }
+}
+/** Resolve the sandbox app's stack names via `cdk ls` (best-effort; [] on error). */
+function listSandboxStackNames(backendPath, cdkEnv) {
+    try {
+        const out = execFileSync('npm', 
+        // Single-quote backendPath: CDK re-runs the --app string through a shell,
+        // so an unquoted path containing a space would split.
+        ['exec', 'cdk', '--', 'ls', '--context', 'sandboxMode=true', '--app', `npm exec tsx -- -C cdk '${backendPath}'`], 
+        // Capture stderr so a genuine synth/`cdk ls` failure can be logged below
+        // rather than being indistinguishable from "app has zero stacks".
+        { env: cdkEnv, encoding: 'utf-8', stdio: ['ignore', 'pipe', 'pipe'] });
+        return out
+            .split('\n')
+            .map((s) => s.trim())
+            .filter(Boolean);
+    }
+    catch (e) {
+        // Warn (don't stay silent): if this returns [] on a real failure, the retry
+        // skips bucket-emptying — the exact silent-no-op this change exists to avoid.
+        const detail = e.stderr || e.message;
+        console.warn(`  ⚠️  could not list sandbox stacks via 'cdk ls'; skipping bucket-emptying: ${detail}`);
+        return [];
+    }
+}
+/**
+ * Retry `cdk destroy`, clearing the two known teardown blockers before each
+ * retry: (1) non-empty versioned S3 buckets — `cdk destroy` relies on each
+ * bucket's autoDeleteObjects Lambda, which never provisioned if the CREATE
+ * failed, so the bucket blocks the delete and the stack (+ its IAM roles) leaks;
+ * we empty them out-of-band — and (2) VPC ENIs that take 60-120s to detach (the
+ * backoff). Stack names are resolved once and reused (topology is stable across
+ * attempts). Factored out with injectable deps so this behavior is unit-testable.
+ */
+export async function runDestroyWithRetries(deps) {
+    const retryDelays = deps.retryDelays ?? [60_000, 120_000];
+    let stackNames;
+    for (let attempt = 0;; attempt++) {
+        try {
+            deps.runDestroy();
+            console.log(attempt === 0 ? '\n✅ Sandbox destroyed!' : '\n✅ Sandbox destroyed on retry!');
+            return;
+        }
+        catch (error) {
+            if (attempt < retryDelays.length) {
+                if (stackNames === undefined)
+                    stackNames = deps.listStackNames();
+                if (stackNames.length > 0) {
+                    console.log('\n🧹 Emptying versioned S3 buckets before retry...');
+                    await deps.emptyBuckets(stackNames);
+                }
+                const delaySec = retryDelays[attempt] / 1000;
+                console.log(`\n⏳ Stack deletion failed. Retrying in ${delaySec}s (waiting for resource cleanup)...`);
+                await deps.sleep(retryDelays[attempt]);
+            }
+            else {
+                console.error('\n❌ Destroy failed after retries.');
+                throw error;
+            }
+        }
+    }
+}
 export async function destroySandbox(backendPath) {
     return trackCommand('sandbox:destroy', async () => {
         console.log("🗑️  Destroying sandbox...");
@@ -217,32 +448,16 @@ export async function destroySandbox(backendPath) {
             "exec", "cdk", "--", "destroy",
             "--force",
             "--context", "sandboxMode=true",
-            "--app", `npm exec tsx -- -C cdk ${backendPath}`,
+            // Single-quote backendPath: CDK re-runs the --app string through a shell,
+            // so an unquoted path containing a space would split.
+            "--app", `npm exec tsx -- -C cdk '${backendPath}'`,
         ];
         const cdkEnv = { ...process.env, NODE_OPTIONS: "--conditions=cdk", ...getCdkTelemetryEnv('sandbox') };
-        // Retry with backoff for VPC-dependent resources (e.g. Aurora clusters).
-        // CloudFormation deletes the cluster first, but its ENIs take 60-120s to
-        // detach from the VPC subnets asynchronously. The initial destroy fails
-        // because the subnets still have attached ENIs; retrying after the cleanup
-        // window lets the VPC delete succeed.
-        const retryDelays = [60_000, 120_000]; // 1min, then 2min
-        for (let attempt = 0;; attempt++) {
-            try {
-                runSync("npm", cdkArgs, { stdio: "inherit", env: cdkEnv });
-                console.log(attempt === 0 ? "\n✅ Sandbox destroyed!" : "\n✅ Sandbox destroyed on retry!");
-                return;
-            }
-            catch (error) {
-                if (attempt < retryDelays.length) {
-                    const delaySec = retryDelays[attempt] / 1000;
-                    console.log(`\n⏳ Stack deletion failed. Retrying in ${delaySec}s (waiting for resource cleanup)...`);
-                    await new Promise(r => setTimeout(r, retryDelays[attempt]));
-                }
-                else {
-                    console.error("\n❌ Destroy failed after retries.");
-                    throw error;
-                }
-            }
-        }
+        await runDestroyWithRetries({
+            runDestroy: () => runSync('npm', cdkArgs, { stdio: 'inherit', env: cdkEnv }),
+            listStackNames: () => listSandboxStackNames(backendPath, cdkEnv),
+            emptyBuckets: emptySandboxBuckets,
+            sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
+        });
     });
 }

package/dist/scripts/secret.d.ts

@@ -0,0 +1,15 @@
+/** Set (create or overwrite) a Blocks secret. */
+export declare function setSecret(key: string, value: string, opts?: {
+    stage?: string;
+}): Promise<void>;
+/** List Blocks secret keys (names only). */
+export declare function listSecrets(opts?: {
+    stage?: string;
+}): Promise<string[]>;
+/** Remove a Blocks secret. */
+export declare function removeSecret(key: string, opts?: {
+    stage?: string;
+}): Promise<boolean>;
+/** CLI dispatcher for `blocks secret <set|list|remove> …`. */
+export declare function runSecretCli(argv: string[]): Promise<void>;
+//# sourceMappingURL=secret.d.ts.map

package/dist/scripts/secret.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"secret.d.ts","sourceRoot":"","sources":["../../src/scripts/secret.ts"],"names":[],"mappings":"AAgBA,iDAAiD;AACjD,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAElG;AAED,4CAA4C;AAC5C,wBAAgB,WAAW,CAAC,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAE5E;AAED,8BAA8B;AAC9B,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAEzF;AAED,8DAA8D;AAC9D,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAE1D"}