@fastagent-sh/fastagent 0.21.1 → 0.22.0

package/dist/deploy/agentcore/run.js

@@ -1,41 +1,31 @@
-/**
- * `fastagent deploy agentcore --run` — drive the AWS CLI + Docker to completion. The middle of the
- * deploy the plain runbook hands to the operator; `--run` executes it so a coding agent runs ONE
- * command. Idempotent (ECR check-then-act; `cloudformation deploy` converges the stack) and
- * resumable: it STOPS at a human gate with one actionable line and a non-zero exit.
- *
- * TWO runners, one seam ({@link CliRunner}): `aws` (identity, ECR, CloudFormation) and `docker`
- * (buildx). AgentCore is the ONE host whose image builds on the operator's machine — the platform
- * requires linux/arm64 in the account's ECR and has no remote builder — so a missing Docker/buildx
- * is a first-class gate, not an incidental failure.
- *
- * Secrets ride CloudFormation NoEcho parameters. `--parameter-overrides` on argv would put the
- * values in the process listing (the same reason Fly imports secrets over stdin), so they go through
- * a caller-provided temp parameters file (`file://…`, mode 0600, deleted by the caller) — the write
- * is injected to keep this module pure and the security-sensitive wiring testable.
- */
+/** `fastagent deploy agentcore --run` — drive the AWS CLI + Docker to completion. */
 import { RESERVED_PATHS } from "../../channels/agentcore-protocol.js";
 import { registerWebhooks } from "../channel-ingress.js";
+import { awsCli, awsJson } from "./aws-cli.js";
 import { createHash } from "node:crypto";
 import { Buffer } from "node:buffer";
-import { AUTH_SEED_CHUNK_SIZE, AUTH_SEED_MAX_CHUNKS, cfnParamName, forwarderSource, ingressSessionId, stateBucketName, } from "./plan.js";
+import { CARRIERS, CARRIER_CHUNK_SIZE, CARRIER_MAX_CHUNKS, MOUNT, agentcoreRepoName, agentcoreStackName, carrierChunk, deploymentBucketName, forwarderSource, ingressSessionId, } from "./plan.js";
 import { zipSingleFile } from "./zip.js";
-/** How long the post-deploy probe waits for the fresh session (image pull + microVM boot + snapshot
- *  restore + channel construction) before gating with the last answer. */
-const PROBE_TIMEOUT_MS = 120_000;
-const PROBE_INTERVAL_MS = 3_000;
+import { encodeCarriedEnv, missingValuesGate } from "../secrets.js";
 /**
- * Drive the forwarder's reserved probe path until it answers, and read the runtime's
- * STRUCTURED verdict. The path answers on every forwarder topology (a schedule-only URL refuses
- * ordinary public traffic, so a plain `GET /health` would 404 there), and the verdict rides a
- * transport-200 JSON body `{ ok, error? }` — the ordinary webhook relay folds a non-200 transport
- * into an opaque 502, which would strip the very diagnostics this probe exists to carry.
- *
- * Outcome policy: `ok:true` verifies the deploy; `ok:false` gates IMMEDIATELY with the runtime's own
- * error text (construction rejections are cached per session, so polling cannot change the answer);
- * anything else (unroutable URL, forwarder 4xx/5xx, malformed body) is retried to the deadline —
- * that budget's job is absorbing cold-start provisioning — and then gates with the last answer seen.
+ * Stack statuses that hold NO agent memory: a create that never succeeded, its rollback, a change set never
+ * executed, or a stack already gone. Everything else — including `UPDATE_ROLLBACK_COMPLETE` and any status AWS adds
+ * later — is treated as carrying state, so an unrecognized status over-warns instead of quietly promising nothing is
+ * lost.
  */
+const EMPTY_STACK_STATUSES = new Set([
+    "CREATE_FAILED",
+    "CREATE_IN_PROGRESS",
+    "ROLLBACK_IN_PROGRESS",
+    "ROLLBACK_FAILED",
+    "ROLLBACK_COMPLETE",
+    "REVIEW_IN_PROGRESS",
+    "DELETE_COMPLETE",
+]);
+/** Budget for image pull, storage initialization and channel construction. */
+const PROBE_TIMEOUT_MS = 240_000;
+const PROBE_INTERVAL_MS = 3_000;
+/** Drive the forwarder's reserved probe path until it answers, and read the runtime's STRUCTURED verdict. */
 async function probeRuntime(probeUrl, auth, fetchImpl, timeoutMs = PROBE_TIMEOUT_MS, intervalMs = PROBE_INTERVAL_MS) {
     const deadline = Date.now() + timeoutMs;
     let last;
@@ -54,7 +44,7 @@ async function probeRuntime(probeUrl, auth, fetchImpl, timeoutMs = PROBE_TIMEOUT
                     verdict = JSON.parse(bodyText);
                 }
                 catch {
-                    /* malformed — fall through to retry with it as the last answer */
+                    // malformed — fall through to retry with it as the last answer
                 }
                 if (verdict?.ok === true)
                     return { ok: true };
@@ -67,7 +57,7 @@ async function probeRuntime(probeUrl, auth, fetchImpl, timeoutMs = PROBE_TIMEOUT
             last = `${res.status}${firstLine ? ` ${firstLine}` : ""}`;
         }
         catch {
-            /* not routable yet (Function URL DNS, cold start) — keep polling until the deadline */
+            // not routable yet (Function URL DNS, cold start) — keep polling until the deadline
         }
         if (Date.now() >= deadline) {
             return {
@@ -81,90 +71,79 @@ async function probeRuntime(probeUrl, auth, fetchImpl, timeoutMs = PROBE_TIMEOUT
     }
 }
 /** Stack outputs (`describe-stacks --query "Stacks[0].Outputs"`) → { OutputKey: OutputValue }. */
-export function parseStackOutputs(stdout) {
-    try {
-        const arr = JSON.parse(stdout);
-        if (!Array.isArray(arr))
-            return {};
-        const out = {};
-        for (const o of arr) {
-            if (typeof o?.OutputKey === "string" && typeof o?.OutputValue === "string")
-                out[o.OutputKey] = o.OutputValue;
-        }
-        return out;
-    }
-    catch {
+/**
+ * `Stacks[0].Outputs` as a map, or `undefined` when the document is not that list at all.
+ *
+ * `null` IS AN ANSWER, and an empty map is what it says: that is what JMESPath prints for a stack with no
+ * Outputs section, i.e. a template edited or generated without one. Reading it as unreadable blamed
+ * `aws cloudformation describe-stacks` — a command that had just succeeded — instead of letting the caller's
+ * own "no RuntimeArn, regenerate with --force" gate say what to do. Distinct from empty stdout, which is the
+ * CLI printing nothing at all (aws-cli.ts, point 4).
+ */
+export function pickStackOutputs(parsed) {
+    if (parsed === null)
         return {};
+    if (!Array.isArray(parsed))
+        return undefined;
+    const out = {};
+    for (const o of parsed) {
+        if (typeof o?.OutputKey === "string" && typeof o?.OutputValue === "string")
+            out[o.OutputKey] = o.OutputValue;
     }
+    return out;
+}
+/** The secrets `--run` mints for this host, each with the template parameter of its own. */
+const MINTED_PARAMS = {
+    FASTAGENT_INGRESS_SECRET: "FastagentIngressSecret",
+    FASTAGENT_WAKE_SECRET: "FastagentWakeSecret",
+};
+/** What each carrier holds for this deploy: the credential seed, and every other variable as one encoded object. */
+function carrierValues(secrets) {
+    const { FASTAGENT_AUTH_SEED: seed = "", ...rest } = secrets;
+    const carried = Object.fromEntries(Object.entries(rest).filter(([name]) => !(name in MINTED_PARAMS)));
+    return { FASTAGENT_AUTH_SEED: seed, FASTAGENT_ENV: encodeCarriedEnv(carried) };
 }
-/** The `--parameter-overrides file://` payload: a JSON array of "Key=Value" strings. The auth seed
- *  is CHUNKED across FastagentAuthSeed(2…) — AgentCore env values cap at 2048 chars and a real OAuth
- *  auth.json's base64 exceeds it; `start` reassembles (collectAuthSeed). Every chunk is emitted on
- *  every deploy, including empty trailing chunks, so CloudFormation cannot retain stale values. */
+/** The `--parameter-overrides file://` payload: a JSON array of "Key=Value" strings. */
 export function paramsFileContent(imageUri, secrets, forwarder) {
-    const params = [`ImageUri=${imageUri}`];
-    if (forwarder)
-        params.push(`StateBucket=${forwarder.bucket}`, `ForwarderS3Key=${forwarder.key}`);
-    for (const [k, v] of Object.entries(secrets)) {
-        if (k !== "FASTAGENT_AUTH_SEED")
-            params.push(`${cfnParamName(k)}=${v}`);
+    const params = [`ImageUri=${imageUri}`, `ForwarderBucket=${forwarder.bucket}`, `ForwarderS3Key=${forwarder.key}`];
+    for (const [name, param] of Object.entries(MINTED_PARAMS)) {
+        if (secrets[name] !== undefined)
+            params.push(`${param}=${secrets[name]}`);
     }
-    const seed = secrets.FASTAGENT_AUTH_SEED ?? "";
-    for (let i = 0; i < AUTH_SEED_MAX_CHUNKS; i++) {
-        const param = i === 0 ? "FastagentAuthSeed" : `FastagentAuthSeed${i + 1}`;
-        params.push(`${param}=${seed.slice(i * AUTH_SEED_CHUNK_SIZE, (i + 1) * AUTH_SEED_CHUNK_SIZE)}`);
-    }
-    return `${JSON.stringify(params)}\n`;
-}
-/** Parse the runtime's checkpoint acknowledgement without treating malformed output as success. */
-export function parseCheckpointReply(stdout) {
-    try {
-        const parsed = JSON.parse(stdout.trim());
-        if (parsed === null ||
-            typeof parsed !== "object" ||
-            !("written" in parsed) ||
-            typeof parsed.written !== "boolean") {
-            return undefined;
+    // Every chunk is written, empty ones included: an omitted parameter keeps its PREVIOUS value on a stack update.
+    const values = carrierValues(secrets);
+    for (const carrier of CARRIERS) {
+        const value = values[carrier.env];
+        for (let i = 0; i < CARRIER_MAX_CHUNKS; i++) {
+            params.push(`${carrierChunk(carrier, i).param}=${value.slice(i * CARRIER_CHUNK_SIZE, (i + 1) * CARRIER_CHUNK_SIZE)}`);
         }
-        return {
-            written: parsed.written,
-            reason: "reason" in parsed && typeof parsed.reason === "string" ? parsed.reason : undefined,
-        };
-    }
-    catch {
-        return undefined;
     }
+    return `${JSON.stringify(params)}\n`;
 }
-/**
- * Run the deploy through `aws` + `docker`. `log` reports progress; the injected registrars perform
- * post-deploy webhook steps from the builder machine against the forwarder's Function URL. Every
- * gate is fail-visible; `writeSecretFile` is the caller's 0600-temp-file seam (see the header).
- */
+/** Run the deploy through `aws` + `docker`. */
 export async function deployAgentcoreRun(plan, aws, docker, log, writeSecretFile, writeForwarderZip, registrars, 
 /** Injected in tests; the probe itself stays inside the run so no deploy can skip it. */
 probe = {}) {
     const gate = (g) => ({ ok: false, gate: g });
-    const stack = `fastagent-${plan.name}`;
-    const repo = `fastagent/${plan.name}`;
-    // 1. Identity + region — the two facts everything downstream (registry hostname, stack region)
-    //    hangs on. `sts get-caller-identity` succeeds with any working credential source.
-    const identity = await aws(["sts", "get-caller-identity", "--output", "json"], { capture: true });
-    if (identity.code === 127) {
-        return gate("aws CLI not found — install AWS CLI v2: https://docs.aws.amazon.com/cli/, then re-run");
-    }
-    if (identity.code !== 0) {
-        return gate("no working AWS credentials — run `aws configure` (or set AWS_ACCESS_KEY_ID/…), then re-run");
-    }
-    let account;
-    try {
-        const parsed = JSON.parse(identity.stdout);
-        if (typeof parsed.Account !== "string")
-            throw new Error("no Account");
-        account = parsed.Account;
-    }
-    catch {
-        return gate("could not read the account id from `aws sts get-caller-identity` — see the output above");
+    const stack = agentcoreStackName(plan.name);
+    const repo = agentcoreRepoName(plan.name);
+    // 1.
+    const cli = awsCli(aws);
+    const identity = await cli.read(["sts", "get-caller-identity", "--output", "json"], awsJson((parsed) => {
+        const { Account, Arn } = (parsed ?? {});
+        return typeof Account === "string"
+            ? { account: Account, principal: typeof Arn === "string" ? Arn : undefined }
+            : undefined;
+    }));
+    if (!("ok" in identity)) {
+        if ("absent" in identity)
+            return gate("`aws sts get-caller-identity` found no caller — run `aws configure`");
+        if (identity.code === 127)
+            return gate(`${identity.unreadable}, then re-run`);
+        // QUOTED: expired token, a missing profile and a proxy that eats STS are three different next actions.
+        return gate(`no working AWS credentials (${identity.unreadable}) — run \`aws configure\`, then re-run`);
     }
+    const { account, principal } = identity.ok;
     let region = plan.region;
     if (!region) {
         const fromConfig = await aws(["configure", "get", "region"], { capture: true });
@@ -173,10 +152,7 @@ probe = {}) {
     if (!region) {
         return gate("no AWS region configured — set AWS_REGION (or `aws configure set region <region>`), then re-run");
     }
-    // 2. Docker + buildx — this host builds LOCALLY (linux/arm64 into the account's ECR; AgentCore has
-    //    no remote builder), so their absence is a first-class gate with the install pointer. ANY
-    //    non-zero gates BEFORE side effects: `docker version` with the daemon down exits non-127, and
-    //    letting it through would create the ECR repo and then fail the build with a generic error.
+    // 2.
     const dockerVersion = await docker(["version"], { capture: true });
     if (dockerVersion.code === 127) {
         return gate("docker not found — install Docker (https://docs.docker.com/get-docker/), then re-run");
@@ -188,29 +164,91 @@ probe = {}) {
         return gate("docker buildx not available — the image must be linux/arm64 (cross-built); install buildx, then re-run");
     }
     // 3. Gate missing required secret VALUES before any side effect (no half-created infra).
-    if (plan.missingSecrets.length > 0) {
-        return gate(`no local value for: ${plan.missingSecrets.join(", ")} — set them in .env (or the environment) and re-run`);
-    }
-    // 3b. AgentCore env values cap at 2048 chars. The auth seed is chunked (paramsFileContent) up to
-    //     its ceiling; any OTHER oversized value has no chunk lane — gate it instead of a cryptic
-    //     CloudFormation "maxLength" failure mid-deploy.
-    const seed = plan.secrets.FASTAGENT_AUTH_SEED;
-    if (seed && seed.length > AUTH_SEED_CHUNK_SIZE * AUTH_SEED_MAX_CHUNKS) {
-        return gate(`your auth.json is too large to carry (${seed.length} chars base64 > ${AUTH_SEED_CHUNK_SIZE * AUTH_SEED_MAX_CHUNKS}) — ` +
+    const missingValues = missingValuesGate(plan.missingSecrets, plan.valueFile);
+    if (missingValues)
+        return gate(missingValues);
+    // 3b.
+    const capacity = CARRIER_CHUNK_SIZE * CARRIER_MAX_CHUNKS;
+    const carried = carrierValues(plan.secrets);
+    if (carried.FASTAGENT_AUTH_SEED.length > capacity) {
+        return gate(`your auth.json is too large to carry (${carried.FASTAGENT_AUTH_SEED.length} chars base64 > ${capacity}) — ` +
             `slim it (keep only the model's credential), or set a provider API key in .env instead`);
     }
-    for (const [k, v] of Object.entries(plan.secrets)) {
-        if (k !== "FASTAGENT_AUTH_SEED" && v.length > 2048) {
-            return gate(`secret ${k} is ${v.length} chars — AgentCore environment values cap at 2048; shorten it`);
-        }
+    if (carried.FASTAGENT_ENV.length > capacity) {
+        return gate(`the variables in ${plan.valueFile} are too large to carry (${carried.FASTAGENT_ENV.length} chars encoded > ` +
+            `${capacity}) — move what the deployment does not need out of that file`);
     }
-    // 4. ECR repository — check-then-act. A FAILED describe that isn't "not found" would misreport the
-    //    create, but ECR's not-found also exits non-zero — so try describe, and on failure attempt the
-    //    create; a create failing for a REAL reason (permissions) still gates with its own message.
+    // Name what travels from the value file onto the runtime.
+    const secretNames = Object.keys(plan.secrets);
+    if (secretNames.length > 0)
+        log(`carrying ${secretNames.length} secret(s): ${secretNames.join(", ")}`);
+    // 3c. Say what this deploy is ABOUT TO TOUCH while it can still be stopped for free: until here the account and
+    // region only surfaced several hundred log lines in, inside the ECR image URI.
+    const source = plan.region ? "the environment" : "aws configure";
+    log(`account ${account}${principal ? ` (${principal})` : ""}, region ${region} (from ${source}), image ${repo}:${plan.tag}`);
+    if (principal?.endsWith(":root")) {
+        log(`warn: deploying as the account root user — this stack's IAM roles are created under it; prefer an IAM role`);
+    }
+    // Warn, never gate: an aws CLI too old to know the service, or a role without ListAgentRuntimes, would make a gate
+    // refuse a valid deploy. The point is to say it BEFORE the multi-minute arm64 build, not to be authoritative.
+    // `--region` explicitly: every other region-dependent call here writes it out (the ECR registry URI, the bucket's
+    // LocationConstraint), and the warning below names ${region} — the probe must be about the SAME region it names.
+    const service = await cli.present([
+        "bedrock-agentcore-control",
+        "list-agent-runtimes",
+        "--max-items",
+        "1",
+        "--region",
+        region,
+    ]);
+    if (!("ok" in service)) {
+        const why = "unreadable" in service ? service.unreadable.split("\n")[0] : "";
+        log(`warn: could not confirm AgentCore is available in ${region}${why ? ` (${why})` : ""} — if it is not, ` +
+            `cloudformation deploy fails minutes from now, after the image build: ` +
+            `https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/agentcore-regions.html`);
+    }
+    // 3d. Read the stack here, for the two questions that both depend on it: does this deploy destroy the agent's
+    // memory (below), and is there a failed first create to clear (step 7)? The destructive one is only worth saying
+    // while the multi-minute build has not run yet, and step 7 reuses this answer unless it was still in flight.
+    // An unanswered question must not read as "a first deploy, nothing to lose": `sts get-caller-identity` succeeding
+    // says nothing about whether this role can read CloudFormation. That three-way answer is aws-cli.ts's.
+    const readStackStatus = () => cli.read([
+        "cloudformation",
+        "describe-stacks",
+        "--stack-name",
+        stack,
+        "--query",
+        "Stacks[0].StackStatus",
+        "--output",
+        "text",
+    ], (stdout) => stdout.trim() || undefined);
+    const stackStatus = await readStackStatus();
+    const status = "ok" in stackStatus ? stackStatus.ok : "";
+    // The only answers the build can invalidate: one still in flight (a first create rolling back is exactly what step 7
+    // exists for, and minutes of arm64 build are long enough for it to settle), and one we never got.
+    const settling = status.endsWith("_IN_PROGRESS") || "unreadable" in stackStatus;
+    // Warn, never gate — same as the region probe above: a role without this read, or an older CLI, must not refuse a
+    // legitimate deploy.
+    if ("unreadable" in stackStatus) {
+        log(`warn: could not read stack ${stack} (${stackStatus.unreadable}) — if it exists, this deploy resets its ` +
+            `managed SessionStorage (${MOUNT}) and a failed first create will not be cleared`);
+    }
+    else if (status !== "" && !EMPTY_STACK_STATUSES.has(status)) {
+        log(`warn: this is a REDEPLOY and AWS resets managed SessionStorage (${MOUNT}) on every runtime version update — ` +
+            `sessions, channel state and pending wake-ups start blank` +
+            // Only the carried auth.json is re-seeded; a provider API key deployment has no such step to blame.
+            `${carried.FASTAGENT_AUTH_SEED ? ", and the model credential is re-seeded from FASTAGENT_AUTH_SEED" : ""}. ` +
+            `Cross-deploy memory needs a real volume: \`deploy fly\` or \`deploy railway\`.`);
+    }
+    // 4.
     const registry = `${account}.dkr.ecr.${region}.amazonaws.com`;
     const image = `${registry}/${repo}:${plan.tag}`;
-    const described = await aws(["ecr", "describe-repositories", "--repository-names", repo], { capture: true });
-    if (described.code === 0) {
+    const described = await cli.present(["ecr", "describe-repositories", "--repository-names", repo]);
+    if ("unreadable" in described) {
+        // NOT the create branch: a denial here must name this step, not the create that would follow.
+        return gate(`could not read ECR repository ${repo} (${described.unreadable}) — fix that, then re-run`);
+    }
+    if ("ok" in described) {
         log(`ECR repository ${repo} exists — skipping create`);
     }
     else {
@@ -219,84 +257,43 @@ probe = {}) {
             return gate("`aws ecr create-repository` failed — see the output above; fix and re-run");
         }
     }
-    // 4b. The deployment bucket + the forwarder package. The bucket is created OUTSIDE the stack, on
-    //     purpose and unlike everything else here: it holds the agent's STATE SNAPSHOT, and AgentCore
-    //     wipes the /mnt/state mount on every runtime version update (i.e. every deploy). Keeping it
-    //     out of CloudFormation means a `delete-stack` — or a rolled-back create — cannot take the
-    //     agent's sessions, channel state and pending wake-ups with it.
-    let forwarderParams;
-    if (plan.topology.forwarder) {
-        const bucket = stateBucketName(plan.name, account);
-        if ((await aws(["s3api", "head-bucket", "--bucket", bucket], { capture: true })).code !== 0) {
-            log(`creating deployment bucket ${bucket}…`);
-            // us-east-1 is the ONE region that must not carry a LocationConstraint (the API rejects it).
-            const createArgs = ["s3api", "create-bucket", "--bucket", bucket];
-            if (region !== "us-east-1")
-                createArgs.push("--create-bucket-configuration", `LocationConstraint=${region}`);
-            if ((await aws(createArgs)).code !== 0) {
-                return gate(`\`aws s3api create-bucket --bucket ${bucket}\` failed — see the output above; fix and re-run`);
-            }
-        }
-        // CONVERGE the properties on EVERY deploy, not just at creation. They are what makes the bucket
-        // safe (nothing public) and recoverable (a bad write is not the end of the agent's memory); doing
-        // them only in the create branch means a run that failed halfway leaves a bucket that looks
-        // finished forever after, and ignoring the exit codes means "deployed" would be reported over a
-        // world-readable or unversioned store of the agent's credentials.
-        const converge = [
-            {
-                label: "block public access",
-                args: [
-                    "s3api",
-                    "put-public-access-block",
-                    "--bucket",
-                    bucket,
-                    "--public-access-block-configuration",
-                    "BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true",
-                ],
-            },
-            {
-                label: "enable versioning",
-                args: ["s3api", "put-bucket-versioning", "--bucket", bucket, "--versioning-configuration", "Status=Enabled"],
-            },
-            {
-                label: "set the snapshot lifecycle",
-                args: [
-                    "s3api",
-                    "put-bucket-lifecycle-configuration",
-                    "--bucket",
-                    bucket,
-                    "--lifecycle-configuration",
-                    JSON.stringify({
-                        Rules: [
-                            {
-                                ID: "fastagent-expire-old-snapshots",
-                                Status: "Enabled",
-                                Filter: { Prefix: "state/" },
-                                NoncurrentVersionExpiration: { NoncurrentDays: 7 },
-                            },
-                        ],
-                    }),
-                ],
-            },
-        ];
-        for (const step of converge) {
-            // `capture` keeps the CLI's JSON off the deploy log: the exit code is the signal, and a failure
-            // gates with the step's name below.
-            if ((await aws(step.args, { capture: true })).code !== 0) {
-                return gate(`could not ${step.label} on ${bucket} — refusing to store agent state in it; fix and re-run`);
-            }
-        }
-        // Content-hashed key: CloudFormation rolls the function only when a parameter VALUE changes, so
-        // identical source must map to an identical key (hence the deterministic zip) and changed source
-        // to a new one.
-        const zip = zipSingleFile("index.js", Buffer.from(forwarderSource()));
-        const key = `forwarder/${createHash("sha256").update(zip).digest("hex").slice(0, 16)}.zip`;
-        const zipPath = await writeForwarderZip(zip);
-        if ((await aws(["s3", "cp", zipPath, `s3://${bucket}/${key}`])).code !== 0) {
-            return gate("uploading the forwarder package to S3 failed — see the output above; fix and re-run");
+    // The forwarder package must exist before CloudFormation can create its Lambda.
+    const bucket = deploymentBucketName(plan.name, account);
+    const head = await cli.present(["s3api", "head-bucket", "--bucket", bucket]);
+    // Same shape as the repository above: a 403 is not a 404, and creating on top of it fails with a message
+    // about the wrong thing.
+    if ("unreadable" in head) {
+        return gate(`could not read deployment bucket ${bucket} (${head.unreadable}) — fix that, then re-run`);
+    }
+    if ("absent" in head) {
+        log(`creating deployment bucket ${bucket}…`);
+        // us-east-1 is the ONE region that must not carry a LocationConstraint (the API rejects it).
+        const createArgs = ["s3api", "create-bucket", "--bucket", bucket];
+        if (region !== "us-east-1")
+            createArgs.push("--create-bucket-configuration", `LocationConstraint=${region}`);
+        if ((await aws(createArgs)).code !== 0) {
+            return gate(`\`aws s3api create-bucket --bucket ${bucket}\` failed — see the output above; fix and re-run`);
         }
-        forwarderParams = { bucket, key };
     }
+    if ((await aws([
+        "s3api",
+        "put-public-access-block",
+        "--bucket",
+        bucket,
+        "--public-access-block-configuration",
+        "BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true",
+    ], { capture: true })).code !== 0) {
+        return gate(`could not block public access on ${bucket}; fix and re-run`);
+    }
+    // Content-hashed key: CloudFormation rolls the function only when a parameter VALUE changes, so identical source
+    // must map to an identical key (hence the deterministic zip) and changed source to a new one.
+    const zip = zipSingleFile("index.js", Buffer.from(forwarderSource()));
+    const key = `forwarder/${createHash("sha256").update(zip).digest("hex").slice(0, 16)}.zip`;
+    const zipPath = await writeForwarderZip(zip);
+    if ((await aws(["s3", "cp", zipPath, `s3://${bucket}/${key}`])).code !== 0) {
+        return gate("uploading the forwarder package to S3 failed — see the output above; fix and re-run");
+    }
+    const forwarderParams = { bucket, key };
     // 5. Registry login — the password flows stdout→stdin between the two runners, never argv.
     const password = await aws(["ecr", "get-login-password"], { capture: true });
     if (password.code !== 0)
@@ -307,29 +304,14 @@ probe = {}) {
     // 6. Build (linux/arm64) + push in one step.
     log(`building + pushing ${image} (linux/arm64)…`);
     const buildArgs = ["buildx", "build", "--platform", "linux/arm64", "-t", image, "--push"];
-    if (plan.dockerfilePath)
-        buildArgs.push("-f", plan.dockerfilePath);
+    buildArgs.push("-f", plan.dockerfilePath);
     buildArgs.push(".");
     if ((await docker(buildArgs)).code !== 0) {
         return gate("`docker buildx build` failed — see the output above; fix and re-run");
     }
-    // 7. Deploy the stack. Secret values ride the temp params file (file://), never argv.
-    //    --no-fail-on-empty-changeset: a re-run whose only change already applied must not gate.
-    //    Self-heal the one un-resumable state first: a FAILED first create leaves the stack in
-    //    ROLLBACK_COMPLETE, which CloudFormation refuses to update — without this, "fix and re-run"
-    //    (our own gate advice) would dead-end on a different error. Nothing real is lost by deleting:
-    //    a ROLLBACK_COMPLETE stack holds no live resources.
-    const status = await aws([
-        "cloudformation",
-        "describe-stacks",
-        "--stack-name",
-        stack,
-        "--query",
-        "Stacks[0].StackStatus",
-        "--output",
-        "text",
-    ], { capture: true });
-    if (status.code === 0 && status.stdout.trim() === "ROLLBACK_COMPLETE") {
+    // 7. The 3d answer stands unless it was still settling then — a second read is the narrow exception, not the rule.
+    const beforeDeploy = settling ? await readStackStatus() : stackStatus;
+    if ("ok" in beforeDeploy && beforeDeploy.ok === "ROLLBACK_COMPLETE") {
         log(`stack ${stack} is ROLLBACK_COMPLETE (a failed first create) — deleting it before re-creating…`);
         if ((await aws(["cloudformation", "delete-stack", "--stack-name", stack])).code !== 0) {
             return gate("`aws cloudformation delete-stack` failed — see the output above");
@@ -358,66 +340,18 @@ probe = {}) {
             `(aws cloudformation describe-stack-events --stack-name ${stack}), fix, and re-run`);
     }
     // 8. Outputs — the runtime ARN (the data plane) and the forwarder URL (the webhook surface).
-    const outputsQuery = await aws(["cloudformation", "describe-stacks", "--stack-name", stack, "--query", "Stacks[0].Outputs", "--output", "json"], { capture: true });
-    if (outputsQuery.code !== 0)
-        return gate("`aws cloudformation describe-stacks` failed — see the output above");
-    const outputs = parseStackOutputs(outputsQuery.stdout);
+    const outputsRead = await cli.read(["cloudformation", "describe-stacks", "--stack-name", stack, "--query", "Stacks[0].Outputs", "--output", "json"], awsJson(pickStackOutputs));
+    if (!("ok" in outputsRead)) {
+        const why = "absent" in outputsRead ? `${stack} is not there` : outputsRead.unreadable;
+        return gate(`\`aws cloudformation describe-stacks\` failed after a successful deploy (${why})`);
+    }
+    const outputs = outputsRead.ok;
     const runtimeArn = outputs.RuntimeArn;
     if (!runtimeArn)
         return gate("stack has no RuntimeArn output — was the template edited? Regenerate with --force");
     const url = outputs.ForwarderUrl?.replace(/\/$/, ""); // registrars append /<path>; no double slash
-    // 8b. Restart the ingress session so the new image serves IMMEDIATELY. A live session keeps its
-    //     old compute until the idle timeout or the max compute lifetime (8 h) — without this, a
-    //     redeploy "succeeds" while an actively-chatting session keeps answering from the PREVIOUS
-    //     image (the exact silent trap the first real deploy hit). Failure is advisory, never a gate:
-    //     on a first deploy the session does not exist yet, and the stop is an immediacy optimization
-    //     — the platform's reclaim gets there eventually. An in-flight turn on the old compute is cut;
-    //     the checkpoint above is what lets a replaying channel re-run it. Only when a forwarder exists
-    //     (the ingress session is the forwarder's session; pure-invoke deployments have none).
-    // Keyed on the FORWARDER: every current forwarder has a callback URL for state-capability refresh,
-    // and every forwarder topology has an ingress session whose next event would otherwise land on compute still
-    // running the previous image.
-    if (plan.topology.forwarder) {
-        // CHECKPOINT FIRST. The stop cuts whatever turn is running, and that turn's durable intent was
-        // written to a mount the version update erases — so without this flush "replay re-runs it" would
-        // be false: the intent never reaches S3 and the message is simply gone. Best-effort: a session
-        // that is not up has nothing to lose, and a failure here must not block the (already applied)
-        // deploy — it only downgrades the promise, so say so.
-        const checkpointPayloadPath = await writeSecretFile(`${JSON.stringify({ kind: "checkpoint", auth: plan.secrets.FASTAGENT_INGRESS_SECRET })}\n`);
-        const checkpoint = await aws([
-            "bedrock-agentcore",
-            "invoke-agent-runtime",
-            "--agent-runtime-arn",
-            runtimeArn,
-            "--runtime-session-id",
-            ingressSessionId(plan.name),
-            "--payload",
-            `file://${checkpointPayloadPath}`,
-            "--cli-binary-format",
-            "raw-in-base64-out",
-            "/dev/stdout",
-        ], { capture: true, captureStderr: true });
-        // Report what the container ACTUALLY did. This line is the only signal an operator has about
-        // whether an in-flight turn survived the deploy, so a blanket "checkpointed" — printed even when
-        // nothing was written — would be worse than no line at all.
-        if (checkpoint.code !== 0) {
-            log("note: could not reach the ingress session to checkpoint — if a turn was in flight it is lost " +
-                "rather than replayed (see the output above)");
-        }
-        else {
-            const reply = parseCheckpointReply(checkpoint.stdout);
-            if (reply?.written) {
-                log("checkpointed the ingress session (an interrupted turn can be replayed)");
-            }
-            else if (reply) {
-                // The ordinary case: the session was already idle-reclaimed, so its snapshot was written when
-                // it settled and there is nothing in flight to lose.
-                log(`note: nothing to checkpoint${reply.reason ? ` — ${reply.reason}` : " (no session was running)"}`);
-            }
-            else {
-                log("warn: ingress session returned an invalid checkpoint response — could not verify the state snapshot");
-            }
-        }
+    // A live session keeps its previous image. Stop the fixed writer before verifying the new release.
+    {
         log("stopping the ingress session so the new image serves immediately…");
         const stopCommand = [
             "bedrock-agentcore",
@@ -427,21 +361,19 @@ probe = {}) {
             "--runtime-session-id",
             ingressSessionId(plan.name),
         ];
-        const stopped = await aws(stopCommand, { capture: true, captureStderr: true });
-        if (stopped.code !== 0) {
-            // Classify, don't guess: "no session yet" (first deploy — expected, quiet note) vs a REAL stop
-            // failure (permissions/CLI/network — the old image may keep serving, say so loudly with the
-            // manual command). Not a gate: the deploy itself succeeded, and stop is an immediacy
-            // optimization — the platform's reclaim converges regardless.
-            const stderr = stopped.stderr ?? "";
-            if (/ResourceNotFound|not\s*found|does not exist/i.test(stderr)) {
+        const stopped = await cli.write(stopCommand);
+        if (!("done" in stopped)) {
+            // Classify, don't guess: "no session yet" (first deploy — expected, quiet note) vs a REAL stop failure
+            // (permissions/CLI/network), which must stop verification against the previous image. Both words are
+            // aws-cli.ts's — this file had its own third spelling of "already gone".
+            const noSession = "absent" in stopped;
+            const stderr = noSession ? "" : stopped.refused;
+            if (noSession) {
                 log("note: no ingress session to stop (first deploy, or already reclaimed)");
             }
             else {
-                // A GATE, not a warning: the probe below reaches the SAME fixed session id, so a session
-                // still running the previous image would answer it and the deploy would claim to have
-                // verified a serving path it never touched. Unable to guarantee the session is fresh =
-                // unable to verify = stop.
+                // A GATE, not a warning: the probe below reaches the SAME fixed session id, so a session still running the
+                // previous image would answer it and the deploy would claim to have verified a serving path it never touched.
                 const firstLine = stderr.trim().split("\n")[0];
                 return gate(`could not stop the ingress session — it may still be serving the PREVIOUS image, so the ` +
                     `deploy cannot verify the new one${firstLine ? ` (${firstLine})` : ""}. ` +
@@ -449,31 +381,20 @@ probe = {}) {
             }
         }
     }
-    // 8c. Every forwarder topology MUST carry the ForwarderUrl output — schedule-only and
-    //     selfSchedule-only deployments included, since the probe below is their only construction
-    //     check (there is no boot-time failStartup on this host). A missing output means an edited
-    //     template; skipping the probe silently would let such a deploy report success unverified.
-    //     Only a pure-invoke deployment (no forwarder) legitimately has no URL and nothing to probe.
-    if (plan.topology.forwarder && !url) {
+    // 8c.
+    if (!url) {
         return gate("this deployment needs the forwarder but the stack has no ForwarderUrl output — regenerate the " +
             "template with --force");
     }
-    // 8d. Warm + verify the NEW serving path end to end, BEFORE registration: the probe wakes a fresh
-    //     session on the new image through the forwarder's reserved path, which restores the state
-    //     snapshot and constructs the channels — construction is deferred to exactly that moment
-    //     (channels/agentcore.ts), so this is where a bad credential, a broken channels/ module, or an
-    //     unrestorable snapshot surfaces AT DEPLOY TIME with the runtime's own error text.
+    // Verify storage initialization and channel construction before registering webhooks.
     if (url) {
-        log("probing the deployed runtime (state restore + channel construction)…");
+        log("probing the deployed runtime (workspace initialization + channel construction)…");
         const verdict = await probeRuntime(`${url}${RESERVED_PATHS.probe}`, plan.secrets.FASTAGENT_INGRESS_SECRET ?? "", probe.fetchImpl ?? fetch, probe.timeoutMs, probe.intervalMs);
         if (!verdict.ok)
             return gate(verdict.gate);
-        log("runtime verified (state restored, channels constructed)");
+        log("runtime verified (workspace ready, channels constructed)");
     }
-    // 9. Post-deploy webhook registration — same registrar seam as every host, pointed at the
-    //    forwarder's Function URL. Gate policy is the shared registration-gate kernel.
-    // No long-connection channels reach here: `deploy.ts` gates them before the driver runs (AgentCore
-    // has no resident process to hold a connection), so the deploy's channels are all webhook ones.
+    // 9.
     const registrationGateMsg = url
         ? await registerWebhooks({
             baseUrl: url,