@fastagent-sh/fastagent 0.21.1 → 0.22.1

package/dist/deploy/agentcore/aws-cli.d.ts

@@ -0,0 +1,91 @@
+/**
+ * ONE AWS CLI invocation, and what its result MEANS.
+ *
+ * The CLI says everything through an exit code plus prose on a stream the caller must opt into capturing — and
+ * the three answers a teardown needs are not separable from the code alone: **it is there**, **it is gone**,
+ * **I could not find out**. The third one is not a detail. "Gone" ends the command successfully; "could not
+ * find out" must stop it or report a leak, because the resource may well still be running and billing.
+ *
+ * WHY THIS IS A MODULE and not an expression at each call site: `destroy.ts` had eleven of those call sites,
+ * and five review rounds found the same defect at a new one each time — a denial read as an absence (three
+ * separate rounds, four sites), an empty paginated listing read as broken JSON, and gates that said "see the
+ * output above" while the flags they passed had piped that output away. One requirement, N places, every time.
+ *
+ * So the four decisions that kept being re-derived live here, once:
+ *
+ * 1. **The stdio flags.** No call site has to know `spawnRunner`'s mapping (`capture`/`captureStderr` true means
+ *    pipe, i.e. NOT the terminal). Getting that wrong is what made "see the output above" point at a blank
+ *    screen for seven gates.
+ * 2. **{@link ABSENT}** — the one description of what "already gone" looks like.
+ * 3. **The text travels with the failure.** A caller that cannot reach what AWS said cannot write a message
+ *    that omits it; `AccessDeniedException` vs `ThrottlingException` vs `ExpiredToken` are different operator
+ *    actions, and that prose is the only thing telling them apart.
+ * 4. **What "no results" actually looks like**, which is not what a fake suggested. MEASURED against AWS CLI
+ *    2.36.47 (ap-southeast-1, 2026-09-22): an empty bucket answers
+ *    `{"RequestCharged": null, "Prefix": ""}`, `scheduler list-schedules` with no match answers
+ *    `{"Schedules": []}`, and `describe-log-groups --query` with no match answers `[]`. None of them prints
+ *    nothing — so empty stdout is output we could not read, and {@link awsList} says so rather than reporting
+ *    an empty list.
+ *
+ * WHAT DOES NOT GO THROUGH IT: a call whose output an operator watches (`docker buildx build`, `s3 cp`,
+ * `cloudformation deploy`) keeps its streams on the terminal — capturing a multi-minute arm64 build to replay it
+ * at the end is a regression, and it is why `run.ts` can say "see the output above" where `destroy.ts` cannot.
+ * That is a stdio choice, not a second classifier: those calls ask "did this work", never "is it there".
+ * Nor does `aws configure get region`, whose nonzero exit means "not set" and whose only consumer is the next
+ * line's "no region configured" gate.
+ *
+ * What is deliberately NOT here: what to DO about each answer. Whether an unreadable read aborts the teardown
+ * or degrades it is a per-resource policy (the stack's probe decides the report's honesty; a log-group listing
+ * only supplies names), and that belongs at the call site where the reason is visible.
+ */
+import type { CliRunner } from "../runner.ts";
+/**
+ * A read's answer. `unreadable` carries `said` (what AWS printed) and `code`, which only the very first call of
+ * a command needs — `127` is "the aws CLI is not installed" rather than anything about the resource.
+ */
+type AwsRead<T> = {
+    ok: T;
+} | {
+    absent: true;
+} | {
+    unreadable: string;
+    code: number;
+};
+/** A write's answer. `absent` is the goal state of a teardown, not a failure. */
+type AwsWrite = {
+    done: true;
+} | {
+    absent: true;
+} | {
+    refused: string;
+};
+export interface AwsCli {
+    /**
+     * Run a read and classify it. `parse` turns stdout into the value the caller wanted and returns `undefined`
+     * for output it cannot read — which becomes `unreadable`, never a thrown `SyntaxError`: an exception escaping
+     * here reaches the operator as a Node stack trace, since `src/cli.ts` has no catch-all.
+     */
+    read<T>(args: string[], parse: (stdout: string) => T | undefined): Promise<AwsRead<T>>;
+    /** A read whose only question is whether the thing is there. */
+    present(args: string[]): Promise<AwsRead<true>>;
+    write(args: string[]): Promise<AwsWrite>;
+}
+export declare function awsCli(aws: CliRunner): AwsCli;
+/**
+ * A JSON read: `pick` says what it wants out of the parsed document, and this owns the two answers it must not
+ * produce.
+ *
+ * NOT AN EXCEPTION — `JSON.parse` on output the CLI never promised threw
+ * `SyntaxError: Unexpected end of JSON input` out of a command whose CLI has no catch-all, i.e. a Node stack
+ * trace where one actionable line belonged.
+ *
+ * AND NOT AN EMPTY LIST. The AWS CLI prints a real document for a list with no results (header, point 4), so empty
+ * stdout is output we failed to read, never "there is nothing there". It needs no branch of its own:
+ * `JSON.parse("")` throws like any other unreadable output, and {@link awsCli} names it `(no output)`.
+ *
+ * ONE FUNCTION for lists and single values.
+ */
+export declare function awsJson<T>(pick: (parsed: unknown) => T | undefined): (stdout: string) => T | undefined;
+/** `logGroups[].logGroupName` as names — the shape both `logs` and `destroy` ask CloudWatch for. */
+export declare const parseLogGroupNames: (stdout: string) => string[] | undefined;
+export {};

package/dist/deploy/agentcore/aws-cli.js

@@ -0,0 +1,64 @@
+/** The most of AWS's own words any message carries. */
+const SAID_LIMIT = 300;
+/** An AWS CLI failure that means the thing is already gone, which is what a teardown is trying to achieve. */
+const ABSENT = /does not exist|NotFound|not found|NoSuchBucket|NoSuchEntity|ResourceNotFoundException/i;
+export function awsCli(aws) {
+    const read = async (args, parse) => {
+        const { code, stdout, stderr } = await aws(args, { capture: true, captureStderr: true });
+        // `(no output)` because a message ending in a colon and nothing else tells an operator less than the fact
+        // that AWS printed nothing at all — which, given point 4, is itself the anomaly.
+        const said = ((stderr ?? "").trim() || stdout.trim()).slice(0, SAID_LIMIT) || "(no output)";
+        if (code === 127) {
+            return { unreadable: "aws CLI not found — install AWS CLI v2: https://docs.aws.amazon.com/cli/", code };
+        }
+        if (code !== 0)
+            return ABSENT.test(stderr ?? "") ? { absent: true } : { unreadable: said, code };
+        const parsed = parse(stdout);
+        return parsed === undefined ? { unreadable: said, code } : { ok: parsed };
+    };
+    return {
+        read,
+        present: (args) => read(args, () => true),
+        write: async (args) => {
+            // CAPTURED, both streams. Every write that goes through here is a short API call whose success output is
+            // noise (`stop-runtime-session` answers with its session JSON) — the long ones an operator watches are
+            // exactly the ones that do not go through this module, see the header.
+            const { code, stderr } = await aws(args, { capture: true, captureStderr: true });
+            if (code === 0)
+                return { done: true };
+            if (ABSENT.test(stderr ?? ""))
+                return { absent: true };
+            return { refused: (stderr ?? "").trim().slice(0, SAID_LIMIT) };
+        },
+    };
+}
+/**
+ * A JSON read: `pick` says what it wants out of the parsed document, and this owns the two answers it must not
+ * produce.
+ *
+ * NOT AN EXCEPTION — `JSON.parse` on output the CLI never promised threw
+ * `SyntaxError: Unexpected end of JSON input` out of a command whose CLI has no catch-all, i.e. a Node stack
+ * trace where one actionable line belonged.
+ *
+ * AND NOT AN EMPTY LIST. The AWS CLI prints a real document for a list with no results (header, point 4), so empty
+ * stdout is output we failed to read, never "there is nothing there". It needs no branch of its own:
+ * `JSON.parse("")` throws like any other unreadable output, and {@link awsCli} names it `(no output)`.
+ *
+ * ONE FUNCTION for lists and single values.
+ */
+export function awsJson(pick) {
+    return (stdout) => {
+        try {
+            return pick(JSON.parse(stdout));
+        }
+        catch {
+            return undefined;
+        }
+    };
+}
+/** `logGroups[].logGroupName` as names — the shape both `logs` and `destroy` ask CloudWatch for. */
+export const parseLogGroupNames = awsJson((parsed) => {
+    if (!Array.isArray(parsed) || !parsed.every((v) => typeof v === "string"))
+        throw new Error("not a name list");
+    return parsed;
+});

package/dist/deploy/agentcore/destroy.d.ts

@@ -0,0 +1,51 @@
+/**
+ * `destroy agentcore`: removing what a deploy created, INCLUDING the parts the stack does not own.
+ *
+ * `aws cloudformation delete-stack` is the obvious move and it leaves three of the four behind, each for its own
+ * reason:
+ *
+ * - the S3 artifact bucket and the ECR repository must exist BEFORE the stack (it reads the forwarder's zip and
+ *   the runtime's image from them), so they cannot be stack resources;
+ * - the forwarder's log group is created by AWS on the Lambda's first WRITE, so no template ever mentions it —
+ *   and the deploy put a 14-day retention on it, i.e. it holds data;
+ * - a wake alarm is minted at RUNTIME by the container. `ActionAfterCompletion: DELETE` only runs once one
+ *   FIRES, so anything stopped between the wake call and the fire leaves a schedule that retries into a deleted
+ *   Lambda for weeks.
+ *
+ * This file is the RESOURCE LIST, the order the dependencies demand, and one policy decision per resource for
+ * what to do when AWS cannot answer. What an AWS answer MEANS — there, gone, or unreadable — belongs to
+ * {@link awsCli} and is not re-decided here; that repetition is what eleven call sites of it cost.
+ *
+ * Every deletion is ATTEMPTED even after an earlier one fails, and "already gone" is the goal state rather than
+ * an error — a teardown that stops at the first miss is one that cannot finish a half-deleted deployment. The
+ * exception is the stack, whose wait gates the rest: the image and the forwarder package below it are what a
+ * retry needs.
+ */
+import type { CliRunner } from "../runner.ts";
+export interface AgentcoreDestroyPlan {
+    /** Deployment base name — stack `fastagent-<name>`, bucket `fa-<name>-<account>`, repo `fastagent/<name>`. */
+    name: string;
+    /** Without it, report what is out there and delete nothing. */
+    run: boolean;
+}
+export type AgentcoreDestroyOutcome = 
+/** `found` is what the reads saw; `removed` what the writes did; `kept` what this command refused to delete. */
+{
+    ok: true;
+    found: string[];
+    removed: string[];
+    kept: string[];
+}
+/**
+ * The same three ride along on a failure: a half-finished teardown is exactly when "what is out there" and
+ * "what is already gone" matter most, and the kept bucket is this command's only deliberate decision — none
+ * of them may vanish from the report because something else failed.
+ */
+ | {
+    ok: false;
+    gate: string;
+    found: string[];
+    removed: string[];
+    kept: string[];
+};
+export declare function destroyAgentcoreDeployment(plan: AgentcoreDestroyPlan, runner: CliRunner, announce?: (message: string) => void): Promise<AgentcoreDestroyOutcome>;

package/dist/deploy/agentcore/destroy.js

@@ -0,0 +1,273 @@
+import { awsCli, awsJson, parseLogGroupNames } from "./aws-cli.js";
+import { agentcoreRepoName, agentcoreStackName, deploymentBucketName, forwarderLogGroup, runtimeLogGroupPrefix, toRuntimeName, wakeAlarmPrefix, } from "./plan.js";
+/** S3 deletes at most 1000 keys per call. */
+const DELETE_BATCH = 1000;
+/**
+ * Every object AND delete marker in the bucket — both, because a versioned bucket refuses `delete-bucket` while
+ * either is non-empty, which is how a hand-cleanup of this exact account hit
+ * `BucketNotEmpty ... You must delete all versions in the bucket`.
+ *
+ * AN EMPTY BUCKET is an ordinary answer, and a real one: `create-bucket` happens before the upload into it, and
+ * a previous destroy that got through `delete-objects` and failed on `delete-bucket` leaves exactly one — the
+ * half-deleted deployment this command exists to finish. AWS says it with a document that simply has neither
+ * key (measured: `{"RequestCharged": null, "Prefix": ""}`), which is why `??` here and not a special case.
+ */
+const parseVersions = awsJson((parsed) => {
+    const { Versions, DeleteMarkers } = parsed;
+    return [...(Versions ?? []), ...(DeleteMarkers ?? [])];
+});
+const parseScheduleNames = awsJson((parsed) => (parsed.Schedules ?? []).flatMap((s) => typeof s.Name === "string" ? [s.Name] : []));
+const parseAccountId = awsJson((parsed) => {
+    const { Account } = parsed;
+    return typeof Account === "string" ? Account : undefined;
+});
+export async function destroyAgentcoreDeployment(plan, runner, announce = () => { }) {
+    const aws = awsCli(runner);
+    const gate = (g, parts = {}) => ({
+        ok: false,
+        gate: g,
+        found: parts.found ?? [],
+        removed: parts.removed ?? [],
+        kept: parts.kept ?? [],
+    });
+    const stack = agentcoreStackName(plan.name);
+    const repo = agentcoreRepoName(plan.name);
+    const identity = await aws.read(["sts", "get-caller-identity", "--output", "json"], parseAccountId);
+    if (!("ok" in identity)) {
+        if ("absent" in identity)
+            return gate("`aws sts get-caller-identity` found no caller — check your profile");
+        if (identity.code === 127)
+            return gate(`${identity.unreadable}, then re-run`);
+        // A MISSING REGION FAILS HERE TOO, and blaming the credentials sends the operator to fix the wrong thing.
+        return gate(/region/i.test(identity.unreadable)
+            ? "no AWS region configured — set AWS_REGION (or `aws configure set region <region>`), then re-run"
+            : `no working AWS credentials — run \`aws configure\` (or set AWS_ACCESS_KEY_ID/…), then re-run: ${identity.unreadable}`);
+    }
+    const account = identity.ok;
+    const bucket = deploymentBucketName(plan.name, account);
+    // EVERY RESOURCE BELOW IS REGIONAL, and none of them says so. A default profile pointing somewhere other than
+    // the deploy's region answers "nothing in this account", which an operator reads as "already clean" — so the
+    // region is resolved and REPORTED the way `run.ts` does it, and its absence is its own gate rather than a
+    // credential complaint. `aws configure get region` because the CLI does not take it from `AWS_REGION`.
+    const configured = await aws.read(["configure", "get", "region"], (stdout) => stdout.trim());
+    const fromConfig = "ok" in configured ? configured.ok : "";
+    const region = process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION ?? fromConfig;
+    if (!region) {
+        return gate("no AWS region configured — set AWS_REGION (or `aws configure set region <region>`), then re-run");
+    }
+    // BEFORE THE READS, not after: every gate below names the account or the region, and the operator pointed at
+    // the wrong profile has to be able to see that from the line above the failure.
+    announce(`account ${account}, region ${region}`);
+    // ---- What is out there. Each read gets its own policy for "AWS could not answer". ----
+    // The alarm ids are minted inside the container, so the PREFIX is all we can ask for. Unreadable is a gate
+    // here: an alarm nobody deleted keeps firing at a Lambda that is about to stop existing.
+    const prefix = wakeAlarmPrefix(plan.name);
+    const listed = await aws.read(["scheduler", "list-schedules", "--name-prefix", prefix, "--output", "json"], parseScheduleNames);
+    if ("unreadable" in listed)
+        return gate(`could not list wake alarms under ${prefix}: ${listed.unreadable}`);
+    // THE FULL SHAPE, not just the prefix. The forwarder mints every alarm as prefix + sha256(wakeId)[:16]
+    // (plan.ts / forwarder.js), and a prefix alone is ambiguous between sibling agents: a workspace literally
+    // named `<name>-wk-abc` produces `fa-<name>-wk-abc-wk-<hash>`, which starts with THIS deployment's prefix.
+    // Deleting it would take a live deployment's pending wake-ups, and a wake-up is not re-created.
+    const isMintedAlarm = (alarm) => alarm.startsWith(prefix) && /^[0-9a-f]{16}$/.test(alarm.slice(prefix.length));
+    const alarms = ("ok" in listed ? listed.ok : []).filter(isMintedAlarm);
+    // A GATE for these three, because they decide whether the REPORT is true. `delete-stack` answers 0 for a
+    // stack that does not exist, so "deleted: stack X" for a stack nobody deployed — or "nothing left to delete"
+    // over a runtime that is still billing — is the false signal this repo refuses.
+    const stackRead = await aws.present(["cloudformation", "describe-stacks", "--stack-name", stack, "--output", "json"]);
+    if ("unreadable" in stackRead) {
+        return gate(`could not tell whether stack ${stack} exists in ${region}: ${stackRead.unreadable}`);
+    }
+    const stackExists = "ok" in stackRead;
+    const bucketRead = await aws.read(["s3api", "list-object-versions", "--bucket", bucket, "--output", "json"], parseVersions);
+    if ("unreadable" in bucketRead) {
+        return gate(`could not read the object listing for bucket ${bucket} in ${region}: ${bucketRead.unreadable}`);
+    }
+    const bucketExists = "ok" in bucketRead;
+    const keys = ("ok" in bucketRead ? bucketRead.ok : []).map((v) => v.Key);
+    // A bucket that holds anything but the forwarder's zips is NOT ours to delete. Today's deploy puts nothing
+    // else there (agent state lives on the AgentCore storage mount), but an older one kept `state/snapshot.json.gz`
+    // in it, and in this account those snapshots were the only copy left of two retired agents.
+    const foreign = [...new Set(keys.filter((key) => !key.startsWith("forwarder/")))];
+    const repoRead = await aws.present(["ecr", "describe-repositories", "--repository-names", repo, "--output", "json"]);
+    if ("unreadable" in repoRead) {
+        return gate(`could not tell whether repository ${repo} exists in ${region}: ${repoRead.unreadable}`);
+    }
+    const repoExists = "ok" in repoRead;
+    // BOTH LOG GROUPS. AWS creates each on the first WRITE, so neither is a stack resource — same reason, twice —
+    // and the runtime's holds the agent's own stdout/stderr, i.e. what it said in every conversation. `deploy`'s
+    // runbook already knows there are two: it sets a retention on each.
+    //
+    // THE RUNTIME'S GROUP IS NAMED WITHOUT THE STACK. MEASURED (ap-southeast-1, 2026-09-22): a deploy of
+    // `destroy-probe-yehg` produced `/aws/bedrock-agentcore/runtimes/destroy_probe_yehg-6oR1zq7pBZ-DEFAULT`, so
+    // the id is `<AgentRuntimeName>-<suffix>` and the name is `toRuntimeName(plan.name)`, the same derivation the
+    // template deployed. That account also held ELEVEN of these groups from past deployments whose every other
+    // resource was long gone — this is the leak, not a hypothetical one. Reading the name out of the
+    // stack's `RuntimeArn` instead meant that the operator whose first move was `aws cloudformation delete-stack`
+    // (which `docs/deploy.md` says it will be) got a clean-looking teardown with the group holding every line the
+    // agent ever printed still sitting there. The trailing `-` keeps `probe-` off `probe2-…`.
+    const forwarderGroup = forwarderLogGroup(plan.name);
+    const runtimePrefix = runtimeLogGroupPrefix(toRuntimeName(plan.name));
+    const logGroupsUnder = async (prefix) => aws.read([
+        "logs",
+        "describe-log-groups",
+        "--log-group-name-prefix",
+        prefix,
+        "--query",
+        "logGroups[].logGroupName",
+        "--output",
+        "json",
+    ], parseLogGroupNames);
+    const forwarderRead = await logGroupsUnder(forwarderGroup);
+    const runtimeRead = await logGroupsUnder(runtimePrefix);
+    // NOT A GATE, unlike the three above: these two only supply NAMES, and refusing to delete anything because a
+    // listing failed turned a role without `logs:DescribeLogGroups` — or one `ThrottlingException` — into a
+    // permanent no-op with a Bedrock runtime and a Lambda billing behind it.
+    //
+    // FAILURE vs WARNING is decided by whether the resource SURVIVES it, not by whether a call failed. The
+    // forwarder's group is named exactly, so a `--run` deletes it by name and the failed listing changed nothing
+    // — reporting `1 resource(s) survived` for a teardown that removed everything is the same false signal as the
+    // reverse, and `test/live/env.ts` treats it as a leak. In the inventory, where no delete follows, the same
+    // unreadable listing DOES mean the answer is incomplete.
+    const failures = [];
+    const warnings = [];
+    const named = (read) => ("ok" in read ? read.ok : []);
+    if ("unreadable" in forwarderRead) {
+        (plan.run ? warnings : failures).push(plan.run
+            ? `could not list log groups under ${forwarderGroup} (${forwarderRead.unreadable}) — deleted it by name instead`
+            : `listing log groups under ${forwarderGroup} (${forwarderRead.unreadable}) — cannot confirm whether it exists`);
+    }
+    if ("unreadable" in runtimeRead) {
+        // A failure in BOTH modes — this one carries a suffix only AWS knows, so nothing downstream can address it
+        // — but an inventory deletes nothing, so saying it "was NOT deleted" there names the wrong problem.
+        failures.push(`listing log groups under ${runtimePrefix} (${runtimeRead.unreadable}) — the runtime's log group holds ` +
+            (plan.run ? `the agent's stdout and was NOT deleted` : `the agent's stdout and cannot be named`));
+    }
+    for (const warning of warnings)
+        announce(warning);
+    const listedGroups = [...named(forwarderRead).filter((name) => name === forwarderGroup), ...named(runtimeRead)];
+    const logGroups = "unreadable" in forwarderRead ? [forwarderGroup, ...listedGroups] : listedGroups;
+    const found = [];
+    if (alarms.length > 0)
+        found.push(`${alarms.length} wake alarm(s) under ${prefix}`);
+    if (stackExists)
+        found.push(`stack ${stack}`);
+    if (bucketExists)
+        found.push(`bucket ${bucket} (${keys.length} object version(s))`);
+    if (repoExists)
+        found.push(`repository ${repo}`);
+    for (const name of listedGroups)
+        found.push(`log group ${name}`);
+    // The one thing this command refuses to delete.
+    const kept = foreign.length > 0
+        ? [
+            `bucket ${bucket} — it holds ${foreign.join(", ")}, which no deploy of this version writes. Read it, ` +
+                `then \`aws s3 rb s3://${bucket} --force\` if you want it gone`,
+        ]
+        : [];
+    if (!plan.run) {
+        return failures.length > 0
+            ? gate(`${failures.length} read(s) failed:\n  ${failures.join("\n  ")}`, { found, kept })
+            : { ok: true, found, removed: [], kept };
+    }
+    // ---- The deletions, in the order the dependencies demand. ----
+    const removed = [];
+    /** One deletion and its bookkeeping: what got deleted, and what refused to be. */
+    const attempt = async (label, args) => {
+        const result = await aws.write(args);
+        if ("done" in result) {
+            removed.push(label);
+            return true;
+        }
+        if ("absent" in result)
+            return true; // already gone: the goal state
+        failures.push(`${label}: ${result.refused}`);
+        return false;
+    };
+    const swept = new Set();
+    const sweepAlarms = async (names) => {
+        for (const alarm of names.filter((name) => !swept.has(name))) {
+            swept.add(alarm);
+            await attempt(`wake alarm ${alarm}`, ["scheduler", "delete-schedule", "--name", alarm]);
+        }
+    };
+    await sweepAlarms(alarms);
+    if (stackExists) {
+        announce(`deleting stack ${stack} (this waits for CloudFormation)…`);
+        const asked = await aws.write(["cloudformation", "delete-stack", "--stack-name", stack]);
+        if ("refused" in asked) {
+            return gate(`stack ${stack} refused to delete: ${asked.refused}\n` +
+                `  the image and the forwarder package it was built from are untouched, so a retry has what it needs.`, { found, removed, kept });
+        }
+        // THE WAIT DECIDES whether anything below may run, AND whether the stack counts as deleted at all.
+        // `delete-stack` returning 0 only means CloudFormation accepted the request: a DELETE_FAILED stack still
+        // holds a billing Bedrock runtime and Lambda, and the image and the forwarder zip below are what it was
+        // created FROM — deleting those while it stands only makes the operator's retry worse. Recording it as
+        // removed before this point printed "deleted: stack X" directly above "did not finish deleting".
+        const waited = await aws.write(["cloudformation", "wait", "stack-delete-complete", "--stack-name", stack]);
+        if ("refused" in waited) {
+            return gate(`stack ${stack} did not finish deleting: ${waited.refused}\n` +
+                `  its runtime and Lambda are still billing. The image and the forwarder package it was built ` +
+                `from are untouched, so a retry has what it needs.`, { found, removed, kept });
+        }
+        removed.push(`stack ${stack}`);
+        // AGAIN, NOW. The container served the whole deletion and holds `scheduler:CreateSchedule`, so a wake-up
+        // taken in those minutes minted an alarm after the first sweep read the list — an alarm whose target is the
+        // Lambda we just deleted, retrying into nothing for weeks.
+        const after = await aws.read(["scheduler", "list-schedules", "--name-prefix", prefix, "--output", "json"], parseScheduleNames);
+        if ("unreadable" in after) {
+            failures.push(`re-listing wake alarms under ${prefix}: ${after.unreadable}`);
+        }
+        else {
+            await sweepAlarms(("ok" in after ? after.ok : []).filter(isMintedAlarm));
+        }
+    }
+    if (bucketExists && foreign.length === 0)
+        await purgeBucket(bucket, aws, attempt, failures);
+    if (repoExists) {
+        await attempt(`repository ${repo}`, ["ecr", "delete-repository", "--repository-name", repo, "--force"]);
+    }
+    for (const name of logGroups) {
+        await attempt(`log group ${name}`, ["logs", "delete-log-group", "--log-group-name", name]);
+    }
+    if (failures.length > 0) {
+        return gate(`${failures.length} resource(s) survived:\n  ${failures.join("\n  ")}`, { found, removed, kept });
+    }
+    return { ok: true, found, removed, kept };
+}
+/**
+ * Empty the bucket, then delete it. The emptying is not optional even for a bucket that was never versioned:
+ * `delete-bucket` refuses a non-empty one, and the forwarder's zips are always in there.
+ */
+async function purgeBucket(bucket, aws, attempt, failures) {
+    // RE-READ rather than reusing the inventory: minutes of `stack-delete-complete` sit between the two. And the
+    // same three answers as everywhere — a failed read must not be reported as `nothing left to delete`.
+    const listed = await aws.read(["s3api", "list-object-versions", "--bucket", bucket, "--output", "json"], parseVersions);
+    if ("absent" in listed)
+        return; // gone between the inventory and now: the goal state
+    if ("unreadable" in listed) {
+        failures.push(`bucket ${bucket}: could not list its objects, so it was left in place (${listed.unreadable})`);
+        return;
+    }
+    // THE SAME RULE ON WHAT IT JUST READ. The decision to delete this bucket was made against the inventory,
+    // minutes of `stack-delete-complete` ago, and this listing is the one whose contents actually get deleted.
+    // Nothing in the deployment can write here in that window (the template grants the runtime and the forwarder
+    // no `s3:*`), so this catches a person or another tool — and a batch delete is not reversible.
+    const foreign = [...new Set(listed.ok.map((v) => v.Key).filter((key) => !key.startsWith("forwarder/")))];
+    if (foreign.length > 0) {
+        failures.push(`bucket ${bucket} — ${foreign.join(", ")} appeared in it since the inventory, so it was left in place`);
+        return;
+    }
+    for (let at = 0; at < listed.ok.length; at += DELETE_BATCH) {
+        const batch = listed.ok.slice(at, at + DELETE_BATCH).map((v) => ({ Key: v.Key, VersionId: v.VersionId }));
+        await attempt(`${batch.length} object version(s) in ${bucket}`, [
+            "s3api",
+            "delete-objects",
+            "--bucket",
+            bucket,
+            "--delete",
+            JSON.stringify({ Objects: batch, Quiet: true }),
+        ]);
+    }
+    await attempt(`bucket ${bucket}`, ["s3api", "delete-bucket", "--bucket", bucket]);
+}

package/dist/deploy/agentcore/forwarder.js

@@ -6,7 +6,7 @@
 // what runs (and what the manual runbook zips) — not an input you can change.
 // Webhooks (Function URL) and EventBridge Scheduler fires are forwarded as envelopes to the
 // AgentCore Runtime over SigV4 InvokeAgentRuntime, all on ONE fixed ingress session (fastagent
-// channel state is single-writer; one session = at most one microVM). With selfSchedule, this
+// channel state is single-writer; one session = at most one microVM). This
 // Lambda also OWNS the wake alarms: the container POSTs its pending wake-ups to /__fastagent/
 // wake-alarm (shared secret) and each becomes a self-deleting one-shot EventBridge schedule that
 // pokes this Lambda — which wakes the container, whose wake pump fires the due entry.
@@ -17,13 +17,6 @@ const { BedrockAgentCoreClient, InvokeAgentRuntimeCommand } = require("@aws-sdk/
 const client = new BedrockAgentCoreClient({});
 let ownUrl; // self-resolved once per cold start; rides on every envelope for the wake-alarm callback
 
-// Presigned S3 URLs for the container's state snapshot. AgentCore wipes the /mnt/state mount on
-// every runtime version update (= every deploy), so the durable copy lives in S3 — but the
-// container is given NO AWS credentials by the platform, so the only reachable form is a URL that
-// carries its own authorization. SigV4 query signing, node:crypto only (no SDK, nothing to install).
-const enc = (s) => encodeURIComponent(s).replace(/[!'()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
-const hmac = (key, data) => crypto.createHmac("sha256", key).update(data).digest();
-
 // Every shared-secret gate on this public URL goes through here. Constant-time, and an unset
 // expected secret NEVER matches — otherwise a topology that did not configure one would accept a
 // request that sent none. Non-string input is coerced to "" first: Buffer.from(8) allocates eight
@@ -34,39 +27,8 @@ const secretEq = (given, expected) => {
   return b.length > 0 && a.length === b.length && crypto.timingSafeEqual(a, b);
 };
 
-function presign(method, seconds) {
-  const bucket = process.env.STATE_BUCKET,
-    key = process.env.STATE_KEY,
-    region = process.env.AWS_REGION;
-  const host = `${bucket}.s3.${region}.amazonaws.com`;
-  const stamp = new Date().toISOString().replace(/[-:]/g, "").replace(/\.\d+/, "");
-  const scope = `${stamp.slice(0, 8)}/${region}/s3/aws4_request`;
-  const pairs = [
-    ["X-Amz-Algorithm", "AWS4-HMAC-SHA256"],
-    ["X-Amz-Credential", `${process.env.AWS_ACCESS_KEY_ID}/${scope}`],
-    ["X-Amz-Date", stamp],
-    ["X-Amz-Expires", String(seconds)],
-    ["X-Amz-SignedHeaders", "host"],
-  ];
-  if (process.env.AWS_SESSION_TOKEN) pairs.push(["X-Amz-Security-Token", process.env.AWS_SESSION_TOKEN]);
-  // The canonical query must be byte-identical to the one on the wire — build it ONCE, reuse below.
-  const query = pairs
-    .map(([k, v]) => [enc(k), enc(v)])
-    .sort((a, b) => (a[0] < b[0] ? -1 : 1))
-    .map((p) => p.join("="))
-    .join("&");
-  const path = `/${key.split("/").map(enc).join("/")}`;
-  const canonical = [method, path, query, `host:${host}\n`, "host", "UNSIGNED-PAYLOAD"].join("\n");
-  const sts = ["AWS4-HMAC-SHA256", stamp, scope, crypto.createHash("sha256").update(canonical).digest("hex")].join(
-    "\n",
-  );
-  let k = hmac(`AWS4${process.env.AWS_SECRET_ACCESS_KEY}`, stamp.slice(0, 8));
-  for (const part of [region, "s3", "aws4_request"]) k = hmac(k, part);
-  return `https://${host}${path}?${query}&X-Amz-Signature=${hmac(k, sts).toString("hex")}`;
-}
-
 async function invoke(envelope) {
-  if ((process.env.WAKE_SECRET || process.env.STATE_REFRESH_SECRET) && !ownUrl) {
+  if (process.env.WAKE_SECRET && !ownUrl) {
     const { LambdaClient, GetFunctionUrlConfigCommand } = require("@aws-sdk/client-lambda");
     ownUrl = (
       await new LambdaClient({}).send(
@@ -77,22 +39,6 @@ async function invoke(envelope) {
   if (ownUrl) envelope.wake = { url: ownUrl };
   // Authenticates this envelope as coming from the forwarder (see the template's FastagentIngressSecret).
   envelope.auth = process.env.INGRESS_SECRET;
-  // Keep each capability short-lived. Function-URL deployments also carry an authenticated refresh
-  // endpoint, so a background turn settling hours after its webhook never depends on the temporary
-  // Lambda credentials that signed the original pair still being alive.
-  if (process.env.STATE_BUCKET)
-    envelope.state = {
-      getUrl: presign("GET", 3600),
-      putUrl: presign("PUT", 3600),
-      ...(ownUrl && process.env.STATE_REFRESH_SECRET
-        ? {
-            refresh: {
-              url: `${ownUrl.replace(/\/$/, "")}/__fastagent/state-urls`,
-              auth: process.env.STATE_REFRESH_SECRET,
-            },
-          }
-        : {}),
-    };
   const res = await client.send(
     new InvokeAgentRuntimeCommand({
       agentRuntimeArn: process.env.RUNTIME_ARN,
@@ -151,35 +97,43 @@ async function syncAlarms(alarms, ctx) {
   return failed;
 }
 
+/**
+ * Invoke the container, and leave ONE line under `label` whichever way it ends — including the way that never came
+ * back at all.
+ *
+ * Lambda already logs an unhandled throw, so the failure is not invisible; what it is not is ATTRIBUTABLE. An
+ * operator (or a probe) filtering this log group for `routine-fire <name>` sees nothing and concludes the clock
+ * never fired, when what actually happened is that the clock fired and the call died. Those need different fixes
+ * and looked identical from outside — that mistake cost a real debugging session.
+ */
+async function invokeLogged(label, envelope) {
+  try {
+    return await invoke(envelope);
+  } catch (e) {
+    console.log(`${label}: invoke failed: ${e?.message || e}`);
+    throw e; // the miss still lands as a Lambda error, which is what makes EventBridge retry it
+  }
+}
+
 exports.handler = async (event, ctx) => {
   // EventBridge wake-up poke: the invocation itself wakes the container; its pump does the rest.
   if (event?.wakePoke) {
-    const r = await invoke({ kind: "wake-poke" });
+    const r = await invokeLogged("wake-poke", { kind: "wake-poke" });
     console.log(`wake-poke: ${r.status}`);
     return { status: r.status };
   }
   // EventBridge Scheduler fire — throw on failure so the miss lands in CloudWatch, never silently.
   if (event?.scheduleFire) {
-    const { name, slot } = event.scheduleFire;
-    const r = await invoke({ kind: "schedule-fire", name, slot });
+    const { name, occurrence } = event.scheduleFire;
+    const label = `routine-fire ${name} (${occurrence})`;
+    const r = await invokeLogged(label, { kind: "routine-fire", name, occurrence });
     const out = r.body.toString();
-    console.log(`schedule-fire ${name} (${slot}): ${r.status} ${out}`);
-    if (r.status >= 400) throw new Error(`schedule-fire ${name} failed: ${r.status} ${out}`);
+    console.log(`${label}: ${r.status} ${out}`);
+    if (r.status >= 400) throw new Error(`routine-fire ${name} failed: ${r.status} ${out}`);
     return { status: r.status };
   }
   const http = event?.requestContext?.http;
   if (!http) throw new Error("unrecognized event shape");
-  // Refresh the snapshot capabilities with THIS Lambda invocation's current temporary credentials.
-  // The container may settle long after the webhook Lambda (and its credentials) expired.
-  if (event.rawPath === "/__fastagent/state-urls") {
-    const req = JSON.parse(event.isBase64Encoded ? Buffer.from(event.body, "base64").toString() : event.body || "{}");
-    if (!secretEq(req.auth, process.env.STATE_REFRESH_SECRET)) return { statusCode: 403, body: "forbidden\n" };
-    return {
-      statusCode: 200,
-      headers: { "content-type": "application/json" },
-      body: JSON.stringify({ getUrl: presign("GET", 3600), putUrl: presign("PUT", 3600) }),
-    };
-  }
   // The container's wake-alarm callback (reserved path, shared secret) — handled HERE, never forwarded.
   if (event.rawPath === "/__fastagent/wake-alarm") {
     const req = JSON.parse(event.isBase64Encoded ? Buffer.from(event.body, "base64").toString() : event.body || "{}");
@@ -190,13 +144,8 @@ exports.handler = async (event, ctx) => {
     if (failed > 0) return { statusCode: 500, body: `${failed} alarm(s) failed\n` };
     return { statusCode: 200, body: "ok\n" };
   }
-  // The deploy driver's probe (reserved path, ingress secret): wake the runtime through the SAME
-  // trusted envelope pipeline (state URLs included — a direct InvokeAgentRuntime call could not mint
-  // them, and would make the runtime construct against a pre-restore mount) and pass its structured
-  // transport-200 verdict back VERBATIM. The ordinary webhook path below folds a non-200 transport
-  // into an opaque 502, which would strip exactly the diagnostics the probe exists to carry — and it
-  // sits BEFORE the WEBHOOKS_ENABLED gate so schedule-only topologies (whose URLs refuse ordinary
-  // public traffic) are probeable too.
+  // Preserve the trusted probe's structured verdict instead of folding its diagnostics into a 502.
+  // Schedule-only topologies also need this path, before the ordinary webhook gate.
   if (event.rawPath === "/__fastagent/probe") {
     const req = JSON.parse(event.isBase64Encoded ? Buffer.from(event.body, "base64").toString() : event.body || "{}");
     if (!secretEq(req.auth, process.env.INGRESS_SECRET)) return { statusCode: 403, body: "forbidden\n" };