@deployfoundation/foundation-deploy 0.2.6 → 0.2.8

package/README.md

@@ -47,7 +47,7 @@ bun run packages/infra/bin/foundation-deploy.ts <command> --instance <path>
 
 | Command | What it does |
 | --- | --- |
-| `deploy` | Build and push the agent image, `cdk deploy --all`, refresh the runtime secret, sync config. |
+| `deploy` | Build and push the agent image, `cdk deploy --all`, refresh the runtime secret (and `cdk deploy` once more when that changed it, so no container boots on the old one), sync config. |
 | `post-deploy` | Smoke DEFAULT, probe the persistent mount, promote `live`, smoke `live`. |
 | `config:sync` | Upload the runtime config and Foundation's product skills to S3. |
 | `setup` | First-deploy orchestration, idempotent. |

package/dist/bin/app.js

@@ -7,7 +7,7 @@ import {
   FoundationNetwork,
   FoundationPipeline,
   NewsletterStack
-} from "../chunk-my0awr1g.js";
+} from "../chunk-revzxtba.js";
 import {
   adminsFor,
   instanceNames,

package/dist/bin/foundation-deploy.js

@@ -87,6 +87,7 @@ async function extractSkillsTarball(tarball, dir, opts = {}) {
 }
 
 // src/deploy/deploy.ts
+import { createHash } from "node:crypto";
 import { existsSync as existsSync2 } from "node:fs";
 import { createRequire } from "node:module";
 import { join as join3 } from "node:path";
@@ -1030,7 +1031,8 @@ async function cdkDeploy(ctx, opts) {
     ...opts.alarmEmail === "" ? [] : ["-c", `alarmEmail=${opts.alarmEmail}`],
     ...opts.phase1 === true ? ["-c", "deployRuntime=false"] : [],
     ...opts.tag === undefined || opts.release !== undefined ? [] : ["-c", `agentImageTag=${opts.tag}`],
-    ...opts.release === undefined ? [] : releaseContext(opts.release)
+    ...opts.release === undefined ? [] : releaseContext(opts.release),
+    ...opts.runtimeSecretDigest === undefined ? [] : ["-c", `runtimeSecretDigest=${opts.runtimeSecretDigest}`]
   ];
   await run([
     ...cdkCommand(),
@@ -1130,6 +1132,28 @@ async function syncRuntimeSecret(ctx) {
   await putSecretJson(ctx, secretRuntime, value);
   return Object.keys(value);
 }
+function runtimeSecretDigest(value) {
+  return createHash("sha256").update(value).digest("hex").slice(0, 16);
+}
+async function currentRuntimeSecretDigest(ctx) {
+  if (ctx.dryRun === true)
+    return;
+  try {
+    return runtimeSecretDigest(await readSecretString(ctx, ctx.names.secretRuntime));
+  } catch {
+    return;
+  }
+}
+async function refreshRuntimeSecret(before, refresh) {
+  const log = refresh.log ?? ((line) => console.log(line));
+  log(`  keys: ${(await refresh.sync()).join(", ")}`);
+  const after = await refresh.digest();
+  if (after === undefined || after === before)
+    return "unchanged";
+  log("▶ the runtime secret changed: cdk deploy once more, so no container boots on the old one");
+  await refresh.redeploy(after);
+  return "rolled";
+}
 async function promoteLive(ctx) {
   const dry = ctx.dryRun === true;
   const runner = cliRunner(ctx);
@@ -1184,19 +1208,26 @@ async function deploy(ctx, opts) {
     console.log(`  ${await buildAndPushImage(ctx, { tag })}`);
   }
   console.log(`▶ cdk deploy --all${phase1 ? " (phase 1: no runtime)" : release !== undefined ? ` (release ${release.version})` : ` (image ${tag})`}`);
-  await cdkDeploy(ctx, {
+  const deployed = await currentRuntimeSecretDigest(ctx);
+  const deployStacks = (runtimeSecretDigest2) => cdkDeploy(ctx, {
     admins,
     alarmEmail,
     tag,
     phase1,
-    ...release === undefined ? {} : { release }
+    ...release === undefined ? {} : { release },
+    ...runtimeSecretDigest2 === undefined ? {} : { runtimeSecretDigest: runtimeSecretDigest2 }
   });
+  await deployStacks(deployed);
   if (ctx.instance.integrations.knock) {
     console.log(`▶ Knock OAuth public-client registration (${ctx.names.secretKnockOauthClient})`);
     console.log(`  ${await syncKnockOAuthClient(ctx)}`);
   }
   console.log(`▶ runtime secret refresh (${ctx.names.secretRuntime})`);
-  console.log(`  keys: ${(await syncRuntimeSecret(ctx)).join(", ")}`);
+  await refreshRuntimeSecret(phase1 ? undefined : deployed, {
+    sync: () => syncRuntimeSecret(ctx),
+    digest: async () => phase1 ? undefined : currentRuntimeSecretDigest(ctx),
+    redeploy: (digest) => deployStacks(digest)
+  });
   console.log("▶ config sync");
   const bucket = await configSync(ctx, {
     runtimeConfigPath: customization.runtimeConfigPath,
@@ -1819,9 +1850,20 @@ async function setup(ctx, opts = {}) {
   console.log(`  ${repositoryUri}:${tag}`);
   await buildAndPushImage(ctx, { tag, repositoryUri });
   step(8, "phase 2: deploy the AgentCore runtime");
-  await cdkDeploy(ctx, { admins, alarmEmail, tag });
+  const phase2 = (runtimeSecretDigest2) => cdkDeploy(ctx, {
+    admins,
+    alarmEmail,
+    tag,
+    ...runtimeSecretDigest2 === undefined ? {} : { runtimeSecretDigest: runtimeSecretDigest2 }
+  });
+  const written = await currentRuntimeSecretDigest(ctx);
+  await phase2(written);
   step(9, `refresh ${ctx.names.secretRuntime} with the runtime arn`);
-  await writeRuntimeSecret(ctx);
+  await refreshRuntimeSecret(written, {
+    sync: () => syncRuntimeSecret(ctx),
+    digest: () => currentRuntimeSecretDigest(ctx),
+    redeploy: phase2
+  });
   step(10, "create the pinned `live` endpoint");
   await ensureLiveEndpoint(ctx);
   step(11, `seed ${ctx.names.secretCodex}`);

package/dist/{chunk-my0awr1g.js → chunk-revzxtba.js}

@@ -41,6 +41,7 @@ class FoundationAgent extends cdk.Stack {
     const display = props.instance.displayName;
     const deployRuntime = String(this.node.tryGetContext("deployRuntime") ?? "true") !== "false";
     const imageTag = this.node.tryGetContext("agentImageTag");
+    const secretDigest = runtimeSecretDigestContext(this.node.tryGetContext("runtimeSecretDigest"));
     this.repository = new ecr.Repository(this, "AgentRepository", {
       repositoryName: names.ecrRepo,
       imageScanOnPush: true,
@@ -124,7 +125,8 @@ class FoundationAgent extends cdk.Stack {
         EnvironmentVariables: {
           RUNTIME_SECRET_ID: props.secrets.runtime.secretName,
           AWS_REGION: this.region,
-          FOUNDATION_WORKSPACE_ROOT: "/workspace"
+          FOUNDATION_WORKSPACE_ROOT: "/workspace",
+          ...secretDigest === undefined ? {} : { FOUNDATION_RUNTIME_SECRET_DIGEST: secretDigest }
         },
         Tags: { project: props.instance.naming.secretPrefix }
       }
@@ -661,6 +663,14 @@ class FoundationAgent extends cdk.Stack {
 function secretArnPattern(secret) {
   return secret.secretFullArn ?? `${secret.secretArn}-??????`;
 }
+function runtimeSecretDigestContext(value) {
+  if (value === undefined || value === "")
+    return;
+  const digest = String(value);
+  if (!/^[0-9a-f]{16}$/.test(digest))
+    throw new Error(`runtimeSecretDigest must be 16 lowercase hex characters, got "${digest}"`);
+  return digest;
+}
 
 // src/stacks/api-stack.ts
 import { readFileSync } from "node:fs";

package/dist/src/index.js

@@ -9,7 +9,7 @@ import {
   FoundationPipeline,
   NewsletterStack,
   deployStatements
-} from "../chunk-my0awr1g.js";
+} from "../chunk-revzxtba.js";
 import {
   BUN_VERSION,
   LAMBDA_ENTRY_POINTS,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@deployfoundation/foundation-deploy",
-  "version": "0.2.6",
+  "version": "0.2.8",
   "description": "Deploys one Foundation instance from a published, signed release: the CDK app and the deploy tool.",
   "license": "UNLICENSED",
   "private": false,

package/src/deploy/deploy.ts

@@ -11,6 +11,7 @@
  * `--phase1` deploys everything except the runtime (`-c deployRuntime=false`);
  * a plain `deploy` afterwards builds, pushes and creates it.
  */
+import { createHash } from "node:crypto";
 import { existsSync } from "node:fs";
 import { createRequire } from "node:module";
 import { join } from "node:path";
@@ -18,7 +19,15 @@ import {
   parseKnockOAuthClient,
   registerKnockPublicClient,
 } from "@deployfoundation/foundation-connectors";
-import { type AwsContext, aws, cdkEnv, putSecretJson, readSecretJson, stackOutput } from "./aws.ts";
+import {
+  type AwsContext,
+  aws,
+  cdkEnv,
+  putSecretJson,
+  readSecretJson,
+  readSecretString,
+  stackOutput,
+} from "./aws.ts";
 import { configSync } from "./config-sync.ts";
 import { adminsCsvFromFile } from "./config.ts";
 import {
@@ -206,6 +215,8 @@ export interface CdkDeployOptions {
   phase1?: boolean;
   /** A verified release; its manifest is what the app reads its artifacts from. */
   release?: ResolvedRelease;
+  /** {@link runtimeSecretDigest} of the runtime secret the new version should boot with. */
+  runtimeSecretDigest?: string;
 }
 
 export async function cdkDeploy(ctx: AwsContext, opts: CdkDeployOptions): Promise<void> {
@@ -225,6 +236,9 @@ export async function cdkDeploy(ctx: AwsContext, opts: CdkDeployOptions): Promis
       ? []
       : ["-c", `agentImageTag=${opts.tag}`]),
     ...(opts.release === undefined ? [] : releaseContext(opts.release)),
+    ...(opts.runtimeSecretDigest === undefined
+      ? []
+      : ["-c", `runtimeSecretDigest=${opts.runtimeSecretDigest}`]),
   ];
   await run(
     [
@@ -410,6 +424,68 @@ export async function syncRuntimeSecret(ctx: AwsContext): Promise<string[]> {
   return Object.keys(value);
 }
 
+/**
+ * A short, one-way digest of the runtime secret as stored. It ends up in the
+ * runtime's environment (see the Agent stack), so it is truncated SHA-256 of
+ * a document full of high-entropy tokens, never anything reversible.
+ */
+export function runtimeSecretDigest(value: string): string {
+  return createHash("sha256").update(value).digest("hex").slice(0, 16);
+}
+
+/** The digest of the runtime secret now, or undefined when there is none yet (or in a dry run). */
+export async function currentRuntimeSecretDigest(ctx: AwsContext): Promise<string | undefined> {
+  if (ctx.dryRun === true) return undefined;
+  try {
+    return runtimeSecretDigest(await readSecretString(ctx, ctx.names.secretRuntime));
+  } catch {
+    return undefined;
+  }
+}
+
+export interface RuntimeSecretRefresh {
+  /** Rewrite the secret from the stack outputs; returns the key names. */
+  sync: () => Promise<string[]>;
+  /** The secret's digest as it stands now. */
+  digest: () => Promise<string | undefined>;
+  /** `cdk deploy` again with this digest, which cuts a new runtime version. */
+  redeploy: (digest: string) => Promise<void>;
+  log?: (line: string) => void;
+}
+
+/**
+ * Refresh the runtime secret, and when that changed it, roll the runtime.
+ *
+ * A container reads the secret once, at boot, and AgentCore boots a pool of
+ * them ahead of demand as soon as a version exists. The secret is composed from
+ * stack outputs (the runtime's own ARN among them), so it can only be written
+ * AFTER `cdk deploy` created the version, by which time that pool has already
+ * booted on the old value: a first deploy's placeholder, or an upgrade's stale
+ * outputs. Those containers answer with the fallback config until they
+ * recycle, and every session they serve in the meantime is broken. Found by
+ * migrating Sky, where a dozen of them failed smoke after smoke.
+ *
+ * Writing first is impossible, and `update-agent-runtime` from the CLI would
+ * resend an incomplete definition (the CLI does not carry the S3 Files mount),
+ * so the new version comes from CloudFormation: `cdk deploy` once more with the
+ * new digest in the runtime's environment. An upgrade whose outputs did not
+ * change produces the same digest and costs nothing.
+ *
+ * `before` is the digest the version just deployed was given.
+ */
+export async function refreshRuntimeSecret(
+  before: string | undefined,
+  refresh: RuntimeSecretRefresh,
+): Promise<"unchanged" | "rolled"> {
+  const log = refresh.log ?? ((line: string) => console.log(line));
+  log(`  keys: ${(await refresh.sync()).join(", ")}`);
+  const after = await refresh.digest();
+  if (after === undefined || after === before) return "unchanged";
+  log("▶ the runtime secret changed: cdk deploy once more, so no container boots on the old one");
+  await refresh.redeploy(after);
+  return "rolled";
+}
+
 /**
  * What the workflow does after a deploy, for a deploy run from a laptop: smoke
  * the version just created on DEFAULT, move the pinned `live` endpoint onto
@@ -543,20 +619,29 @@ export async function deploy(ctx: AwsContext, opts: DeployOptions): Promise<void
           : ` (image ${tag})`
     }`,
   );
-  await cdkDeploy(ctx, {
-    admins,
-    alarmEmail,
-    tag,
-    phase1,
-    ...(release === undefined ? {} : { release }),
-  });
+  const deployed = await currentRuntimeSecretDigest(ctx);
+  const deployStacks = (runtimeSecretDigest: string | undefined) =>
+    cdkDeploy(ctx, {
+      admins,
+      alarmEmail,
+      tag,
+      phase1,
+      ...(release === undefined ? {} : { release }),
+      ...(runtimeSecretDigest === undefined ? {} : { runtimeSecretDigest }),
+    });
+  await deployStacks(deployed);
   if (ctx.instance.integrations.knock) {
     console.log(`▶ Knock OAuth public-client registration (${ctx.names.secretKnockOauthClient})`);
     console.log(`  ${await syncKnockOAuthClient(ctx)}`);
   }
 
   console.log(`▶ runtime secret refresh (${ctx.names.secretRuntime})`);
-  console.log(`  keys: ${(await syncRuntimeSecret(ctx)).join(", ")}`);
+  await refreshRuntimeSecret(phase1 ? undefined : deployed, {
+    sync: () => syncRuntimeSecret(ctx),
+    // Phase 1 has no runtime to roll.
+    digest: async () => (phase1 ? undefined : currentRuntimeSecretDigest(ctx)),
+    redeploy: (digest) => deployStacks(digest),
+  });
 
   console.log("▶ config sync");
   const bucket = await configSync(ctx, {

package/src/deploy/setup.ts

@@ -31,6 +31,8 @@ import {
   buildAndPushImage,
   cdkCommand,
   cdkDeploy,
+  currentRuntimeSecretDigest,
+  refreshRuntimeSecret,
   syncKnockOAuthClient,
   syncRuntimeSecret,
 } from "./deploy.ts";
@@ -375,12 +377,25 @@ export async function setup(
   await buildAndPushImage(ctx, { tag, repositoryUri });
 
   step(8, "phase 2: deploy the AgentCore runtime");
-  await cdkDeploy(ctx, { admins, alarmEmail, tag });
+  const phase2 = (runtimeSecretDigest: string | undefined) =>
+    cdkDeploy(ctx, {
+      admins,
+      alarmEmail,
+      tag,
+      ...(runtimeSecretDigest === undefined ? {} : { runtimeSecretDigest }),
+    });
+  const written = await currentRuntimeSecretDigest(ctx);
+  await phase2(written);
 
   // Only FOUNDATION_AGENT_RUNTIME_ARN needs phase 2, but it is what a routine's
   // schedule targets, so the secret is composed once more now that it exists.
+  // That changes it, and phase 2's containers booted without it: roll them.
   step(9, `refresh ${ctx.names.secretRuntime} with the runtime arn`);
-  await writeRuntimeSecret(ctx);
+  await refreshRuntimeSecret(written, {
+    sync: () => syncRuntimeSecret(ctx),
+    digest: () => currentRuntimeSecretDigest(ctx),
+    redeploy: phase2,
+  });
 
   step(10, "create the pinned `live` endpoint");
   await ensureLiveEndpoint(ctx);

package/src/stacks/agent-stack.ts

@@ -108,6 +108,7 @@ export class FoundationAgent extends cdk.Stack {
 
     const deployRuntime = String(this.node.tryGetContext("deployRuntime") ?? "true") !== "false";
     const imageTag: string | undefined = this.node.tryGetContext("agentImageTag");
+    const secretDigest = runtimeSecretDigestContext(this.node.tryGetContext("runtimeSecretDigest"));
 
     this.repository = new ecr.Repository(this, "AgentRepository", {
       repositoryName: names.ecrRepo,
@@ -237,6 +238,12 @@ export class FoundationAgent extends cdk.Stack {
           RUNTIME_SECRET_ID: props.secrets.runtime.secretName,
           AWS_REGION: this.region,
           FOUNDATION_WORKSPACE_ROOT: "/workspace",
+          // Which runtime secret the containers of this version boot with.
+          // Nothing reads it: it is here so that a deploy whose secret refresh
+          // changed the secret can cut a new version through CloudFormation
+          // (the only path that carries every property, the mount included),
+          // and no pre-warmed container keeps serving the old value.
+          ...(secretDigest === undefined ? {} : { FOUNDATION_RUNTIME_SECRET_DIGEST: secretDigest }),
         },
         // The App-level tag aspect cannot reach a raw CfnResource.
         Tags: { project: props.instance.naming.secretPrefix },
@@ -1004,3 +1011,16 @@ export class FoundationAgent extends cdk.Stack {
 function secretArnPattern(secret: secretsmanager.ISecret): string {
   return secret.secretFullArn ?? `${secret.secretArn}-??????`;
 }
+
+/**
+ * `-c runtimeSecretDigest=<16 hex>`, from `runtimeSecretDigest()` in the
+ * deploy tool. Absent means "not known", which a first `cdk deploy` of a
+ * brand-new instance is: its secret does not exist yet.
+ */
+function runtimeSecretDigestContext(value: unknown): string | undefined {
+  if (value === undefined || value === "") return undefined;
+  const digest = String(value);
+  if (!/^[0-9a-f]{16}$/.test(digest))
+    throw new Error(`runtimeSecretDigest must be 16 lowercase hex characters, got "${digest}"`);
+  return digest;
+}