@deployfoundation/foundation-deploy 0.1.0

package/src/deploy/aws.ts

@@ -0,0 +1,265 @@
+/**
+ * `aws` CLI wrappers for the deploy scripts. The CLI (not an SDK) so that
+ * every call is one a human can copy out of OPERATIONS.md and re-run.
+ *
+ * Secret values travel through a short-lived 0600 temp file, so they never
+ * appear in an argv the process table can show.
+ */
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  type Instance,
+  type InstanceContext,
+  type InstanceNames,
+  instanceNames,
+  loadInstanceFromArgs,
+} from "./instance.ts";
+import { run, runCapture } from "./sh.ts";
+
+export interface AwsContext {
+  /** Empty string means "no `--profile` flag" — use the default credential chain. */
+  profile: string;
+  region: string;
+  /** Read-only calls still run; mutating helpers print instead. */
+  dryRun?: boolean;
+  /** Which deployment this command targets. Every name below is derived from it. */
+  instance: Instance;
+  /** Where the instance file and its runtime config live. */
+  paths: InstanceContext;
+  /** `instanceNames(ctx.instance)`, computed once. */
+  names: InstanceNames;
+}
+
+/**
+ * Pick the AWS profile, or `""` for "pass no `--profile` at all".
+ *
+ * Locally a named profile is how credentials are found; in a pipeline the
+ * credentials come from the build role in the environment and naming a profile
+ * that does not exist there would break every call. `FOUNDATION_NO_PROFILE=1` (or
+ * `--profile ""`) forces the default chain explicitly.
+ */
+export function resolveProfile(
+  fallback: string,
+  explicit?: string,
+  env: Record<string, string | undefined> = process.env,
+): string {
+  if (env.FOUNDATION_NO_PROFILE === "1") return "";
+  if (explicit !== undefined) return explicit;
+  if (env.AWS_PROFILE !== undefined && env.AWS_PROFILE !== "") return env.AWS_PROFILE;
+  if (env.GITHUB_ACTIONS === "true") return "";
+  return fallback;
+}
+
+/**
+ * Build the context every helper below takes. The instance supplies the
+ * account, region and profile defaults; a flag or the environment may still
+ * override the last two.
+ */
+export function awsContext(
+  overrides: Partial<Omit<AwsContext, "names" | "instance">> & {
+    /**
+     * The environment the profile and region fall back to. Injectable so a
+     * test is not at the mercy of the machine it runs on: `GITHUB_ACTIONS`
+     * and `AWS_REGION` both change the answer, so a test that means "just the
+     * instance file" has to be able to say so by passing `{}`.
+     */
+    env?: Record<string, string | undefined>;
+  } = {},
+): AwsContext {
+  const env = overrides.env ?? process.env;
+  const paths = overrides.paths ?? loadInstanceFromArgs();
+  const instance = paths.instance;
+  return {
+    profile: resolveProfile(instance.aws.profile, overrides.profile, env),
+    region: overrides.region ?? env.AWS_REGION ?? instance.aws.region,
+    dryRun: overrides.dryRun ?? false,
+    instance,
+    paths,
+    names: instanceNames(instance),
+  };
+}
+
+/**
+ * What the plain `aws` wrappers need. Instance-free, so a helper that only
+ * shells out to the CLI cannot accidentally depend on a resource name.
+ */
+export type AwsCallContext = Pick<AwsContext, "profile" | "region" | "dryRun">;
+
+/** The full argv for an `aws` call, `--profile` included only when set. */
+export function argv(ctx: AwsCallContext, args: string[]): string[] {
+  const profile = ctx.profile === "" ? [] : ["--profile", ctx.profile];
+  return ["aws", ...args, ...profile, "--region", ctx.region];
+}
+
+/** Child-process env for `cdk`: no `AWS_PROFILE` when the default chain is in use. */
+export function cdkEnv(ctx: AwsCallContext): Record<string, string> {
+  return {
+    ...(ctx.profile === "" ? {} : { AWS_PROFILE: ctx.profile }),
+    AWS_REGION: ctx.region,
+    CDK_DEFAULT_REGION: ctx.region,
+  };
+}
+
+/** Run an `aws` subcommand and return trimmed stdout. */
+export async function aws(ctx: AwsCallContext, args: string[], stdin?: string): Promise<string> {
+  return runCapture(argv(ctx, args), { stdin });
+}
+
+/** Run a mutating `aws` subcommand, honouring `ctx.dryRun`. */
+export async function awsMutate(
+  ctx: AwsCallContext,
+  args: string[],
+  stdin?: string,
+): Promise<void> {
+  if (ctx.dryRun === true) {
+    console.log(`  $ ${argv(ctx, args).join(" ")}${stdin === undefined ? "" : " < (stdin)"}`);
+    return;
+  }
+  await run(argv(ctx, args), { stdin });
+}
+
+/** One CloudFormation stack output, by OutputKey. */
+export async function stackOutput(
+  ctx: AwsCallContext,
+  stack: string,
+  key: string,
+): Promise<string> {
+  const value = await aws(ctx, [
+    "cloudformation",
+    "describe-stacks",
+    "--stack-name",
+    stack,
+    "--query",
+    `Stacks[0].Outputs[?OutputKey=='${key}'].OutputValue`,
+    "--output",
+    "text",
+  ]);
+  if (value === "" || value === "None")
+    throw new Error(`stack ${stack} has no output ${key} — has it been deployed?`);
+  return value;
+}
+
+export async function stackExists(ctx: AwsCallContext, stack: string): Promise<boolean> {
+  try {
+    await aws(ctx, [
+      "cloudformation",
+      "describe-stacks",
+      "--stack-name",
+      stack,
+      "--output",
+      "text",
+    ]);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export async function secretExists(ctx: AwsCallContext, secretId: string): Promise<boolean> {
+  try {
+    await aws(ctx, [
+      "secretsmanager",
+      "describe-secret",
+      "--secret-id",
+      secretId,
+      "--output",
+      "text",
+    ]);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export async function readSecretString(ctx: AwsCallContext, secretId: string): Promise<string> {
+  return aws(ctx, [
+    "secretsmanager",
+    "get-secret-value",
+    "--secret-id",
+    secretId,
+    "--query",
+    "SecretString",
+    "--output",
+    "text",
+  ]);
+}
+
+export async function readSecretJson<T>(ctx: AwsCallContext, secretId: string): Promise<T> {
+  return JSON.parse(await readSecretString(ctx, secretId)) as T;
+}
+
+/**
+ * Overwrite a secret's value. The JSON never touches argv (visible in `ps` and
+ * shell history): it goes through a 0600 file in a private temp directory that
+ * is removed as soon as the CLI returns. (`file:///dev/stdin` was used before,
+ * but on Linux the CLI cannot open it when stdin is a socket, as under Bun.)
+ */
+export async function putSecretString(
+  ctx: AwsCallContext,
+  secretId: string,
+  value: string,
+): Promise<void> {
+  await throughSecretFile(value, (file) =>
+    awsMutate(ctx, [
+      "secretsmanager",
+      "put-secret-value",
+      "--secret-id",
+      secretId,
+      "--secret-string",
+      `file://${file}`,
+    ]),
+  );
+}
+
+/**
+ * Create a secret that does not exist yet, with the same argv discipline.
+ * Used by the deploy for secrets no stack owns — the license verification
+ * cache, which only exists for an instance that has a license at all.
+ */
+export async function createSecretString(
+  ctx: AwsCallContext,
+  secretId: string,
+  value: string,
+  description: string,
+): Promise<void> {
+  await throughSecretFile(value, (file) =>
+    awsMutate(ctx, [
+      "secretsmanager",
+      "create-secret",
+      "--name",
+      secretId,
+      "--description",
+      description,
+      "--secret-string",
+      `file://${file}`,
+    ]),
+  );
+}
+
+/** Hand a value to the CLI as a 0600 file in a private temp dir, then remove it. */
+async function throughSecretFile(
+  value: string,
+  body: (file: string) => Promise<void>,
+): Promise<void> {
+  const dir = mkdtempSync(join(tmpdir(), "foundation-secret-"));
+  const file = join(dir, "value.json");
+  try {
+    writeFileSync(file, value, { mode: 0o600 });
+    await body(file);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+export async function putSecretJson(
+  ctx: AwsCallContext,
+  secretId: string,
+  value: Record<string, string>,
+): Promise<void> {
+  await putSecretString(ctx, secretId, JSON.stringify(value));
+}
+
+export async function callerAccountId(ctx: AwsCallContext): Promise<string> {
+  return aws(ctx, ["sts", "get-caller-identity", "--query", "Account", "--output", "text"]);
+}

package/src/deploy/cli.ts

@@ -0,0 +1,212 @@
+/**
+ * `foundation-deploy` — every deploy-time command for one Foundation instance.
+ *
+ * One required input, everywhere: `--instance <path>`, the path to the
+ * deployment's instance YAML. There is no instance name to resolve, no
+ * `instances/` directory, no `FOUNDATION_INSTANCE`, and no default: a deploy that
+ * guessed its target would be a deploy into the wrong account.
+ *
+ * The tool deliberately does not import the agent package. It runs in a
+ * pipeline, not in the agent container, and pulling the agent's dependency
+ * tree into a deploy would make every release of the tool carry a model
+ * runtime it never executes.
+ */
+import { type AwsContext, awsContext } from "./aws.ts";
+import { configSync } from "./config-sync.ts";
+import { deploy } from "./deploy.ts";
+import { githubAppCreate } from "./github-app-create.ts";
+import { instanceBanner, loadInstanceContext, resolveInstanceFilePath } from "./instance.ts";
+import { postDeploy } from "./post-deploy.ts";
+import { releaseRequest, resolveRelease } from "./release.ts";
+import { setup } from "./setup.ts";
+import { slackManifestFor } from "./slack-manifest.ts";
+import { stageCustomization } from "./stage-customization.ts";
+
+export const COMMANDS = [
+  "deploy",
+  "post-deploy",
+  "config:sync",
+  "setup",
+  "github-app-create",
+  "slack-manifest",
+  "stage-customization",
+] as const;
+
+export type Command = (typeof COMMANDS)[number];
+
+export const USAGE = `usage: foundation-deploy <command> --instance <path> [options]
+
+commands
+  deploy               build and push the agent image, cdk deploy --all, refresh
+                       the runtime secret, sync config
+  post-deploy          smoke DEFAULT, probe the mount, promote \`live\`, smoke \`live\`
+  config:sync          upload the runtime config and product skills to S3
+  setup                first-deploy orchestration (idempotent)
+  github-app-create    create the instance's GitHub App from the shipped manifest
+  slack-manifest       print the instance's Slack app manifest as JSON
+  stage-customization  validate and stage a tenant-owned runtime config
+
+options
+  --instance <path>    REQUIRED: path to the deployment's instance YAML
+                       (or set FOUNDATION_INSTANCE_FILE)
+  --profile <p>        AWS profile (default: the instance's aws.profile; "" or
+                       FOUNDATION_NO_PROFILE=1 for the default credential chain)
+  --region <r>         AWS region (default: the instance's aws.region)
+  --alarm-email <a>    DLQ alarm subscriber (default: the instance's aws.alarmEmail)
+  --dry-run            print the commands without running them
+  -h, --help           this message
+
+release options (deploy, config:sync)
+  --release <vX.Y.Z>   deploy a published Foundation release: verify its signed
+                       manifest, then deploy the artifacts it names. A copy of
+                       this tool installed from npm defaults to its OWN version;
+                       a Foundation checkout defaults to building locally.
+  --manifest <ref>     that release's manifest, as a path or an s3:// URI
+  --release-bucket <b> where releases live (default: the Foundation bucket, or
+                       FOUNDATION_RELEASE_BUCKET)
+
+deploy options
+  --tag <tag>          image tag to deploy (default: git short sha, +"-dirty");
+                       ignored in release mode, which pins the image by digest
+  --skip-image         do not build/push; deploy the stacks against --tag
+  --phase1             bootstrap deploy: stacks only, no AgentCore runtime
+
+setup options
+  --seed-codex         copy a local Codex credential into the instance's secret
+  --codex-file <path>  where that credential is (default: beside the instance
+                       file, .foundation-local/codex.json)
+
+github-app-create options
+  --org <org>          GitHub org (default: the instance's github.org)
+  --port <n>           local callback port (default 8765)
+  --secret <id>        Secrets Manager id (default: the instance's github/app)`;
+
+function flag(args: readonly string[], name: string): string | undefined {
+  const index = args.indexOf(name);
+  return index === -1 ? undefined : args[index + 1];
+}
+
+/** The AWS context for a command, with the instance file already resolved. */
+function contextFor(args: readonly string[]): AwsContext {
+  const paths = loadInstanceContext(resolveInstanceFilePath(args, process.env));
+  return awsContext({
+    paths,
+    profile: flag(args, "--profile"),
+    region: flag(args, "--region"),
+    dryRun: args.includes("--dry-run"),
+  });
+}
+
+export async function main(argv: readonly string[] = process.argv.slice(2)): Promise<number> {
+  const [command, ...args] = argv;
+  if (command === undefined || command === "--help" || command === "-h") {
+    console.log(USAGE);
+    return command === undefined ? 1 : 0;
+  }
+  if (args.includes("--help") || args.includes("-h")) {
+    console.log(USAGE);
+    return 0;
+  }
+  if (!(COMMANDS as readonly string[]).includes(command)) {
+    console.error(`unknown command: ${command}\n\n${USAGE}`);
+    return 1;
+  }
+
+  // `slack-manifest` writes a machine-readable document to stdout and must not
+  // print a banner into it; every other command announces its target first.
+  if (command === "slack-manifest") {
+    const paths = loadInstanceContext(resolveInstanceFilePath(args, process.env));
+    console.log(
+      JSON.stringify(
+        slackManifestFor(paths.instance, {
+          ...(process.env.EVENTS_REQUEST_URL === undefined
+            ? {}
+            : { EVENTS_REQUEST_URL: process.env.EVENTS_REQUEST_URL }),
+          ...(process.env.COMMANDS_REQUEST_URL === undefined
+            ? {}
+            : { COMMANDS_REQUEST_URL: process.env.COMMANDS_REQUEST_URL }),
+          ...(process.env.INTERACTIVE_REQUEST_URL === undefined
+            ? {}
+            : { INTERACTIVE_REQUEST_URL: process.env.INTERACTIVE_REQUEST_URL }),
+        }),
+      ),
+    );
+    return 0;
+  }
+
+  if (command === "github-app-create") {
+    const paths = loadInstanceContext(resolveInstanceFilePath(args, process.env));
+    console.log(instanceBanner(paths));
+    const port = flag(args, "--port");
+    await githubAppCreate({
+      paths,
+      ...(flag(args, "--org") === undefined ? {} : { org: flag(args, "--org") as string }),
+      ...(port === undefined ? {} : { port: Number(port) }),
+      ...(flag(args, "--secret") === undefined ? {} : { secret: flag(args, "--secret") as string }),
+      ...(flag(args, "--profile") === undefined
+        ? {}
+        : { profile: flag(args, "--profile") as string }),
+      ...(flag(args, "--region") === undefined ? {} : { region: flag(args, "--region") as string }),
+      dryRun: args.includes("--dry-run"),
+    });
+    return 0;
+  }
+
+  const ctx = contextFor(args);
+  console.log(instanceBanner(ctx.paths));
+  if (ctx.dryRun === true) console.log("dry run — no changes will be made");
+
+  switch (command) {
+    case "deploy": {
+      const tag = flag(args, "--tag");
+      const alarmEmail = flag(args, "--alarm-email");
+      const release = releaseRequest(args);
+      await deploy(ctx, {
+        ...(tag === undefined ? {} : { tag }),
+        skipImage: args.includes("--skip-image"),
+        phase1: args.includes("--phase1"),
+        ...(alarmEmail === undefined ? {} : { alarmEmail }),
+        ...(release === undefined ? {} : { release }),
+      });
+      return 0;
+    }
+    case "post-deploy":
+      await postDeploy(ctx);
+      return 0;
+    case "config:sync": {
+      const customization = stageCustomization({ paths: ctx.paths, dryRun: ctx.dryRun });
+      const bucket = flag(args, "--bucket");
+      // Same rule as `deploy`: the product skills are the release's when a
+      // release is what is deployed, and this checkout's otherwise.
+      const request = releaseRequest(args);
+      const release = request === undefined ? undefined : await resolveRelease(ctx, request);
+      const synced = await configSync(ctx, {
+        ...(bucket === undefined ? {} : { bucket }),
+        ...(customization.runtimeConfigPath === undefined
+          ? {}
+          : { runtimeConfigPath: customization.runtimeConfigPath }),
+        ...(release === undefined ? {} : { release }),
+      });
+      console.log(`config + product skills synced to s3://${synced}/`);
+      return 0;
+    }
+    case "setup": {
+      const codexFile = flag(args, "--codex-file");
+      const alarmEmail = flag(args, "--alarm-email");
+      await setup(ctx, {
+        seedCodex: args.includes("--seed-codex"),
+        ...(codexFile === undefined ? {} : { codexFile }),
+        ...(alarmEmail === undefined ? {} : { alarmEmail }),
+      });
+      return 0;
+    }
+    case "stage-customization": {
+      const result = stageCustomization({ paths: ctx.paths, dryRun: ctx.dryRun });
+      console.log(`  ${result.status}: ${result.runtimeConfigPath}`);
+      return 0;
+    }
+    default:
+      console.error(`unhandled command: ${command}\n\n${USAGE}`);
+      return 1;
+  }
+}

package/src/deploy/config-sync.ts

@@ -0,0 +1,93 @@
+/**
+ * Push runtime configuration and product skills to S3.
+ *
+ * Uploads a validated staged tenant config when one exists, otherwise the
+ * instance repository's own config, then mirrors product skills into the data
+ * bucket. Both are read by the agent at session start, so this needs no
+ * redeploy — except for `admins`, which is also baked into the gateway
+ * Lambda's environment.
+ *
+ * Product skills are part of the product, not of the deployment: they come
+ * from the release's `skills.tar.gz` when one is being deployed, and from this
+ * checkout's `skills/` otherwise. A company's own skills are a source in its
+ * runtime config (a GitHub repo), never a directory in this package.
+ */
+import { existsSync, mkdirSync, mkdtempSync, readdirSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { skillsKey } from "../release/manifest.ts";
+import { type AwsContext, awsMutate, stackOutput } from "./aws.ts";
+import { FOUNDATION_ROOT } from "./paths.ts";
+import type { ResolvedRelease } from "./release.ts";
+import { run } from "./sh.ts";
+
+/**
+ * Foundation's product skills directory, or `undefined` when this checkout
+ * ships none. A directory holding only a README is "no skills": `--delete`
+ * below would otherwise empty the bucket prefix.
+ */
+export function productSkillsDir(foundationRoot: string = FOUNDATION_ROOT): string | undefined {
+  const dir = resolve(foundationRoot, "skills");
+  if (!existsSync(dir)) return undefined;
+  const entries = readdirSync(dir).filter((name) => name !== "README.md");
+  return entries.length > 0 ? dir : undefined;
+}
+
+export interface ConfigSyncOptions {
+  bucket?: string;
+  /** The staged tenant config, when `stageCustomization` produced one. */
+  runtimeConfigPath?: string;
+  /** Overridable for tests. */
+  foundationRoot?: string;
+  /** In release mode the skills come out of the release, not this checkout. */
+  release?: ResolvedRelease;
+  /** Test seam for the download-and-unpack above. */
+  releaseSkillsDir?: (ctx: AwsContext, release: ResolvedRelease) => Promise<string>;
+}
+
+export async function configSync(ctx: AwsContext, opts: ConfigSyncOptions = {}): Promise<string> {
+  if (opts.runtimeConfigPath === undefined && ctx.instance.customization !== undefined)
+    throw new Error(
+      "customized instances require a freshly validated runtimeConfigPath; run through `deploy` or `config:sync`",
+    );
+  const source = opts.runtimeConfigPath ?? ctx.paths.configPath;
+  // A dry run has no credentials to read a stack output with, and printing
+  // the placeholder is what every other dry-run path here does.
+  const bucket =
+    opts.bucket ??
+    (ctx.dryRun === true
+      ? `<${ctx.names.data}.BucketName>`
+      : await stackOutput(ctx, ctx.names.data, "BucketName"));
+  await awsMutate(ctx, ["s3", "cp", source, `s3://${bucket}/${ctx.names.configKey}`]);
+  const skills =
+    opts.release === undefined
+      ? productSkillsDir(opts.foundationRoot)
+      : await (opts.releaseSkillsDir ?? unpackReleaseSkills)(ctx, opts.release);
+  if (skills !== undefined)
+    await awsMutate(ctx, ["s3", "sync", `${skills}/`, `s3://${bucket}/skills/`, "--delete"]);
+  return bucket;
+}
+
+/**
+ * The product skills a release carries: `releases/vX.Y.Z/skills.tar.gz`,
+ * fetched and unpacked into a temp directory the sync then mirrors.
+ *
+ * The tarball's digest was checked against the signed manifest before the
+ * deploy began (`verifyRelease`), so these are the release's skills and not
+ * whatever a bucket happens to hold now.
+ */
+export async function unpackReleaseSkills(
+  ctx: AwsContext,
+  release: ResolvedRelease,
+): Promise<string> {
+  const key = release.manifest?.skills.key ?? skillsKey(release.version);
+  const dir = mkdtempSync(join(tmpdir(), "foundation-skills-"));
+  const tarball = join(dir, "skills.tar.gz");
+  const contents = join(dir, "skills");
+  mkdirSync(contents, { recursive: true });
+  // `awsMutate` rather than a plain read, only so a dry run prints the copy
+  // instead of performing it.
+  await awsMutate(ctx, ["s3", "cp", `s3://${release.bucket}/${key}`, tarball]);
+  await run(["tar", "-xzf", tarball, "-C", contents], { dryRun: ctx.dryRun });
+  return contents;
+}

package/src/deploy/config.ts

@@ -0,0 +1,29 @@
+/**
+ * Reading the bits of the instance's runtime config the deploy needs.
+ *
+ * `admins` is baked into the gateway Lambda's environment at deploy time
+ * (`-c admins=…`), so a change to the list needs a redeploy, not just a config
+ * sync. The file is named by the instance file's `config:` key, resolved
+ * against the instance file's own directory — never a path this package knows.
+ */
+import { readFileSync } from "node:fs";
+import { parse } from "yaml";
+
+/** `admins` from a runtime config document, as the CSV the CDK context expects. */
+export function adminsCsv(yamlText: string, source = "runtime config"): string {
+  const doc = parse(yamlText) as { admins?: unknown } | null;
+  const admins = doc?.admins;
+  if (!Array.isArray(admins) || admins.length === 0)
+    throw new Error(`${source}: \`admins\` must be a non-empty list of Slack user ids`);
+  return admins
+    .map((a) => {
+      if (typeof a !== "string" || a.trim() === "")
+        throw new Error(`${source}: admins entry is not a string: ${JSON.stringify(a)}`);
+      return a.trim();
+    })
+    .join(",");
+}
+
+export function adminsCsvFromFile(path: string): string {
+  return adminsCsv(readFileSync(path, "utf8"), path);
+}