@kungfu-tech/buildchain 3.0.5 → 3.0.6-alpha.1

package/scripts/aws-windows-jit-campaign.mjs

@@ -0,0 +1,202 @@
+#!/usr/bin/env node
+// SPDX-License-Identifier: Apache-2.0
+
+import { spawnSync } from "node:child_process";
+import { pathToFileURL } from "node:url";
+import {
+  createWindowsJitCampaignArmPlan,
+  windowsCampaignArmItems,
+  windowsCampaignKillArgs,
+} from "./aws-windows-jit-campaign-core.mjs";
+
+function arg(name, fallback = "") {
+  const index = process.argv.indexOf(`--${name}`);
+  return index === -1 ? fallback : process.argv[index + 1] || "";
+}
+
+function aws(plan, serviceArgs) {
+  const profile = arg("aws-profile");
+  const result = spawnSync(
+    "aws",
+    [
+      ...(profile ? ["--profile", profile] : []),
+      "--region",
+      plan.aws.region,
+      ...serviceArgs,
+    ],
+    { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 },
+  );
+  if (result.status !== 0) {
+    const detail = String(result.stderr || result.stdout || "")
+      .trim()
+      .slice(0, 2000);
+    throw new Error(
+      `AWS campaign mutation failed${detail ? `: ${detail}` : ""}`,
+    );
+  }
+  return result.stdout ? JSON.parse(result.stdout) : {};
+}
+
+function armPlan() {
+  return createWindowsJitCampaignArmPlan({
+    campaignId: arg("campaign-id"),
+    sourceSha: arg("source-sha"),
+    stateTable: arg("state-table"),
+    region: arg("region", "us-east-1"),
+    armedAt: arg("armed-at", new Date().toISOString()),
+    expiresAt: arg("expires-at"),
+    phaseSpendBaselineUsd: arg("phase-spend-baseline-usd"),
+    maxAcceptedInstances: arg("max-accepted-instances"),
+  });
+}
+
+function confirm(
+  plan,
+  { phaseSpendBaseline = true, maxAcceptedInstances = true } = {},
+) {
+  if (arg("confirm-campaign-id") !== plan.campaign.id) {
+    throw new Error("--confirm-campaign-id must equal the campaign id");
+  }
+  if (arg("confirm-source-sha") !== plan.source.sha) {
+    throw new Error("--confirm-source-sha must equal the exact source SHA");
+  }
+  if (arg("confirm-state-table") !== plan.aws.stateTable) {
+    throw new Error(
+      "--confirm-state-table must equal the campaign state table",
+    );
+  }
+  if (
+    phaseSpendBaseline &&
+    (!arg("confirm-phase-spend-baseline-usd").trim() ||
+      Number(arg("confirm-phase-spend-baseline-usd")) !==
+        plan.limits.phaseSpendBaselineUsd)
+  ) {
+    throw new Error(
+      "--confirm-phase-spend-baseline-usd must equal the phase spend baseline",
+    );
+  }
+  if (
+    maxAcceptedInstances &&
+    (!arg("confirm-max-accepted-instances").trim() ||
+      Number(arg("confirm-max-accepted-instances")) !==
+        plan.limits.maxAcceptedInstances)
+  ) {
+    throw new Error(
+      "--confirm-max-accepted-instances must equal the campaign slot ceiling",
+    );
+  }
+}
+
+function killSwitchTopic() {
+  const topic = arg("kill-switch-topic");
+  if (
+    !/^arn:aws:sns:us-east-1:\d{12}:kungfu-buildchain-windows-jit-[A-Za-z0-9_-]+$/.test(
+      topic,
+    )
+  ) {
+    throw new Error(
+      "--kill-switch-topic must be the dedicated Windows JIT SNS ARN",
+    );
+  }
+  if (arg("confirm-kill-switch-topic") !== topic) {
+    throw new Error(
+      "--confirm-kill-switch-topic must equal the dedicated kill-switch topic",
+    );
+  }
+  return topic;
+}
+
+export function main() {
+  const mode = process.argv[2] || "plan-arm";
+  if (["plan-arm", "arm-campaign"].includes(mode)) {
+    const plan = armPlan();
+    if (mode === "plan-arm") {
+      process.stdout.write(`${JSON.stringify(plan, null, 2)}\n`);
+      return plan;
+    }
+    confirm(plan);
+    aws(plan, [
+      "dynamodb",
+      "transact-write-items",
+      "--transact-items",
+      JSON.stringify(windowsCampaignArmItems(plan)),
+      "--output",
+      "json",
+    ]);
+    const result = {
+      contract: plan.contract,
+      kind: "campaign-arm-result",
+      status: "armed",
+      campaign: plan.campaign,
+      source: plan.source,
+      aws: plan.aws,
+      limits: plan.limits,
+    };
+    process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+    return result;
+  }
+  if (mode === "kill-campaign") {
+    const now = new Date();
+    const plan = createWindowsJitCampaignArmPlan({
+      campaignId: arg("campaign-id"),
+      sourceSha: arg("source-sha"),
+      stateTable: arg("state-table"),
+      region: arg("region", "us-east-1"),
+      armedAt: now.toISOString(),
+      expiresAt: new Date(now.getTime() + 1000).toISOString(),
+      phaseSpendBaselineUsd: 0,
+    });
+    confirm(plan, {
+      phaseSpendBaseline: false,
+      maxAcceptedInstances: false,
+    });
+    const topic = killSwitchTopic();
+    aws(
+      plan,
+      windowsCampaignKillArgs(
+        plan.aws.stateTable,
+        arg("reason", "operator-kill"),
+        new Date().toISOString(),
+      ),
+    );
+    const notification = aws(plan, [
+      "sns",
+      "publish",
+      "--topic-arn",
+      topic,
+      "--message",
+      JSON.stringify({
+        contract: plan.contract,
+        action: "kill-campaign",
+        campaignId: plan.campaign.id,
+        sourceSha: plan.source.sha,
+      }),
+      "--output",
+      "json",
+    ]);
+    const result = {
+      contract: plan.contract,
+      kind: "campaign-kill-result",
+      status: "killed",
+      campaign: plan.campaign,
+      source: plan.source,
+      killSwitchTopic: topic,
+      notificationMessageId: notification.MessageId || "",
+    };
+    process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+    return result;
+  }
+  throw new Error(`unsupported Windows JIT campaign mode: ${mode}`);
+}
+
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(process.argv[1]).href
+) {
+  try {
+    main();
+  } catch (error) {
+    console.error(`::error::${error.message || error}`);
+    process.exitCode = 1;
+  }
+}

package/scripts/aws-windows-jit-controller-core.mjs

@@ -3,12 +3,13 @@
 import { digest } from "./aws-runner-burst-core.mjs";
 import {
   WINDOWS_EC2_JIT,
+  windowsJitCampaignId,
   windowsJitRunnerLabel,
   windowsJitRunnerLabels,
 } from "./aws-windows-jit-core.mjs";
 
 export const AWS_WINDOWS_JIT_CONTROLLER_CONTRACT =
-  "kungfu-buildchain-aws-windows-jit-controller/v1";
+  "kungfu-buildchain-aws-windows-jit-controller/v2";
 
 function exact(value, pattern, label) {
   const normalized = String(value || "").trim();
@@ -59,8 +60,9 @@ export function createWindowsJitLaunchPlan(values = {}) {
   );
   const jobId = exact(values.jobId, /^\d+$/, "jobId");
   const qualification = qualificationId(values.qualificationId);
+  const campaign = windowsJitCampaignId(values.campaignId);
   const runnerLabel = windowsJitRunnerLabel(values.runnerLabel);
-  const expectedLabel = `${WINDOWS_EC2_JIT.labelPrefix}${qualification}`;
+  const expectedLabel = `${WINDOWS_EC2_JIT.labelPrefix}${campaign}-${qualification}`;
   if (runnerLabel !== expectedLabel) {
     throw new Error(`runnerLabel must be ${expectedLabel}`);
   }
@@ -109,6 +111,11 @@ export function createWindowsJitLaunchPlan(values = {}) {
     /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/,
     "evidenceBucket",
   );
+  const stateTable = exact(
+    values.stateTable,
+    /^kungfu-buildchain-windows-jit(?:-[A-Za-z0-9_.-]+)?$/,
+    "stateTable",
+  );
   const launchedAt = iso(values.launchedAt, "launchedAt");
   const clientToken = `kungfu-${runId}-${runAttempt}-${qualification}`;
   const jitParameterName =
@@ -125,6 +132,7 @@ export function createWindowsJitLaunchPlan(values = {}) {
     tag("kungfu:owner", "buildchain"),
     tag("kungfu:plane", "aws-us-elastic-runner-burst"),
     tag("kungfu:provider", "windows-ec2-jit"),
+    tag("kungfu:campaign-id", campaign),
     tag("kungfu:github-run-id", runId),
     tag("kungfu:github-run-attempt", runAttempt),
     tag("kungfu:qualification-id", qualification),
@@ -141,6 +149,7 @@ export function createWindowsJitLaunchPlan(values = {}) {
     contract: AWS_WINDOWS_JIT_CONTROLLER_CONTRACT,
     kind: "launch-plan",
     repository,
+    campaign: { id: campaign },
     source: { sha: sourceSha, ref: sourceRef },
     github: {
       runId,
@@ -148,6 +157,7 @@ export function createWindowsJitLaunchPlan(values = {}) {
       jobId,
       qualificationId: qualification,
       event: "workflow_dispatch",
+      displayTitle: `AWS Windows JIT ${campaign} ${qualification}`,
     },
     runner: {
       name: runnerName,
@@ -164,6 +174,7 @@ export function createWindowsJitLaunchPlan(values = {}) {
       securityGroupId,
       instanceProfileName,
       evidenceBucket,
+      stateTable,
       jitParameterName,
       launchedAt,
       clientToken,
@@ -188,6 +199,13 @@ export function createWindowsJitLaunchPlan(values = {}) {
       exactSourceRequired: true,
       queuedJobRequired: true,
       activeInstanceCeiling: WINDOWS_EC2_JIT.maxConcurrentInstances,
+      campaignAcceptedInstanceCeiling: WINDOWS_EC2_JIT.maxAcceptedInstances,
+      campaignReservationUsd:
+        (WINDOWS_EC2_JIT.pricePerHourUsd *
+          WINDOWS_EC2_JIT.maximumInstanceLifetimeMinutes) /
+        60,
+      campaignBudgetLimitUsd: WINDOWS_EC2_JIT.budgetLimitUsd,
+      persistentCampaignLedgerRequired: true,
       awsDryRunRequiredBeforeLaunch: true,
       userDataTransport: "fileb://rendered-bootstrap",
       jitConfigTransport: "0600-temporary-file-to-ssm-secure-string",

package/scripts/aws-windows-jit-controller.mjs

@@ -11,6 +11,10 @@ import {
   createWindowsJitLaunchPlan,
   windowsRunInstancesArgs,
 } from "./aws-windows-jit-controller-core.mjs";
+import {
+  windowsCampaignMarkLaunchedArgs,
+  windowsCampaignReservationItems,
+} from "./aws-windows-jit-campaign-core.mjs";
 import { renderWindowsJitBootstrap } from "./aws-windows-jit-core.mjs";
 
 function arg(name, fallback = "") {
@@ -75,6 +79,7 @@ function assertLivePreflight(plan, profile) {
   );
   if (
     run.event !== plan.github.event ||
+    run.display_title !== plan.github.displayTitle ||
     run.head_sha !== plan.source.sha ||
     run.head_repository?.full_name !== plan.repository ||
     !["queued", "in_progress"].includes(run.status)
@@ -159,6 +164,7 @@ function assertLivePreflight(plan, profile) {
     throw new Error("Windows AMI identity or availability mismatch");
   }
   return {
+    budgetGuard: jsonResult(commandResult("/bin/bash", ["scripts/aws-windows-jit-operator.sh", "launch-gate", "--region", plan.aws.region, ...(profile ? ["--aws-profile", profile] : [])]), "provider Budget launch gate"),
     runStatus: run.status,
     jobStatus: job.status,
     activeInstances: activeInstances.length,
@@ -267,6 +273,169 @@ function cleanupFailedLaunch(
   return failures;
 }
 
+function generateJitConfiguration(plan) {
+  const jit = ghJson(
+    [
+      "api",
+      "--method",
+      "POST",
+      `repos/${plan.repository}/actions/runners/generate-jitconfig`,
+      "--input",
+      "-",
+    ],
+    JSON.stringify({
+      name: plan.runner.name,
+      runner_group_id: 1,
+      labels: plan.runner.labels,
+      work_folder: "_work",
+    }),
+    "GitHub JIT configuration",
+  ).encoded_jit_config;
+  if (!jit || typeof jit !== "string") {
+    throw new Error("GitHub JIT configuration was empty");
+  }
+  return jit;
+}
+
+function createJitParameter(plan, profile, parameterInputPath, jit) {
+  fs.writeFileSync(
+    parameterInputPath,
+    JSON.stringify({
+      Name: plan.aws.jitParameterName,
+      Description: `One-shot GitHub Actions JIT config for ${plan.repository} run ${plan.github.runId} attempt ${plan.github.runAttempt}`,
+      Type: "SecureString",
+      Tier: "Advanced",
+      Value: jit,
+      Tags: [
+        { Key: "kungfu:owner", Value: "buildchain" },
+        { Key: "kungfu:plane", Value: "aws-us-elastic-runner-burst" },
+        { Key: "kungfu:provider", Value: "windows-ec2-jit" },
+        { Key: "kungfu:campaign-id", Value: plan.campaign.id },
+        { Key: "kungfu:github-run-id", Value: plan.github.runId },
+        {
+          Key: "kungfu:qualification-id",
+          Value: plan.github.qualificationId,
+        },
+      ],
+    }),
+    { mode: 0o600 },
+  );
+  requireSuccess(
+    commandResult(
+      "aws",
+      awsArgs(plan, profile, [
+        "ssm",
+        "put-parameter",
+        "--cli-input-json",
+        `file://${parameterInputPath}`,
+        "--output",
+        "json",
+      ]),
+    ),
+    "SSM JIT parameter creation",
+  );
+}
+
+function writeBootstrap(plan, bootstrapPath) {
+  const template = fs.readFileSync(
+    path.resolve(
+      "infra/aws-us-elastic-runner-burst-plane/windows-jit-bootstrap.ps1",
+    ),
+    "utf8",
+  );
+  fs.writeFileSync(
+    bootstrapPath,
+    renderWindowsJitBootstrap(template, {
+      region: plan.aws.region,
+      campaignId: plan.campaign.id,
+      jitParameterName: plan.aws.jitParameterName,
+      evidenceBucket: plan.aws.evidenceBucket,
+      runnerLabel: plan.runner.label,
+      sourceSha: plan.source.sha,
+      githubRunId: plan.github.runId,
+      githubRunAttempt: plan.github.runAttempt,
+      amiId: plan.aws.amiId,
+      amiName: plan.aws.amiName,
+      instanceType: plan.aws.instanceType,
+      launchedAt: plan.aws.launchedAt,
+    }),
+    { mode: 0o600 },
+  );
+}
+
+function assertLaunchDryRun(plan, profile, bootstrapPath) {
+  const dryRun = commandResult(
+    "aws",
+    awsArgs(
+      plan,
+      profile,
+      windowsRunInstancesArgs(plan, { bootstrapPath, dryRun: true }),
+    ),
+  );
+  if (
+    dryRun.status === 0 ||
+    !/DryRunOperation/.test(`${dryRun.stdout}\n${dryRun.stderr}`)
+  ) {
+    throw new Error("EC2 RunInstances DryRun did not return DryRunOperation");
+  }
+}
+
+function reserveCampaign(plan, profile) {
+  requireSuccess(
+    commandResult(
+      "aws",
+      awsArgs(plan, profile, [
+        "dynamodb",
+        "transact-write-items",
+        "--transact-items",
+        JSON.stringify(
+          windowsCampaignReservationItems(plan, new Date().toISOString()),
+        ),
+        "--output",
+        "json",
+      ]),
+    ),
+    "Windows campaign atomic reservation",
+  );
+}
+
+function launchInstance(plan, profile, bootstrapPath) {
+  const launched = jsonResult(
+    commandResult(
+      "aws",
+      awsArgs(
+        plan,
+        profile,
+        windowsRunInstancesArgs(plan, { bootstrapPath, dryRun: false }),
+      ),
+    ),
+    "EC2 RunInstances",
+  );
+  const instance = launched.Instances?.[0];
+  if (!/^i-[0-9a-f]+$/.test(String(instance?.InstanceId || ""))) {
+    throw new Error("EC2 RunInstances returned no instance identity");
+  }
+  return instance;
+}
+
+function markCampaignLaunched(plan, profile, instanceId) {
+  requireSuccess(
+    commandResult(
+      "aws",
+      awsArgs(
+        plan,
+        profile,
+        windowsCampaignMarkLaunchedArgs(
+          plan,
+          instanceId,
+          new Date().toISOString(),
+        ),
+      ),
+    ),
+    "Windows campaign launch ledger update",
+  );
+}
+
 export function executeWindowsJitLaunch(plan, { profile = "" } = {}) {
   if (plan?.contract !== AWS_WINDOWS_JIT_CONTROLLER_CONTRACT) {
     throw new Error("Windows JIT launch plan contract is invalid");
@@ -285,124 +454,23 @@ export function executeWindowsJitLaunch(plan, { profile = "" } = {}) {
   let failure;
   const cleanupFailures = [];
   try {
-    const jit = ghJson(
-      [
-        "api",
-        "--method",
-        "POST",
-        `repos/${plan.repository}/actions/runners/generate-jitconfig`,
-        "--input",
-        "-",
-      ],
-      JSON.stringify({
-        name: plan.runner.name,
-        runner_group_id: 1,
-        labels: plan.runner.labels,
-        work_folder: "_work",
-      }),
-      "GitHub JIT configuration",
-    ).encoded_jit_config;
-    if (!jit || typeof jit !== "string") {
-      throw new Error("GitHub JIT configuration was empty");
-    }
-    fs.writeFileSync(
-      parameterInputPath,
-      JSON.stringify({
-        Name: plan.aws.jitParameterName,
-        Description: `One-shot GitHub Actions JIT config for ${plan.repository} run ${plan.github.runId} attempt ${plan.github.runAttempt}`,
-        Type: "SecureString",
-        Tier: "Advanced",
-        Value: jit,
-        Tags: [
-          { Key: "kungfu:owner", Value: "buildchain" },
-          { Key: "kungfu:plane", Value: "aws-us-elastic-runner-burst" },
-          { Key: "kungfu:provider", Value: "windows-ec2-jit" },
-          { Key: "kungfu:github-run-id", Value: plan.github.runId },
-          {
-            Key: "kungfu:qualification-id",
-            Value: plan.github.qualificationId,
-          },
-        ],
-      }),
-      { mode: 0o600 },
-    );
-    requireSuccess(
-      commandResult(
-        "aws",
-        awsArgs(plan, profile, [
-          "ssm",
-          "put-parameter",
-          "--cli-input-json",
-          `file://${parameterInputPath}`,
-          "--output",
-          "json",
-        ]),
-      ),
-      "SSM JIT parameter creation",
-    );
+    const jit = generateJitConfiguration(plan);
+    createJitParameter(plan, profile, parameterInputPath, jit);
     parameterCreated = true;
-    fs.writeFileSync(
-      bootstrapPath,
-      renderWindowsJitBootstrap(
-        fs.readFileSync(
-          path.resolve(
-            "infra/aws-us-elastic-runner-burst-plane/windows-jit-bootstrap.ps1",
-          ),
-          "utf8",
-        ),
-        {
-          region: plan.aws.region,
-          jitParameterName: plan.aws.jitParameterName,
-          evidenceBucket: plan.aws.evidenceBucket,
-          runnerLabel: plan.runner.label,
-          sourceSha: plan.source.sha,
-          githubRunId: plan.github.runId,
-          githubRunAttempt: plan.github.runAttempt,
-          amiId: plan.aws.amiId,
-          amiName: plan.aws.amiName,
-          instanceType: plan.aws.instanceType,
-          launchedAt: plan.aws.launchedAt,
-        },
-      ),
-      { mode: 0o600 },
-    );
-    const dryRun = commandResult(
-      "aws",
-      awsArgs(
-        plan,
-        profile,
-        windowsRunInstancesArgs(plan, { bootstrapPath, dryRun: true }),
-      ),
-    );
-    if (
-      dryRun.status === 0 ||
-      !/DryRunOperation/.test(`${dryRun.stdout}\n${dryRun.stderr}`)
-    ) {
-      throw new Error("EC2 RunInstances DryRun did not return DryRunOperation");
-    }
+    writeBootstrap(plan, bootstrapPath);
+    assertLaunchDryRun(plan, profile, bootstrapPath);
+    reserveCampaign(plan, profile);
     launchAttempted = true;
-    const launched = jsonResult(
-      commandResult(
-        "aws",
-        awsArgs(
-          plan,
-          profile,
-          windowsRunInstancesArgs(plan, { bootstrapPath, dryRun: false }),
-        ),
-      ),
-      "EC2 RunInstances",
-    );
-    const instance = launched.Instances?.[0];
-    if (!/^i-[0-9a-f]+$/.test(String(instance?.InstanceId || ""))) {
-      throw new Error("EC2 RunInstances returned no instance identity");
-    }
+    const instance = launchInstance(plan, profile, bootstrapPath);
     launchSucceeded = true;
+    markCampaignLaunched(plan, profile, instance.InstanceId);
     result = {
       schemaVersion: 1,
       contract: AWS_WINDOWS_JIT_CONTROLLER_CONTRACT,
       kind: "launch-result",
       status: "launched",
       source: plan.source,
+      campaign: plan.campaign,
       github: plan.github,
       runner: plan.runner,
       aws: {
@@ -452,6 +520,7 @@ function planFromArgs(execute) {
     runAttempt: arg("run-attempt", "1"),
     jobId: arg("job-id"),
     qualificationId: arg("qualification-id"),
+    campaignId: arg("campaign-id"),
     runnerLabel: arg("runner-label"),
     runnerName: arg("runner-name"),
     sourceSha: arg("source-sha"),
@@ -464,6 +533,7 @@ function planFromArgs(execute) {
     securityGroupId: arg("security-group-id"),
     instanceProfileName: arg("instance-profile-name"),
     evidenceBucket: arg("evidence-bucket"),
+    stateTable: arg("state-table"),
     jitParameterName: arg("jit-parameter"),
     launchedAt: arg("launched-at", new Date().toISOString()),
   });
@@ -482,6 +552,12 @@ export function main() {
   if (arg("confirm-run-id") !== plan.github.runId) {
     throw new Error("--confirm-run-id must equal the exact GitHub run id");
   }
+  if (arg("confirm-campaign-id") !== plan.campaign.id) {
+    throw new Error("--confirm-campaign-id must equal the campaign id");
+  }
+  if (arg("confirm-state-table") !== plan.aws.stateTable) {
+    throw new Error("--confirm-state-table must equal the campaign state table");
+  }
   const result = executeWindowsJitLaunch(plan, {
     profile: arg("aws-profile"),
   });