@kungfu-tech/buildchain 3.0.5 → 3.0.6-alpha.0

package/scripts/aws-macos-jit-job-controller.mjs

@@ -0,0 +1,401 @@
+// SPDX-License-Identifier: Apache-2.0
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { AWS_MACOS_JIT_CONTROLLER_CONTRACT } from "./aws-macos-jit-controller-core.mjs";
+import {
+  assertOwnership,
+  awsArgs,
+  commandResult,
+  ghJson,
+  jsonResult,
+  requireSuccess,
+} from "./aws-macos-jit-controller-runtime.mjs";
+import { renderMacosJitBootstrap } from "./aws-macos-jit-core.mjs";
+
+function assertExactJobPreflight(plan, profile) {
+  const run = ghJson(
+    ["api", `repos/${plan.repository}/actions/runs/${plan.github.runId}`],
+    undefined,
+    "GitHub run preflight",
+  );
+  if (
+    run.event !== plan.github.event ||
+    run.head_sha !== plan.source.sha ||
+    run.head_repository?.full_name !== plan.repository ||
+    !["queued", "in_progress"].includes(run.status)
+  ) {
+    throw new Error("GitHub run is not the trusted exact-source dispatch");
+  }
+  const job = ghJson(
+    ["api", `repos/${plan.repository}/actions/jobs/${plan.github.jobId}`],
+    undefined,
+    "GitHub job preflight",
+  );
+  if (job.status !== "queued") {
+    throw new Error(
+      `GitHub job must be queued, observed ${job.status || "unknown"}`,
+    );
+  }
+  if (
+    !Array.isArray(job.labels) ||
+    !plan.runner.labels.every((label) => job.labels.includes(label))
+  ) {
+    throw new Error("GitHub job labels do not match the macOS JIT job plan");
+  }
+  const host = jsonResult(
+    commandResult(
+      "aws",
+      awsArgs(plan, profile, [
+        "ec2",
+        "describe-hosts",
+        "--host-ids",
+        plan.aws.hostId,
+        "--output",
+        "json",
+      ]),
+    ),
+    "macOS campaign host preflight",
+  ).Hosts?.[0];
+  if (
+    !host ||
+    host.State !== "available" ||
+    host.AvailabilityZone !== plan.aws.availabilityZone ||
+    host.HostProperties?.InstanceType !== plan.aws.instanceType
+  ) {
+    throw new Error("macOS campaign host identity or state mismatch");
+  }
+  assertOwnership(host.Tags, plan);
+  const instance = jsonResult(
+    commandResult(
+      "aws",
+      awsArgs(plan, profile, [
+        "ec2",
+        "describe-instances",
+        "--instance-ids",
+        plan.aws.instanceId,
+        "--output",
+        "json",
+      ]),
+    ),
+    "macOS campaign instance preflight",
+  ).Reservations?.[0]?.Instances?.[0];
+  if (
+    !instance ||
+    instance.State?.Name !== "running" ||
+    instance.Placement?.HostId !== plan.aws.hostId ||
+    instance.ImageId !== plan.aws.amiId ||
+    instance.InstanceType !== plan.aws.instanceType
+  ) {
+    throw new Error("macOS campaign instance identity or state mismatch");
+  }
+  assertOwnership(instance.Tags, plan);
+  const parameters = jsonResult(
+    commandResult(
+      "aws",
+      awsArgs(plan, profile, [
+        "ssm",
+        "describe-parameters",
+        "--parameter-filters",
+        `Key=Name,Option=Equals,Values=${plan.aws.jitParameterName}`,
+        "--output",
+        "json",
+      ]),
+    ),
+    "macOS JIT parameter preflight",
+  );
+  if ((parameters.Parameters || []).length !== 0) {
+    throw new Error("macOS JIT parameter already exists");
+  }
+  return {
+    runStatus: run.status,
+    jobStatus: job.status,
+    hostState: host.State,
+    instanceState: instance.State.Name,
+  };
+}
+
+function deleteRunnerRegistration(plan) {
+  const runners = ghJson(
+    ["api", `repos/${plan.repository}/actions/runners?per_page=100`],
+    undefined,
+    "GitHub runner cleanup lookup",
+  ).runners;
+  const runner = (runners || []).find(
+    (entry) => entry.name === plan.runner.name,
+  );
+  if (!runner) return false;
+  requireSuccess(
+    commandResult("gh", [
+      "api",
+      "--method",
+      "DELETE",
+      `repos/${plan.repository}/actions/runners/${runner.id}`,
+    ]),
+    "GitHub runner cleanup",
+  );
+  return true;
+}
+
+function waitForSsmOnline(plan, profile, timeoutMs = 15 * 60 * 1000) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    const response = jsonResult(
+      commandResult(
+        "aws",
+        awsArgs(plan, profile, [
+          "ssm",
+          "describe-instance-information",
+          "--filters",
+          `Key=InstanceIds,Values=${plan.aws.instanceId}`,
+          "--output",
+          "json",
+        ]),
+      ),
+      "macOS SSM online preflight",
+    );
+    const found = (response.InstanceInformationList || []).find(
+      (entry) =>
+        entry.InstanceId === plan.aws.instanceId &&
+        entry.PingStatus === "Online",
+    );
+    if (found) return found;
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5000);
+  }
+  throw new Error("macOS campaign instance did not become SSM Online in time");
+}
+
+function cleanupFailedCommand(plan, profile, parameterCreated) {
+  const failures = [];
+  const attempt = (cleanup) => {
+    try {
+      cleanup();
+    } catch (error) {
+      failures.push(error.message || String(error));
+    }
+  };
+  if (parameterCreated) {
+    attempt(() =>
+      requireSuccess(
+        commandResult(
+          "aws",
+          awsArgs(plan, profile, [
+            "ssm",
+            "delete-parameter",
+            "--name",
+            plan.aws.jitParameterName,
+          ]),
+        ),
+        "macOS JIT parameter cleanup",
+      ),
+    );
+  }
+  attempt(() => deleteRunnerRegistration(plan));
+  return failures;
+}
+
+function createJitParameter(plan, profile, parameterInputPath) {
+  const jit = ghJson(
+    [
+      "api",
+      "--method",
+      "POST",
+      `repos/${plan.repository}/actions/runners/generate-jitconfig`,
+      "--input",
+      "-",
+    ],
+    JSON.stringify({
+      name: plan.runner.name,
+      runner_group_id: 1,
+      labels: plan.runner.labels,
+      work_folder: "_work",
+    }),
+    "GitHub JIT configuration",
+  ).encoded_jit_config;
+  if (!jit || typeof jit !== "string") {
+    throw new Error("GitHub JIT configuration was empty");
+  }
+  fs.writeFileSync(
+    parameterInputPath,
+    JSON.stringify({
+      Name: plan.aws.jitParameterName,
+      Description: `One-shot macOS GitHub Actions JIT config for ${plan.repository} run ${plan.github.runId}`,
+      Type: "SecureString",
+      Tier: "Advanced",
+      Value: jit,
+      Tags: [
+        { Key: "kungfu:owner", Value: "buildchain" },
+        { Key: "kungfu:plane", Value: "aws-us-elastic-runner-burst" },
+        { Key: "kungfu:provider", Value: "macos-ec2-jit" },
+        { Key: "kungfu:campaign-id", Value: plan.campaign.id },
+        { Key: "kungfu:github-run-id", Value: plan.github.runId },
+        { Key: "kungfu:qualification-id", Value: plan.github.qualificationId },
+      ],
+    }),
+    { mode: 0o600 },
+  );
+  requireSuccess(
+    commandResult(
+      "aws",
+      awsArgs(plan, profile, [
+        "ssm",
+        "put-parameter",
+        "--cli-input-json",
+        `file://${parameterInputPath}`,
+        "--output",
+        "json",
+      ]),
+    ),
+    "macOS JIT parameter creation",
+  );
+}
+
+function sendBootstrap(plan, profile, { bootstrapPath, commandInputPath }) {
+  fs.writeFileSync(
+    bootstrapPath,
+    renderMacosJitBootstrap(
+      fs.readFileSync(
+        path.resolve(
+          "infra/aws-us-elastic-runner-burst-plane/macos-jit-bootstrap.sh",
+        ),
+        "utf8",
+      ),
+      {
+        region: plan.aws.region,
+        jitParameterName: plan.aws.jitParameterName,
+        evidenceBucket: plan.aws.evidenceBucket,
+        runnerLabel: plan.runner.label,
+        sourceSha: plan.source.sha,
+        githubRunId: plan.github.runId,
+        githubRunAttempt: plan.github.runAttempt,
+        amiId: plan.aws.amiId,
+        amiName: plan.aws.amiName,
+        hostId: plan.aws.hostId,
+        instanceType: plan.aws.instanceType,
+        hostAllocatedAt: plan.aws.hostAllocatedAt,
+      },
+    ),
+    { mode: 0o600 },
+  );
+  const ssm = waitForSsmOnline(plan, profile);
+  const remotePath = `/private/tmp/kungfu-macos-jit-${plan.github.runId}-${plan.github.qualificationId}.sh`;
+  const bootstrapBase64 = fs.readFileSync(bootstrapPath).toString("base64");
+  fs.writeFileSync(
+    commandInputPath,
+    JSON.stringify({
+      DocumentName: "AWS-RunShellScript",
+      InstanceIds: [plan.aws.instanceId],
+      Comment: `kungfu macOS JIT ${plan.campaign.id} ${plan.github.qualificationId}`,
+      TimeoutSeconds: 10800,
+      Parameters: {
+        commands: [
+          `printf '%s' '${bootstrapBase64}' | base64 --decode > '${remotePath}'`,
+          `chmod 700 '${remotePath}'`,
+          `sudo /bin/bash '${remotePath}'`,
+          `status=$?; rm -f '${remotePath}'; exit $status`,
+        ],
+        executionTimeout: ["10800"],
+      },
+    }),
+    { mode: 0o600 },
+  );
+  const sent = jsonResult(
+    commandResult(
+      "aws",
+      awsArgs(plan, profile, [
+        "ssm",
+        "send-command",
+        "--cli-input-json",
+        `file://${commandInputPath}`,
+        "--output",
+        "json",
+      ]),
+    ),
+    "macOS JIT SSM command",
+  );
+  const commandId = sent.Command?.CommandId;
+  if (!/^[0-9a-f-]{36}$/.test(String(commandId || ""))) {
+    throw new Error("SSM SendCommand returned no command identity");
+  }
+  return { commandId, ssmAgentVersion: ssm.AgentVersion };
+}
+
+function dispatchMacosJitJob(plan, profile, files, state) {
+  createJitParameter(plan, profile, files.parameterInputPath);
+  state.parameterCreated = true;
+  requireSuccess(
+    commandResult(
+      "aws",
+      awsArgs(plan, profile, [
+        "ec2",
+        "create-tags",
+        "--resources",
+        plan.aws.instanceId,
+        "--tags",
+        `Key=kungfu:jit-parameter,Value=${plan.aws.jitParameterName}`,
+      ]),
+    ),
+    "macOS campaign instance JIT tag update",
+  );
+  return sendBootstrap(plan, profile, files);
+}
+
+export function executeMacosJitJob(plan, { profile = "" } = {}) {
+  if (
+    plan?.contract !== AWS_MACOS_JIT_CONTROLLER_CONTRACT ||
+    plan.kind !== "job-run-plan"
+  ) {
+    throw new Error("macOS JIT job plan contract is invalid");
+  }
+  const preflight = assertExactJobPreflight(plan, profile);
+  const tempRoot = fs.mkdtempSync(
+    path.join(os.tmpdir(), "buildchain-macos-jit-controller-"),
+  );
+  fs.chmodSync(tempRoot, 0o700);
+  const files = {
+    parameterInputPath: path.join(tempRoot, "ssm-parameter.json"),
+    bootstrapPath: path.join(tempRoot, "bootstrap.sh"),
+    commandInputPath: path.join(tempRoot, "ssm-command.json"),
+  };
+  const state = { parameterCreated: false };
+  try {
+    const sent = dispatchMacosJitJob(plan, profile, files, state);
+    return {
+      schemaVersion: 1,
+      contract: AWS_MACOS_JIT_CONTROLLER_CONTRACT,
+      kind: "job-command-result",
+      status: "command-sent",
+      campaign: plan.campaign,
+      source: plan.source,
+      github: plan.github,
+      runner: plan.runner,
+      aws: {
+        region: plan.aws.region,
+        hostId: plan.aws.hostId,
+        instanceId: plan.aws.instanceId,
+        ...sent,
+        jitParameterName: plan.aws.jitParameterName,
+      },
+      preflight,
+      planDigest: plan.digest,
+    };
+  } catch (error) {
+    const failures = cleanupFailedCommand(
+      plan,
+      profile,
+      state.parameterCreated,
+    );
+    if (failures.length) {
+      throw new Error(
+        `${error.message || error}; cleanup failed: ${failures.join("; ")}`,
+      );
+    }
+    throw error;
+  } finally {
+    for (const file of Object.values(files)) {
+      if (fs.existsSync(file)) fs.unlinkSync(file);
+    }
+    fs.rmdirSync(tempRoot);
+  }
+}

package/scripts/aws-windows-jit-campaign-core.mjs

@@ -0,0 +1,296 @@
+// SPDX-License-Identifier: Apache-2.0
+
+import {
+  WINDOWS_EC2_JIT,
+  windowsJitCampaignId,
+} from "./aws-windows-jit-core.mjs";
+
+export const AWS_WINDOWS_JIT_CAMPAIGN_CONTRACT =
+  "kungfu-buildchain-aws-windows-jit-campaign/v1";
+
+function exact(value, pattern, label) {
+  const normalized = String(value || "").trim();
+  if (!pattern.test(normalized)) throw new Error(`${label} is invalid`);
+  return normalized;
+}
+
+function exactSha(value) {
+  return exact(value, /^[0-9a-f]{40}$/i, "sourceSha").toLowerCase();
+}
+
+function epoch(value, label) {
+  const milliseconds = new Date(value || "").getTime();
+  if (!Number.isFinite(milliseconds)) {
+    throw new Error(`${label} must be an ISO timestamp`);
+  }
+  return Math.floor(milliseconds / 1000);
+}
+
+function tableName(value) {
+  return exact(
+    value,
+    /^kungfu-buildchain-windows-jit(?:-[A-Za-z0-9_.-]+)?$/,
+    "stateTable",
+  );
+}
+
+function number(value) {
+  return { N: String(value) };
+}
+
+function money(value, label) {
+  const normalized = String(value ?? "").trim();
+  if (!normalized) {
+    throw new Error(`${label} is required`);
+  }
+  const parsed = Number(normalized);
+  if (!Number.isFinite(parsed) || parsed < 0) {
+    throw new Error(`${label} must be a non-negative finite number`);
+  }
+  return Math.round(parsed * 100_000_000) / 100_000_000;
+}
+
+function acceptedInstances(value) {
+  const normalized = String(value ?? "").trim();
+  if (!normalized) return WINDOWS_EC2_JIT.maxAcceptedInstances;
+  const parsed = Number(normalized);
+  if (
+    !Number.isInteger(parsed) ||
+    parsed < 1 ||
+    parsed > WINDOWS_EC2_JIT.maxAcceptedInstances
+  ) {
+    throw new Error(
+      `maxAcceptedInstances must be an integer from 1 through ${WINDOWS_EC2_JIT.maxAcceptedInstances}`,
+    );
+  }
+  return parsed;
+}
+
+function string(value) {
+  return { S: String(value) };
+}
+
+function runKey(plan) {
+  return `RUN#${plan.github.runId}#${plan.github.runAttempt}#${plan.github.qualificationId}`;
+}
+
+export function createWindowsJitCampaignArmPlan(values = {}) {
+  const armedAt = epoch(values.armedAt, "armedAt");
+  const expiresAt = epoch(values.expiresAt, "expiresAt");
+  if (expiresAt <= armedAt || expiresAt - armedAt > 86400) {
+    throw new Error("campaign expiry must be within 24 hours after arming");
+  }
+  const reservationUsd =
+    (WINDOWS_EC2_JIT.pricePerHourUsd *
+      WINDOWS_EC2_JIT.maximumInstanceLifetimeMinutes) /
+    60;
+  const phaseSpendBaselineUsd = money(
+    values.phaseSpendBaselineUsd,
+    "phaseSpendBaselineUsd",
+  );
+  const maxAcceptedInstances = acceptedInstances(values.maxAcceptedInstances);
+  const campaignReservationCeilingUsd = reservationUsd * maxAcceptedInstances;
+  const campaignSafetyCeilingUsd =
+    campaignReservationCeilingUsd +
+    reservationUsd * WINDOWS_EC2_JIT.maxConcurrentInstances;
+  const remainingPhaseBudgetUsd =
+    WINDOWS_EC2_JIT.budgetLimitUsd - phaseSpendBaselineUsd;
+  if (campaignSafetyCeilingUsd >= remainingPhaseBudgetUsd) {
+    throw new Error(
+      "campaign safety envelope must remain below the remaining Windows phase budget",
+    );
+  }
+  return {
+    schemaVersion: 1,
+    contract: AWS_WINDOWS_JIT_CAMPAIGN_CONTRACT,
+    kind: "campaign-arm-plan",
+    campaign: {
+      id: windowsJitCampaignId(values.campaignId),
+      armedAt,
+      expiresAt,
+    },
+    source: { sha: exactSha(values.sourceSha) },
+    aws: {
+      region: exact(
+        values.region || WINDOWS_EC2_JIT.region,
+        /^us-[a-z]+-\d$/,
+        "region",
+      ),
+      stateTable: tableName(values.stateTable),
+    },
+    limits: {
+      maxAcceptedInstances,
+      reservationUsd,
+      budgetLimitUsd: WINDOWS_EC2_JIT.budgetLimitUsd,
+      phaseSpendBaselineUsd,
+      remainingPhaseBudgetUsd,
+      campaignReservationCeilingUsd,
+      campaignSafetyCeilingUsd,
+      reservationLimitUsd: remainingPhaseBudgetUsd - reservationUsd,
+    },
+  };
+}
+
+export function windowsCampaignArmItems(plan) {
+  if (plan?.contract !== AWS_WINDOWS_JIT_CAMPAIGN_CONTRACT) {
+    throw new Error("Windows JIT campaign arm plan contract is invalid");
+  }
+  const table = plan.aws.stateTable;
+  const campaignPk = `CAMPAIGN#${plan.campaign.id}`;
+  return [
+    {
+      Put: {
+        TableName: table,
+        Item: {
+          pk: string("CONTROL"),
+          state: string("ARMED"),
+          campaign_id: string(plan.campaign.id),
+          source_sha: string(plan.source.sha),
+          phase_spend_baseline_usd: number(plan.limits.phaseSpendBaselineUsd),
+          budget_limit_usd: number(plan.limits.budgetLimitUsd),
+          armed_at: number(plan.campaign.armedAt),
+          expires_epoch: number(plan.campaign.expiresAt),
+        },
+        ConditionExpression: "attribute_not_exists(pk)",
+      },
+    },
+    {
+      Put: {
+        TableName: table,
+        Item: {
+          pk: string(campaignPk),
+          state: string("ARMED"),
+          source_sha: string(plan.source.sha),
+          accepted_instances: number(0),
+          reserved_usd: number(0),
+          max_accepted_instances: number(plan.limits.maxAcceptedInstances),
+          reservation_usd: number(plan.limits.reservationUsd),
+          budget_limit_usd: number(plan.limits.budgetLimitUsd),
+          phase_spend_baseline_usd: number(plan.limits.phaseSpendBaselineUsd),
+          remaining_phase_budget_usd: number(
+            plan.limits.remainingPhaseBudgetUsd,
+          ),
+          campaign_reservation_ceiling_usd: number(
+            plan.limits.campaignReservationCeilingUsd,
+          ),
+          campaign_safety_ceiling_usd: number(
+            plan.limits.campaignSafetyCeilingUsd,
+          ),
+          reservation_limit_usd: number(plan.limits.reservationLimitUsd),
+          armed_at: number(plan.campaign.armedAt),
+          expires_epoch: number(plan.campaign.expiresAt),
+        },
+        ConditionExpression: "attribute_not_exists(pk)",
+      },
+    },
+  ];
+}
+
+export function windowsCampaignReservationItems(plan, observedAt) {
+  const now = epoch(observedAt, "observedAt");
+  const runPk = runKey(plan);
+  const campaignPk = `CAMPAIGN#${plan.campaign.id}`;
+  const reservation = plan.safety.campaignReservationUsd;
+  return [
+    {
+      ConditionCheck: {
+        TableName: plan.aws.stateTable,
+        Key: { pk: string("CONTROL") },
+        ConditionExpression:
+          "#state = :armed AND campaign_id = :campaign AND source_sha = :source AND expires_epoch >= :now",
+        ExpressionAttributeNames: { "#state": "state" },
+        ExpressionAttributeValues: {
+          ":armed": string("ARMED"),
+          ":campaign": string(plan.campaign.id),
+          ":source": string(plan.source.sha),
+          ":now": number(now),
+        },
+      },
+    },
+    {
+      Put: {
+        TableName: plan.aws.stateTable,
+        Item: {
+          pk: string(runPk),
+          state: string("RESERVED"),
+          campaign_id: string(plan.campaign.id),
+          source_sha: string(plan.source.sha),
+          github_run_id: string(plan.github.runId),
+          github_run_attempt: string(plan.github.runAttempt),
+          qualification_id: string(plan.github.qualificationId),
+          reserved_usd: number(reservation),
+          observed_at: number(now),
+          expires_at: number(now + 1209600),
+        },
+        ConditionExpression: "attribute_not_exists(pk)",
+      },
+    },
+    {
+      Update: {
+        TableName: plan.aws.stateTable,
+        Key: { pk: string(campaignPk) },
+        UpdateExpression:
+          "ADD accepted_instances :one, reserved_usd :reservation SET updated_at = :now",
+        ConditionExpression:
+          "#state = :armed AND source_sha = :source AND accepted_instances < max_accepted_instances AND reserved_usd <= reservation_limit_usd",
+        ExpressionAttributeNames: { "#state": "state" },
+        ExpressionAttributeValues: {
+          ":armed": string("ARMED"),
+          ":source": string(plan.source.sha),
+          ":one": number(1),
+          ":reservation": number(reservation),
+          ":now": number(now),
+        },
+      },
+    },
+  ];
+}
+
+export function windowsCampaignMarkLaunchedArgs(plan, instanceId, observedAt) {
+  return [
+    "dynamodb",
+    "update-item",
+    "--table-name",
+    plan.aws.stateTable,
+    "--key",
+    JSON.stringify({ pk: string(runKey(plan)) }),
+    "--update-expression",
+    "SET #state = :launched, instance_id = :instance, launched_at = :now",
+    "--condition-expression",
+    "#state = :reserved",
+    "--expression-attribute-names",
+    JSON.stringify({ "#state": "state" }),
+    "--expression-attribute-values",
+    JSON.stringify({
+      ":launched": string("LAUNCHED"),
+      ":reserved": string("RESERVED"),
+      ":instance": string(instanceId),
+      ":now": number(epoch(observedAt, "observedAt")),
+    }),
+    "--output",
+    "json",
+  ];
+}
+
+export function windowsCampaignKillArgs(stateTable, reason, observedAt) {
+  return [
+    "dynamodb",
+    "update-item",
+    "--table-name",
+    tableName(stateTable),
+    "--key",
+    JSON.stringify({ pk: string("CONTROL") }),
+    "--update-expression",
+    "SET #state = :killed, reason = :reason, killed_at = :now",
+    "--expression-attribute-names",
+    JSON.stringify({ "#state": "state" }),
+    "--expression-attribute-values",
+    JSON.stringify({
+      ":killed": string("KILLED"),
+      ":reason": string(exact(reason, /^[a-z0-9][a-z0-9-]{2,63}$/, "reason")),
+      ":now": number(epoch(observedAt, "observedAt")),
+    }),
+    "--output",
+    "json",
+  ];
+}