@kungfu-tech/buildchain 3.0.9-alpha.1 → 3.0.9-alpha.10

package/scripts/aws-macos-jit-controller.mjs

@@ -6,6 +6,7 @@ import {
   AWS_MACOS_JIT_CONTROLLER_CONTRACT,
   createMacosJitCampaignPlan,
   createMacosJitClosePlan,
+  createMacosJitInstanceRehydratePlan,
   createMacosJitJobPlan,
   createMacosJitSourceRebindPlan,
   macosAllocateHostsArgs,
@@ -13,6 +14,7 @@ import {
   macosRunInstancesArgs,
 } from "./aws-macos-jit-controller-core.mjs";
 import { MACOS_EC2_JIT_REGIONS } from "./aws-macos-jit-core.mjs";
+import { executeMacosJitInstanceRehydrate } from "./aws-macos-jit-instance-rehydrate.mjs";
 import { executeMacosJitJob } from "./aws-macos-jit-job-controller.mjs";
 import { executeMacosJitSourceRebind } from "./aws-macos-jit-source-rebind.mjs";
 import {
@@ -36,6 +38,19 @@ function flag(name) {
   return process.argv.includes(`--${name}`);
 }
 
+function jsonArrayArg(name) {
+  let parsed;
+  try {
+    parsed = JSON.parse(arg(name));
+  } catch {
+    throw new Error(`--${name} must be valid JSON`);
+  }
+  if (!Array.isArray(parsed)) {
+    throw new Error(`--${name} must be a JSON array`);
+  }
+  return parsed;
+}
+
 function assertCampaignLaunchPreflight(plan, profile) {
   const launchGate = assertMacosBudgetLaunchGate(plan, profile);
   const commit = ghJson(
@@ -493,6 +508,20 @@ function commonValues(execute) {
   };
 }
 
+function confirmHistoricalPreflightFailures(plan) {
+  const confirmed = flag("confirm-historical-preflight-failure-runs-json")
+    ? jsonArrayArg("confirm-historical-preflight-failure-runs-json")
+    : [];
+  if (
+    JSON.stringify(confirmed) !==
+    JSON.stringify(plan.github.historicalPreflightFailureRuns)
+  ) {
+    throw new Error(
+      "--confirm-historical-preflight-failure-runs-json must equal the exact historical preflight failure inventory",
+    );
+  }
+}
+
 function confirm(plan) {
   if (arg("confirm-source-sha") !== plan.source.sha) {
     throw new Error("--confirm-source-sha must equal the exact source SHA");
@@ -515,6 +544,115 @@ function confirm(plan) {
     if (!flag("confirm-zero-allocation")) {
       throw new Error("--confirm-zero-allocation is required");
     }
+    if (plan.github.priorRunPolicy === "terminal-failure") {
+      const confirmed = jsonArrayArg(
+        "confirm-terminal-failure-run-ids-json",
+      ).map(String);
+      if (
+        JSON.stringify(confirmed) !==
+        JSON.stringify(plan.github.terminalFailureRunIds)
+      ) {
+        throw new Error(
+          "--confirm-terminal-failure-run-ids-json must equal the exact failed run inventory",
+        );
+      }
+      const confirmedStartupFailures = flag("confirm-startup-failure-runs-json")
+        ? jsonArrayArg("confirm-startup-failure-runs-json")
+        : [];
+      if (
+        JSON.stringify(confirmedStartupFailures) !==
+        JSON.stringify(plan.github.startupFailureRuns)
+      ) {
+        throw new Error(
+          "--confirm-startup-failure-runs-json must equal the exact startup failure inventory",
+        );
+      }
+      const confirmedHistoricalTerminalFailures = flag(
+        "confirm-historical-terminal-failure-runs-json",
+      )
+        ? jsonArrayArg("confirm-historical-terminal-failure-runs-json")
+        : [];
+      if (
+        JSON.stringify(confirmedHistoricalTerminalFailures) !==
+        JSON.stringify(plan.github.historicalTerminalFailureRuns)
+      ) {
+        throw new Error(
+          "--confirm-historical-terminal-failure-runs-json must equal the exact historical terminal failure inventory",
+        );
+      }
+      const confirmedSuccessfulRuns = flag("confirm-successful-run-ids-json")
+        ? jsonArrayArg("confirm-successful-run-ids-json").map(String)
+        : [];
+      if (
+        JSON.stringify(confirmedSuccessfulRuns) !==
+        JSON.stringify(plan.github.successfulRunIds)
+      ) {
+        throw new Error(
+          "--confirm-successful-run-ids-json must equal the exact successful run inventory",
+        );
+      }
+    } else if (plan.github.priorRunPolicy === "preflight-failure") {
+      const confirmed = jsonArrayArg(
+        "confirm-preflight-failure-run-ids-json",
+      ).map(String);
+      if (
+        JSON.stringify(confirmed) !==
+        JSON.stringify(plan.github.preflightFailureRunIds)
+      ) {
+        throw new Error(
+          "--confirm-preflight-failure-run-ids-json must equal the exact preflight failure inventory",
+        );
+      }
+      const confirmedStartupFailures = flag("confirm-startup-failure-runs-json")
+        ? jsonArrayArg("confirm-startup-failure-runs-json")
+        : [];
+      if (
+        JSON.stringify(confirmedStartupFailures) !==
+        JSON.stringify(plan.github.startupFailureRuns)
+      ) {
+        throw new Error(
+          "--confirm-startup-failure-runs-json must equal the exact startup failure inventory",
+        );
+      }
+      const confirmedHistoricalTerminalFailures = flag(
+        "confirm-historical-terminal-failure-runs-json",
+      )
+        ? jsonArrayArg("confirm-historical-terminal-failure-runs-json")
+        : [];
+      if (
+        JSON.stringify(confirmedHistoricalTerminalFailures) !==
+        JSON.stringify(plan.github.historicalTerminalFailureRuns)
+      ) {
+        throw new Error(
+          "--confirm-historical-terminal-failure-runs-json must equal the exact historical terminal failure inventory",
+        );
+      }
+      const confirmedSuccessfulRuns = flag("confirm-successful-run-ids-json")
+        ? jsonArrayArg("confirm-successful-run-ids-json").map(String)
+        : [];
+      if (
+        JSON.stringify(confirmedSuccessfulRuns) !==
+        JSON.stringify(plan.github.successfulRunIds)
+      ) {
+        throw new Error(
+          "--confirm-successful-run-ids-json must equal the exact successful run inventory",
+        );
+      }
+    }
+    confirmHistoricalPreflightFailures(plan);
+  }
+  if (plan.kind === "instance-rehydrate-plan") {
+    if (arg("confirm-host-id") !== plan.aws.hostId) {
+      throw new Error("--confirm-host-id must equal the exact host id");
+    }
+    if (arg("confirm-replaces-instance-id") !== plan.aws.replacesInstanceId) {
+      throw new Error(
+        "--confirm-replaces-instance-id must equal replacesInstanceId",
+      );
+    }
+    if (!flag("confirm-no-host-allocation")) {
+      throw new Error("--confirm-no-host-allocation is required");
+    }
   }
 }
 
@@ -541,7 +679,32 @@ export function main() {
       executeMacosJitCampaignLaunch(plan, { profile: arg("aws-profile") }),
     );
   }
-  if (["plan-rebind", "rebind-campaign"].includes(mode)) {
+  if (["plan-rehydrate", "rehydrate-instance"].includes(mode)) {
+    const plan = createMacosJitInstanceRehydratePlan({
+      ...commonValues(execute),
+      hostId: arg("host-id"),
+      hostAllocatedAt: arg("host-allocated-at"),
+      replacesInstanceId: arg("replaces-instance-id"),
+      rehydrationId: arg("rehydration-id"),
+    });
+    return emit(plan, mode, execute, () =>
+      executeMacosJitInstanceRehydrate(plan, {
+        profile: arg("aws-profile"),
+      }),
+    );
+  }
+  if (
+    [
+      "plan-rebind",
+      "rebind-campaign",
+      "plan-rebind-after-failure",
+      "rebind-campaign-after-failure",
+      "plan-rebind-after-preflight-failure",
+      "rebind-campaign-after-preflight-failure",
+    ].includes(mode)
+  ) {
+    const afterFailure = mode.endsWith("after-failure");
+    const afterPreflightFailure = mode.endsWith("after-preflight-failure");
     const plan = createMacosJitSourceRebindPlan({
       ...commonValues(execute),
       previousSourceSha: arg("previous-source-sha"),
@@ -549,8 +712,41 @@ export function main() {
       workflowId: arg("workflow-id"),
       hostId: arg("host-id"),
       instanceId: arg("instance-id"),
+      priorRunPolicy: afterPreflightFailure
+        ? "preflight-failure"
+        : afterFailure
+          ? "terminal-failure"
+          : "unused",
+      terminalFailureRunIds: afterFailure
+        ? jsonArrayArg("terminal-failure-run-ids-json")
+        : [],
+      preflightFailureRunIds: afterPreflightFailure
+        ? jsonArrayArg("preflight-failure-run-ids-json")
+        : [],
+      startupFailureRuns:
+        (afterFailure || afterPreflightFailure) &&
+        flag("startup-failure-runs-json")
+          ? jsonArrayArg("startup-failure-runs-json")
+          : [],
+      historicalTerminalFailureRuns:
+        (afterFailure || afterPreflightFailure) &&
+        flag("historical-terminal-failure-runs-json")
+          ? jsonArrayArg("historical-terminal-failure-runs-json")
+          : [],
+      historicalPreflightFailureRuns:
+        (afterFailure || afterPreflightFailure) &&
+        flag("historical-preflight-failure-runs-json")
+          ? jsonArrayArg("historical-preflight-failure-runs-json")
+          : [],
+      successfulRunIds:
+        (afterFailure || afterPreflightFailure) &&
+        flag("successful-run-ids-json")
+          ? jsonArrayArg("successful-run-ids-json")
+          : [],
     });
-    return emit(plan, mode, execute, () => executeMacosJitSourceRebind(plan, { profile: arg("aws-profile") }));
+    return emit(plan, mode, execute, () =>
+      executeMacosJitSourceRebind(plan, { profile: arg("aws-profile") }),
+    );
   }
   if (["plan-job", "run-job"].includes(mode)) {
     const plan = createMacosJitJobPlan({

package/scripts/aws-macos-jit-instance-rehydrate.mjs

@@ -0,0 +1,260 @@
+// SPDX-License-Identifier: Apache-2.0
+
+import {
+  AWS_MACOS_JIT_CONTROLLER_CONTRACT,
+  macosRunInstancesArgs,
+} from "./aws-macos-jit-controller-core.mjs";
+import { MACOS_EC2_JIT_REGIONS } from "./aws-macos-jit-core.mjs";
+import {
+  assertDryRun,
+  assertMacosBudgetLaunchGate,
+  assertOwnership,
+  awsArgs,
+  awsJson,
+  commandResult,
+  ghJson,
+  requireSuccess,
+} from "./aws-macos-jit-controller-runtime.mjs";
+
+function assertInstanceRehydratePreflight(plan, profile) {
+  const launchGate = assertMacosBudgetLaunchGate(plan, profile);
+  const commit = ghJson(
+    ["api", `repos/${plan.repository}/commits/${plan.source.sha}`],
+    undefined,
+    "GitHub exact-source preflight",
+  );
+  if (String(commit.sha || "").toLowerCase() !== plan.source.sha) {
+    throw new Error("GitHub did not resolve the exact campaign source SHA");
+  }
+  const host = awsJson(
+    plan,
+    profile,
+    [
+      "ec2",
+      "describe-hosts",
+      "--host-ids",
+      plan.aws.hostId,
+      "--output",
+      "json",
+    ],
+    "macOS rehydration host preflight",
+  ).Hosts?.[0];
+  if (
+    !host ||
+    host.State !== "available" ||
+    host.AvailabilityZone !== plan.aws.availabilityZone ||
+    host.HostProperties?.InstanceType !== plan.aws.instanceType ||
+    (host.Instances || []).length !== 0 ||
+    new Date(host.AllocationTime).toISOString() !== plan.aws.hostAllocatedAt
+  ) {
+    throw new Error("macOS rehydration host identity or capacity mismatch");
+  }
+  assertOwnership(host.Tags, plan);
+
+  const regionalPlans = Object.keys(MACOS_EC2_JIT_REGIONS).map((region) => ({
+    ...plan,
+    aws: { ...plan.aws, region },
+  }));
+  const otherHosts = regionalPlans.flatMap((regionalPlan) =>
+    (
+      awsJson(
+        regionalPlan,
+        profile,
+        [
+          "ec2",
+          "describe-hosts",
+          "--filter",
+          "Name=tag:kungfu:plane,Values=aws-us-elastic-runner-burst",
+          "Name=tag:kungfu:provider,Values=macos-ec2-jit",
+          "Name=state,Values=available,pending,under-assessment",
+          "--output",
+          "json",
+        ],
+        `macOS rehydration host ceiling in ${regionalPlan.aws.region}`,
+      ).Hosts || []
+    ).filter((candidate) => candidate.HostId !== plan.aws.hostId),
+  );
+  if (otherHosts.length !== 0) {
+    throw new Error("macOS rehydration found another active Dedicated Host");
+  }
+  const activeInstances = regionalPlans.flatMap((regionalPlan) => {
+    const active = awsJson(
+      regionalPlan,
+      profile,
+      [
+        "ec2",
+        "describe-instances",
+        "--filters",
+        "Name=tag:kungfu:plane,Values=aws-us-elastic-runner-burst",
+        "Name=tag:kungfu:provider,Values=macos-ec2-jit",
+        "Name=instance-state-name,Values=pending,running,stopping,stopped,shutting-down",
+        "--output",
+        "json",
+      ],
+      `active macOS JIT instance preflight in ${regionalPlan.aws.region}`,
+    );
+    return (active.Reservations || []).flatMap(
+      (reservation) => reservation.Instances || [],
+    );
+  });
+  if (activeInstances.length !== 0) {
+    throw new Error("macOS rehydration requires zero active JIT instances");
+  }
+  const runners = ghJson(
+    ["api", `repos/${plan.repository}/actions/runners?per_page=100`],
+    undefined,
+    "macOS rehydration runner preflight",
+  ).runners;
+  if (
+    (runners || []).some((runner) =>
+      (runner.labels || []).some((label) =>
+        String(label.name || label).startsWith("aws-us-ec2-macos-jit-"),
+      ),
+    )
+  ) {
+    throw new Error("macOS rehydration requires zero JIT runner residue");
+  }
+  const image = awsJson(
+    plan,
+    profile,
+    [
+      "ec2",
+      "describe-images",
+      "--image-ids",
+      plan.aws.amiId,
+      "--output",
+      "json",
+    ],
+    "macOS rehydration AMI preflight",
+  ).Images?.[0];
+  if (
+    !image ||
+    image.Name !== plan.aws.amiName ||
+    image.Architecture !== "arm64_mac" ||
+    image.State !== "available"
+  ) {
+    throw new Error("macOS AMI identity or availability mismatch");
+  }
+  const subnet = awsJson(
+    plan,
+    profile,
+    [
+      "ec2",
+      "describe-subnets",
+      "--subnet-ids",
+      plan.aws.subnetId,
+      "--output",
+      "json",
+    ],
+    "macOS rehydration subnet preflight",
+  ).Subnets?.[0];
+  if (!subnet || subnet.AvailabilityZone !== plan.aws.availabilityZone) {
+    throw new Error(
+      "macOS subnet and Dedicated Host availability zone mismatch",
+    );
+  }
+  return {
+    ...launchGate,
+    exactCommit: plan.source.sha,
+    hostId: host.HostId,
+    hostState: host.State,
+    activeInstances: 0,
+    otherActiveHosts: 0,
+    amiOwnerId: image.OwnerId,
+    subnetAvailabilityZone: subnet.AvailabilityZone,
+  };
+}
+
+export function executeMacosJitInstanceRehydrate(plan, { profile = "" } = {}) {
+  if (
+    plan?.contract !== AWS_MACOS_JIT_CONTROLLER_CONTRACT ||
+    plan.kind !== "instance-rehydrate-plan"
+  ) {
+    throw new Error("macOS JIT instance rehydrate plan contract is invalid");
+  }
+  const preflight = assertInstanceRehydratePreflight(plan, profile);
+  assertDryRun(
+    commandResult(
+      "aws",
+      awsArgs(
+        plan,
+        profile,
+        macosRunInstancesArgs(plan, { hostId: plan.aws.hostId, dryRun: true }),
+      ),
+    ),
+    "EC2 same-host RunInstances DryRun",
+  );
+  const launched = awsJson(
+    plan,
+    profile,
+    macosRunInstancesArgs(plan, { hostId: plan.aws.hostId }),
+    "EC2 same-host RunInstances",
+  );
+  const instanceId = launched.Instances?.[0]?.InstanceId;
+  if (!/^i-[0-9a-f]+$/.test(String(instanceId || ""))) {
+    throw new Error("EC2 same-host RunInstances returned no instance identity");
+  }
+  requireSuccess(
+    commandResult(
+      "aws",
+      awsArgs(plan, profile, [
+        "ec2",
+        "wait",
+        "instance-running",
+        "--instance-ids",
+        instanceId,
+      ]),
+    ),
+    "macOS rehydrated instance-running waiter",
+  );
+  const instance = awsJson(
+    plan,
+    profile,
+    [
+      "ec2",
+      "describe-instances",
+      "--instance-ids",
+      instanceId,
+      "--output",
+      "json",
+    ],
+    "macOS rehydrated instance readback",
+  ).Reservations?.[0]?.Instances?.[0];
+  if (
+    !instance ||
+    instance.State?.Name !== "running" ||
+    instance.Placement?.HostId !== plan.aws.hostId ||
+    instance.ImageId !== plan.aws.amiId ||
+    instance.InstanceType !== plan.aws.instanceType
+  ) {
+    throw new Error("macOS rehydrated instance identity or placement mismatch");
+  }
+  assertOwnership(instance.Tags, plan);
+  return {
+    schemaVersion: 1,
+    contract: AWS_MACOS_JIT_CONTROLLER_CONTRACT,
+    kind: "instance-rehydrate-result",
+    status: "rehydrated",
+    repository: plan.repository,
+    campaign: plan.campaign,
+    source: plan.source,
+    replacement: {
+      rehydrationId: plan.aws.rehydrationId,
+      replacesInstanceId: plan.aws.replacesInstanceId,
+    },
+    aws: {
+      region: plan.aws.region,
+      availabilityZone: plan.aws.availabilityZone,
+      hostId: plan.aws.hostId,
+      hostAllocatedAt: plan.aws.hostAllocatedAt,
+      instanceId,
+      instanceType: instance.InstanceType,
+      imageId: instance.ImageId,
+      launchTime: new Date(instance.LaunchTime).toISOString(),
+    },
+    preflight,
+    dryRun: "RunInstances:DryRunOperation",
+    hostAllocated: false,
+    planDigest: plan.digest,
+  };
+}