@kungfu-tech/buildchain 3.0.9-alpha.1 → 3.0.9-alpha.11

package/scripts/aws-macos-jit-controller-core.mjs

@@ -75,6 +75,103 @@ function qualificationId(value) {
   return exact(value, /^mac-(?:smoke-0[12]|full-01)$/, "qualificationId");
 }
 
+function terminalFailureRunIds(value) {
+  if (!Array.isArray(value) || value.length === 0) {
+    throw new Error("terminalFailureRunIds must be a non-empty array");
+  }
+  const normalized = value.map((entry) =>
+    exact(entry, /^[1-9]\d*$/, "terminalFailureRunId"),
+  );
+  if (new Set(normalized).size !== normalized.length) {
+    throw new Error("terminalFailureRunIds must be unique");
+  }
+  return [...normalized].sort((left, right) => Number(left) - Number(right));
+}
+
+function preflightFailureRunIds(value) {
+  if (!Array.isArray(value) || value.length === 0) {
+    throw new Error("preflightFailureRunIds must be a non-empty array");
+  }
+  const normalized = value.map((entry) =>
+    exact(entry, /^[1-9]\d*$/, "preflightFailureRunId"),
+  );
+  if (new Set(normalized).size !== normalized.length) {
+    throw new Error("preflightFailureRunIds must be unique");
+  }
+  return [...normalized].sort((left, right) => Number(left) - Number(right));
+}
+
+function successfulRunIds(value) {
+  if (value === undefined) return [];
+  if (!Array.isArray(value)) {
+    throw new Error("successfulRunIds must be an array");
+  }
+  const normalized = value.map((entry) =>
+    exact(entry, /^[1-9]\d*$/, "successfulRunId"),
+  );
+  if (new Set(normalized).size !== normalized.length) {
+    throw new Error("successfulRunIds must be unique");
+  }
+  return [...normalized].sort((left, right) => Number(left) - Number(right));
+}
+
+function startupFailureRuns(value) {
+  if (value === undefined) return [];
+  if (!Array.isArray(value)) {
+    throw new Error("startupFailureRuns must be an array");
+  }
+  const normalized = value.map((entry) => ({
+    runId: exact(entry?.runId, /^[1-9]\d*$/, "startupFailureRunId"),
+    sourceSha: exactSha(entry?.sourceSha),
+  }));
+  if (
+    new Set(normalized.map(({ runId }) => runId)).size !== normalized.length
+  ) {
+    throw new Error("startupFailureRuns must use unique run ids");
+  }
+  return [...normalized].sort(
+    (left, right) => Number(left.runId) - Number(right.runId),
+  );
+}
+
+function historicalTerminalFailureRuns(value) {
+  if (value === undefined) return [];
+  if (!Array.isArray(value)) {
+    throw new Error("historicalTerminalFailureRuns must be an array");
+  }
+  const normalized = value.map((entry) => ({
+    runId: exact(entry?.runId, /^[1-9]\d*$/, "historicalTerminalFailureRunId"),
+    sourceSha: exactSha(entry?.sourceSha),
+  }));
+  if (
+    new Set(normalized.map(({ runId }) => runId)).size !== normalized.length
+  ) {
+    throw new Error("historicalTerminalFailureRuns must use unique run ids");
+  }
+  return [...normalized].sort(
+    (left, right) => Number(left.runId) - Number(right.runId),
+  );
+}
+
+function historicalPreflightFailureRuns(value) {
+  if (value === undefined) return [];
+  if (!Array.isArray(value)) {
+    throw new Error("historicalPreflightFailureRuns must be an array");
+  }
+  const normalized = value.map((entry) => ({
+    runId: exact(entry?.runId, /^[1-9]\d*$/, "historicalPreflightFailureRunId"),
+    sourceSha: exactSha(entry?.sourceSha),
+  }));
+  if (
+    new Set(normalized.map(({ runId }) => runId)).size !== normalized.length
+  ) {
+    throw new Error("historicalPreflightFailureRuns must use unique run ids");
+  }
+  return [...normalized].sort(
+    (left, right) => Number(left.runId) - Number(right.runId),
+  );
+}
+
 function commonAws(values) {
   const region = exact(
     values.region || MACOS_EC2_JIT.region,
@@ -203,6 +300,87 @@ export function createMacosJitCampaignPlan(values = {}) {
   return { ...plan, digest: digest(plan) };
 }
 
+export function createMacosJitInstanceRehydratePlan(values = {}) {
+  const id = campaignId(values.campaignId);
+  const boundSource = source(values);
+  const aws = commonAws(values);
+  const regionConfig = macosJitRegionConfig(aws.region);
+  const tags = ownershipTags({ id, sourceSha: boundSource.sha });
+  const rehydrationId = exact(
+    values.rehydrationId,
+    /^r[1-9]\d{0,2}$/,
+    "rehydrationId",
+  );
+  const instanceClientToken = `kungfu-mac-rehydrate-${id}-${rehydrationId}`;
+  if (instanceClientToken.length > 64) {
+    throw new Error("rehydration client token exceeds the EC2 limit");
+  }
+  const plan = {
+    schemaVersion: 1,
+    contract: AWS_MACOS_JIT_CONTROLLER_CONTRACT,
+    kind: "instance-rehydrate-plan",
+    repository: repository(values.repository),
+    account: {
+      id: exact(values.accountId, /^\d{12}$/, "accountId"),
+    },
+    github: {
+      workflowId: MACOS_EC2_JIT.workflowId,
+      requiredState: "disabled_manually",
+    },
+    campaign: { id },
+    source: boundSource,
+    aws: {
+      ...aws,
+      hostId: exact(values.hostId, /^h-[0-9a-f]+$/, "hostId"),
+      hostAllocatedAt: iso(values.hostAllocatedAt, "hostAllocatedAt"),
+      replacesInstanceId: exact(
+        values.replacesInstanceId,
+        /^i-[0-9a-f]+$/,
+        "replacesInstanceId",
+      ),
+      rehydrationId,
+      instanceTags: tags,
+      volumeTags: tags,
+      instanceClientToken,
+      rootVolume: {
+        deviceName: "/dev/sda1",
+        deleteOnTermination: true,
+        encrypted: true,
+        volumeSizeGiB: 200,
+        volumeType: "gp3",
+      },
+      metadata: {
+        httpEndpoint: "enabled",
+        httpTokens: "required",
+        httpPutResponseHopLimit: 1,
+        instanceMetadataTags: "disabled",
+      },
+    },
+    safety: {
+      applyMode: values.execute === true ? "execute" : "dry-run",
+      exactSourceRequired: true,
+      existingCampaignHostRequired: true,
+      noHostAllocation: true,
+      zeroActiveInstanceRequired: true,
+      zeroRunnerResidueRequired: true,
+      awsDryRunRequiredBeforeLaunch: true,
+      cleanupOwner: "scheduled-card-scoped-reaper",
+      budget: {
+        name: regionConfig.budgetName,
+        limitUsd: MACOS_EC2_JIT.budgetLimitUsd,
+        metrics: ["UnblendedCost"],
+        dimensionFilter: {
+          usageTypes: regionConfig.budgetUsageTypes,
+          operation: MACOS_EC2_JIT.budgetOperation,
+          regions: regionConfig.budgetRegions,
+        },
+        requiredActualThresholds: [80, 95],
+      },
+    },
+  };
+  return { ...plan, digest: digest(plan) };
+}
+
 export function createMacosJitSourceRebindPlan(values = {}) {
   const id = campaignId(values.campaignId);
   const boundSource = source(values);
@@ -219,6 +397,48 @@ export function createMacosJitSourceRebindPlan(values = {}) {
   if (workflowId !== MACOS_EC2_JIT.workflowId) {
     throw new Error(`workflowId must be ${MACOS_EC2_JIT.workflowId}`);
   }
+  const priorRunPolicy = values.priorRunPolicy || "unused";
+  if (
+    !["unused", "terminal-failure", "preflight-failure"].includes(
+      priorRunPolicy,
+    )
+  ) {
+    throw new Error(
+      "priorRunPolicy must be unused, terminal-failure, or preflight-failure",
+    );
+  }
+  const terminalRunIds =
+    priorRunPolicy === "terminal-failure"
+      ? terminalFailureRunIds(values.terminalFailureRunIds)
+      : [];
+  const preflightRunIds =
+    priorRunPolicy === "preflight-failure"
+      ? preflightFailureRunIds(values.preflightFailureRunIds)
+      : [];
+  const startupRuns = startupFailureRuns(values.startupFailureRuns);
+  const historicalTerminalRuns = historicalTerminalFailureRuns(
+    values.historicalTerminalFailureRuns,
+  );
+  const historicalPreflightRuns = historicalPreflightFailureRuns(
+    values.historicalPreflightFailureRuns,
+  );
+  const successfulRuns = successfulRunIds(values.successfulRunIds);
+  if (priorRunPolicy === "unused" && startupRuns.length !== 0) {
+    throw new Error("startupFailureRuns require a failed priorRunPolicy");
+  }
+  if (priorRunPolicy === "unused" && historicalTerminalRuns.length !== 0) {
+    throw new Error(
+      "historicalTerminalFailureRuns require a failed priorRunPolicy",
+    );
+  }
+  if (priorRunPolicy === "unused" && historicalPreflightRuns.length !== 0) {
+    throw new Error(
+      "historicalPreflightFailureRuns require a failed priorRunPolicy",
+    );
+  }
+  if (priorRunPolicy === "unused" && successfulRuns.length !== 0) {
+    throw new Error("successfulRunIds require a failed priorRunPolicy");
+  }
   const plan = {
     schemaVersion: 1,
     contract: AWS_MACOS_JIT_CONTROLLER_CONTRACT,
@@ -233,6 +453,13 @@ export function createMacosJitSourceRebindPlan(values = {}) {
     github: {
       workflowId,
       requiredState: "disabled_manually",
+      priorRunPolicy,
+      terminalFailureRunIds: terminalRunIds,
+      preflightFailureRunIds: preflightRunIds,
+      startupFailureRuns: startupRuns,
+      historicalTerminalFailureRuns: historicalTerminalRuns,
+      historicalPreflightFailureRuns: historicalPreflightRuns,
+      successfulRunIds: successfulRuns,
     },
     aws: {
       ...aws,
@@ -246,8 +473,16 @@ export function createMacosJitSourceRebindPlan(values = {}) {
       forwardOnlySource: true,
       sameSourceRefRequired: true,
       workflowDisabledRequired: true,
-      zeroPriorJobsRequired: true,
-      zeroPriorArtifactsRequired: true,
+      zeroPriorJobsRequired: priorRunPolicy === "unused",
+      zeroPriorArtifactsRequired: priorRunPolicy === "unused",
+      terminalFailureEvidencePreserved: priorRunPolicy === "terminal-failure",
+      preflightFailureEvidencePreserved: priorRunPolicy === "preflight-failure",
+      startupFailureEvidencePreserved: startupRuns.length !== 0,
+      historicalTerminalFailureEvidencePreserved:
+        historicalTerminalRuns.length !== 0,
+      historicalPreflightFailureEvidencePreserved:
+        historicalPreflightRuns.length !== 0,
+      successfulRunEvidencePreserved: successfulRuns.length !== 0,
       zeroJitResidueRequired: true,
       zeroEvidenceRequired: true,
       exactCampaignResourcesRequired: true,
@@ -301,9 +536,9 @@ export function macosAllocateHostsArgs(plan) {
 export function macosRunInstancesArgs(plan, { hostId, dryRun = false } = {}) {
   if (
     plan?.contract !== AWS_MACOS_JIT_CONTROLLER_CONTRACT ||
-    plan.kind !== "campaign-launch-plan"
+    !["campaign-launch-plan", "instance-rehydrate-plan"].includes(plan.kind)
   ) {
-    throw new Error("macOS JIT campaign launch plan contract is invalid");
+    throw new Error("macOS JIT instance launch plan contract is invalid");
   }
   const resolvedHostId = exact(hostId, /^h-[0-9a-f]+$/, "hostId");
   const args = [

package/scripts/aws-macos-jit-controller.mjs

@@ -6,6 +6,7 @@ import {
   AWS_MACOS_JIT_CONTROLLER_CONTRACT,
   createMacosJitCampaignPlan,
   createMacosJitClosePlan,
+  createMacosJitInstanceRehydratePlan,
   createMacosJitJobPlan,
   createMacosJitSourceRebindPlan,
   macosAllocateHostsArgs,
@@ -13,6 +14,7 @@ import {
   macosRunInstancesArgs,
 } from "./aws-macos-jit-controller-core.mjs";
 import { MACOS_EC2_JIT_REGIONS } from "./aws-macos-jit-core.mjs";
+import { executeMacosJitInstanceRehydrate } from "./aws-macos-jit-instance-rehydrate.mjs";
 import { executeMacosJitJob } from "./aws-macos-jit-job-controller.mjs";
 import { executeMacosJitSourceRebind } from "./aws-macos-jit-source-rebind.mjs";
 import {
@@ -36,6 +38,19 @@ function flag(name) {
   return process.argv.includes(`--${name}`);
 }
 
+function jsonArrayArg(name) {
+  let parsed;
+  try {
+    parsed = JSON.parse(arg(name));
+  } catch {
+    throw new Error(`--${name} must be valid JSON`);
+  }
+  if (!Array.isArray(parsed)) {
+    throw new Error(`--${name} must be a JSON array`);
+  }
+  return parsed;
+}
+
 function assertCampaignLaunchPreflight(plan, profile) {
   const launchGate = assertMacosBudgetLaunchGate(plan, profile);
   const commit = ghJson(
@@ -493,6 +508,20 @@ function commonValues(execute) {
   };
 }
 
+function confirmHistoricalPreflightFailures(plan) {
+  const confirmed = flag("confirm-historical-preflight-failure-runs-json")
+    ? jsonArrayArg("confirm-historical-preflight-failure-runs-json")
+    : [];
+  if (
+    JSON.stringify(confirmed) !==
+    JSON.stringify(plan.github.historicalPreflightFailureRuns)
+  ) {
+    throw new Error(
+      "--confirm-historical-preflight-failure-runs-json must equal the exact historical preflight failure inventory",
+    );
+  }
+}
+
 function confirm(plan) {
   if (arg("confirm-source-sha") !== plan.source.sha) {
     throw new Error("--confirm-source-sha must equal the exact source SHA");
@@ -515,6 +544,115 @@ function confirm(plan) {
     if (!flag("confirm-zero-allocation")) {
       throw new Error("--confirm-zero-allocation is required");
     }
+    if (plan.github.priorRunPolicy === "terminal-failure") {
+      const confirmed = jsonArrayArg(
+        "confirm-terminal-failure-run-ids-json",
+      ).map(String);
+      if (
+        JSON.stringify(confirmed) !==
+        JSON.stringify(plan.github.terminalFailureRunIds)
+      ) {
+        throw new Error(
+          "--confirm-terminal-failure-run-ids-json must equal the exact failed run inventory",
+        );
+      }
+      const confirmedStartupFailures = flag("confirm-startup-failure-runs-json")
+        ? jsonArrayArg("confirm-startup-failure-runs-json")
+        : [];
+      if (
+        JSON.stringify(confirmedStartupFailures) !==
+        JSON.stringify(plan.github.startupFailureRuns)
+      ) {
+        throw new Error(
+          "--confirm-startup-failure-runs-json must equal the exact startup failure inventory",
+        );
+      }
+      const confirmedHistoricalTerminalFailures = flag(
+        "confirm-historical-terminal-failure-runs-json",
+      )
+        ? jsonArrayArg("confirm-historical-terminal-failure-runs-json")
+        : [];
+      if (
+        JSON.stringify(confirmedHistoricalTerminalFailures) !==
+        JSON.stringify(plan.github.historicalTerminalFailureRuns)
+      ) {
+        throw new Error(
+          "--confirm-historical-terminal-failure-runs-json must equal the exact historical terminal failure inventory",
+        );
+      }
+      const confirmedSuccessfulRuns = flag("confirm-successful-run-ids-json")
+        ? jsonArrayArg("confirm-successful-run-ids-json").map(String)
+        : [];
+      if (
+        JSON.stringify(confirmedSuccessfulRuns) !==
+        JSON.stringify(plan.github.successfulRunIds)
+      ) {
+        throw new Error(
+          "--confirm-successful-run-ids-json must equal the exact successful run inventory",
+        );
+      }
+    } else if (plan.github.priorRunPolicy === "preflight-failure") {
+      const confirmed = jsonArrayArg(
+        "confirm-preflight-failure-run-ids-json",
+      ).map(String);
+      if (
+        JSON.stringify(confirmed) !==
+        JSON.stringify(plan.github.preflightFailureRunIds)
+      ) {
+        throw new Error(
+          "--confirm-preflight-failure-run-ids-json must equal the exact preflight failure inventory",
+        );
+      }
+      const confirmedStartupFailures = flag("confirm-startup-failure-runs-json")
+        ? jsonArrayArg("confirm-startup-failure-runs-json")
+        : [];
+      if (
+        JSON.stringify(confirmedStartupFailures) !==
+        JSON.stringify(plan.github.startupFailureRuns)
+      ) {
+        throw new Error(
+          "--confirm-startup-failure-runs-json must equal the exact startup failure inventory",
+        );
+      }
+      const confirmedHistoricalTerminalFailures = flag(
+        "confirm-historical-terminal-failure-runs-json",
+      )
+        ? jsonArrayArg("confirm-historical-terminal-failure-runs-json")
+        : [];
+      if (
+        JSON.stringify(confirmedHistoricalTerminalFailures) !==
+        JSON.stringify(plan.github.historicalTerminalFailureRuns)
+      ) {
+        throw new Error(
+          "--confirm-historical-terminal-failure-runs-json must equal the exact historical terminal failure inventory",
+        );
+      }
+      const confirmedSuccessfulRuns = flag("confirm-successful-run-ids-json")
+        ? jsonArrayArg("confirm-successful-run-ids-json").map(String)
+        : [];
+      if (
+        JSON.stringify(confirmedSuccessfulRuns) !==
+        JSON.stringify(plan.github.successfulRunIds)
+      ) {
+        throw new Error(
+          "--confirm-successful-run-ids-json must equal the exact successful run inventory",
+        );
+      }
+    }
+    confirmHistoricalPreflightFailures(plan);
+  }
+  if (plan.kind === "instance-rehydrate-plan") {
+    if (arg("confirm-host-id") !== plan.aws.hostId) {
+      throw new Error("--confirm-host-id must equal the exact host id");
+    }
+    if (arg("confirm-replaces-instance-id") !== plan.aws.replacesInstanceId) {
+      throw new Error(
+        "--confirm-replaces-instance-id must equal replacesInstanceId",
+      );
+    }
+    if (!flag("confirm-no-host-allocation")) {
+      throw new Error("--confirm-no-host-allocation is required");
+    }
   }
 }
 
@@ -541,7 +679,32 @@ export function main() {
       executeMacosJitCampaignLaunch(plan, { profile: arg("aws-profile") }),
     );
   }
-  if (["plan-rebind", "rebind-campaign"].includes(mode)) {
+  if (["plan-rehydrate", "rehydrate-instance"].includes(mode)) {
+    const plan = createMacosJitInstanceRehydratePlan({
+      ...commonValues(execute),
+      hostId: arg("host-id"),
+      hostAllocatedAt: arg("host-allocated-at"),
+      replacesInstanceId: arg("replaces-instance-id"),
+      rehydrationId: arg("rehydration-id"),
+    });
+    return emit(plan, mode, execute, () =>
+      executeMacosJitInstanceRehydrate(plan, {
+        profile: arg("aws-profile"),
+      }),
+    );
+  }
+  if (
+    [
+      "plan-rebind",
+      "rebind-campaign",
+      "plan-rebind-after-failure",
+      "rebind-campaign-after-failure",
+      "plan-rebind-after-preflight-failure",
+      "rebind-campaign-after-preflight-failure",
+    ].includes(mode)
+  ) {
+    const afterFailure = mode.endsWith("after-failure");
+    const afterPreflightFailure = mode.endsWith("after-preflight-failure");
     const plan = createMacosJitSourceRebindPlan({
       ...commonValues(execute),
       previousSourceSha: arg("previous-source-sha"),
@@ -549,8 +712,41 @@ export function main() {
       workflowId: arg("workflow-id"),
       hostId: arg("host-id"),
       instanceId: arg("instance-id"),
+      priorRunPolicy: afterPreflightFailure
+        ? "preflight-failure"
+        : afterFailure
+          ? "terminal-failure"
+          : "unused",
+      terminalFailureRunIds: afterFailure
+        ? jsonArrayArg("terminal-failure-run-ids-json")
+        : [],
+      preflightFailureRunIds: afterPreflightFailure
+        ? jsonArrayArg("preflight-failure-run-ids-json")
+        : [],
+      startupFailureRuns:
+        (afterFailure || afterPreflightFailure) &&
+        flag("startup-failure-runs-json")
+          ? jsonArrayArg("startup-failure-runs-json")
+          : [],
+      historicalTerminalFailureRuns:
+        (afterFailure || afterPreflightFailure) &&
+        flag("historical-terminal-failure-runs-json")
+          ? jsonArrayArg("historical-terminal-failure-runs-json")
+          : [],
+      historicalPreflightFailureRuns:
+        (afterFailure || afterPreflightFailure) &&
+        flag("historical-preflight-failure-runs-json")
+          ? jsonArrayArg("historical-preflight-failure-runs-json")
+          : [],
+      successfulRunIds:
+        (afterFailure || afterPreflightFailure) &&
+        flag("successful-run-ids-json")
+          ? jsonArrayArg("successful-run-ids-json")
+          : [],
     });
-    return emit(plan, mode, execute, () => executeMacosJitSourceRebind(plan, { profile: arg("aws-profile") }));
+    return emit(plan, mode, execute, () =>
+      executeMacosJitSourceRebind(plan, { profile: arg("aws-profile") }),
+    );
   }
   if (["plan-job", "run-job"].includes(mode)) {
     const plan = createMacosJitJobPlan({