@kungfu-tech/buildchain 3.0.9-alpha.0 → 3.0.9-alpha.10

package/packages/core/release-train.js

@@ -8,6 +8,8 @@ export const RELEASE_TRAIN_TRANSITION_CONTRACT =
   "kungfu-buildchain-release-train-transition/v1";
 export const RELEASE_TRAIN_OBSERVATION_CONTRACT =
   "kungfu-buildchain-release-train-observation/v1";
+export const RELEASE_BLOCKER_REPAIR_CONTRACT =
+  "kungfu-buildchain-release-blocker-repair/v1";
 export const LEGACY_DEV_ALPHA_CANDIDATE_STATE_SCHEMA =
   "kungfu-buildchain-dev-alpha-candidate-state/v1";
 
@@ -22,6 +24,7 @@ export const RELEASE_TRAIN_STATES = Object.freeze([
 ]);
 
 export const RELEASE_TRAIN_SUPERSESSION_CAUSES = Object.freeze([
+  "release-blocker-repair",
   "incompatible-semantics",
   "alpha-base-incompatibility",
   "invalid-authority",
@@ -518,3 +521,220 @@ export function readReleaseTrain(input) {
   }
   throw new Error("unsupported Release Train record");
 }
+
+function releaseBlockerLanding(input, label, { allowConflict = false } = {}) {
+  const status = text(input?.status, `${label}.status`);
+  if (status !== "landed" && !(allowConflict && status === "conflict")) {
+    throw new Error(
+      `${label}.status must be ${allowConflict ? "landed or conflict" : "landed"}`,
+    );
+  }
+  const landing = {
+    status,
+    baseSha: exactSha(input.baseSha, `${label}.baseSha`),
+    landedSha:
+      status === "landed"
+        ? exactSha(input.landedSha, `${label}.landedSha`)
+        : "",
+    patchRoot: contentRoot(input.patchRoot, `${label}.patchRoot`),
+    evidenceRoot: contentRoot(input.evidenceRoot, `${label}.evidenceRoot`),
+  };
+  return landing;
+}
+
+function releaseBlockerPublication(patchRoot, devLanding) {
+  const eligible =
+    devLanding.status === "landed" && devLanding.patchRoot === patchRoot;
+  const body = {
+    eligible,
+    reason:
+      devLanding.status === "conflict"
+        ? "dev-forward-port-conflict"
+        : eligible
+          ? "exact-cut-dev-patch-settled"
+          : "cut-dev-patch-root-mismatch",
+    requiredPatchRoot: patchRoot,
+    observedDevPatchRoot: devLanding.patchRoot,
+    devLandingStatus: devLanding.status,
+  };
+  return { ...body, gateRoot: releaseTrainRoot(body) };
+}
+
+function buildReleaseBlockerRepair(activeTrainInput, input = {}) {
+  const activeTrain = validateReleaseTrain(activeTrainInput);
+  if (activeTrain.state.status !== "repair-required") {
+    throw new Error(
+      "release-blocker repair requires the active train to be repair-required",
+    );
+  }
+  const expectedStateRoot = contentRoot(
+    input.expectedStateRoot,
+    "expectedStateRoot",
+  );
+  if (expectedStateRoot !== activeTrain.state.stateRoot) {
+    throw new Error("release-blocker repair compare-and-swap failed");
+  }
+  const patchRoot = contentRoot(input.patchRoot, "patchRoot");
+  const blockerRoot = contentRoot(input.blockerRoot, "blockerRoot");
+  const authorityRoots = sortedRoots(input.authorityRoots, "authorityRoots");
+  const createdAt = timestamp(input.createdAt, "createdAt");
+  const cutLanding = releaseBlockerLanding(
+    {
+      status: "landed",
+      baseSha: activeTrain.releaseCut.candidateSha,
+      landedSha: input.cutCandidateSha,
+      patchRoot: input.cutPatchRoot,
+      evidenceRoot: input.cutLandingEvidenceRoot,
+    },
+    "cutLanding",
+  );
+  if (cutLanding.patchRoot !== patchRoot) {
+    throw new Error(
+      "cut landing must preserve the declared release-blocker patch root",
+    );
+  }
+  const devLanding = releaseBlockerLanding(
+    {
+      status: input.devLandingStatus,
+      baseSha: input.devBaseSha,
+      landedSha: input.devLandingSha,
+      patchRoot: input.devPatchRoot,
+      evidenceRoot: input.devLandingEvidenceRoot,
+    },
+    "devLanding",
+    { allowConflict: true },
+  );
+  const priorCut = activeTrain.releaseCut;
+  const successorTrain = createReleaseTrain({
+    repository: priorCut.repository,
+    sourceBranch: priorCut.sourceBranch,
+    targetBranch: priorCut.targetBranch,
+    originDevSha: priorCut.originDevSha,
+    candidateSha: cutLanding.landedSha,
+    candidateTreeSha: exactSha(
+      input.cutCandidateTreeSha,
+      "cutCandidateTreeSha",
+    ),
+    alphaBaseSha: priorCut.alphaBaseSha,
+    buildchainRuntimeSha: priorCut.buildchainRuntimeSha,
+    generation: priorCut.generation + 1,
+    authorityRoots: [
+      ...new Set([...priorCut.authorityRoots, ...authorityRoots]),
+    ].sort(),
+    supersession: {
+      cause: "release-blocker-repair",
+      priorCutRoot: priorCut.cutRoot,
+    },
+    createdAt,
+  });
+  const supersededTrain = transitionReleaseTrain(activeTrain, {
+    to: "superseded",
+    expectedStateRoot,
+    event: "release-blocker-successor-cut",
+    reason:
+      "a rooted release-blocker repair advanced the active candidate generation",
+    authorityRoots,
+    supersessionCause: "release-blocker-repair",
+    replacementCutRoot: successorTrain.releaseCut.cutRoot,
+    replacementCandidateSha: successorTrain.releaseCut.candidateSha,
+    recordedAt: createdAt,
+  });
+  const publication = releaseBlockerPublication(patchRoot, devLanding);
+  const body = {
+    schemaVersion: 1,
+    contract: RELEASE_BLOCKER_REPAIR_CONTRACT,
+    blockerRoot,
+    patchRoot,
+    priorTrain: activeTrain,
+    supersededTrain,
+    successorTrain,
+    cutLanding,
+    devLanding,
+    candidateBuild: {
+      eligible: true,
+      candidateSha: successorTrain.releaseCut.candidateSha,
+      candidateTreeSha: successorTrain.releaseCut.candidateTreeSha,
+      generation: successorTrain.releaseCut.generation,
+      cutRoot: successorTrain.releaseCut.cutRoot,
+      reason: "rooted-cut-landing",
+    },
+    publication,
+    authorityRoots,
+    createdAt,
+  };
+  return { ...body, repairRoot: releaseTrainRoot(body) };
+}
+
+export function createReleaseBlockerRepair(activeTrainInput, input = {}) {
+  return buildReleaseBlockerRepair(activeTrainInput, input);
+}
+
+export function validateReleaseBlockerRepair(input) {
+  if (
+    input?.contract !== RELEASE_BLOCKER_REPAIR_CONTRACT ||
+    Number(input?.schemaVersion) !== 1
+  ) {
+    throw new Error(
+      `release-blocker repair must use ${RELEASE_BLOCKER_REPAIR_CONTRACT}`,
+    );
+  }
+  const rebuilt = buildReleaseBlockerRepair(input.priorTrain, {
+    expectedStateRoot: input.priorTrain?.state?.stateRoot,
+    blockerRoot: input.blockerRoot,
+    patchRoot: input.patchRoot,
+    cutCandidateSha: input.cutLanding?.landedSha,
+    cutCandidateTreeSha: input.successorTrain?.releaseCut?.candidateTreeSha,
+    cutPatchRoot: input.cutLanding?.patchRoot,
+    cutLandingEvidenceRoot: input.cutLanding?.evidenceRoot,
+    devLandingStatus: input.devLanding?.status,
+    devBaseSha: input.devLanding?.baseSha,
+    devLandingSha: input.devLanding?.landedSha,
+    devPatchRoot: input.devLanding?.patchRoot,
+    devLandingEvidenceRoot: input.devLanding?.evidenceRoot,
+    authorityRoots: input.authorityRoots,
+    createdAt: input.createdAt,
+  });
+  if (input.repairRoot !== rebuilt.repairRoot) {
+    throw new Error("release-blocker repair root does not match its content");
+  }
+  if (releaseTrainRoot(input) !== releaseTrainRoot(rebuilt)) {
+    throw new Error(
+      "release-blocker repair content drifted from its rooted contract",
+    );
+  }
+  return clone(input);
+}
+
+export function settleReleaseBlockerDevLanding(repairInput, input = {}) {
+  const repair = validateReleaseBlockerRepair(repairInput);
+  if (
+    contentRoot(input.expectedRepairRoot, "expectedRepairRoot") !==
+    repair.repairRoot
+  ) {
+    throw new Error(
+      "release-blocker repair settlement compare-and-swap failed",
+    );
+  }
+  const patchRoot = contentRoot(input.patchRoot, "patchRoot");
+  if (patchRoot !== repair.patchRoot) {
+    throw new Error(
+      "dev landing settlement must preserve the release-blocker patch root",
+    );
+  }
+  return buildReleaseBlockerRepair(repair.priorTrain, {
+    expectedStateRoot: repair.priorTrain.state.stateRoot,
+    blockerRoot: repair.blockerRoot,
+    patchRoot: repair.patchRoot,
+    cutCandidateSha: repair.cutLanding.landedSha,
+    cutCandidateTreeSha: repair.successorTrain.releaseCut.candidateTreeSha,
+    cutPatchRoot: repair.cutLanding.patchRoot,
+    cutLandingEvidenceRoot: repair.cutLanding.evidenceRoot,
+    devLandingStatus: "landed",
+    devBaseSha: input.devBaseSha,
+    devLandingSha: input.devLandingSha,
+    devPatchRoot: patchRoot,
+    devLandingEvidenceRoot: input.devLandingEvidenceRoot,
+    authorityRoots: repair.authorityRoots,
+    createdAt: repair.createdAt,
+  });
+}

package/scripts/aws-macos-jit-controller-core.mjs

@@ -75,6 +75,103 @@ function qualificationId(value) {
   return exact(value, /^mac-(?:smoke-0[12]|full-01)$/, "qualificationId");
 }
 
+function terminalFailureRunIds(value) {
+  if (!Array.isArray(value) || value.length === 0) {
+    throw new Error("terminalFailureRunIds must be a non-empty array");
+  }
+  const normalized = value.map((entry) =>
+    exact(entry, /^[1-9]\d*$/, "terminalFailureRunId"),
+  );
+  if (new Set(normalized).size !== normalized.length) {
+    throw new Error("terminalFailureRunIds must be unique");
+  }
+  return [...normalized].sort((left, right) => Number(left) - Number(right));
+}
+
+function preflightFailureRunIds(value) {
+  if (!Array.isArray(value) || value.length === 0) {
+    throw new Error("preflightFailureRunIds must be a non-empty array");
+  }
+  const normalized = value.map((entry) =>
+    exact(entry, /^[1-9]\d*$/, "preflightFailureRunId"),
+  );
+  if (new Set(normalized).size !== normalized.length) {
+    throw new Error("preflightFailureRunIds must be unique");
+  }
+  return [...normalized].sort((left, right) => Number(left) - Number(right));
+}
+
+function successfulRunIds(value) {
+  if (value === undefined) return [];
+  if (!Array.isArray(value)) {
+    throw new Error("successfulRunIds must be an array");
+  }
+  const normalized = value.map((entry) =>
+    exact(entry, /^[1-9]\d*$/, "successfulRunId"),
+  );
+  if (new Set(normalized).size !== normalized.length) {
+    throw new Error("successfulRunIds must be unique");
+  }
+  return [...normalized].sort((left, right) => Number(left) - Number(right));
+}
+
+function startupFailureRuns(value) {
+  if (value === undefined) return [];
+  if (!Array.isArray(value)) {
+    throw new Error("startupFailureRuns must be an array");
+  }
+  const normalized = value.map((entry) => ({
+    runId: exact(entry?.runId, /^[1-9]\d*$/, "startupFailureRunId"),
+    sourceSha: exactSha(entry?.sourceSha),
+  }));
+  if (
+    new Set(normalized.map(({ runId }) => runId)).size !== normalized.length
+  ) {
+    throw new Error("startupFailureRuns must use unique run ids");
+  }
+  return [...normalized].sort(
+    (left, right) => Number(left.runId) - Number(right.runId),
+  );
+}
+
+function historicalTerminalFailureRuns(value) {
+  if (value === undefined) return [];
+  if (!Array.isArray(value)) {
+    throw new Error("historicalTerminalFailureRuns must be an array");
+  }
+  const normalized = value.map((entry) => ({
+    runId: exact(entry?.runId, /^[1-9]\d*$/, "historicalTerminalFailureRunId"),
+    sourceSha: exactSha(entry?.sourceSha),
+  }));
+  if (
+    new Set(normalized.map(({ runId }) => runId)).size !== normalized.length
+  ) {
+    throw new Error("historicalTerminalFailureRuns must use unique run ids");
+  }
+  return [...normalized].sort(
+    (left, right) => Number(left.runId) - Number(right.runId),
+  );
+}
+
+function historicalPreflightFailureRuns(value) {
+  if (value === undefined) return [];
+  if (!Array.isArray(value)) {
+    throw new Error("historicalPreflightFailureRuns must be an array");
+  }
+  const normalized = value.map((entry) => ({
+    runId: exact(entry?.runId, /^[1-9]\d*$/, "historicalPreflightFailureRunId"),
+    sourceSha: exactSha(entry?.sourceSha),
+  }));
+  if (
+    new Set(normalized.map(({ runId }) => runId)).size !== normalized.length
+  ) {
+    throw new Error("historicalPreflightFailureRuns must use unique run ids");
+  }
+  return [...normalized].sort(
+    (left, right) => Number(left.runId) - Number(right.runId),
+  );
+}
+
 function commonAws(values) {
   const region = exact(
     values.region || MACOS_EC2_JIT.region,
@@ -203,6 +300,87 @@ export function createMacosJitCampaignPlan(values = {}) {
   return { ...plan, digest: digest(plan) };
 }
 
+export function createMacosJitInstanceRehydratePlan(values = {}) {
+  const id = campaignId(values.campaignId);
+  const boundSource = source(values);
+  const aws = commonAws(values);
+  const regionConfig = macosJitRegionConfig(aws.region);
+  const tags = ownershipTags({ id, sourceSha: boundSource.sha });
+  const rehydrationId = exact(
+    values.rehydrationId,
+    /^r[1-9]\d{0,2}$/,
+    "rehydrationId",
+  );
+  const instanceClientToken = `kungfu-mac-rehydrate-${id}-${rehydrationId}`;
+  if (instanceClientToken.length > 64) {
+    throw new Error("rehydration client token exceeds the EC2 limit");
+  }
+  const plan = {
+    schemaVersion: 1,
+    contract: AWS_MACOS_JIT_CONTROLLER_CONTRACT,
+    kind: "instance-rehydrate-plan",
+    repository: repository(values.repository),
+    account: {
+      id: exact(values.accountId, /^\d{12}$/, "accountId"),
+    },
+    github: {
+      workflowId: MACOS_EC2_JIT.workflowId,
+      requiredState: "disabled_manually",
+    },
+    campaign: { id },
+    source: boundSource,
+    aws: {
+      ...aws,
+      hostId: exact(values.hostId, /^h-[0-9a-f]+$/, "hostId"),
+      hostAllocatedAt: iso(values.hostAllocatedAt, "hostAllocatedAt"),
+      replacesInstanceId: exact(
+        values.replacesInstanceId,
+        /^i-[0-9a-f]+$/,
+        "replacesInstanceId",
+      ),
+      rehydrationId,
+      instanceTags: tags,
+      volumeTags: tags,
+      instanceClientToken,
+      rootVolume: {
+        deviceName: "/dev/sda1",
+        deleteOnTermination: true,
+        encrypted: true,
+        volumeSizeGiB: 200,
+        volumeType: "gp3",
+      },
+      metadata: {
+        httpEndpoint: "enabled",
+        httpTokens: "required",
+        httpPutResponseHopLimit: 1,
+        instanceMetadataTags: "disabled",
+      },
+    },
+    safety: {
+      applyMode: values.execute === true ? "execute" : "dry-run",
+      exactSourceRequired: true,
+      existingCampaignHostRequired: true,
+      noHostAllocation: true,
+      zeroActiveInstanceRequired: true,
+      zeroRunnerResidueRequired: true,
+      awsDryRunRequiredBeforeLaunch: true,
+      cleanupOwner: "scheduled-card-scoped-reaper",
+      budget: {
+        name: regionConfig.budgetName,
+        limitUsd: MACOS_EC2_JIT.budgetLimitUsd,
+        metrics: ["UnblendedCost"],
+        dimensionFilter: {
+          usageTypes: regionConfig.budgetUsageTypes,
+          operation: MACOS_EC2_JIT.budgetOperation,
+          regions: regionConfig.budgetRegions,
+        },
+        requiredActualThresholds: [80, 95],
+      },
+    },
+  };
+  return { ...plan, digest: digest(plan) };
+}
+
 export function createMacosJitSourceRebindPlan(values = {}) {
   const id = campaignId(values.campaignId);
   const boundSource = source(values);
@@ -219,6 +397,48 @@ export function createMacosJitSourceRebindPlan(values = {}) {
   if (workflowId !== MACOS_EC2_JIT.workflowId) {
     throw new Error(`workflowId must be ${MACOS_EC2_JIT.workflowId}`);
   }
+  const priorRunPolicy = values.priorRunPolicy || "unused";
+  if (
+    !["unused", "terminal-failure", "preflight-failure"].includes(
+      priorRunPolicy,
+    )
+  ) {
+    throw new Error(
+      "priorRunPolicy must be unused, terminal-failure, or preflight-failure",
+    );
+  }
+  const terminalRunIds =
+    priorRunPolicy === "terminal-failure"
+      ? terminalFailureRunIds(values.terminalFailureRunIds)
+      : [];
+  const preflightRunIds =
+    priorRunPolicy === "preflight-failure"
+      ? preflightFailureRunIds(values.preflightFailureRunIds)
+      : [];
+  const startupRuns = startupFailureRuns(values.startupFailureRuns);
+  const historicalTerminalRuns = historicalTerminalFailureRuns(
+    values.historicalTerminalFailureRuns,
+  );
+  const historicalPreflightRuns = historicalPreflightFailureRuns(
+    values.historicalPreflightFailureRuns,
+  );
+  const successfulRuns = successfulRunIds(values.successfulRunIds);
+  if (priorRunPolicy === "unused" && startupRuns.length !== 0) {
+    throw new Error("startupFailureRuns require a failed priorRunPolicy");
+  }
+  if (priorRunPolicy === "unused" && historicalTerminalRuns.length !== 0) {
+    throw new Error(
+      "historicalTerminalFailureRuns require a failed priorRunPolicy",
+    );
+  }
+  if (priorRunPolicy === "unused" && historicalPreflightRuns.length !== 0) {
+    throw new Error(
+      "historicalPreflightFailureRuns require a failed priorRunPolicy",
+    );
+  }
+  if (priorRunPolicy === "unused" && successfulRuns.length !== 0) {
+    throw new Error("successfulRunIds require a failed priorRunPolicy");
+  }
   const plan = {
     schemaVersion: 1,
     contract: AWS_MACOS_JIT_CONTROLLER_CONTRACT,
@@ -233,6 +453,13 @@ export function createMacosJitSourceRebindPlan(values = {}) {
     github: {
       workflowId,
       requiredState: "disabled_manually",
+      priorRunPolicy,
+      terminalFailureRunIds: terminalRunIds,
+      preflightFailureRunIds: preflightRunIds,
+      startupFailureRuns: startupRuns,
+      historicalTerminalFailureRuns: historicalTerminalRuns,
+      historicalPreflightFailureRuns: historicalPreflightRuns,
+      successfulRunIds: successfulRuns,
     },
     aws: {
       ...aws,
@@ -246,8 +473,16 @@ export function createMacosJitSourceRebindPlan(values = {}) {
       forwardOnlySource: true,
       sameSourceRefRequired: true,
       workflowDisabledRequired: true,
-      zeroPriorJobsRequired: true,
-      zeroPriorArtifactsRequired: true,
+      zeroPriorJobsRequired: priorRunPolicy === "unused",
+      zeroPriorArtifactsRequired: priorRunPolicy === "unused",
+      terminalFailureEvidencePreserved: priorRunPolicy === "terminal-failure",
+      preflightFailureEvidencePreserved: priorRunPolicy === "preflight-failure",
+      startupFailureEvidencePreserved: startupRuns.length !== 0,
+      historicalTerminalFailureEvidencePreserved:
+        historicalTerminalRuns.length !== 0,
+      historicalPreflightFailureEvidencePreserved:
+        historicalPreflightRuns.length !== 0,
+      successfulRunEvidencePreserved: successfulRuns.length !== 0,
       zeroJitResidueRequired: true,
       zeroEvidenceRequired: true,
       exactCampaignResourcesRequired: true,
@@ -301,9 +536,9 @@ export function macosAllocateHostsArgs(plan) {
 export function macosRunInstancesArgs(plan, { hostId, dryRun = false } = {}) {
   if (
     plan?.contract !== AWS_MACOS_JIT_CONTROLLER_CONTRACT ||
-    plan.kind !== "campaign-launch-plan"
+    !["campaign-launch-plan", "instance-rehydrate-plan"].includes(plan.kind)
   ) {
-    throw new Error("macOS JIT campaign launch plan contract is invalid");
+    throw new Error("macOS JIT instance launch plan contract is invalid");
   }
   const resolvedHostId = exact(hostId, /^h-[0-9a-f]+$/, "hostId");
   const args = [