@kungfu-tech/buildchain 3.0.9-alpha.2 → 3.0.9-alpha.4

package/packages/core/paper-agent-entry.js

@@ -13,6 +13,7 @@ import {
   workCheck,
 } from "./paper-repository.js";
 import { mergePublicationRehearsalAgentInstructions } from "./publication-rehearsal-projection.js";
+import { mergeNextDevelopmentAgentInstructions } from "./next-development-projection.js";
 
 export const PAPER_AGENT_ENTRY_CONTRACT = "kungfu-buildchain-paper-agent-entry";
 export const PAPER_AGENT_ENTRY_SCHEMA_VERSION = 1;
@@ -168,10 +169,12 @@ export function paperAgentEntryFiles({
     [PAPER_PATHS.agentEntry, jsonText(entry)],
     [
       PAPER_PATHS.agentInstructions,
-      mergePublicationRehearsalAgentInstructions(
-        mergePaperAgentEntryInstructions(currentAgents, {
-          developmentRef: resolvedDevelopmentRef,
-        }),
+      mergeNextDevelopmentAgentInstructions(
+        mergePublicationRehearsalAgentInstructions(
+          mergePaperAgentEntryInstructions(currentAgents, {
+            developmentRef: resolvedDevelopmentRef,
+          }),
+        ),
       ),
     ],
   ]);

package/packages/core/paper.js

@@ -51,6 +51,11 @@ import {
   projectPublicationRehearsalToml,
   publicationRehearsalWorkflow,
 } from "./publication-rehearsal-projection.js";
+import {
+  appendNextDevelopmentToml,
+  nextDevelopmentWorkflowHeader,
+  projectNextDevelopmentToml,
+} from "./next-development-projection.js";
 
 export { PAPER_PATHS, resolvePaperRepository } from "./paper-repository.js";
 export {
@@ -356,7 +361,7 @@ immutable_base_url = ${tomlString(joinUrl(siteBaseUrl, "archive").replace(/\/$/,
 registry_path = ".buildchain/publication/publication-registry.json"
 `
     : "";
-  return appendPublicationRehearsalToml(`schema = 1
+  return appendNextDevelopmentToml(appendPublicationRehearsalToml(`schema = 1
 
 [project]
 type = "publication-artifact"
@@ -390,14 +395,14 @@ command = "make pdf"
 
 [lifecycle.verify]
 command = "make check"
-`);
+`));
 }
 
 function scaffoldBuildWorkflow(
   buildchainSha,
   { artifactName = "paper-publication" } = {},
 ) {
-  return `name: Build
+  return `${nextDevelopmentWorkflowHeader()}name: Build
 
 on:
   workflow_dispatch:
@@ -425,7 +430,7 @@ jobs:
 }
 
 function scaffoldVerifyWorkflow(buildchainSha) {
-  return `name: Verify
+  return `${nextDevelopmentWorkflowHeader()}name: Verify
 
 on:
   pull_request:
@@ -456,7 +461,7 @@ function scaffoldReleaseWorkflow(
   const passportInput = releasePassportProductName
     ? `      release-passport-product-name: ${JSON.stringify(releasePassportProductName)}\n`
     : "";
-  return `name: Paper Release
+  return `${nextDevelopmentWorkflowHeader()}name: Paper Release
 
 on:
   workflow_dispatch:
@@ -1017,8 +1022,10 @@ function migrationFiles({
   const files = new Map([
     [
       PAPER_PATHS.config,
-      projectPublicationRehearsalToml(
-        fs.readFileSync(path.resolve(cwd, PAPER_PATHS.config), "utf8"),
+      projectNextDevelopmentToml(
+        projectPublicationRehearsalToml(
+          fs.readFileSync(path.resolve(cwd, PAPER_PATHS.config), "utf8"),
+        ),
       ),
     ],
     [PAPER_PATHS.contractLock, contractLockText],

package/packages/core/release-passport-contract.js

@@ -414,7 +414,7 @@ export function collectKfdAdopterReleaseEvidence({ manifest, gateResults = [], c
   }
   const manifestGate = createKfdAdopterManifestGate({
     manifest, packageArtifactRoot: installedKfdPackageArtifactRoot(), gateResults,
-    expectedSourceSha: sourceSha, checkedAt,
+    authorityPath: "kfd-adopter-manifest.json", expectedSourceSha: sourceSha, checkedAt,
   });
   const legacyProjection = createKfdLegacySupportMatrixProjection({ manifest, manifestGate });
   if (comparisonMatrix) {

package/scripts/aws-macos-jit-controller-core.mjs

@@ -75,6 +75,19 @@ function qualificationId(value) {
   return exact(value, /^mac-(?:smoke-0[12]|full-01)$/, "qualificationId");
 }
 
+function terminalFailureRunIds(value) {
+  if (!Array.isArray(value) || value.length === 0) {
+    throw new Error("terminalFailureRunIds must be a non-empty array");
+  }
+  const normalized = value.map((entry) =>
+    exact(entry, /^[1-9]\d*$/, "terminalFailureRunId"),
+  );
+  if (new Set(normalized).size !== normalized.length) {
+    throw new Error("terminalFailureRunIds must be unique");
+  }
+  return [...normalized].sort((left, right) => Number(left) - Number(right));
+}
+
 function commonAws(values) {
   const region = exact(
     values.region || MACOS_EC2_JIT.region,
@@ -203,6 +216,87 @@ export function createMacosJitCampaignPlan(values = {}) {
   return { ...plan, digest: digest(plan) };
 }
 
+export function createMacosJitInstanceRehydratePlan(values = {}) {
+  const id = campaignId(values.campaignId);
+  const boundSource = source(values);
+  const aws = commonAws(values);
+  const regionConfig = macosJitRegionConfig(aws.region);
+  const tags = ownershipTags({ id, sourceSha: boundSource.sha });
+  const rehydrationId = exact(
+    values.rehydrationId,
+    /^r[1-9]\d{0,2}$/,
+    "rehydrationId",
+  );
+  const instanceClientToken = `kungfu-mac-rehydrate-${id}-${rehydrationId}`;
+  if (instanceClientToken.length > 64) {
+    throw new Error("rehydration client token exceeds the EC2 limit");
+  }
+  const plan = {
+    schemaVersion: 1,
+    contract: AWS_MACOS_JIT_CONTROLLER_CONTRACT,
+    kind: "instance-rehydrate-plan",
+    repository: repository(values.repository),
+    account: {
+      id: exact(values.accountId, /^\d{12}$/, "accountId"),
+    },
+    github: {
+      workflowId: MACOS_EC2_JIT.workflowId,
+      requiredState: "disabled_manually",
+    },
+    campaign: { id },
+    source: boundSource,
+    aws: {
+      ...aws,
+      hostId: exact(values.hostId, /^h-[0-9a-f]+$/, "hostId"),
+      hostAllocatedAt: iso(values.hostAllocatedAt, "hostAllocatedAt"),
+      replacesInstanceId: exact(
+        values.replacesInstanceId,
+        /^i-[0-9a-f]+$/,
+        "replacesInstanceId",
+      ),
+      rehydrationId,
+      instanceTags: tags,
+      volumeTags: tags,
+      instanceClientToken,
+      rootVolume: {
+        deviceName: "/dev/sda1",
+        deleteOnTermination: true,
+        encrypted: true,
+        volumeSizeGiB: 200,
+        volumeType: "gp3",
+      },
+      metadata: {
+        httpEndpoint: "enabled",
+        httpTokens: "required",
+        httpPutResponseHopLimit: 1,
+        instanceMetadataTags: "disabled",
+      },
+    },
+    safety: {
+      applyMode: values.execute === true ? "execute" : "dry-run",
+      exactSourceRequired: true,
+      existingCampaignHostRequired: true,
+      noHostAllocation: true,
+      zeroActiveInstanceRequired: true,
+      zeroRunnerResidueRequired: true,
+      awsDryRunRequiredBeforeLaunch: true,
+      cleanupOwner: "scheduled-card-scoped-reaper",
+      budget: {
+        name: regionConfig.budgetName,
+        limitUsd: MACOS_EC2_JIT.budgetLimitUsd,
+        metrics: ["UnblendedCost"],
+        dimensionFilter: {
+          usageTypes: regionConfig.budgetUsageTypes,
+          operation: MACOS_EC2_JIT.budgetOperation,
+          regions: regionConfig.budgetRegions,
+        },
+        requiredActualThresholds: [80, 95],
+      },
+    },
+  };
+  return { ...plan, digest: digest(plan) };
+}
+
 export function createMacosJitSourceRebindPlan(values = {}) {
   const id = campaignId(values.campaignId);
   const boundSource = source(values);
@@ -219,6 +313,14 @@ export function createMacosJitSourceRebindPlan(values = {}) {
   if (workflowId !== MACOS_EC2_JIT.workflowId) {
     throw new Error(`workflowId must be ${MACOS_EC2_JIT.workflowId}`);
   }
+  const priorRunPolicy = values.priorRunPolicy || "unused";
+  if (!["unused", "terminal-failure"].includes(priorRunPolicy)) {
+    throw new Error("priorRunPolicy must be unused or terminal-failure");
+  }
+  const terminalRunIds =
+    priorRunPolicy === "terminal-failure"
+      ? terminalFailureRunIds(values.terminalFailureRunIds)
+      : [];
   const plan = {
     schemaVersion: 1,
     contract: AWS_MACOS_JIT_CONTROLLER_CONTRACT,
@@ -233,6 +335,8 @@ export function createMacosJitSourceRebindPlan(values = {}) {
     github: {
       workflowId,
       requiredState: "disabled_manually",
+      priorRunPolicy,
+      terminalFailureRunIds: terminalRunIds,
     },
     aws: {
       ...aws,
@@ -246,8 +350,9 @@ export function createMacosJitSourceRebindPlan(values = {}) {
       forwardOnlySource: true,
       sameSourceRefRequired: true,
       workflowDisabledRequired: true,
-      zeroPriorJobsRequired: true,
-      zeroPriorArtifactsRequired: true,
+      zeroPriorJobsRequired: priorRunPolicy === "unused",
+      zeroPriorArtifactsRequired: priorRunPolicy === "unused",
+      terminalFailureEvidencePreserved: priorRunPolicy === "terminal-failure",
       zeroJitResidueRequired: true,
       zeroEvidenceRequired: true,
       exactCampaignResourcesRequired: true,
@@ -301,9 +406,9 @@ export function macosAllocateHostsArgs(plan) {
 export function macosRunInstancesArgs(plan, { hostId, dryRun = false } = {}) {
   if (
     plan?.contract !== AWS_MACOS_JIT_CONTROLLER_CONTRACT ||
-    plan.kind !== "campaign-launch-plan"
+    !["campaign-launch-plan", "instance-rehydrate-plan"].includes(plan.kind)
   ) {
-    throw new Error("macOS JIT campaign launch plan contract is invalid");
+    throw new Error("macOS JIT instance launch plan contract is invalid");
   }
   const resolvedHostId = exact(hostId, /^h-[0-9a-f]+$/, "hostId");
   const args = [

package/scripts/aws-macos-jit-controller.mjs

@@ -6,6 +6,7 @@ import {
   AWS_MACOS_JIT_CONTROLLER_CONTRACT,
   createMacosJitCampaignPlan,
   createMacosJitClosePlan,
+  createMacosJitInstanceRehydratePlan,
   createMacosJitJobPlan,
   createMacosJitSourceRebindPlan,
   macosAllocateHostsArgs,
@@ -13,6 +14,7 @@ import {
   macosRunInstancesArgs,
 } from "./aws-macos-jit-controller-core.mjs";
 import { MACOS_EC2_JIT_REGIONS } from "./aws-macos-jit-core.mjs";
+import { executeMacosJitInstanceRehydrate } from "./aws-macos-jit-instance-rehydrate.mjs";
 import { executeMacosJitJob } from "./aws-macos-jit-job-controller.mjs";
 import { executeMacosJitSourceRebind } from "./aws-macos-jit-source-rebind.mjs";
 import {
@@ -36,6 +38,19 @@ function flag(name) {
   return process.argv.includes(`--${name}`);
 }
 
+function jsonArrayArg(name) {
+  let parsed;
+  try {
+    parsed = JSON.parse(arg(name));
+  } catch {
+    throw new Error(`--${name} must be valid JSON`);
+  }
+  if (!Array.isArray(parsed)) {
+    throw new Error(`--${name} must be a JSON array`);
+  }
+  return parsed;
+}
+
 function assertCampaignLaunchPreflight(plan, profile) {
   const launchGate = assertMacosBudgetLaunchGate(plan, profile);
   const commit = ghJson(
@@ -515,6 +530,32 @@ function confirm(plan) {
     if (!flag("confirm-zero-allocation")) {
       throw new Error("--confirm-zero-allocation is required");
     }
+    if (plan.github.priorRunPolicy === "terminal-failure") {
+      const confirmed = jsonArrayArg(
+        "confirm-terminal-failure-run-ids-json",
+      ).map(String);
+      if (
+        JSON.stringify(confirmed) !==
+        JSON.stringify(plan.github.terminalFailureRunIds)
+      ) {
+        throw new Error(
+          "--confirm-terminal-failure-run-ids-json must equal the exact failed run inventory",
+        );
+      }
+    }
+  }
+  if (plan.kind === "instance-rehydrate-plan") {
+    if (arg("confirm-host-id") !== plan.aws.hostId) {
+      throw new Error("--confirm-host-id must equal the exact host id");
+    }
+    if (arg("confirm-replaces-instance-id") !== plan.aws.replacesInstanceId) {
+      throw new Error(
+        "--confirm-replaces-instance-id must equal replacesInstanceId",
+      );
+    }
+    if (!flag("confirm-no-host-allocation")) {
+      throw new Error("--confirm-no-host-allocation is required");
+    }
   }
 }
 
@@ -541,7 +582,29 @@ export function main() {
       executeMacosJitCampaignLaunch(plan, { profile: arg("aws-profile") }),
     );
   }
-  if (["plan-rebind", "rebind-campaign"].includes(mode)) {
+  if (["plan-rehydrate", "rehydrate-instance"].includes(mode)) {
+    const plan = createMacosJitInstanceRehydratePlan({
+      ...commonValues(execute),
+      hostId: arg("host-id"),
+      hostAllocatedAt: arg("host-allocated-at"),
+      replacesInstanceId: arg("replaces-instance-id"),
+      rehydrationId: arg("rehydration-id"),
+    });
+    return emit(plan, mode, execute, () =>
+      executeMacosJitInstanceRehydrate(plan, {
+        profile: arg("aws-profile"),
+      }),
+    );
+  }
+  if (
+    [
+      "plan-rebind",
+      "rebind-campaign",
+      "plan-rebind-after-failure",
+      "rebind-campaign-after-failure",
+    ].includes(mode)
+  ) {
+    const afterFailure = mode.endsWith("after-failure");
     const plan = createMacosJitSourceRebindPlan({
       ...commonValues(execute),
       previousSourceSha: arg("previous-source-sha"),
@@ -549,8 +612,14 @@ export function main() {
       workflowId: arg("workflow-id"),
       hostId: arg("host-id"),
       instanceId: arg("instance-id"),
+      priorRunPolicy: afterFailure ? "terminal-failure" : "unused",
+      terminalFailureRunIds: afterFailure
+        ? jsonArrayArg("terminal-failure-run-ids-json")
+        : [],
     });
-    return emit(plan, mode, execute, () => executeMacosJitSourceRebind(plan, { profile: arg("aws-profile") }));
+    return emit(plan, mode, execute, () =>
+      executeMacosJitSourceRebind(plan, { profile: arg("aws-profile") }),
+    );
   }
   if (["plan-job", "run-job"].includes(mode)) {
     const plan = createMacosJitJobPlan({

package/scripts/aws-macos-jit-instance-rehydrate.mjs

@@ -0,0 +1,260 @@
+// SPDX-License-Identifier: Apache-2.0
+
+import {
+  AWS_MACOS_JIT_CONTROLLER_CONTRACT,
+  macosRunInstancesArgs,
+} from "./aws-macos-jit-controller-core.mjs";
+import { MACOS_EC2_JIT_REGIONS } from "./aws-macos-jit-core.mjs";
+import {
+  assertDryRun,
+  assertMacosBudgetLaunchGate,
+  assertOwnership,
+  awsArgs,
+  awsJson,
+  commandResult,
+  ghJson,
+  requireSuccess,
+} from "./aws-macos-jit-controller-runtime.mjs";
+
+function assertInstanceRehydratePreflight(plan, profile) {
+  const launchGate = assertMacosBudgetLaunchGate(plan, profile);
+  const commit = ghJson(
+    ["api", `repos/${plan.repository}/commits/${plan.source.sha}`],
+    undefined,
+    "GitHub exact-source preflight",
+  );
+  if (String(commit.sha || "").toLowerCase() !== plan.source.sha) {
+    throw new Error("GitHub did not resolve the exact campaign source SHA");
+  }
+  const host = awsJson(
+    plan,
+    profile,
+    [
+      "ec2",
+      "describe-hosts",
+      "--host-ids",
+      plan.aws.hostId,
+      "--output",
+      "json",
+    ],
+    "macOS rehydration host preflight",
+  ).Hosts?.[0];
+  if (
+    !host ||
+    host.State !== "available" ||
+    host.AvailabilityZone !== plan.aws.availabilityZone ||
+    host.HostProperties?.InstanceType !== plan.aws.instanceType ||
+    (host.Instances || []).length !== 0 ||
+    new Date(host.AllocationTime).toISOString() !== plan.aws.hostAllocatedAt
+  ) {
+    throw new Error("macOS rehydration host identity or capacity mismatch");
+  }
+  assertOwnership(host.Tags, plan);
+
+  const regionalPlans = Object.keys(MACOS_EC2_JIT_REGIONS).map((region) => ({
+    ...plan,
+    aws: { ...plan.aws, region },
+  }));
+  const otherHosts = regionalPlans.flatMap((regionalPlan) =>
+    (
+      awsJson(
+        regionalPlan,
+        profile,
+        [
+          "ec2",
+          "describe-hosts",
+          "--filter",
+          "Name=tag:kungfu:plane,Values=aws-us-elastic-runner-burst",
+          "Name=tag:kungfu:provider,Values=macos-ec2-jit",
+          "Name=state,Values=available,pending,under-assessment",
+          "--output",
+          "json",
+        ],
+        `macOS rehydration host ceiling in ${regionalPlan.aws.region}`,
+      ).Hosts || []
+    ).filter((candidate) => candidate.HostId !== plan.aws.hostId),
+  );
+  if (otherHosts.length !== 0) {
+    throw new Error("macOS rehydration found another active Dedicated Host");
+  }
+  const activeInstances = regionalPlans.flatMap((regionalPlan) => {
+    const active = awsJson(
+      regionalPlan,
+      profile,
+      [
+        "ec2",
+        "describe-instances",
+        "--filters",
+        "Name=tag:kungfu:plane,Values=aws-us-elastic-runner-burst",
+        "Name=tag:kungfu:provider,Values=macos-ec2-jit",
+        "Name=instance-state-name,Values=pending,running,stopping,stopped,shutting-down",
+        "--output",
+        "json",
+      ],
+      `active macOS JIT instance preflight in ${regionalPlan.aws.region}`,
+    );
+    return (active.Reservations || []).flatMap(
+      (reservation) => reservation.Instances || [],
+    );
+  });
+  if (activeInstances.length !== 0) {
+    throw new Error("macOS rehydration requires zero active JIT instances");
+  }
+  const runners = ghJson(
+    ["api", `repos/${plan.repository}/actions/runners?per_page=100`],
+    undefined,
+    "macOS rehydration runner preflight",
+  ).runners;
+  if (
+    (runners || []).some((runner) =>
+      (runner.labels || []).some((label) =>
+        String(label.name || label).startsWith("aws-us-ec2-macos-jit-"),
+      ),
+    )
+  ) {
+    throw new Error("macOS rehydration requires zero JIT runner residue");
+  }
+  const image = awsJson(
+    plan,
+    profile,
+    [
+      "ec2",
+      "describe-images",
+      "--image-ids",
+      plan.aws.amiId,
+      "--output",
+      "json",
+    ],
+    "macOS rehydration AMI preflight",
+  ).Images?.[0];
+  if (
+    !image ||
+    image.Name !== plan.aws.amiName ||
+    image.Architecture !== "arm64_mac" ||
+    image.State !== "available"
+  ) {
+    throw new Error("macOS AMI identity or availability mismatch");
+  }
+  const subnet = awsJson(
+    plan,
+    profile,
+    [
+      "ec2",
+      "describe-subnets",
+      "--subnet-ids",
+      plan.aws.subnetId,
+      "--output",
+      "json",
+    ],
+    "macOS rehydration subnet preflight",
+  ).Subnets?.[0];
+  if (!subnet || subnet.AvailabilityZone !== plan.aws.availabilityZone) {
+    throw new Error(
+      "macOS subnet and Dedicated Host availability zone mismatch",
+    );
+  }
+  return {
+    ...launchGate,
+    exactCommit: plan.source.sha,
+    hostId: host.HostId,
+    hostState: host.State,
+    activeInstances: 0,
+    otherActiveHosts: 0,
+    amiOwnerId: image.OwnerId,
+    subnetAvailabilityZone: subnet.AvailabilityZone,
+  };
+}
+
+export function executeMacosJitInstanceRehydrate(plan, { profile = "" } = {}) {
+  if (
+    plan?.contract !== AWS_MACOS_JIT_CONTROLLER_CONTRACT ||
+    plan.kind !== "instance-rehydrate-plan"
+  ) {
+    throw new Error("macOS JIT instance rehydrate plan contract is invalid");
+  }
+  const preflight = assertInstanceRehydratePreflight(plan, profile);
+  assertDryRun(
+    commandResult(
+      "aws",
+      awsArgs(
+        plan,
+        profile,
+        macosRunInstancesArgs(plan, { hostId: plan.aws.hostId, dryRun: true }),
+      ),
+    ),
+    "EC2 same-host RunInstances DryRun",
+  );
+  const launched = awsJson(
+    plan,
+    profile,
+    macosRunInstancesArgs(plan, { hostId: plan.aws.hostId }),
+    "EC2 same-host RunInstances",
+  );
+  const instanceId = launched.Instances?.[0]?.InstanceId;
+  if (!/^i-[0-9a-f]+$/.test(String(instanceId || ""))) {
+    throw new Error("EC2 same-host RunInstances returned no instance identity");
+  }
+  requireSuccess(
+    commandResult(
+      "aws",
+      awsArgs(plan, profile, [
+        "ec2",
+        "wait",
+        "instance-running",
+        "--instance-ids",
+        instanceId,
+      ]),
+    ),
+    "macOS rehydrated instance-running waiter",
+  );
+  const instance = awsJson(
+    plan,
+    profile,
+    [
+      "ec2",
+      "describe-instances",
+      "--instance-ids",
+      instanceId,
+      "--output",
+      "json",
+    ],
+    "macOS rehydrated instance readback",
+  ).Reservations?.[0]?.Instances?.[0];
+  if (
+    !instance ||
+    instance.State?.Name !== "running" ||
+    instance.Placement?.HostId !== plan.aws.hostId ||
+    instance.ImageId !== plan.aws.amiId ||
+    instance.InstanceType !== plan.aws.instanceType
+  ) {
+    throw new Error("macOS rehydrated instance identity or placement mismatch");
+  }
+  assertOwnership(instance.Tags, plan);
+  return {
+    schemaVersion: 1,
+    contract: AWS_MACOS_JIT_CONTROLLER_CONTRACT,
+    kind: "instance-rehydrate-result",
+    status: "rehydrated",
+    repository: plan.repository,
+    campaign: plan.campaign,
+    source: plan.source,
+    replacement: {
+      rehydrationId: plan.aws.rehydrationId,
+      replacesInstanceId: plan.aws.replacesInstanceId,
+    },
+    aws: {
+      region: plan.aws.region,
+      availabilityZone: plan.aws.availabilityZone,
+      hostId: plan.aws.hostId,
+      hostAllocatedAt: plan.aws.hostAllocatedAt,
+      instanceId,
+      instanceType: instance.InstanceType,
+      imageId: instance.ImageId,
+      launchTime: new Date(instance.LaunchTime).toISOString(),
+    },
+    preflight,
+    dryRun: "RunInstances:DryRunOperation",
+    hostAllocated: false,
+    planDigest: plan.digest,
+  };
+}