@kungfu-tech/buildchain 3.0.9-alpha.3 → 3.0.9-alpha.5

package/scripts/aws-macos-jit-source-rebind.mjs

@@ -28,7 +28,137 @@ function priorOwnershipPlan(plan) {
   };
 }
 
-function campaignRunsWithNoWork(plan) {
+function assertRunMatchesPolicy(policy, run, jobs, jobCount, artifactCount) {
+  if (policy === "unused") {
+    if (jobCount !== 0 || artifactCount !== 0) {
+      throw new Error(
+        `macOS campaign run ${run.id} already has jobs or artifacts`,
+      );
+    }
+    return;
+  }
+  if (policy === "preflight-failure") {
+    const jobList = jobs.jobs || [];
+    const failedJobs = jobList.filter((job) => job.conclusion === "failure");
+    const nonFailedJobs = jobList.filter((job) => job.conclusion !== "failure");
+    const hasJitLabel = jobList.some((job) =>
+      (job.labels || []).some((label) =>
+        String(label.name || label).startsWith(MACOS_JIT_RUNNER_LABEL_PREFIX),
+      ),
+    );
+    if (
+      run.status !== "completed" ||
+      run.conclusion !== "failure" ||
+      jobCount === 0 ||
+      artifactCount === 0 ||
+      jobList.some((job) => job.status !== "completed") ||
+      failedJobs.length !== 1 ||
+      !String(failedJobs[0].name || "").endsWith(" / Trust gate") ||
+      nonFailedJobs.some((job) => job.conclusion !== "skipped") ||
+      hasJitLabel
+    ) {
+      throw new Error(
+        `macOS campaign run ${run.id} is not an exact preflight Trust gate failure`,
+      );
+    }
+    return;
+  }
+  if (
+    run.status !== "completed" ||
+    run.conclusion !== "failure" ||
+    jobCount === 0 ||
+    artifactCount === 0 ||
+    (jobs.jobs || []).some((job) => job.status !== "completed")
+  ) {
+    throw new Error(
+      `macOS campaign run ${run.id} is not a terminal failed run with retained evidence`,
+    );
+  }
+}
+
+function assertExactStartupFailure(run, jobCount, artifactCount) {
+  if (
+    run.status !== "completed" ||
+    run.conclusion !== "startup_failure" ||
+    jobCount !== 0 ||
+    artifactCount !== 0
+  ) {
+    throw new Error(
+      `macOS campaign run ${run.id} is not an exact startup failure`,
+    );
+  }
+}
+
+function classifyCampaignRun(plan, run) {
+  const startupFailure = plan.github.startupFailureRuns.find(
+    ({ runId }) => runId === String(run.id),
+  );
+  const historicalTerminalFailure =
+    plan.github.historicalTerminalFailureRuns.find(
+      ({ runId }) => runId === String(run.id),
+    );
+  const preflightFailure = plan.github.preflightFailureRunIds.includes(
+    String(run.id),
+  );
+  return {
+    startupFailure,
+    historicalTerminalFailure,
+    expectedSourceSha:
+      startupFailure?.sourceSha ||
+      historicalTerminalFailure?.sourceSha ||
+      plan.previousSource.sha,
+    classification: startupFailure
+      ? "startup-failure"
+      : historicalTerminalFailure
+        ? "historical-terminal-failure"
+        : preflightFailure
+          ? "preflight-failure"
+          : plan.github.priorRunPolicy,
+  };
+}
+
+function exactReceiptInventory(receipts, classification, withSource = false) {
+  return receipts
+    .filter((receipt) => receipt.classification === classification)
+    .map((receipt) =>
+      withSource
+        ? { runId: receipt.runId, sourceSha: receipt.sourceSha }
+        : receipt.runId,
+    )
+    .sort(
+      (left, right) =>
+        Number(withSource ? left.runId : left) -
+        Number(withSource ? right.runId : right),
+    );
+}
+
+function assertExactReceiptInventories(plan, receipts) {
+  if (plan.github.priorRunPolicy === "unused") return;
+  const inventories = [
+    ["terminal-failure", plan.github.terminalFailureRunIds, false],
+    ["preflight-failure", plan.github.preflightFailureRunIds, false],
+    ["startup-failure", plan.github.startupFailureRuns, true],
+    [
+      "historical-terminal-failure",
+      plan.github.historicalTerminalFailureRuns,
+      true,
+    ],
+  ];
+  for (const [classification, expected, withSource] of inventories) {
+    const observed = exactReceiptInventory(
+      receipts,
+      classification,
+      withSource,
+    );
+    if (JSON.stringify(observed) !== JSON.stringify(expected)) {
+      throw new Error(
+        `macOS campaign ${classification} inventory does not match the confirmed runs`,
+      );
+    }
+  }
+}
+
+function campaignRunReceipts(plan) {
   const branch = sourceRefName(plan.source.ref);
   const response = ghJson(
     [
@@ -46,7 +176,8 @@ function campaignRunsWithNoWork(plan) {
   }
   const receipts = [];
   for (const run of matching) {
-    if (run.head_sha !== plan.previousSource.sha) {
+    const classification = classifyCampaignRun(plan, run);
+    if (run.head_sha !== classification.expectedSourceSha) {
       throw new Error(
         `macOS campaign run ${run.id} is bound to an unexpected source`,
       );
@@ -68,19 +199,37 @@ function campaignRunsWithNoWork(plan) {
     const artifactCount = Number(
       artifacts.total_count || (artifacts.artifacts || []).length || 0,
     );
-    if (jobCount !== 0 || artifactCount !== 0) {
-      throw new Error(
-        `macOS campaign run ${run.id} already has jobs or artifacts`,
+    if (classification.startupFailure) {
+      assertExactStartupFailure(run, jobCount, artifactCount);
+    } else {
+      assertRunMatchesPolicy(
+        classification.historicalTerminalFailure
+          ? "terminal-failure"
+          : plan.github.priorRunPolicy,
+        run,
+        jobs,
+        jobCount,
+        artifactCount,
       );
     }
     receipts.push({
       runId: String(run.id),
+      sourceSha: String(run.head_sha || ""),
+      classification: classification.classification,
       status: String(run.status || "unknown"),
       conclusion: String(run.conclusion || ""),
       jobCount,
       artifactCount,
+      artifacts: (artifacts.artifacts || []).map((artifact) => ({
+        id: String(artifact.id),
+        name: String(artifact.name || ""),
+        sizeInBytes: Number(artifact.size_in_bytes || 0),
+        digest: String(artifact.digest || ""),
+        expired: artifact.expired === true,
+      })),
     });
   }
+  assertExactReceiptInventories(plan, receipts);
   return receipts;
 }
 
@@ -286,8 +435,17 @@ function assertZeroCampaignResidue(plan, profile) {
   if (matchingRunners.length !== 0) {
     throw new Error("macOS campaign already has a registered JIT runner");
   }
-  const evidenceObjectCounts = {};
-  for (const sourceSha of [plan.previousSource.sha, plan.source.sha]) {
+  const evidenceObjects = {};
+  const historicalTerminalSources =
+    plan.github.historicalTerminalFailureRuns.map(({ sourceSha }) => sourceSha);
+  const evidenceSources = [
+    ...new Set([
+      plan.previousSource.sha,
+      ...historicalTerminalSources,
+      plan.source.sha,
+    ]),
+  ];
+  for (const sourceSha of evidenceSources) {
     const evidence = awsJson(
       plan,
       profile,
@@ -299,27 +457,44 @@ function assertZeroCampaignResidue(plan, profile) {
         "--prefix",
         `macos/${sourceSha}/`,
         "--max-keys",
-        "1",
+        "1000",
         "--output",
         "json",
       ],
       `macOS campaign evidence inventory for ${sourceSha}`,
     );
+    if (evidence.IsTruncated === true) {
+      throw new Error(
+        `macOS campaign source ${sourceSha} evidence inventory is incomplete`,
+      );
+    }
+    const objects = (evidence.Contents || []).map((object) => ({
+      key: String(object.Key || ""),
+      size: Number(object.Size || 0),
+      etag: String(object.ETag || ""),
+    }));
     const count = Number(
       evidence.KeyCount || (evidence.Contents || []).length || 0,
     );
-    if (count !== 0) {
+    const terminalEvidenceSource =
+      (sourceSha === plan.previousSource.sha &&
+        plan.github.priorRunPolicy === "terminal-failure") ||
+      historicalTerminalSources.includes(sourceSha);
+    if (
+      (!terminalEvidenceSource && count !== 0) ||
+      (terminalEvidenceSource && count === 0)
+    ) {
       throw new Error(
-        `macOS campaign source ${sourceSha} already has bootstrap evidence`,
+        `macOS campaign source ${sourceSha} bootstrap evidence does not match the rebind policy`,
       );
     }
-    evidenceObjectCounts[sourceSha] = count;
+    evidenceObjects[sourceSha] = objects;
   }
   return {
-    priorRuns: campaignRunsWithNoWork(plan),
+    priorRuns: campaignRunReceipts(plan),
     jitParameterCount: parameters.length,
     runnerCount: matchingRunners.length,
-    evidenceObjectCounts,
+    evidenceObjects,
   };
 }
 
@@ -480,7 +655,12 @@ export function executeMacosJitSourceRebind(plan, { profile = "" } = {}) {
     schemaVersion: 1,
     contract: AWS_MACOS_JIT_CONTROLLER_CONTRACT,
     kind: "campaign-source-rebind-result",
-    status: "rebound-zero-allocation",
+    status:
+      plan.github.priorRunPolicy === "terminal-failure"
+        ? "rebound-after-terminal-failure-zero-allocation"
+        : plan.github.priorRunPolicy === "preflight-failure"
+          ? "rebound-after-preflight-failure-zero-allocation"
+          : "rebound-zero-allocation",
     repository: plan.repository,
     campaign: plan.campaign,
     previousSource: plan.previousSource,

package/scripts/buildchain-cli-help.mjs

@@ -129,11 +129,15 @@ export const BUILDCHAIN_USAGE = `Usage:
                              [--from-config | --workflow <required-workflow.yml>...] [--cwd <dir>]
                              [--check-response-timeout-minutes <n>]
                              [--max-entries-to-build <n>] [--apply]
-  buildchain dev warrant <submit|select|heartbeat|recover|close|cancel-queued|observe>
+  buildchain dev warrant <submit|select|heartbeat|qualify|recover|close|cancel-queued|observe>
                              --repository <owner/repo> --branch <dev/vN/vN.M>
                              [--execute] [--output <file>] [--json]
-  buildchain dev proof <source|verify-source|classify|replay|integration|verify-integration>
+  buildchain dev proof <source|verify-source|classify|native|verify-native|classify-native|verify-native-reuse|replay|integration|verify-integration>
                              [--output <file>] [--json]
+  buildchain dev two-phase --repository <owner/repo> --branch <dev/vN/vN.M>
+                             --pull-request <n> --expected-head <sha>
+                             --warrant-result <file> [--native-proof <file>]
+                             [--native-command <command>]
   buildchain log <info|warn|error> --event <name> [--phase <phase>]
                  [--component <name>] [--source <name>] [--attribute key=value]...
                  [--path <jsonl>] [--json]

package/scripts/check-internal-architecture.mjs

@@ -131,6 +131,7 @@ function repositoryJavaScriptFiles(root) {
     .filter((file) => [".js", ".mjs", ".cjs"].includes(path.extname(file)))
     .filter((file) => !file.startsWith("tests/"))
     .filter((file) => !/(?:^|\/)tests?\//u.test(file))
+    .filter((file) => !file.startsWith(".buildchain/"))
     .filter((file) => !/^actions\/[^/]+\/dist\//u.test(file))
     .sort();
 }

package/scripts/dev-delivery-native-run.mjs

@@ -0,0 +1,187 @@
+#!/usr/bin/env node
+import { spawn } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+
+import { devDeliveryContentRoot } from "../packages/core/dev-delivery-warrant.js";
+import { runDevDeliveryCommand } from "./dev-delivery-warrant.mjs";
+
+function flag(args, name, fallback = "") {
+  const index = args.indexOf(`--${name}`);
+  return index === -1 ? fallback : args[index + 1] || "";
+}
+
+function positiveInteger(value, label, fallback) {
+  const parsed = Number(value || fallback);
+  if (!Number.isInteger(parsed) || parsed < 1) {
+    throw new Error(`${label} must be a positive integer`);
+  }
+  return parsed;
+}
+
+function exactRoot(value, label) {
+  const normalized = String(value || "")
+    .trim()
+    .toLowerCase();
+  if (!/^sha256:[0-9a-f]{64}$/u.test(normalized)) {
+    throw new Error(`${label} must be a sha256 content root`);
+  }
+  return normalized;
+}
+
+export async function runNativeWithHeartbeat({
+  command,
+  cwd = process.cwd(),
+  heartbeat,
+  executionBinding = {},
+  intervalMs = 30_000,
+  spawnImpl = spawn,
+  now = () => new Date().toISOString(),
+} = {}) {
+  if (!String(command || "").trim()) {
+    throw new Error("native command is required");
+  }
+  if (typeof heartbeat !== "function") {
+    throw new Error("heartbeat callback is required");
+  }
+  const startedAt = now();
+  try {
+    await heartbeat();
+  } catch (error) {
+    throw new Error(`native heartbeat failed: ${error.message}`);
+  }
+  let heartbeatCount = 1;
+  let heartbeatError = null;
+  let heartbeatRunning = false;
+  const child = spawnImpl("bash", ["-lc", command], {
+    cwd,
+    env: process.env,
+    stdio: "inherit",
+  });
+
+  const beat = async () => {
+    if (heartbeatRunning || heartbeatError) return;
+    heartbeatRunning = true;
+    try {
+      await heartbeat();
+      heartbeatCount += 1;
+    } catch (error) {
+      heartbeatError = error;
+      child.kill("SIGTERM");
+    } finally {
+      heartbeatRunning = false;
+    }
+  };
+  const timer = setInterval(beat, intervalMs);
+  const exit = await new Promise((resolve, reject) => {
+    child.once("error", reject);
+    child.once("exit", (code, signal) => resolve({ code, signal }));
+  }).finally(() => clearInterval(timer));
+  while (heartbeatRunning) {
+    await new Promise((resolve) => setTimeout(resolve, 10));
+  }
+  if (heartbeatError) {
+    throw new Error(`native heartbeat failed: ${heartbeatError.message}`);
+  }
+  if (exit.code !== 0) {
+    throw new Error(
+      `native command failed with ${exit.signal || `exit ${exit.code}`}`,
+    );
+  }
+  await beat();
+  const completedAt = now();
+  const receipt = {
+    schema: "kungfu.buildchain.native-heartbeat-run-receipt/v1",
+    outcome: "succeeded",
+    commandRoot: devDeliveryContentRoot({ command }),
+    executionBindingRoot: devDeliveryContentRoot(executionBinding),
+    startedAt,
+    completedAt,
+    heartbeatCount,
+  };
+  return { ...receipt, receiptRoot: devDeliveryContentRoot(receipt) };
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  const repository = flag(args, "repository", process.env.GITHUB_REPOSITORY);
+  const branch = flag(
+    args,
+    "branch",
+    process.env.BUILDCHAIN_DEV_DELIVERY_BRANCH || process.env.GITHUB_BASE_REF,
+  );
+  const command = flag(
+    args,
+    "command",
+    process.env.BUILDCHAIN_DEV_DELIVERY_NATIVE_COMMAND,
+  );
+  const cwd = path.resolve(
+    flag(args, "working-directory", process.env.GITHUB_WORKSPACE),
+  );
+  const fencingToken = exactRoot(
+    flag(
+      args,
+      "fencing-token",
+      process.env.BUILDCHAIN_DEV_DELIVERY_FENCING_TOKEN,
+    ),
+    "fencingToken",
+  );
+  const leaseGeneration = positiveInteger(
+    flag(
+      args,
+      "lease-generation",
+      process.env.BUILDCHAIN_DEV_DELIVERY_LEASE_GENERATION,
+    ),
+    "leaseGeneration",
+  );
+  const leaseSeconds = positiveInteger(
+    flag(
+      args,
+      "lease-seconds",
+      process.env.BUILDCHAIN_DEV_DELIVERY_LEASE_SECONDS,
+    ),
+    "leaseSeconds",
+    3600,
+  );
+  const heartbeatSeconds = positiveInteger(
+    flag(
+      args,
+      "heartbeat-seconds",
+      process.env.BUILDCHAIN_DEV_DELIVERY_HEARTBEAT_SECONDS,
+    ),
+    "heartbeatSeconds",
+    Math.max(15, Math.floor(leaseSeconds / 3)),
+  );
+  if (heartbeatSeconds >= leaseSeconds) {
+    throw new Error("heartbeatSeconds must be less than leaseSeconds");
+  }
+  const output = path.resolve(
+    flag(args, "output", ".buildchain/dev-delivery/native-heartbeat-run.json"),
+  );
+  const result = await runNativeWithHeartbeat({
+    command,
+    cwd,
+    intervalMs: heartbeatSeconds * 1000,
+    heartbeat: async () => {
+      await runDevDeliveryCommand({
+        command: "heartbeat",
+        repository,
+        branch,
+        fencingToken,
+        leaseGeneration,
+        leaseSeconds,
+        execute: true,
+      });
+    },
+  });
+  fs.mkdirSync(path.dirname(output), { recursive: true });
+  fs.writeFileSync(output, `${JSON.stringify(result, null, 2)}\n`);
+  process.stdout.write(`Native heartbeat run: ${result.receiptRoot}\n`);
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  main().catch((error) => {
+    console.error(`buildchain dev native: ${error.message}`);
+    process.exit(1);
+  });
+}

package/scripts/dev-delivery-proof.mjs

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import fs from "node:fs";
 import path from "node:path";
-import { classifyDevDeliveryDelta, createIntegrationDeliveryProof, createProjectCutReplayPlan, createProjectCutReplayProof, createSourceQualificationProof, verifyIntegrationDeliveryProof, verifyProjectCutReplayProof, verifySourceQualificationProof } from "../packages/core/dev-delivery-warrant.js";
+import { classifyDevDeliveryDelta, createIntegrationDeliveryProof, createNativeProofReuseDecision, createNativeQualificationProof, createProjectCutReplayPlan, createProjectCutReplayProof, createSourceQualificationProof, verifyIntegrationDeliveryProof, verifyNativeProofReuseDecision, verifyNativeQualificationProof, verifyProjectCutReplayProof, verifySourceQualificationProof } from "../packages/core/dev-delivery-warrant.js";
 
 function flag(args, name, fallback = "") {
   const index = args.indexOf(`--${name}`);
@@ -52,6 +52,7 @@ export function devDeliveryProofCliOptions(args = [], environment = process.env)
     affectedPaths: flag(rest, "affected-paths-json", environment.BUILDCHAIN_DEV_DELIVERY_AFFECTED_PATHS || "[]"),
     shardEvidenceRoots: flag(rest, "shard-evidence-roots-json", environment.BUILDCHAIN_DEV_DELIVERY_SHARD_EVIDENCE_ROOTS || "[]"),
     qualifiedAt: flag(rest, "qualified-at", environment.BUILDCHAIN_DEV_DELIVERY_QUALIFIED_AT),
+    qualifiedBase: flag(rest, "qualified-base", environment.BUILDCHAIN_DEV_DELIVERY_QUALIFIED_BASE),
     sourceProofPath: flag(rest, "source-proof", environment.BUILDCHAIN_DEV_DELIVERY_SOURCE_PROOF),
     sourceProofRoot: flag(rest, "source-proof-root", environment.BUILDCHAIN_DEV_DELIVERY_SOURCE_PROOF_ROOT),
     warrantResultPath: flag(rest, "warrant-result", environment.BUILDCHAIN_DEV_DELIVERY_WARRANT_RESULT),
@@ -117,6 +118,57 @@ export function runDevDeliveryProofCommand(options) {
       },
     });
   }
+  if (options.command === "native") {
+    return createNativeQualificationProof({
+      repository: options.repository,
+      protectedBase: options.protectedBase,
+      sourceIdentityRoot: options.sourceIdentityRoot,
+      sourcePatchRoot: options.sourcePatchRoot,
+      planRoot: options.planRoot,
+      closureRoot: options.closureRoot,
+      dependencyRoot: options.dependencyRoot,
+      toolchainRoot: options.toolchainRoot,
+      qualifiedBase: options.qualifiedBase,
+      affectedPaths: jsonList(options.affectedPaths, "affected paths"),
+      shardEvidenceRoots: jsonList(options.shardEvidenceRoots, "shard evidence roots", { required: true }),
+      qualifiedAt: options.qualifiedAt,
+    });
+  }
+  if (options.command === "verify-native") {
+    return verifyNativeQualificationProof(jsonFile(options.sourceProofPath, "native proof"));
+  }
+  if (options.command === "classify-native") {
+    return createNativeProofReuseDecision({
+      proof: jsonFile(options.sourceProofPath, "native proof"),
+      current: {
+        sourceIdentityRoot: options.sourceIdentityRoot,
+        sourcePatchRoot: options.sourcePatchRoot,
+        planRoot: options.planRoot,
+        closureRoot: options.closureRoot,
+        dependencyRoot: options.dependencyRoot,
+        toolchainRoot: options.toolchainRoot,
+        currentBase: options.currentBase,
+        graphKnown: options.graphKnown,
+        changedPaths: jsonList(options.changedPaths, "changed paths"),
+      },
+    });
+  }
+  if (options.command === "verify-native-reuse") {
+    return verifyNativeProofReuseDecision(jsonFile(options.qualificationReceiptPath, "native reuse decision"), {
+      proof: jsonFile(options.sourceProofPath, "native proof"),
+      current: {
+        sourceIdentityRoot: options.sourceIdentityRoot,
+        sourcePatchRoot: options.sourcePatchRoot,
+        planRoot: options.planRoot,
+        closureRoot: options.closureRoot,
+        dependencyRoot: options.dependencyRoot,
+        toolchainRoot: options.toolchainRoot,
+        currentBase: options.currentBase,
+        graphKnown: options.graphKnown,
+        changedPaths: jsonList(options.changedPaths, "changed paths"),
+      },
+    });
+  }
   if (options.command === "replay") {
     return createProjectCutReplayPlan({
       repository: options.repository,
@@ -168,7 +220,7 @@ export function runDevDeliveryProofCommand(options) {
 }
 
 function usage() {
-  return "Usage:\n  buildchain dev proof <source|verify-source|classify|replay|replay-proof|verify-replay|integration|verify-integration> [options] [--output FILE] [--json]\n";
+  return "Usage:\n  buildchain dev proof <source|verify-source|classify|native|verify-native|classify-native|verify-native-reuse|replay|replay-proof|verify-replay|integration|verify-integration> [options] [--output FILE] [--json]\n";
 }
 
 async function main() {