@kungfu-tech/buildchain 3.0.2 → 3.0.3-alpha.1

package/scripts/aws-macos-jit.mjs

@@ -0,0 +1,123 @@
+#!/usr/bin/env node
+import fs from "node:fs";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import {
+  createMacosJitEvidence,
+  macosEc2JitPlan,
+  renderMacosJitBootstrap,
+  verifyMacosEc2JitQualification,
+} from "./aws-macos-jit-core.mjs";
+
+function arg(name, fallback = "") {
+  const index = process.argv.indexOf(`--${name}`);
+  return index === -1 ? fallback : process.argv[index + 1] || "";
+}
+
+function readJson(file) {
+  return JSON.parse(fs.readFileSync(path.resolve(file), "utf8"));
+}
+
+function writeJson(file, value) {
+  const resolved = path.resolve(file);
+  fs.mkdirSync(path.dirname(resolved), { recursive: true });
+  fs.writeFileSync(resolved, `${JSON.stringify(value, null, 2)}\n`);
+  return resolved;
+}
+
+export function main() {
+  const mode = process.argv[2] || "plan";
+  if (mode === "plan") {
+    const result = macosEc2JitPlan();
+    process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+    return result;
+  }
+  if (mode === "evidence") {
+    const result = createMacosJitEvidence({
+      repository: process.env.GITHUB_REPOSITORY,
+      sourceSha:
+        process.env.BUILDCHAIN_EXPECTED_SOURCE_SHA || process.env.GITHUB_SHA,
+      sourceRef:
+        process.env.BUILDCHAIN_EXPECTED_SOURCE_REF || process.env.GITHUB_REF,
+      githubRunId: process.env.GITHUB_RUN_ID,
+      githubRunAttempt: process.env.GITHUB_RUN_ATTEMPT,
+      githubJob: process.env.GITHUB_JOB,
+      runnerName: process.env.RUNNER_NAME,
+      runnerLabels: JSON.parse(
+        process.env.BUILDCHAIN_RUNNER_LABELS_JSON || "[]",
+      ),
+      hostId: process.env.AWS_EC2_MAC_HOST_ID,
+      instanceId: process.env.AWS_EC2_INSTANCE_ID,
+      instanceType: process.env.AWS_EC2_INSTANCE_TYPE,
+      amiId: process.env.AWS_EC2_AMI_ID,
+      amiName: process.env.AWS_EC2_AMI_NAME,
+      availabilityZone: process.env.AWS_EC2_AVAILABILITY_ZONE,
+      hostAllocatedAt: process.env.AWS_EC2_MAC_HOST_ALLOCATED_AT,
+      instanceLaunchedAt: process.env.AWS_EC2_LAUNCHED_AT,
+      runnerStartedAt: process.env.AWS_EC2_RUNNER_STARTED_AT,
+      runnerExitedAt:
+        process.env.AWS_EC2_RUNNER_EXITED_AT || new Date().toISOString(),
+      cacheMode: process.env.BUILDCHAIN_CHECKOUT_CACHE_MODE || "off",
+    });
+    const output = writeJson(
+      process.env.BUILDCHAIN_MACOS_JIT_EVIDENCE_PATH ||
+        ".buildchain/aws-macos-jit.json",
+      result,
+    );
+    process.stdout.write(`${output}\n`);
+    return result;
+  }
+  if (mode === "render-bootstrap") {
+    const template = arg(
+      "template",
+      "infra/aws-us-elastic-runner-burst-plane/macos-jit-bootstrap.sh",
+    );
+    const output = arg("output");
+    if (!output) throw new Error("render-bootstrap requires --output");
+    const result = renderMacosJitBootstrap(
+      fs.readFileSync(path.resolve(template), "utf8"),
+      {
+        region: arg("region", "us-east-1"),
+        jitParameterName: arg("jit-parameter"),
+        evidenceBucket: arg("evidence-bucket"),
+        runnerLabel: arg("runner-label"),
+        sourceSha: arg("source-sha"),
+        githubRunId: arg("github-run-id"),
+        githubRunAttempt: arg("github-run-attempt", "1"),
+        amiId: arg("ami-id"),
+        amiName: arg("ami-name"),
+        hostId: arg("host-id"),
+        instanceType: arg("instance-type", "mac2.metal"),
+        hostAllocatedAt: arg("host-allocated-at"),
+      },
+    );
+    const resolved = path.resolve(output);
+    fs.mkdirSync(path.dirname(resolved), { recursive: true });
+    fs.writeFileSync(resolved, result);
+    process.stdout.write(`${resolved}\n`);
+    return resolved;
+  }
+  if (mode === "verify") {
+    const input = arg("input");
+    if (!input) throw new Error("verify requires --input");
+    const result = verifyMacosEc2JitQualification(readJson(input));
+    const output = arg("output");
+    if (output) writeJson(output, result);
+    process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+    if (!result.qualifying) process.exitCode = 1;
+    return result;
+  }
+  throw new Error(`unsupported aws-macos-jit mode: ${mode}`);
+}
+
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(process.argv[1]).href
+) {
+  try {
+    main();
+  } catch (error) {
+    console.error(`::error::${error.message || error}`);
+    process.exitCode = 1;
+  }
+}

package/scripts/aws-runner-burst-core.mjs

@@ -0,0 +1,248 @@
+import crypto from "node:crypto";
+
+export const AWS_RUNNER_BURST_CONTRACT =
+  "kungfu-buildchain-aws-runner-burst/v1";
+
+export const LINUX_CODEBUILD_POC = Object.freeze({
+  phase: "linux-codebuild-poc-under-usd-50",
+  region: "us-east-1",
+  project: "kungfu-buildchain-linux-burst-poc",
+  computeType: "BUILD_GENERAL1_XLARGE",
+  pricePerMinuteUsd: 0.08,
+  timeoutMinutes: 40,
+  maxConcurrentBuilds: 2,
+  maxAcceptedBuilds: 12,
+  budgetLimitUsd: 49,
+  minimumAcceptedJobs: 10,
+  minimumObservedConcurrency: 2,
+  maximumP95QueueStartSeconds: 300,
+});
+
+function exactSha(value, label) {
+  const normalized = String(value || "").trim().toLowerCase();
+  if (!/^[0-9a-f]{40}$/.test(normalized)) {
+    throw new Error(`${label} must be an exact 40-character Git SHA`);
+  }
+  return normalized;
+}
+
+function finiteNumber(value, label) {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed) || parsed < 0) {
+    throw new Error(`${label} must be a non-negative finite number`);
+  }
+  return parsed;
+}
+
+function percentile(values, quantile) {
+  if (values.length === 0) return Number.POSITIVE_INFINITY;
+  const ordered = [...values].sort((left, right) => left - right);
+  const index = Math.max(
+    0,
+    Math.min(ordered.length - 1, Math.ceil(quantile * ordered.length) - 1),
+  );
+  return ordered[index];
+}
+
+export function canonicalJson(value) {
+  if (Array.isArray(value)) {
+    return `[${value.map(canonicalJson).join(",")}]`;
+  }
+  if (value && typeof value === "object") {
+    return `{${Object.keys(value)
+      .sort()
+      .map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`)
+      .join(",")}}`;
+  }
+  return JSON.stringify(value);
+}
+
+export function digest(value) {
+  return `sha256:${crypto
+    .createHash("sha256")
+    .update(canonicalJson(value))
+    .digest("hex")}`;
+}
+
+export function linuxCodeBuildPlan(overrides = {}) {
+  const config = { ...LINUX_CODEBUILD_POC, ...overrides };
+  const money = (value) => Math.round(value * 100) / 100;
+  const maximumCommittedComputeUsd = money(
+    config.pricePerMinuteUsd *
+      config.timeoutMinutes *
+      config.maxAcceptedBuilds,
+  );
+  const maximumRaceStopUsd = money(
+    config.pricePerMinuteUsd *
+      config.timeoutMinutes *
+      config.maxConcurrentBuilds,
+  );
+  const maximumBoundedSpendUsd = money(
+    maximumCommittedComputeUsd + maximumRaceStopUsd,
+  );
+  if (maximumBoundedSpendUsd >= config.budgetLimitUsd) {
+    throw new Error(
+      `bounded CodeBuild spend ${maximumBoundedSpendUsd.toFixed(2)} must remain below budget ${config.budgetLimitUsd.toFixed(2)}`,
+    );
+  }
+  const plan = {
+    schemaVersion: 1,
+    contract: AWS_RUNNER_BURST_CONTRACT,
+    kind: "phase-plan",
+    config,
+    costEnvelope: {
+      currency: "USD",
+      maximumCommittedComputeUsd,
+      maximumRaceStopUsd,
+      maximumBoundedSpendUsd,
+      assumption:
+        "At most maxConcurrentBuilds over-cap builds can race; the envelope charges both for their full timeout.",
+    },
+    invariants: {
+      trustedExactSourceOnly: true,
+      forkPullRequestsRejectedBeforeCloudRunnerSelection: true,
+      ephemeralSingleJobCompute: true,
+      signingAndPublicationCredentialsForbidden: true,
+      staticAwsCredentialsForbidden: true,
+      githubLongLivedCredentialsForbidden: true,
+      staleOrMissingCostTelemetryFailsClosed: true,
+      zeroIdleCompute: true,
+    },
+  };
+  return { ...plan, digest: digest(plan) };
+}
+
+export function createRunnerEvidence({
+  provider,
+  project,
+  repository,
+  sourceSha,
+  sourceRef,
+  runId,
+  runAttempt,
+  job,
+  codeBuildBuildId,
+  codeBuildBuildArn,
+  codeBuildInitiator,
+  observedAt = new Date().toISOString(),
+} = {}) {
+  if (provider !== "aws-codebuild") {
+    throw new Error("runner evidence provider must be aws-codebuild");
+  }
+  if (project !== LINUX_CODEBUILD_POC.project) {
+    throw new Error(`runner evidence project must be ${LINUX_CODEBUILD_POC.project}`);
+  }
+  if (!String(repository || "").includes("/")) {
+    throw new Error("runner evidence repository must be owner/name");
+  }
+  const evidence = {
+    schemaVersion: 1,
+    contract: AWS_RUNNER_BURST_CONTRACT,
+    kind: "runner-evidence",
+    phase: LINUX_CODEBUILD_POC.phase,
+    provider,
+    project,
+    repository,
+    source: {
+      sha: exactSha(sourceSha, "sourceSha"),
+      ref: String(sourceRef || "").trim(),
+    },
+    github: {
+      runId: String(runId || "").trim(),
+      runAttempt: String(runAttempt || "").trim(),
+      job: String(job || "").trim(),
+    },
+    aws: {
+      region: process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || "",
+      buildId: String(codeBuildBuildId || "").trim(),
+      buildArn: String(codeBuildBuildArn || "").trim(),
+      initiator: String(codeBuildInitiator || "").trim(),
+    },
+    observedAt: new Date(observedAt).toISOString(),
+  };
+  for (const [label, value] of Object.entries({
+    runId: evidence.github.runId,
+    runAttempt: evidence.github.runAttempt,
+    job: evidence.github.job,
+    buildId: evidence.aws.buildId,
+    buildArn: evidence.aws.buildArn,
+  })) {
+    if (!value) throw new Error(`runner evidence ${label} is required`);
+  }
+  return { ...evidence, digest: digest(evidence) };
+}
+
+export function verifyLinuxCodeBuildQualification({
+  jobs = [],
+  actualIncrementalSpendUsd,
+  idleBuilds = [],
+  activeCloudResources = [],
+  telemetryObservedAt,
+  observedAt = new Date().toISOString(),
+} = {}) {
+  const plan = linuxCodeBuildPlan();
+  const issues = [];
+  const now = new Date(observedAt).getTime();
+  const telemetryTime = new Date(telemetryObservedAt || "").getTime();
+  if (
+    !Number.isFinite(telemetryTime) ||
+    telemetryTime > now ||
+    now - telemetryTime > 6 * 60 * 60 * 1000
+  ) {
+    issues.push("cost-telemetry-missing-or-stale");
+  }
+  const spend = finiteNumber(
+    actualIncrementalSpendUsd,
+    "actualIncrementalSpendUsd",
+  );
+  if (spend >= plan.config.budgetLimitUsd) {
+    issues.push("actual-spend-not-below-budget");
+  }
+  const accepted = jobs.filter(
+    (job) =>
+      job?.trusted === true &&
+      job?.exactSource === true &&
+      job?.status === "succeeded",
+  );
+  if (accepted.length < plan.config.minimumAcceptedJobs) {
+    issues.push("insufficient-trusted-exact-source-jobs");
+  }
+  const queueSeconds = accepted.map((job) =>
+    finiteNumber(job.queueStartSeconds, "job.queueStartSeconds"),
+  );
+  const p95QueueStartSeconds = percentile(queueSeconds, 0.95);
+  if (p95QueueStartSeconds > plan.config.maximumP95QueueStartSeconds) {
+    issues.push("queue-start-p95-exceeds-five-minutes");
+  }
+  const peakConcurrency = jobs.reduce(
+    (peak, job) => Math.max(peak, Number(job?.observedConcurrency || 0)),
+    0,
+  );
+  if (peakConcurrency < plan.config.minimumObservedConcurrency) {
+    issues.push("concurrency-two-not-observed");
+  }
+  if (idleBuilds.length > 0) issues.push("idle-builds-remain");
+  if (activeCloudResources.length > 0) {
+    issues.push("active-cloud-resources-remain");
+  }
+  const result = {
+    schemaVersion: 1,
+    contract: AWS_RUNNER_BURST_CONTRACT,
+    kind: "phase-verification",
+    phase: plan.config.phase,
+    status: issues.length === 0 ? "passed" : "failed",
+    qualifying: issues.length === 0,
+    metrics: {
+      acceptedJobs: accepted.length,
+      peakConcurrency,
+      p95QueueStartSeconds,
+      actualIncrementalSpendUsd: spend,
+      idleBuilds: idleBuilds.length,
+      activeCloudResources: activeCloudResources.length,
+    },
+    issues,
+    planDigest: plan.digest,
+    observedAt: new Date(observedAt).toISOString(),
+  };
+  return { ...result, digest: digest(result) };
+}

package/scripts/aws-runner-burst.mjs

@@ -0,0 +1,79 @@
+#!/usr/bin/env node
+import fs from "node:fs";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import {
+  createRunnerEvidence,
+  linuxCodeBuildPlan,
+  verifyLinuxCodeBuildQualification,
+} from "./aws-runner-burst-core.mjs";
+
+function readArg(name, fallback = "") {
+  const index = process.argv.indexOf(`--${name}`);
+  return index === -1 ? fallback : process.argv[index + 1] || "";
+}
+
+function writeJson(outputPath, value) {
+  const resolved = path.resolve(outputPath);
+  fs.mkdirSync(path.dirname(resolved), { recursive: true });
+  fs.writeFileSync(resolved, `${JSON.stringify(value, null, 2)}\n`);
+  return resolved;
+}
+
+function readJson(inputPath) {
+  return JSON.parse(fs.readFileSync(path.resolve(inputPath), "utf8"));
+}
+
+export function main() {
+  const mode = process.argv[2] || "plan";
+  if (mode === "plan") {
+    const result = linuxCodeBuildPlan();
+    process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+    return result;
+  }
+  if (mode === "evidence") {
+    const result = createRunnerEvidence({
+      provider: process.env.BUILDCHAIN_BURST_PROVIDER,
+      project: process.env.BUILDCHAIN_BURST_PROJECT,
+      repository: process.env.BUILDCHAIN_BURST_SOURCE_REPOSITORY,
+      sourceSha: process.env.BUILDCHAIN_BURST_SOURCE_SHA,
+      sourceRef: process.env.BUILDCHAIN_BURST_SOURCE_REF,
+      runId: process.env.BUILDCHAIN_BURST_RUN_ID,
+      runAttempt: process.env.BUILDCHAIN_BURST_RUN_ATTEMPT,
+      job: process.env.BUILDCHAIN_BURST_JOB,
+      codeBuildBuildId: process.env.CODEBUILD_BUILD_ID,
+      codeBuildBuildArn: process.env.CODEBUILD_BUILD_ARN,
+      codeBuildInitiator: process.env.CODEBUILD_INITIATOR,
+    });
+    const output = writeJson(
+      process.env.BUILDCHAIN_BURST_EVIDENCE_PATH ||
+        ".buildchain/aws-runner-burst.json",
+      result,
+    );
+    process.stdout.write(`${output}\n`);
+    return result;
+  }
+  if (mode === "verify-linux") {
+    const inputPath = readArg("input");
+    if (!inputPath) throw new Error("verify-linux requires --input");
+    const result = verifyLinuxCodeBuildQualification(readJson(inputPath));
+    const outputPath = readArg("output");
+    if (outputPath) writeJson(outputPath, result);
+    process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+    if (!result.qualifying) process.exitCode = 1;
+    return result;
+  }
+  throw new Error(`unsupported aws-runner-burst mode: ${mode}`);
+}
+
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(process.argv[1]).href
+) {
+  try {
+    main();
+  } catch (error) {
+    console.error(`::error::${error.message || error}`);
+    process.exitCode = 1;
+  }
+}