@kungfu-tech/buildchain 4.1.3-alpha.1 → 4.1.3-alpha.3

package/packages/core/publication/pipeline/sealed-products.js

@@ -0,0 +1,79 @@
+import fs from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { recordDigest } from "../../release/discussion/envelope.js";
+import {
+  publicationFile,
+  publicationPath,
+  writeImmutablePublicationFile,
+} from "./files.js";
+
+export async function retainPipelineProducts(archive, qualified, bundles) {
+  const retained = [];
+  for (const artifact of qualified.artifacts) {
+    const bundle = bundles.find(
+      ({ manifest }) => manifest.root === artifact.manifestRoot,
+    );
+    if (!bundle)
+      throw new Error("Qualified product has no exact source manifest");
+    const file = publicationFile(
+      publicationPath(bundle.directory, artifact.file),
+    );
+    if (file.digest !== artifact.digest || file.size !== artifact.size)
+      throw new Error("Qualified product changed before durable retention");
+    const handle = await archive.put(file.bytes, {
+      name: `${artifact.digest.slice(7)}-${path.basename(artifact.file)}`,
+      mediaType: "application/octet-stream",
+    });
+    await verifiedBytes(archive, handle, artifact);
+    retained.push({ id: artifact.id, handle });
+  }
+  const body = {
+    schema: "buildchain.pipeline-sealed-products/v1",
+    qualificationRoot: qualified.root,
+    products: retained,
+  };
+  return { ...body, root: recordDigest(body) };
+}
+
+async function verifiedBytes(archive, handle, artifact) {
+  if (handle.digest !== artifact.digest || handle.size !== artifact.size)
+    throw new Error("Retained product handle does not bind qualified bytes");
+  const bytes = Buffer.from(await archive.read(handle));
+  if (
+    bytes.length !== artifact.size ||
+    `sha256:${createHash("sha256").update(bytes).digest("hex")}` !==
+      artifact.digest
+  )
+    throw new Error("Retained product readback differs from qualified bytes");
+  return bytes;
+}
+
+export async function restorePipelineProducts(
+  archive,
+  qualified,
+  sealed,
+  directory,
+) {
+  const { root, ...body } = sealed;
+  if (
+    root !== recordDigest(body) ||
+    sealed.qualificationRoot !== qualified.root ||
+    recordDigest(sealed.products.map(({ id }) => id)) !==
+      recordDigest(qualified.artifacts.map(({ id }) => id))
+  )
+    throw new Error(
+      "Retained products must cover the exact qualification inventory",
+    );
+  fs.mkdirSync(directory, { recursive: true });
+  for (const artifact of qualified.artifacts) {
+    const { handle } = sealed.products.find(({ id }) => id === artifact.id);
+    const bytes = await verifiedBytes(archive, handle, artifact);
+    const target = path.resolve(directory, artifact.file);
+    if (!target.startsWith(`${path.resolve(directory)}${path.sep}`))
+      throw new Error("Retained product path escapes its directory");
+    writeImmutablePublicationFile(target, bytes);
+    publicationPath(directory, artifact.file);
+  }
+  return directory;
+}

package/packages/core/publication/pipeline/settle.js

@@ -0,0 +1,43 @@
+import fs from "node:fs";
+import { publicationContext, uniquePublicationMaterial } from "./context.js";
+import { distributePipelineProducts } from "./distribution.js";
+import { nextPipelineDevelopment } from "./next-development.js";
+
+export async function settlePipelinePublication(
+  context,
+  host,
+  directory,
+  environment,
+) {
+  const { session, journal } = await publicationContext(context, host);
+  if (session.observed.phases.publish?.payload.state !== "success")
+    throw new Error(
+      "Publication must retain success before distribution and next-development",
+    );
+  fs.mkdirSync(directory, { recursive: true });
+  const retained = await uniquePublicationMaterial(
+    journal,
+    "publication/qualified/",
+  );
+  if (session.observed.phases.distribution?.payload.state !== "success") {
+    const receipt = await distributePipelineProducts(
+      context,
+      host,
+      journal,
+      retained,
+      environment,
+      directory,
+    );
+    await journal.record("publication/distribution-complete", receipt, {
+      phase: "distribution",
+      state: "success",
+    });
+  }
+  const next = await nextPipelineDevelopment(context, host, journal);
+  await journal.record("publication/next-development-result", next, {
+    phase: "next-development",
+    state: next.state,
+  });
+  await host.project(session);
+  return next;
+}

package/packages/core/publication/pipeline/signing.js

@@ -0,0 +1,140 @@
+import fs from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { execFileSync } from "node:child_process";
+import {
+  canonicalJson,
+  recordDigest,
+} from "../../release/discussion/envelope.js";
+import { verifyPipelineQualification } from "./documents.js";
+import { writeImmutablePublicationFile } from "./files.js";
+
+export const PIPELINE_PRODUCT_PREDICATE =
+  "https://buildchain.libkungfu.dev/attestations/pipeline-products/v1";
+const digest = (bytes) => createHash("sha256").update(bytes).digest("hex");
+
+export function preparePipelineSigning({
+  plan,
+  materialization,
+  qualified,
+  directory,
+  evaluatedAt,
+}) {
+  verifyPipelineQualification({
+    plan,
+    materialization,
+    qualified,
+    evaluatedAt,
+  });
+  const predicate = {
+    schema: "buildchain.pipeline-product-attestation/v1",
+    planRoot: plan.root,
+    materializationRoot: materialization.root,
+    qualificationRoot: qualified.root,
+    source: qualified.source,
+    protectedSource: plan.source,
+    publisher: plan.publisher,
+    runtime: plan.runtime,
+    provider: {
+      runId: qualified.build.runId,
+      runAttempt: qualified.build.runAttempt,
+      source: qualified.build.providerSource,
+    },
+  };
+  const subjectBytes = Buffer.from(`${canonicalJson(qualified)}\n`);
+  const subjectPath = writeImmutablePublicationFile(
+    path.join(directory, "qualified-products.json"),
+    subjectBytes,
+  );
+  const predicatePath = writeImmutablePublicationFile(
+    path.join(directory, "predicate.json"),
+    `${canonicalJson(predicate)}\n`,
+  );
+  return {
+    subjectPath,
+    predicatePath,
+    predicate,
+    subjectDigest: digest(subjectBytes),
+    predicateType: PIPELINE_PRODUCT_PREDICATE,
+  };
+}
+
+export function verifyPipelineSigning({
+  plan,
+  materialization,
+  qualified,
+  bundlePath,
+  directory,
+  token,
+  execute = execFileSync,
+  evaluatedAt,
+}) {
+  const prepared = preparePipelineSigning({
+    plan,
+    materialization,
+    qualified,
+    directory,
+    evaluatedAt,
+  });
+  const output = execute(
+    "gh",
+    [
+      "attestation",
+      "verify",
+      prepared.subjectPath,
+      "--repo",
+      qualified.source.repository,
+      "--signer-workflow",
+      `${plan.publisher.repository}/${plan.publisher.workflow}`,
+      "--signer-digest",
+      plan.publisher.workflowSha,
+      "--source-digest",
+      qualified.build.providerSource,
+      "--predicate-type",
+      PIPELINE_PRODUCT_PREDICATE,
+      "--bundle",
+      bundlePath,
+      "--deny-self-hosted-runners",
+      "--format",
+      "json",
+    ],
+    {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "pipe"],
+      maxBuffer: 16 * 1024 * 1024,
+      env: { ...process.env, GH_TOKEN: token },
+    },
+  );
+  const results = JSON.parse(output);
+  const statements = Array.isArray(results)
+    ? results
+        .map((result) => result.verificationResult?.statement)
+        .filter(Boolean)
+    : [];
+  if (
+    statements.length !== 1 ||
+    statements[0].predicateType !== prepared.predicateType ||
+    recordDigest(statements[0].predicate) !==
+      recordDigest(prepared.predicate) ||
+    statements[0].subject?.length !== 1 ||
+    statements[0].subject[0].digest?.sha256 !== prepared.subjectDigest
+  )
+    throw new Error(
+      "Cryptographically verified attestation does not bind the exact publication predicate and subject",
+    );
+  if (digest(fs.readFileSync(prepared.subjectPath)) !== prepared.subjectDigest)
+    throw new Error("Signed publication subject changed during verification");
+  const bundleBytes = fs.readFileSync(bundlePath);
+  return {
+    schema: "buildchain.pipeline-product-signing/v1",
+    verified: true,
+    planRoot: plan.root,
+    materializationRoot: materialization.root,
+    qualificationRoot: qualified.root,
+    subjectDigest: `sha256:${prepared.subjectDigest}`,
+    predicateRoot: recordDigest(prepared.predicate),
+    bundleDigest: `sha256:${digest(bundleBytes)}`,
+    signer: plan.publisher,
+    providerSource: qualified.build.providerSource,
+  };
+}

package/packages/core/publication/pipeline/source-plan.js

@@ -0,0 +1,83 @@
+import { planPipelinePublication } from "./plan.js";
+
+export async function createPipelinePublicationPlan(
+  session,
+  publisherSha,
+  host,
+  version,
+) {
+  const observed = session.observed;
+  const attempt = observed.attempt;
+  const integration = await host.integration.observe({
+    ...observed.history.at(-1),
+    intent: session.intent,
+  });
+  const source = await host.source.source(
+    integration.mergeCommit,
+    observed.history.at(-1).generation.source.configPath,
+  );
+  const route = source.plan.channels.find(
+    (entry) =>
+      entry.to === session.intent.source.targetBranch &&
+      entry.operation !== "develop",
+  );
+  if (!route)
+    throw new Error(
+      "Protected source does not declare this publication channel",
+    );
+  const input = await version.inspect(source.identity, source.plan.version);
+  const runtimeCommit = await host.request(
+    `/repos/${host.runtime.repository}/git/commits/${host.runtime.sha}`,
+  );
+  if (
+    runtimeCommit.sha !== host.runtime.sha ||
+    !/^[0-9a-f]{40}$/u.test(runtimeCommit.tree?.sha || "")
+  )
+    throw new Error("Publication runtime has no exact source tree");
+  const definition = {
+    repository: "kungfu-systems/buildchain",
+    workflow: ".github/workflows/.release-pipeline-products.yml",
+    workflowSha: publisherSha,
+    job: "apply",
+  };
+  const parameters = {
+    attempt,
+    generation: observed.generation,
+    source: source.identity,
+    intentSource: observed.history.at(-1).generation.source,
+    runtime: {
+      repository: host.runtime.repository,
+      commit: runtimeCommit.sha,
+      tree: runtimeCommit.tree.sha,
+    },
+    publisher: definition,
+    contract: source.plan,
+    route,
+    version: input.version,
+    sourceTimestamp: input.sourceTimestamp,
+  };
+  let plan = planPipelinePublication(parameters);
+  const tag = await host.request(
+    `/repos/${host.repository}/git/ref/tags/${encodeURIComponent(plan.tag)}`,
+    { allow404: true },
+  );
+  const channel = `v${plan.version.split(".")[0]}${plan.channel === "alpha" ? "-alpha" : ""}`;
+  const previous = await host.request(
+    `/repos/${host.repository}/git/ref/tags/${channel}`,
+    { allow404: true },
+  );
+  if (
+    previous &&
+    (previous.object?.type !== "commit" ||
+      !/^[0-9a-f]{40}$/u.test(previous.object.sha))
+  )
+    throw new Error(
+      "Publication channel requires an exact lightweight commit ref",
+    );
+  plan = planPipelinePublication({
+    ...parameters,
+    expectedTagSha: tag?.object?.sha || null,
+    previousChannelCommit: previous?.object?.sha || null,
+  });
+  return plan;
+}

package/packages/core/publication/pipeline/stable-entry.js

@@ -0,0 +1,215 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { readBusinessAttempt } from "../../workflow/attempt/reader.js";
+import { verifyRootedPublication } from "./documents.js";
+import { verifyPipelinePublicationPlan } from "./plan.js";
+import {
+  pipelineEntryRuns,
+  pipelineEntryRun,
+  pipelineEntryCheck,
+} from "../../providers/github/pipeline-entry-runs.js";
+
+async function nativeBuild(check, run, consumer, host) {
+  const loaded = await host.index.resolve(check.attempt);
+  const observed = readBusinessAttempt(loaded.snapshot);
+  const selected = observed.history.find(
+    (item) => item.identity.id === check.attempt,
+  );
+  if (
+    observed.intent.repository !== host.repository ||
+    !selected ||
+    selected.identity.requestKey.startsWith("recover:") ||
+    recordDigest(selected.generation.source) !== recordDigest(consumer.source)
+  )
+    throw new Error(
+      "Published entry check points to another source or a recovery attempt",
+    );
+  const phase = selected.phases.build;
+  if (phase?.payload.state !== "success" || check.outcome !== "success")
+    return {
+      status: "missing",
+      reason: "latest-published-entry-build-failed",
+      attempt: check.attempt,
+    };
+  if (
+    phase.payload.writer.runId !== String(run.id) ||
+    phase.payload.writer.runAttempt !== String(run.run_attempt) ||
+    phase.runtime.repository !== "kungfu-systems/buildchain" ||
+    phase.runtime.sha !== consumer.selection.ref
+  )
+    throw new Error(
+      "Published entry native build changed its actual runtime or execution",
+    );
+  const references = phase.payload.materials.filter((item) =>
+    item.id.startsWith("build/provider-"),
+  );
+  if (references.length !== 1)
+    throw new Error(
+      "Published entry requires one original normal build receipt",
+    );
+  const session = {
+    intent: loaded.snapshot.intent,
+    observed: { ...observed, history: [selected] },
+  };
+  const readback = await host.materialStore(session).read(references[0]);
+  verifyRootedPublication(readback, "buildchain.pipeline-build-readback/v1");
+  if (
+    readback.runId !== run.id ||
+    readback.runAttempt !== run.run_attempt ||
+    readback.outcome !== "success" ||
+    recordDigest(readback.source) !== recordDigest(consumer.source) ||
+    recordDigest(readback.entry) !== recordDigest(consumer.entry) ||
+    check.summary !==
+      `Exact product jobs: ${readback.root}\nAttempt: ${check.attempt}`
+  )
+    throw new Error("Published entry check and native build material disagree");
+  return {
+    readback,
+    phase,
+    reference: references[0],
+    attempt: check.attempt,
+    intent: observed.intent.id,
+  };
+}
+
+function completion(run, jobs, native, source) {
+  const recorder = jobs.filter(
+    (job) => String(job.id) === native.phase.payload.writer.jobId,
+  );
+  if (
+    recorder.length !== 1 ||
+    recorder[0].status !== "completed" ||
+    recorder[0].conclusion !== "success" ||
+    recorder[0].run_id !== run.id ||
+    recorder[0].run_attempt !== run.run_attempt ||
+    !(
+      recorder[0].name === "Record product build" ||
+      recorder[0].name.endsWith(" / Record product build")
+    )
+  )
+    throw new Error(
+      "Published entry native receipt lacks its exact completed recorder job",
+    );
+  const selected = [...native.readback.jobs, recorder[0]];
+  const start = Date.parse(source.candidate.publishedAt),
+    completions = [];
+  for (const job of selected) {
+    const started = Date.parse(job.started_at),
+      ended = Date.parse(job.completed_at);
+    if (
+      !Number.isFinite(started) ||
+      !Number.isFinite(ended) ||
+      started < start ||
+      ended < started
+    )
+      throw new Error(
+        "Published entry jobs must execute after the exact Alpha publication",
+      );
+    completions.push(ended);
+  }
+  return new Date(Math.max(...completions)).toISOString();
+}
+
+async function inspectEntryRun(listed, plan, source, host) {
+  const { run, jobs } = await host.runs.read(listed.id, listed.run_attempt);
+  const consumer = await pipelineEntryRun(run, plan, source, host);
+  if (!consumer) return null;
+  if (
+    !jobs.some((job) => /(?:^| \/ )Build product \([^)]+\)$/u.test(job.name)) &&
+    run.conclusion === "success"
+  )
+    return null;
+  const check = await pipelineEntryCheck(run, host);
+  if (!check)
+    return {
+      status: "missing",
+      reason: "latest-published-entry-build-unrecorded",
+      runId: run.id,
+    };
+  const native = await nativeBuild(check, run, consumer, host);
+  if (native.status === "missing") return native;
+  const platforms = [
+    ...new Set(plan.outputs.map(({ platform }) => platform)),
+  ].sort();
+  const fresh = await host.runs.build(
+    run.id,
+    run.run_attempt,
+    consumer.source,
+    platforms,
+  );
+  if (recordDigest(fresh) !== recordDigest(native.readback))
+    throw new Error(
+      "Published entry original jobs changed or omit declared platforms",
+    );
+  const completedAt = completion(run, jobs, native, source);
+  const again = await host.runs.read(run.id, run.run_attempt);
+  const repeated = await nativeBuild(
+    await pipelineEntryCheck(again.run, host),
+    again.run,
+    consumer,
+    host,
+  );
+  if (
+    recordDigest(repeated) !== recordDigest(native) ||
+    recordDigest(again) !== recordDigest({ run, jobs })
+  )
+    throw new Error(
+      "Published entry evidence changed during final provider readback",
+    );
+  const body = {
+    schema: "buildchain.pipeline-stable-entry/v1",
+    planRoot: plan.root,
+    sourceRoot: source.root,
+    consumer,
+    native,
+    runId: run.id,
+    runAttempt: run.run_attempt,
+    canary: {
+      id: "published-entry",
+      source: "public-build",
+      status: "success",
+      repository: host.repository,
+      candidateSha: source.candidate.sha,
+      completedAt,
+      attestor: "github-actions[bot]",
+      runtimeRef: consumer.selection.ref,
+      evidenceUrl: `https://github.com/${host.repository}/actions/runs/${run.id}/attempts/${run.run_attempt}`,
+    },
+  };
+  return { ...body, root: recordDigest(body) };
+}
+
+export async function readPipelineStableEntry(plan, source, host) {
+  verifyPipelinePublicationPlan(plan);
+  verifyRootedPublication(source, "buildchain.pipeline-stable-source/v1");
+  if (
+    plan.channel !== "stable" ||
+    source.planRoot !== plan.root ||
+    host.repository !== plan.source.repository ||
+    source.contractRoot !== plan.contractRoot ||
+    source.candidate.sha !== plan.intentSource.commit ||
+    !Number.isFinite(Date.parse(source.candidate.publishedAt))
+  )
+    throw new Error(
+      "Published entry collection requires its exact admitted Alpha plan",
+    );
+  const inventory = await pipelineEntryRuns(host, source.candidate.publishedAt);
+  for (const run of inventory) {
+    const result = await inspectEntryRun(run, plan, source, host);
+    if (result) {
+      if (
+        result.canary &&
+        recordDigest(
+          await pipelineEntryRuns(host, source.candidate.publishedAt),
+        ) !== recordDigest(inventory)
+      )
+        throw new Error(
+          "Published entry run inventory changed during qualification",
+        );
+      return result;
+    }
+  }
+  return {
+    status: "missing",
+    reason: "no-normal-published-entry-build-for-exact-runtime",
+  };
+}