@kungfu-tech/buildchain 4.1.3-alpha.0 → 4.1.3-alpha.2

package/packages/core/publication/pipeline/sealed-products.js

@@ -0,0 +1,79 @@
+import fs from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { recordDigest } from "../../release/discussion/envelope.js";
+import {
+  publicationFile,
+  publicationPath,
+  writeImmutablePublicationFile,
+} from "./files.js";
+
+export async function retainPipelineProducts(archive, qualified, bundles) {
+  const retained = [];
+  for (const artifact of qualified.artifacts) {
+    const bundle = bundles.find(
+      ({ manifest }) => manifest.root === artifact.manifestRoot,
+    );
+    if (!bundle)
+      throw new Error("Qualified product has no exact source manifest");
+    const file = publicationFile(
+      publicationPath(bundle.directory, artifact.file),
+    );
+    if (file.digest !== artifact.digest || file.size !== artifact.size)
+      throw new Error("Qualified product changed before durable retention");
+    const handle = await archive.put(file.bytes, {
+      name: `${artifact.digest.slice(7)}-${path.basename(artifact.file)}`,
+      mediaType: "application/octet-stream",
+    });
+    await verifiedBytes(archive, handle, artifact);
+    retained.push({ id: artifact.id, handle });
+  }
+  const body = {
+    schema: "buildchain.pipeline-sealed-products/v1",
+    qualificationRoot: qualified.root,
+    products: retained,
+  };
+  return { ...body, root: recordDigest(body) };
+}
+
+async function verifiedBytes(archive, handle, artifact) {
+  if (handle.digest !== artifact.digest || handle.size !== artifact.size)
+    throw new Error("Retained product handle does not bind qualified bytes");
+  const bytes = Buffer.from(await archive.read(handle));
+  if (
+    bytes.length !== artifact.size ||
+    `sha256:${createHash("sha256").update(bytes).digest("hex")}` !==
+      artifact.digest
+  )
+    throw new Error("Retained product readback differs from qualified bytes");
+  return bytes;
+}
+
+export async function restorePipelineProducts(
+  archive,
+  qualified,
+  sealed,
+  directory,
+) {
+  const { root, ...body } = sealed;
+  if (
+    root !== recordDigest(body) ||
+    sealed.qualificationRoot !== qualified.root ||
+    recordDigest(sealed.products.map(({ id }) => id)) !==
+      recordDigest(qualified.artifacts.map(({ id }) => id))
+  )
+    throw new Error(
+      "Retained products must cover the exact qualification inventory",
+    );
+  fs.mkdirSync(directory, { recursive: true });
+  for (const artifact of qualified.artifacts) {
+    const { handle } = sealed.products.find(({ id }) => id === artifact.id);
+    const bytes = await verifiedBytes(archive, handle, artifact);
+    const target = path.resolve(directory, artifact.file);
+    if (!target.startsWith(`${path.resolve(directory)}${path.sep}`))
+      throw new Error("Retained product path escapes its directory");
+    writeImmutablePublicationFile(target, bytes);
+    publicationPath(directory, artifact.file);
+  }
+  return directory;
+}

package/packages/core/publication/pipeline/settle.js

@@ -0,0 +1,43 @@
+import fs from "node:fs";
+import { publicationContext, uniquePublicationMaterial } from "./context.js";
+import { distributePipelineProducts } from "./distribution.js";
+import { nextPipelineDevelopment } from "./next-development.js";
+
+export async function settlePipelinePublication(
+  context,
+  host,
+  directory,
+  environment,
+) {
+  const { session, journal } = await publicationContext(context, host);
+  if (session.observed.phases.publish?.payload.state !== "success")
+    throw new Error(
+      "Publication must retain success before distribution and next-development",
+    );
+  fs.mkdirSync(directory, { recursive: true });
+  const retained = await uniquePublicationMaterial(
+    journal,
+    "publication/qualified/",
+  );
+  if (session.observed.phases.distribution?.payload.state !== "success") {
+    const receipt = await distributePipelineProducts(
+      context,
+      host,
+      journal,
+      retained,
+      environment,
+      directory,
+    );
+    await journal.record("publication/distribution-complete", receipt, {
+      phase: "distribution",
+      state: "success",
+    });
+  }
+  const next = await nextPipelineDevelopment(context, host, journal);
+  await journal.record("publication/next-development-result", next, {
+    phase: "next-development",
+    state: next.state,
+  });
+  await host.project(session);
+  return next;
+}

package/packages/core/publication/pipeline/signing.js

@@ -0,0 +1,140 @@
+import fs from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { execFileSync } from "node:child_process";
+import {
+  canonicalJson,
+  recordDigest,
+} from "../../release/discussion/envelope.js";
+import { verifyPipelineQualification } from "./documents.js";
+import { writeImmutablePublicationFile } from "./files.js";
+
+export const PIPELINE_PRODUCT_PREDICATE =
+  "https://buildchain.libkungfu.dev/attestations/pipeline-products/v1";
+const digest = (bytes) => createHash("sha256").update(bytes).digest("hex");
+
+export function preparePipelineSigning({
+  plan,
+  materialization,
+  qualified,
+  directory,
+  evaluatedAt,
+}) {
+  verifyPipelineQualification({
+    plan,
+    materialization,
+    qualified,
+    evaluatedAt,
+  });
+  const predicate = {
+    schema: "buildchain.pipeline-product-attestation/v1",
+    planRoot: plan.root,
+    materializationRoot: materialization.root,
+    qualificationRoot: qualified.root,
+    source: qualified.source,
+    protectedSource: plan.source,
+    publisher: plan.publisher,
+    runtime: plan.runtime,
+    provider: {
+      runId: qualified.build.runId,
+      runAttempt: qualified.build.runAttempt,
+      source: qualified.build.providerSource,
+    },
+  };
+  const subjectBytes = Buffer.from(`${canonicalJson(qualified)}\n`);
+  const subjectPath = writeImmutablePublicationFile(
+    path.join(directory, "qualified-products.json"),
+    subjectBytes,
+  );
+  const predicatePath = writeImmutablePublicationFile(
+    path.join(directory, "predicate.json"),
+    `${canonicalJson(predicate)}\n`,
+  );
+  return {
+    subjectPath,
+    predicatePath,
+    predicate,
+    subjectDigest: digest(subjectBytes),
+    predicateType: PIPELINE_PRODUCT_PREDICATE,
+  };
+}
+
+export function verifyPipelineSigning({
+  plan,
+  materialization,
+  qualified,
+  bundlePath,
+  directory,
+  token,
+  execute = execFileSync,
+  evaluatedAt,
+}) {
+  const prepared = preparePipelineSigning({
+    plan,
+    materialization,
+    qualified,
+    directory,
+    evaluatedAt,
+  });
+  const output = execute(
+    "gh",
+    [
+      "attestation",
+      "verify",
+      prepared.subjectPath,
+      "--repo",
+      qualified.source.repository,
+      "--signer-workflow",
+      `${plan.publisher.repository}/${plan.publisher.workflow}`,
+      "--signer-digest",
+      plan.publisher.workflowSha,
+      "--source-digest",
+      qualified.build.providerSource,
+      "--predicate-type",
+      PIPELINE_PRODUCT_PREDICATE,
+      "--bundle",
+      bundlePath,
+      "--deny-self-hosted-runners",
+      "--format",
+      "json",
+    ],
+    {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "pipe"],
+      maxBuffer: 16 * 1024 * 1024,
+      env: { ...process.env, GH_TOKEN: token },
+    },
+  );
+  const results = JSON.parse(output);
+  const statements = Array.isArray(results)
+    ? results
+        .map((result) => result.verificationResult?.statement)
+        .filter(Boolean)
+    : [];
+  if (
+    statements.length !== 1 ||
+    statements[0].predicateType !== prepared.predicateType ||
+    recordDigest(statements[0].predicate) !==
+      recordDigest(prepared.predicate) ||
+    statements[0].subject?.length !== 1 ||
+    statements[0].subject[0].digest?.sha256 !== prepared.subjectDigest
+  )
+    throw new Error(
+      "Cryptographically verified attestation does not bind the exact publication predicate and subject",
+    );
+  if (digest(fs.readFileSync(prepared.subjectPath)) !== prepared.subjectDigest)
+    throw new Error("Signed publication subject changed during verification");
+  const bundleBytes = fs.readFileSync(bundlePath);
+  return {
+    schema: "buildchain.pipeline-product-signing/v1",
+    verified: true,
+    planRoot: plan.root,
+    materializationRoot: materialization.root,
+    qualificationRoot: qualified.root,
+    subjectDigest: `sha256:${prepared.subjectDigest}`,
+    predicateRoot: recordDigest(prepared.predicate),
+    bundleDigest: `sha256:${digest(bundleBytes)}`,
+    signer: plan.publisher,
+    providerSource: qualified.build.providerSource,
+  };
+}

package/packages/core/publication/pipeline/source-plan.js

@@ -0,0 +1,83 @@
+import { planPipelinePublication } from "./plan.js";
+
+export async function createPipelinePublicationPlan(
+  session,
+  publisherSha,
+  host,
+  version,
+) {
+  const observed = session.observed;
+  const attempt = observed.attempt;
+  const integration = await host.integration.observe({
+    ...observed.history.at(-1),
+    intent: session.intent,
+  });
+  const source = await host.source.source(
+    integration.mergeCommit,
+    observed.history.at(-1).generation.source.configPath,
+  );
+  const route = source.plan.channels.find(
+    (entry) =>
+      entry.to === session.intent.source.targetBranch &&
+      entry.operation !== "develop",
+  );
+  if (!route)
+    throw new Error(
+      "Protected source does not declare this publication channel",
+    );
+  const input = await version.inspect(source.identity, source.plan.version);
+  const runtimeCommit = await host.request(
+    `/repos/${host.runtime.repository}/git/commits/${host.runtime.sha}`,
+  );
+  if (
+    runtimeCommit.sha !== host.runtime.sha ||
+    !/^[0-9a-f]{40}$/u.test(runtimeCommit.tree?.sha || "")
+  )
+    throw new Error("Publication runtime has no exact source tree");
+  const definition = {
+    repository: "kungfu-systems/buildchain",
+    workflow: ".github/workflows/.release-pipeline-products.yml",
+    workflowSha: publisherSha,
+    job: "apply",
+  };
+  const parameters = {
+    attempt,
+    generation: observed.generation,
+    source: source.identity,
+    intentSource: observed.history.at(-1).generation.source,
+    runtime: {
+      repository: host.runtime.repository,
+      commit: runtimeCommit.sha,
+      tree: runtimeCommit.tree.sha,
+    },
+    publisher: definition,
+    contract: source.plan,
+    route,
+    version: input.version,
+    sourceTimestamp: input.sourceTimestamp,
+  };
+  let plan = planPipelinePublication(parameters);
+  const tag = await host.request(
+    `/repos/${host.repository}/git/ref/tags/${encodeURIComponent(plan.tag)}`,
+    { allow404: true },
+  );
+  const channel = `v${plan.version.split(".")[0]}${plan.channel === "alpha" ? "-alpha" : ""}`;
+  const previous = await host.request(
+    `/repos/${host.repository}/git/ref/tags/${channel}`,
+    { allow404: true },
+  );
+  if (
+    previous &&
+    (previous.object?.type !== "commit" ||
+      !/^[0-9a-f]{40}$/u.test(previous.object.sha))
+  )
+    throw new Error(
+      "Publication channel requires an exact lightweight commit ref",
+    );
+  plan = planPipelinePublication({
+    ...parameters,
+    expectedTagSha: tag?.object?.sha || null,
+    previousChannelCommit: previous?.object?.sha || null,
+  });
+  return plan;
+}

package/packages/core/publication/pipeline/version.js

@@ -0,0 +1,81 @@
+import { parse, stringify } from "smol-toml";
+import { recordDigest } from "../../release/discussion/envelope.js";
+
+function document(file, bytes) {
+  const text = Buffer.isBuffer(bytes) ? bytes.toString("utf8") : bytes;
+  return file.format === "json" ? JSON.parse(text) : parse(text);
+}
+
+function field(value, key) {
+  const names = key.split(".");
+  let parent = value;
+  for (const name of names.slice(0, -1)) {
+    if (
+      ["__proto__", "prototype", "constructor"].includes(name) ||
+      !Object.hasOwn(parent, name) ||
+      !parent[name] ||
+      typeof parent[name] !== "object"
+    )
+      throw new Error(
+        "Version field does not resolve to an owned document key",
+      );
+    parent = parent[name];
+  }
+  const name = names.at(-1);
+  if (
+    ["__proto__", "prototype", "constructor"].includes(name) ||
+    !Object.hasOwn(parent, name) ||
+    typeof parent[name] !== "string"
+  )
+    throw new Error("Version field must be an existing string");
+  return { parent, name };
+}
+
+export function readPipelineVersion(policy, files) {
+  const versions = policy.files.map((file) => {
+    if (!Object.hasOwn(files, file.path))
+      throw new Error(`Missing declared version file: ${file.path}`);
+    const { parent, name } = field(document(file, files[file.path]), file.key);
+    return parent[name];
+  });
+  if (!versions.length || new Set(versions).size !== 1)
+    throw new Error(
+      "All declared version fields must identify the same candidate version",
+    );
+  return versions[0];
+}
+
+export function materializePipelineVersion(policy, files, version) {
+  const before = readPipelineVersion(policy, files);
+  if (policy.strategy === "anchored" && before !== version)
+    throw new Error("Buildchain cannot rewrite anchored version authority");
+  const documents = new Map();
+  for (const file of policy.files) {
+    if (!documents.has(file.path))
+      documents.set(file.path, {
+        file,
+        value: document(file, files[file.path]),
+      });
+    const entry = documents.get(file.path);
+    if (entry.file.format !== file.format)
+      throw new Error("One version document cannot have conflicting formats");
+    const { parent, name } = field(entry.value, file.key);
+    parent[name] = version;
+  }
+  const changes = [...documents].flatMap(([filePath, { file, value }]) => {
+    if (before === version) return [];
+    const content =
+      file.format === "json"
+        ? `${JSON.stringify(value, null, 2)}\n`
+        : stringify(value);
+    return [
+      {
+        path: filePath,
+        before: recordDigest(String(files[filePath])),
+        content,
+        after: recordDigest(content),
+      },
+    ];
+  });
+  return { before, version, changes };
+}

package/packages/core/publication/publication-reproducibility.js

@@ -1,3 +1,4 @@
+import { readNpmPackResult } from "./npm/pack-result.js";
 import crypto from "node:crypto";
 import fs from "node:fs";
 import os from "node:os";
@@ -564,8 +565,7 @@ function packPublication({ buildRoot, packageDir, source, toolchain }) {
       label: "npm pack reproducibility candidate",
     },
   );
-  const parsed = JSON.parse(output);
-  const result = Array.isArray(parsed) ? parsed[0] : parsed;
+  const result = readNpmPackResult(output);
   if (!result?.filename || !result?.integrity || !result?.shasum) {
     throw new Error(
       "npm pack reproducibility candidate did not report filename, integrity, and shasum",

package/packages/core/release/discussion/binary.js

@@ -1,3 +1,4 @@
+import { releaseCheckpoints } from "./checkpoints.js";
 import { discussionTransport } from "../../providers/github/discussions/transport.js";
 import { releaseDiscussionStore } from "./store.js";
 import { createProgress, decodeRecord } from "./envelope.js";
@@ -46,6 +47,12 @@ export async function observeBinaryDistribution(
   const attemptRecord = state.records.find(
     (record) => record.attempt === state.attempt,
   );
+  Object.assign(session, {
+    attempt: state.attempt,
+    predecessor: attemptRecord.predecessor,
+    runtime,
+    writer,
+  });
   const sequence =
     Math.max(
       -1,
@@ -82,6 +89,10 @@ export async function observeBinaryDistribution(
       await record("failure", 1, {
         code: String(error.code || "execution-failed"),
       });
+      await releaseCheckpoints({ session, store, octokit }).diagnostics(
+        "binary-distribution",
+        error.code,
+      );
     } catch (recordError) {
       error.discussionRecordingError = recordError.message;
     }

package/packages/core/release/discussion/checkpoints.js

@@ -1,3 +1,8 @@
+import {
+  retainAttachment,
+  diagnosticReport,
+  diagnosticLog,
+} from "./evidence.js";
 import fs from "node:fs";
 import path from "node:path";
 import { createProgress, recordDigest } from "./envelope.js";
@@ -10,7 +15,11 @@ export function releaseCheckpoints({ session, store, octokit }) {
     repository: session.intent.repository,
     intentId: session.intent.id,
   });
-  async function checkpoint(node, value) {
+  async function checkpoint(
+    node,
+    value,
+    { attachments = [], label = value.schema || "Recovery checkpoint" } = {},
+  ) {
     const observed = await store.read(session);
     const records = observed.records.filter(
       (record) =>
@@ -21,20 +30,29 @@ export function releaseCheckpoints({ session, store, octokit }) {
     const root = recordDigest(value);
     const duplicate = records.find((record) => record.payload.root === root);
     if (duplicate) return duplicate;
-    const handle = await materials.put(Buffer.from(JSON.stringify(value)));
+    const sequence =
+      Math.max(-1, ...records.map((record) => record.sequence)) + 1;
+    const handle = await retainAttachment(materials, {
+      name: `${node}-checkpoint-${sequence}.json`,
+      mediaType: "application/json",
+      bytes: Buffer.from(JSON.stringify(value, null, 2) + "\n"),
+    });
     const record = createProgress({
       intent: session.intent,
       runtime: session.runtime,
       attempt: session.attempt,
+      writer: session.writer || session.attempt,
       predecessor: session.predecessor,
       kind: "checkpoint",
       node,
       status: "running",
-      sequence: Math.max(-1, ...records.map((record) => record.sequence)) + 1,
+      sequence,
       payload: {
         schema: "buildchain.release-checkpoint/v1",
         root,
         material: handle,
+        label,
+        attachments: [handle, ...attachments],
       },
     });
     await store.append(session, record);
@@ -69,7 +87,30 @@ export function releaseCheckpoints({ session, store, octokit }) {
       throw new Error("Unsupported retained publication reader manifest");
     return materials.read(manifest.reader);
   }
-  return { checkpoint, readCheckpoint, materials, publicationReader };
+  async function diagnostics(node, code) {
+    const report = diagnosticReport(
+      session,
+      await store.read(session),
+      node,
+      code,
+    );
+    const log = await retainAttachment(materials, {
+      name: `${node}-diagnostics.log`,
+      mediaType: "text/plain",
+      bytes: diagnosticLog(report),
+    });
+    return checkpoint(node, report, {
+      attachments: [log],
+      label: `Execution diagnostics: ${report.code}`,
+    });
+  }
+  return {
+    checkpoint,
+    readCheckpoint,
+    materials,
+    publicationReader,
+    diagnostics,
+  };
 }
 
 function assertNoSymlink(file, base) {

package/packages/core/release/discussion/envelope.js

@@ -94,6 +94,7 @@ export function createIntent({
     id: recordDigest(identity),
     ...identity,
     expectedNodes,
+    organization: "attempt-threads/v1",
     runtime,
     source,
   };

package/packages/core/release/discussion/evidence.js

@@ -0,0 +1,51 @@
+// Evidence is declared transaction data, never an implicit upload of workspace
+// files, environment variables, raw exceptions or authenticated HTTP responses.
+export async function retainAttachment(materials, { name, mediaType, bytes }) {
+  if (
+    !/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,159}$/u.test(name) ||
+    !["application/json", "text/plain", "application/octet-stream"].includes(
+      mediaType,
+    )
+  )
+    throw new Error("Invalid transaction attachment descriptor");
+  const handle = await materials.put(bytes, { name, mediaType });
+  if (!handle.downloadUrl)
+    throw new Error("Retained attachment is missing its provider download URL");
+  return { name, mediaType, ...handle };
+}
+
+export function diagnosticReport(session, state, node, code) {
+  return {
+    schema: "buildchain.release-diagnostics/v1",
+    intent: session.intent.id,
+    attempt: session.attempt,
+    node,
+    code: String(code || "execution-failed")
+      .replace(/[^a-zA-Z0-9._-]/gu, "-")
+      .slice(0, 160),
+    events: state.records
+      .filter((record) => record.attempt === session.attempt)
+      .map(({ id, node, kind, status, sequence, writer }) => ({
+        id,
+        node,
+        kind,
+        status,
+        sequence,
+        writer,
+      })),
+  };
+}
+
+export function diagnosticLog(report) {
+  return Buffer.from(
+    [
+      `Release intent ${report.intent}`,
+      `Attempt ${report.attempt}: ${report.node} ${report.code}`,
+      ...report.events.map(
+        (event) =>
+          `${event.node} ${event.kind} ${event.sequence} ${event.status} writer=${event.writer} record=${event.id}`,
+      ),
+      "",
+    ].join("\n"),
+  );
+}