@kungfu-tech/buildchain 4.1.3-alpha.0 → 4.1.3-alpha.10

package/packages/core/publication/pipeline/recovery-prepare.js

@@ -0,0 +1,73 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { retainedRecoveryEvidence } from "../../workflow/pipeline/recovery-session.js";
+import { recoveryPublicationMaterial } from "./recovery-materials.js";
+
+import { importRecoveredPublication } from "./recovery-import.js";
+
+export async function prepareRecoveredPublication(
+  session,
+  publisherSha,
+  host,
+  journal,
+  phase,
+) {
+  if (
+    !session.observed.history.at(-1).identity.requestKey.startsWith("recover:")
+  )
+    return null;
+  const { plan: recoveryPlan, evidence } = await retainedRecoveryEvidence(
+    session,
+    host,
+  );
+  const publication = evidence.publication;
+  if (!publication || publication.mode === "unprepared") return null;
+  const plan = recoveryPublicationMaterial(
+    publication.materials,
+    "publication/plan/",
+    true,
+  );
+  if (recordDigest(host.runtime) !== recordDigest(recoveryPlan.runtime))
+    throw new Error(
+      "Recovered publication executor differs from the admitted repair",
+    );
+  const runtime = await host.request(
+    `/repos/${host.runtime.repository}/git/commits/${host.runtime.sha}`,
+  );
+  if (
+    runtime.sha !== host.runtime.sha ||
+    !/^[0-9a-f]{40}$/u.test(runtime.tree?.sha || "")
+  )
+    throw new Error("Recovery publisher runtime has no exact admitted tree");
+  const execution = {
+    attempt: session.observed.attempt,
+    runId: host.runId,
+    runAttempt: host.runAttempt,
+    publisher: {
+      repository: "kungfu-systems/buildchain",
+      workflow: ".github/workflows/.release-pipeline-products.yml",
+      workflowSha: publisherSha,
+      job: "apply",
+    },
+    runtime: {
+      repository: host.runtime.repository,
+      commit: runtime.sha,
+      tree: runtime.tree.sha,
+    },
+  };
+  await importRecoveredPublication(
+    publication,
+    execution,
+    recoveryPlan.root,
+    journal,
+    phase,
+  );
+  return {
+    planRoot: recoveryPlan.root,
+    predecessor: recoveryPlan.predecessor,
+    originalPlanRoot: plan.root,
+    mode: publication.mode,
+    build: publication.build || null,
+    preserveTransaction: publication.mode === "qualified",
+    execution,
+  };
+}

package/packages/core/publication/pipeline/recovery-qualification.js

@@ -0,0 +1,128 @@
+import path from "node:path";
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { createDomainPublicationQualificationReceipt } from "../publication-qualification.js";
+import { restorePipelineProducts } from "./sealed-products.js";
+import { inspectPipelinePublicationArtifacts } from "./qualification.js";
+import { verifyPipelineQualification } from "./documents.js";
+import { reobservePublicationBuild } from "./recovery-build-readback.js";
+import { requalifyPublicationCapsules } from "./recovery-capsules.js";
+import { readPipelineCaller } from "../../providers/github/pipeline-run-entry.js";
+
+function originalManifests(qualified) {
+  return [...new Set(qualified.artifacts.map((item) => item.manifestRoot))].map(
+    (root) => {
+      const artifacts = qualified.artifacts
+        .filter((item) => item.manifestRoot === root)
+        .map(({ manifestRoot, providerArtifactId, ...item }) => item);
+      const body = {
+        schema: "buildchain.pipeline-publication-products/v1",
+        planRoot: qualified.planRoot,
+        source: qualified.source,
+        platform: artifacts[0].platform,
+        artifacts,
+      };
+      if (recordDigest(body) !== root)
+        throw new Error(
+          "Recovered artifact descriptors cannot reconstruct their original manifest bytes",
+        );
+      return { ...body, root };
+    },
+  );
+}
+
+export async function requalifySealedPublication({
+  context,
+  prepared,
+  originalPlan,
+  originalMaterialization,
+  archive,
+  host,
+  directory,
+  now = new Date(),
+}) {
+  const previous = prepared.qualified;
+  verifyPipelineQualification({
+    plan: originalPlan,
+    materialization: originalMaterialization,
+    qualified: previous,
+    evaluatedAt: previous.qualification.issuedAt,
+  });
+  const { plan, materialization } = context;
+  if (
+    recordDigest(plan.outputs) !== recordDigest(originalPlan.outputs) ||
+    plan.version !== originalPlan.version ||
+    plan.contractRoot !== originalPlan.contractRoot ||
+    recordDigest(materialization.source) !== recordDigest(previous.source)
+  )
+    throw new Error(
+      "Signing recovery cannot change the already sealed product contract or source",
+    );
+  await reobservePublicationBuild(previous.build, previous.source, host);
+  const products = await restorePipelineProducts(
+    archive,
+    previous,
+    prepared.sealed,
+    path.join(directory, "requalified-products"),
+  );
+  for (const manifest of originalManifests(previous))
+    inspectPipelinePublicationArtifacts(products, manifest, plan);
+  const { run } = await host.runs.read(context.runId, context.runAttempt);
+  await readPipelineCaller(
+    run,
+    previous.source.configPath,
+    host.request,
+    host.repository,
+  );
+  const buildBody = {
+    schema: "buildchain.pipeline-publication-requalification/v1",
+    operation: "requalify-sealed-bytes",
+    outcome: "success",
+    planRoot: plan.root,
+    source: previous.source,
+    runId: context.runId,
+    runAttempt: context.runAttempt,
+    providerSource: run.head_sha,
+    predecessorBuild: previous.build,
+    sealedRoot: prepared.sealed.root,
+  };
+  const build = { ...buildBody, root: recordDigest(buildBody) };
+  const qualification = createDomainPublicationQualificationReceipt({
+    ...previous.qualification,
+    candidateRoot: recordDigest({
+      planRoot: plan.root,
+      source: previous.source,
+      artifacts: previous.artifacts,
+    }),
+    issuedAt: now.toISOString(),
+    expiresAt: new Date(now.getTime() + 3600000).toISOString(),
+  });
+  const body = {
+    schema: "buildchain.pipeline-publication-qualification/v1",
+    planRoot: plan.root,
+    source: previous.source,
+    artifacts: previous.artifacts,
+    build,
+    qualification,
+    predecessorRoot: previous.root,
+  };
+  const qualified = { ...body, root: recordDigest(body) };
+  const capsules = requalifyPublicationCapsules(
+    prepared.capsules,
+    qualified,
+    plan,
+  );
+  const { root, ...sealedBody } = prepared.sealed;
+  const sealedValue = {
+    ...sealedBody,
+    qualificationRoot: qualified.root,
+    predecessorRoot: root,
+  };
+  const sealed = { ...sealedValue, root: recordDigest(sealedValue) };
+  return {
+    schema: "buildchain.pipeline-qualification-preparation/v1",
+    contextRoot: recordDigest(context),
+    qualified,
+    capsules,
+    sealed,
+  };
+}

package/packages/core/publication/pipeline/recovery-readback.js

@@ -0,0 +1,56 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+
+export async function observeRecoveryPublicationEffects({
+  effects,
+  receipts,
+  retained,
+  provider,
+  fence,
+}) {
+  const transactionRoot = retained.documents.transaction.transactionRoot;
+  const observations = [];
+  for (const effect of effects) {
+    const { root, ...body } = effect;
+    if (root !== recordDigest(body))
+      throw new Error(
+        "Recovery publication effect changed its immutable declaration",
+      );
+    const prior = receipts.filter((receipt) => receipt.effectId === effect.id);
+    for (const receipt of prior) {
+      const { root: receiptRoot, ...value } = receipt;
+      if (
+        receiptRoot !== recordDigest(value) ||
+        value.schema !== "buildchain.pipeline-publication-effect/v1" ||
+        receipt.transactionRoot !== transactionRoot ||
+        receipt.effectRoot !== effect.root
+      )
+        throw new Error(
+          "Recovery cannot rewrite a prior publication receipt or transaction",
+        );
+    }
+    await fence();
+    const observed = await provider.observe(effect);
+    const matched = provider.matches(effect, observed);
+    if (
+      (!matched && prior.some((item) => item.state === "success")) ||
+      (observed.state !== "absent" &&
+        !matched &&
+        !provider.canApply?.(effect, observed))
+    )
+      throw new Error(
+        "Recovery publication destination conflicts with retained bytes",
+      );
+    observations.push({ effect, observed, matched });
+  }
+  const body = {
+    schema: "buildchain.pipeline-recovery-publication-readback/v1",
+    transactionRoot,
+    qualificationRoot: retained.qualified.root,
+    signingRoot: recordDigest(retained.signing),
+    sealedRoot: retained.sealed.root,
+    effectsRoot: recordDigest(effects),
+    receiptRoots: receipts.map((receipt) => receipt.root),
+    observations,
+  };
+  return { ...body, root: recordDigest(body) };
+}

package/packages/core/publication/pipeline/recovery-signing.js

@@ -0,0 +1,58 @@
+import path from "node:path";
+import { uniquePublicationMaterial } from "./context.js";
+import { requalifySealedPublication } from "./recovery-qualification.js";
+import { preparePipelineSigning } from "./signing.js";
+
+async function originalMaterial(journal, prefix, accepts) {
+  const matches = (await journal.materials(prefix)).filter(accepts);
+  if (matches.length !== 1)
+    throw new Error(
+      `Signing recovery has no unique original material: ${prefix}`,
+    );
+  return matches[0];
+}
+
+export async function prepareRecoveredSigning(
+  context,
+  host,
+  journal,
+  archive,
+  directory,
+) {
+  const prepared = await uniquePublicationMaterial(
+    journal,
+    "publication/predecessor-prepared/",
+  );
+  const originalPlan = await originalMaterial(
+    journal,
+    "publication/predecessor-plan/",
+    (plan) => plan.root === prepared.qualified.planRoot,
+  );
+  const originalMaterialization = await originalMaterial(
+    journal,
+    "publication/predecessor-materialization/",
+    (value) =>
+      value.planRoot === originalPlan.root &&
+      value.source.commit === prepared.qualified.source.commit,
+  );
+  const result = await requalifySealedPublication({
+    context,
+    prepared,
+    originalPlan,
+    originalMaterialization,
+    archive,
+    host,
+    directory,
+  });
+  await journal.fence();
+  await journal.record(
+    `publication/qualification-prepared/${context.runId}-${context.runAttempt}`,
+    result,
+  );
+  return preparePipelineSigning({
+    plan: context.plan,
+    materialization: context.materialization,
+    qualified: result.qualified,
+    directory: path.join(directory, "signing"),
+  });
+}

package/packages/core/publication/pipeline/sealed-products.js

@@ -0,0 +1,79 @@
+import fs from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { recordDigest } from "../../release/discussion/envelope.js";
+import {
+  publicationFile,
+  publicationPath,
+  writeImmutablePublicationFile,
+} from "./files.js";
+
+export async function retainPipelineProducts(archive, qualified, bundles) {
+  const retained = [];
+  for (const artifact of qualified.artifacts) {
+    const bundle = bundles.find(
+      ({ manifest }) => manifest.root === artifact.manifestRoot,
+    );
+    if (!bundle)
+      throw new Error("Qualified product has no exact source manifest");
+    const file = publicationFile(
+      publicationPath(bundle.directory, artifact.file),
+    );
+    if (file.digest !== artifact.digest || file.size !== artifact.size)
+      throw new Error("Qualified product changed before durable retention");
+    const handle = await archive.put(file.bytes, {
+      name: `${artifact.digest.slice(7)}-${path.basename(artifact.file)}`,
+      mediaType: "application/octet-stream",
+    });
+    await verifiedBytes(archive, handle, artifact);
+    retained.push({ id: artifact.id, handle });
+  }
+  const body = {
+    schema: "buildchain.pipeline-sealed-products/v1",
+    qualificationRoot: qualified.root,
+    products: retained,
+  };
+  return { ...body, root: recordDigest(body) };
+}
+
+async function verifiedBytes(archive, handle, artifact) {
+  if (handle.digest !== artifact.digest || handle.size !== artifact.size)
+    throw new Error("Retained product handle does not bind qualified bytes");
+  const bytes = Buffer.from(await archive.read(handle));
+  if (
+    bytes.length !== artifact.size ||
+    `sha256:${createHash("sha256").update(bytes).digest("hex")}` !==
+      artifact.digest
+  )
+    throw new Error("Retained product readback differs from qualified bytes");
+  return bytes;
+}
+
+export async function restorePipelineProducts(
+  archive,
+  qualified,
+  sealed,
+  directory,
+) {
+  const { root, ...body } = sealed;
+  if (
+    root !== recordDigest(body) ||
+    sealed.qualificationRoot !== qualified.root ||
+    recordDigest(sealed.products.map(({ id }) => id)) !==
+      recordDigest(qualified.artifacts.map(({ id }) => id))
+  )
+    throw new Error(
+      "Retained products must cover the exact qualification inventory",
+    );
+  fs.mkdirSync(directory, { recursive: true });
+  for (const artifact of qualified.artifacts) {
+    const { handle } = sealed.products.find(({ id }) => id === artifact.id);
+    const bytes = await verifiedBytes(archive, handle, artifact);
+    const target = path.resolve(directory, artifact.file);
+    if (!target.startsWith(`${path.resolve(directory)}${path.sep}`))
+      throw new Error("Retained product path escapes its directory");
+    writeImmutablePublicationFile(target, bytes);
+    publicationPath(directory, artifact.file);
+  }
+  return directory;
+}

package/packages/core/publication/pipeline/settle.js

@@ -0,0 +1,42 @@
+import fs from "node:fs";
+import { publicationContext, uniquePublicationMaterial } from "./context.js";
+import { distributePipelineProducts } from "./distribution.js";
+import { nextPipelineDevelopment } from "./next-development.js";
+
+export async function settlePipelinePublication(
+  context,
+  host,
+  directory,
+  environment,
+) {
+  const { session, journal } = await publicationContext(context, host);
+  if (session.observed.phases.publish?.payload.state !== "success")
+    throw new Error(
+      "Publication must retain success before distribution and next-development",
+    );
+  fs.mkdirSync(directory, { recursive: true });
+  const retained = await uniquePublicationMaterial(
+    journal,
+    "publication/qualified/",
+  );
+  if (session.observed.phases.distribution?.payload.state !== "success") {
+    const receipt = await distributePipelineProducts(
+      context,
+      host,
+      journal,
+      retained,
+      environment,
+    );
+    await journal.record("publication/distribution-complete", receipt, {
+      phase: "distribution",
+      state: "success",
+    });
+  }
+  const next = await nextPipelineDevelopment(context, host, journal);
+  await journal.record("publication/next-development-result", next, {
+    phase: "next-development",
+    state: next.state,
+  });
+  await host.project(session);
+  return next;
+}

package/packages/core/publication/pipeline/signing.js

@@ -0,0 +1,140 @@
+import fs from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { execFileSync } from "node:child_process";
+import {
+  canonicalJson,
+  recordDigest,
+} from "../../release/discussion/envelope.js";
+import { verifyPipelineQualification } from "./documents.js";
+import { writeImmutablePublicationFile } from "./files.js";
+
+export const PIPELINE_PRODUCT_PREDICATE =
+  "https://buildchain.libkungfu.dev/attestations/pipeline-products/v1";
+const digest = (bytes) => createHash("sha256").update(bytes).digest("hex");
+
+export function preparePipelineSigning({
+  plan,
+  materialization,
+  qualified,
+  directory,
+  evaluatedAt,
+}) {
+  verifyPipelineQualification({
+    plan,
+    materialization,
+    qualified,
+    evaluatedAt,
+  });
+  const predicate = {
+    schema: "buildchain.pipeline-product-attestation/v1",
+    planRoot: plan.root,
+    materializationRoot: materialization.root,
+    qualificationRoot: qualified.root,
+    source: qualified.source,
+    protectedSource: plan.source,
+    publisher: plan.publisher,
+    runtime: plan.runtime,
+    provider: {
+      runId: qualified.build.runId,
+      runAttempt: qualified.build.runAttempt,
+      source: qualified.build.providerSource,
+    },
+  };
+  const subjectBytes = Buffer.from(`${canonicalJson(qualified)}\n`);
+  const subjectPath = writeImmutablePublicationFile(
+    path.join(directory, "qualified-products.json"),
+    subjectBytes,
+  );
+  const predicatePath = writeImmutablePublicationFile(
+    path.join(directory, "predicate.json"),
+    `${canonicalJson(predicate)}\n`,
+  );
+  return {
+    subjectPath,
+    predicatePath,
+    predicate,
+    subjectDigest: digest(subjectBytes),
+    predicateType: PIPELINE_PRODUCT_PREDICATE,
+  };
+}
+
+export function verifyPipelineSigning({
+  plan,
+  materialization,
+  qualified,
+  bundlePath,
+  directory,
+  token,
+  execute = execFileSync,
+  evaluatedAt,
+}) {
+  const prepared = preparePipelineSigning({
+    plan,
+    materialization,
+    qualified,
+    directory,
+    evaluatedAt,
+  });
+  const output = execute(
+    "gh",
+    [
+      "attestation",
+      "verify",
+      prepared.subjectPath,
+      "--repo",
+      qualified.source.repository,
+      "--signer-workflow",
+      `${plan.publisher.repository}/${plan.publisher.workflow}`,
+      "--signer-digest",
+      plan.publisher.workflowSha,
+      "--source-digest",
+      qualified.build.providerSource,
+      "--predicate-type",
+      PIPELINE_PRODUCT_PREDICATE,
+      "--bundle",
+      bundlePath,
+      "--deny-self-hosted-runners",
+      "--format",
+      "json",
+    ],
+    {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "pipe"],
+      maxBuffer: 16 * 1024 * 1024,
+      env: { ...process.env, GH_TOKEN: token },
+    },
+  );
+  const results = JSON.parse(output);
+  const statements = Array.isArray(results)
+    ? results
+        .map((result) => result.verificationResult?.statement)
+        .filter(Boolean)
+    : [];
+  if (
+    statements.length !== 1 ||
+    statements[0].predicateType !== prepared.predicateType ||
+    recordDigest(statements[0].predicate) !==
+      recordDigest(prepared.predicate) ||
+    statements[0].subject?.length !== 1 ||
+    statements[0].subject[0].digest?.sha256 !== prepared.subjectDigest
+  )
+    throw new Error(
+      "Cryptographically verified attestation does not bind the exact publication predicate and subject",
+    );
+  if (digest(fs.readFileSync(prepared.subjectPath)) !== prepared.subjectDigest)
+    throw new Error("Signed publication subject changed during verification");
+  const bundleBytes = fs.readFileSync(bundlePath);
+  return {
+    schema: "buildchain.pipeline-product-signing/v1",
+    verified: true,
+    planRoot: plan.root,
+    materializationRoot: materialization.root,
+    qualificationRoot: qualified.root,
+    subjectDigest: `sha256:${prepared.subjectDigest}`,
+    predicateRoot: recordDigest(prepared.predicate),
+    bundleDigest: `sha256:${digest(bundleBytes)}`,
+    signer: plan.publisher,
+    providerSource: qualified.build.providerSource,
+  };
+}

package/packages/core/publication/pipeline/source-plan.js

@@ -0,0 +1,83 @@
+import { planPipelinePublication } from "./plan.js";
+
+export async function createPipelinePublicationPlan(
+  session,
+  publisherSha,
+  host,
+  version,
+) {
+  const observed = session.observed;
+  const attempt = observed.attempt;
+  const integration = await host.integration.observe({
+    ...observed.history.at(-1),
+    intent: session.intent,
+  });
+  const source = await host.source.source(
+    integration.mergeCommit,
+    observed.history.at(-1).generation.source.configPath,
+  );
+  const route = source.plan.channels.find(
+    (entry) =>
+      entry.to === session.intent.source.targetBranch &&
+      entry.operation !== "develop",
+  );
+  if (!route)
+    throw new Error(
+      "Protected source does not declare this publication channel",
+    );
+  const input = await version.inspect(source.identity, source.plan.version);
+  const runtimeCommit = await host.request(
+    `/repos/${host.runtime.repository}/git/commits/${host.runtime.sha}`,
+  );
+  if (
+    runtimeCommit.sha !== host.runtime.sha ||
+    !/^[0-9a-f]{40}$/u.test(runtimeCommit.tree?.sha || "")
+  )
+    throw new Error("Publication runtime has no exact source tree");
+  const definition = {
+    repository: "kungfu-systems/buildchain",
+    workflow: ".github/workflows/.release-pipeline-products.yml",
+    workflowSha: publisherSha,
+    job: "apply",
+  };
+  const parameters = {
+    attempt,
+    generation: observed.generation,
+    source: source.identity,
+    intentSource: observed.history.at(-1).generation.source,
+    runtime: {
+      repository: host.runtime.repository,
+      commit: runtimeCommit.sha,
+      tree: runtimeCommit.tree.sha,
+    },
+    publisher: definition,
+    contract: source.plan,
+    route,
+    version: input.version,
+    sourceTimestamp: input.sourceTimestamp,
+  };
+  let plan = planPipelinePublication(parameters);
+  const tag = await host.request(
+    `/repos/${host.repository}/git/ref/tags/${encodeURIComponent(plan.tag)}`,
+    { allow404: true },
+  );
+  const channel = `v${plan.version.split(".")[0]}${plan.channel === "alpha" ? "-alpha" : ""}`;
+  const previous = await host.request(
+    `/repos/${host.repository}/git/ref/tags/${channel}`,
+    { allow404: true },
+  );
+  if (
+    previous &&
+    (previous.object?.type !== "commit" ||
+      !/^[0-9a-f]{40}$/u.test(previous.object.sha))
+  )
+    throw new Error(
+      "Publication channel requires an exact lightweight commit ref",
+    );
+  plan = planPipelinePublication({
+    ...parameters,
+    expectedTagSha: tag?.object?.sha || null,
+    previousChannelCommit: previous?.object?.sha || null,
+  });
+  return plan;
+}