@kungfu-tech/buildchain 4.1.3-alpha.2 → 4.1.3-alpha.4

package/packages/core/publication/pipeline/apply.js

@@ -1,8 +1,7 @@
 import path from "node:path";
 import fs from "node:fs";
-import { createHash } from "node:crypto";
 import { recordDigest } from "../../release/discussion/envelope.js";
-import { canonicalJson } from "../../release/discussion/envelope.js";
+import { preparePipelineStableQualification } from "./stable-wait.js";
 import {
   createReleaseReceipt,
   RELEASE_RECEIPT_CONTRACT,
@@ -11,7 +10,10 @@ import { githubPipelineProductRelease } from "../../providers/github/pipeline-pr
 import { pipelineNpmProvider } from "../npm/pipeline-provider.js";
 import { publicationContext, uniquePublicationMaterial } from "./context.js";
 import { restorePipelineProducts } from "./sealed-products.js";
-import { pipelineReleaseDocuments } from "./documents.js";
+import {
+  pipelineReleaseDocuments,
+  pipelineReleaseEvidence,
+} from "./documents.js";
 import { writeImmutablePublicationFile } from "./files.js";
 import { verifyPipelineSigning } from "./signing.js";
 import { pipelinePublicationEffects, applyPipelineEffects } from "./effects.js";
@@ -32,6 +34,17 @@ export async function applyPipelinePublication(
   } = {},
 ) {
   const { journal, archive } = await publicationContext(context, host);
+  const wait = await preparePipelineStableQualification(
+    context.plan,
+    host,
+    journal,
+    {
+      attempt: context.attempt,
+      generation: context.generation,
+      phase: "publish",
+    },
+  );
+  if (wait) return { operation: "wait", wait };
   const retained = await uniquePublicationMaterial(
     journal,
     "publication/qualified/",
@@ -80,20 +93,13 @@ export async function applyPipelinePublication(
     artifacts: qualified.artifacts,
     environment,
   });
-  const evidence = Object.entries({
-    qualification: Buffer.from(`${canonicalJson(qualified)}\n`),
-    capsules: Buffer.from(`${canonicalJson(capsules)}\n`),
-    invocation: Buffer.from(
-      `${canonicalJson(documents.invocation.invocation)}\n`,
-    ),
-    attestation: fs.readFileSync(bundlePath),
-  }).map(([id, bytes]) => ({
-    id,
-    name: `buildchain.${id}.json`,
-    bytes,
-    size: bytes.length,
-    digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}`,
-  }));
+  const evidence = pipelineReleaseEvidence({
+    plan,
+    qualified,
+    capsules,
+    documents,
+    bundle: fs.readFileSync(bundlePath),
+  });
   const github = githubPipelineProductRelease({
     ...host,
     plan,

package/packages/core/publication/pipeline/development-pr.js

@@ -1,4 +1,8 @@
 import { recordDigest } from "../../release/discussion/envelope.js";
+import {
+  preparePipelineVersionContext,
+  retainedPipelineVersionRegeneration,
+} from "./version-context.js";
 
 export async function prepareNextDevelopmentPullRequest(
   context,
@@ -44,8 +48,31 @@ export async function prepareNextDevelopmentPullRequest(
       phase: "next-development",
     });
   }
+  const regeneration = await retainedPipelineVersionRegeneration(
+    journal,
+    request,
+    "development",
+    host.runtime,
+  );
+  if (request.versionPolicy.derived_files?.length && !regeneration)
+    return {
+      state: "waiting",
+      reason: "next-development-version-regeneration-required",
+      versionContext: await preparePipelineVersionContext(
+        request,
+        "development",
+        context,
+        host,
+        journal,
+        context.recovery?.execution.publisher.workflowSha ||
+          plan.publisher.workflowSha,
+      ),
+    };
   await journal.fence();
-  const materialization = await adapter.materializeDevelopment(request);
+  const materialization = await adapter.materializeDevelopment(
+    request,
+    regeneration,
+  );
   const branch = materialization.branch;
   const pulls = await host.request(
     `/repos/${host.repository}/pulls?state=all&head=${encodeURIComponent(`${host.repository.split("/")[0]}:${branch}`)}&base=${encodeURIComponent(plan.developmentBranch)}&per_page=100`,

package/packages/core/publication/pipeline/documents.js

@@ -1,4 +1,8 @@
-import { recordDigest } from "../../release/discussion/envelope.js";
+import { createHash } from "node:crypto";
+import {
+  canonicalJson,
+  recordDigest,
+} from "../../release/discussion/envelope.js";
 import { releaseTailRoot } from "../../release/release-tail-provider-plane.js";
 import {
   RELEASE_INVOCATION_CONTRACT,
@@ -12,6 +16,38 @@ import {
 } from "../publication-qualification.js";
 import { verifyPipelinePublicationPlan } from "./plan.js";
 
+export function pipelineReleaseEvidence({
+  plan,
+  qualified,
+  capsules,
+  documents,
+  bundle,
+}) {
+  verifyPipelinePublicationPlan(plan);
+  if (plan.evidenceVersion !== undefined && plan.evidenceVersion !== 1)
+    throw new Error("Unsupported retained publication evidence version");
+  // An older retained transaction keeps its original evidence inventory.
+  // Adding a disclosure during recovery would change its provider receipt.
+  const values = {
+    ...(plan.evidenceVersion === 1 ? { plan } : {}),
+    qualification: qualified,
+    capsules,
+    invocation: documents.invocation.invocation,
+  };
+  const bytes = Object.entries(values).map(([id, value]) => [
+    id,
+    Buffer.from(`${canonicalJson(value)}\n`),
+  ]);
+  bytes.push(["attestation", Buffer.from(bundle)]);
+  return bytes.map(([id, value]) => ({
+    id,
+    name: `buildchain.${id}.json`,
+    bytes: value,
+    size: value.length,
+    digest: `sha256:${createHash("sha256").update(value).digest("hex")}`,
+  }));
+}
+
 export function verifyRootedPublication(value, schema) {
   const { root, ...body } = value || {};
   if (body.schema !== schema || root !== recordDigest(body))

package/packages/core/publication/pipeline/next-development.js

@@ -46,6 +46,7 @@ export async function nextPipelineDevelopment(context, host, journal) {
       transition,
       adapter,
     );
+    if (selected.versionContext) return selected;
   }
   const pr = await host.request(
     `/repos/${host.repository}/pulls/${selected.number}`,

package/packages/core/publication/pipeline/pack.js

@@ -101,7 +101,13 @@ export function packPipelineProducts({
         ? packageArtifact(directory, staging, item, plan.version, environment)
         : inspectPipelineFileArtifact(directory, item);
     const observed = publicationFile(packed.file);
-    const name = `${item.product}-${platform}-${item.artifact}${packed.suffix}`;
+    const name =
+      item.filename ||
+      `${item.product}-${platform}-${item.artifact}${packed.suffix}`;
+    if (!name.endsWith(packed.suffix))
+      throw new Error(
+        "Declared publication filename does not match the actual artifact format",
+      );
     writeImmutablePublicationFile(
       path.join(output, "payloads", name),
       observed.bytes,

package/packages/core/publication/pipeline/plan.js

@@ -13,6 +13,7 @@ export function pipelineExpectedProducts(plan) {
           kind: artifact.kind,
           directory: product.directory || ".",
           path: artifact.path,
+          ...(artifact.filename ? { filename: artifact.filename } : {}),
           targets: product.targets
             .filter((target) => target.artifacts.includes(artifact.id))
             .map(({ provider, access }) => ({
@@ -31,6 +32,23 @@ export function pipelineExpectedProducts(plan) {
     throw new Error(
       "Publication requires a bounded unique product/platform/artifact inventory",
     );
+  const filenames = outputs.map(
+    (output) =>
+      output.filename ||
+      `${output.product}-${output.platform}-${output.artifact}${
+        output.kind === "npm-package"
+          ? ".tgz"
+          : output.kind === "pdf"
+            ? ".pdf"
+            : [".tar.gz", ".tar.xz", ".tgz", ".zip", ".tar"].find((suffix) =>
+                output.path.endsWith(suffix),
+              )
+      }`,
+  );
+  if (new Set(filenames).size !== filenames.length)
+    throw new Error(
+      "Publication artifact filenames must be unique across every declared platform",
+    );
   return outputs;
 }
 
@@ -79,6 +97,7 @@ export function planPipelinePublication({
     );
   const body = {
     schema: "buildchain.pipeline-publication-plan/v1",
+    evidenceVersion: 1,
     attempt,
     generation,
     source,
@@ -93,6 +112,9 @@ export function planPipelinePublication({
     route,
     sourceTimestamp,
     versionPolicy: contract.version,
+    ...(contract.stable
+      ? { stablePolicy: structuredClone(contract.stable) }
+      : {}),
     developmentBranch: developmentRoutes[0].from,
     expectedTagSha,
     previousChannelCommit,

package/packages/core/publication/pipeline/prepare.js

@@ -1,10 +1,14 @@
+import { claimPipelinePublicationWorker } from "./worker.js";
 import { recordDigest } from "../../release/discussion/envelope.js";
-import { resumePipelineSession } from "../../workflow/pipeline/session.js";
 import { pipelinePlatforms } from "../../workflow/pipeline/platforms.js";
 import { githubPipelineVersion } from "../../providers/github/pipeline-version.js";
 import { createPipelinePublicationPlan } from "./source-plan.js";
-import { pipelinePublicationJournal } from "./journal.js";
+import { preparePipelineStableQualification } from "./stable-wait.js";
 import { prepareRecoveredPublication } from "./recovery-prepare.js";
+import {
+  preparePipelineVersionContext,
+  retainedPipelineVersionRegeneration,
+} from "./version-context.js";
 
 function publicationNextOperation(phase, existing, recovery) {
   if (phase !== "publish") return "settle";
@@ -15,61 +19,14 @@ function publicationNextOperation(phase, existing, recovery) {
 }
 
 export async function preparePipelinePublication(attempt, publisherSha, host) {
-  const { run: execution } = await host.runs.read(host.runId, host.runAttempt);
-  const definitions = (execution.referenced_workflows || []).filter((entry) =>
-    entry.path?.startsWith(
-      "kungfu-systems/buildchain/.github/workflows/.release-pipeline-products.yml@",
-    ),
+  const claimed = await claimPipelinePublicationWorker(
+    attempt,
+    publisherSha,
+    host,
   );
-  if (
-    !/^[0-9a-f]{40}$/u.test(publisherSha || "") ||
-    definitions.length !== 1 ||
-    definitions[0].sha !== publisherSha
-  )
-    throw new Error(
-      "Publisher definition is not the exact hosted reusable workflow",
-    );
-  const session = await resumePipelineSession({ ...host, attempt }, host);
+  if (!claimed) return { operation: "wait" };
+  const { session, journal, phase } = claimed;
   const observed = session.observed;
-  if (observed.phases.merge?.payload.state !== "success")
-    throw new Error(
-      "Product publication requires completed protected integration",
-    );
-  if (observed.status === "complete") return { operation: "wait" };
-  const phase = observed.missing[0];
-  if (!["publish", "distribution", "next-development"].includes(phase))
-    throw new Error("Attempt is not at product publication");
-  const prior = observed.history
-    .at(-1)
-    .events.filter((event) =>
-      event.payload.materials.some((material) =>
-        material.id.startsWith("publication/worker/"),
-      ),
-    )
-    .at(-1)?.payload.writer;
-  if (
-    prior &&
-    (prior.runId !== String(host.runId) ||
-      prior.runAttempt !== String(host.runAttempt))
-  ) {
-    const { run } = await host.runs.read(
-      Number(prior.runId),
-      Number(prior.runAttempt),
-    );
-    if (run.status !== "completed") return { operation: "wait" };
-  }
-  const journal = pipelinePublicationJournal(session, host);
-  const claim = {
-    schema: "buildchain.pipeline-publication-worker/v1",
-    attempt,
-    runId: host.runId,
-    runAttempt: host.runAttempt,
-  };
-  await journal.record("publication/worker", claim, {
-    phase,
-    expectedHead: observed.head,
-  });
-  await journal.fence();
   const recovery = await prepareRecoveredPublication(
     session,
     publisherSha,
@@ -91,6 +48,12 @@ export async function preparePipelinePublication(attempt, publisherSha, host) {
     );
     await journal.record("publication/plan", plan);
   }
+  const wait = await preparePipelineStableQualification(plan, host, journal, {
+    attempt,
+    generation: observed.generation,
+    phase,
+  });
+  if (wait) return { operation: "wait", wait };
   if (
     !recovery &&
     (plan.publisher.workflowSha !== publisherSha ||
@@ -106,8 +69,33 @@ export async function preparePipelinePublication(attempt, publisherSha, host) {
     throw new Error("Publication materialized source is ambiguous");
   let materialization = materializations[0];
   if (!materialization) {
+    const regeneration = await retainedPipelineVersionRegeneration(
+      journal,
+      plan,
+      "publication",
+      host.runtime,
+    );
+    if (plan.versionPolicy.derived_files?.length && !regeneration) {
+      const publication = {
+        attempt,
+        generation: observed.generation,
+        plan,
+        ...(recovery ? { recovery } : {}),
+      };
+      return {
+        operation: "regenerate",
+        context: await preparePipelineVersionContext(
+          plan,
+          "publication",
+          publication,
+          host,
+          journal,
+          publisherSha,
+        ),
+      };
+    }
     await journal.fence();
-    materialization = await version.materialize(plan);
+    materialization = await version.materialize(plan, regeneration);
     const verified = await host.source.source(
       materialization.source.commit,
       plan.source.configPath,

package/packages/core/publication/pipeline/qualification.js

@@ -3,6 +3,7 @@ import {
   publicationArtifactProducer,
   verifyPublicationBuildAggregate,
 } from "./build-segments.js";
+import path from "node:path";
 import { recordDigest } from "../../release/discussion/envelope.js";
 import { createDomainPublicationQualificationReceipt } from "../publication-qualification.js";
 import { verifyPipelinePublicationPlan } from "./plan.js";
@@ -22,6 +23,10 @@ function productDescriptor(artifact) {
 export function inspectPipelinePublicationArtifacts(directory, manifest, plan) {
   verifyPipelineProductFiles(directory, manifest);
   for (const artifact of manifest.artifacts) {
+    if (artifact.filename && path.basename(artifact.file) !== artifact.filename)
+      throw new Error(
+        "Independent qualification rejected the declared download filename",
+      );
     if (artifact.kind === "npm-package") {
       const pkg = readNpmPackageJsonFromTarball(
         publicationPath(directory, artifact.file),

package/packages/core/publication/pipeline/stable-entry.js

@@ -0,0 +1,215 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { readBusinessAttempt } from "../../workflow/attempt/reader.js";
+import { verifyRootedPublication } from "./documents.js";
+import { verifyPipelinePublicationPlan } from "./plan.js";
+import {
+  pipelineEntryRuns,
+  pipelineEntryRun,
+  pipelineEntryCheck,
+} from "../../providers/github/pipeline-entry-runs.js";
+
+async function nativeBuild(check, run, consumer, host) {
+  const loaded = await host.index.resolve(check.attempt);
+  const observed = readBusinessAttempt(loaded.snapshot);
+  const selected = observed.history.find(
+    (item) => item.identity.id === check.attempt,
+  );
+  if (
+    observed.intent.repository !== host.repository ||
+    !selected ||
+    selected.identity.requestKey.startsWith("recover:") ||
+    recordDigest(selected.generation.source) !== recordDigest(consumer.source)
+  )
+    throw new Error(
+      "Published entry check points to another source or a recovery attempt",
+    );
+  const phase = selected.phases.build;
+  if (phase?.payload.state !== "success" || check.outcome !== "success")
+    return {
+      status: "missing",
+      reason: "latest-published-entry-build-failed",
+      attempt: check.attempt,
+    };
+  if (
+    phase.payload.writer.runId !== String(run.id) ||
+    phase.payload.writer.runAttempt !== String(run.run_attempt) ||
+    phase.runtime.repository !== "kungfu-systems/buildchain" ||
+    phase.runtime.sha !== consumer.selection.ref
+  )
+    throw new Error(
+      "Published entry native build changed its actual runtime or execution",
+    );
+  const references = phase.payload.materials.filter((item) =>
+    item.id.startsWith("build/provider-"),
+  );
+  if (references.length !== 1)
+    throw new Error(
+      "Published entry requires one original normal build receipt",
+    );
+  const session = {
+    intent: loaded.snapshot.intent,
+    observed: { ...observed, history: [selected] },
+  };
+  const readback = await host.materialStore(session).read(references[0]);
+  verifyRootedPublication(readback, "buildchain.pipeline-build-readback/v1");
+  if (
+    readback.runId !== run.id ||
+    readback.runAttempt !== run.run_attempt ||
+    readback.outcome !== "success" ||
+    recordDigest(readback.source) !== recordDigest(consumer.source) ||
+    recordDigest(readback.entry) !== recordDigest(consumer.entry) ||
+    check.summary !==
+      `Exact product jobs: ${readback.root}\nAttempt: ${check.attempt}`
+  )
+    throw new Error("Published entry check and native build material disagree");
+  return {
+    readback,
+    phase,
+    reference: references[0],
+    attempt: check.attempt,
+    intent: observed.intent.id,
+  };
+}
+
+function completion(run, jobs, native, source) {
+  const recorder = jobs.filter(
+    (job) => String(job.id) === native.phase.payload.writer.jobId,
+  );
+  if (
+    recorder.length !== 1 ||
+    recorder[0].status !== "completed" ||
+    recorder[0].conclusion !== "success" ||
+    recorder[0].run_id !== run.id ||
+    recorder[0].run_attempt !== run.run_attempt ||
+    !(
+      recorder[0].name === "Record product build" ||
+      recorder[0].name.endsWith(" / Record product build")
+    )
+  )
+    throw new Error(
+      "Published entry native receipt lacks its exact completed recorder job",
+    );
+  const selected = [...native.readback.jobs, recorder[0]];
+  const start = Date.parse(source.candidate.publishedAt),
+    completions = [];
+  for (const job of selected) {
+    const started = Date.parse(job.started_at),
+      ended = Date.parse(job.completed_at);
+    if (
+      !Number.isFinite(started) ||
+      !Number.isFinite(ended) ||
+      started < start ||
+      ended < started
+    )
+      throw new Error(
+        "Published entry jobs must execute after the exact Alpha publication",
+      );
+    completions.push(ended);
+  }
+  return new Date(Math.max(...completions)).toISOString();
+}
+
+async function inspectEntryRun(listed, plan, source, host) {
+  const { run, jobs } = await host.runs.read(listed.id, listed.run_attempt);
+  const consumer = await pipelineEntryRun(run, plan, source, host);
+  if (!consumer) return null;
+  if (
+    !jobs.some((job) => /(?:^| \/ )Build product \([^)]+\)$/u.test(job.name)) &&
+    run.conclusion === "success"
+  )
+    return null;
+  const check = await pipelineEntryCheck(run, host);
+  if (!check)
+    return {
+      status: "missing",
+      reason: "latest-published-entry-build-unrecorded",
+      runId: run.id,
+    };
+  const native = await nativeBuild(check, run, consumer, host);
+  if (native.status === "missing") return native;
+  const platforms = [
+    ...new Set(plan.outputs.map(({ platform }) => platform)),
+  ].sort();
+  const fresh = await host.runs.build(
+    run.id,
+    run.run_attempt,
+    consumer.source,
+    platforms,
+  );
+  if (recordDigest(fresh) !== recordDigest(native.readback))
+    throw new Error(
+      "Published entry original jobs changed or omit declared platforms",
+    );
+  const completedAt = completion(run, jobs, native, source);
+  const again = await host.runs.read(run.id, run.run_attempt);
+  const repeated = await nativeBuild(
+    await pipelineEntryCheck(again.run, host),
+    again.run,
+    consumer,
+    host,
+  );
+  if (
+    recordDigest(repeated) !== recordDigest(native) ||
+    recordDigest(again) !== recordDigest({ run, jobs })
+  )
+    throw new Error(
+      "Published entry evidence changed during final provider readback",
+    );
+  const body = {
+    schema: "buildchain.pipeline-stable-entry/v1",
+    planRoot: plan.root,
+    sourceRoot: source.root,
+    consumer,
+    native,
+    runId: run.id,
+    runAttempt: run.run_attempt,
+    canary: {
+      id: "published-entry",
+      source: "public-build",
+      status: "success",
+      repository: host.repository,
+      candidateSha: source.candidate.sha,
+      completedAt,
+      attestor: "github-actions[bot]",
+      runtimeRef: consumer.selection.ref,
+      evidenceUrl: `https://github.com/${host.repository}/actions/runs/${run.id}/attempts/${run.run_attempt}`,
+    },
+  };
+  return { ...body, root: recordDigest(body) };
+}
+
+export async function readPipelineStableEntry(plan, source, host) {
+  verifyPipelinePublicationPlan(plan);
+  verifyRootedPublication(source, "buildchain.pipeline-stable-source/v1");
+  if (
+    plan.channel !== "stable" ||
+    source.planRoot !== plan.root ||
+    host.repository !== plan.source.repository ||
+    source.contractRoot !== plan.contractRoot ||
+    source.candidate.sha !== plan.intentSource.commit ||
+    !Number.isFinite(Date.parse(source.candidate.publishedAt))
+  )
+    throw new Error(
+      "Published entry collection requires its exact admitted Alpha plan",
+    );
+  const inventory = await pipelineEntryRuns(host, source.candidate.publishedAt);
+  for (const run of inventory) {
+    const result = await inspectEntryRun(run, plan, source, host);
+    if (result) {
+      if (
+        result.canary &&
+        recordDigest(
+          await pipelineEntryRuns(host, source.candidate.publishedAt),
+        ) !== recordDigest(inventory)
+      )
+        throw new Error(
+          "Published entry run inventory changed during qualification",
+        );
+      return result;
+    }
+  }
+  return {
+    status: "missing",
+    reason: "no-normal-published-entry-build-for-exact-runtime",
+  };
+}