@kungfu-tech/buildchain 4.1.3-alpha.0 → 4.1.3-alpha.2

package/packages/core/publication/pipeline/recovery-inspection.js

@@ -0,0 +1,218 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { verifyPipelinePublicationPlan } from "./plan.js";
+import {
+  verifyPipelineQualification,
+  pipelineReleaseDocuments,
+} from "./documents.js";
+import { restorePipelineProducts } from "./sealed-products.js";
+import { verifyPipelineSigning } from "./signing.js";
+import { writeImmutablePublicationFile } from "./files.js";
+import {
+  readRecoveryPublicationMaterials,
+  recoveryPublicationMaterial,
+  publicationLineageMaterial,
+} from "./recovery-materials.js";
+import { planRecoveryPublicationBuild } from "./recovery-build-plan.js";
+import { qualifyRecoveryIntegration } from "../../workflow/pipeline/recovery-integration.js";
+
+async function inspectRetainedBytes(
+  { plan, materialization, retained, archive, host },
+  verifySigning,
+) {
+  const directory = fs.mkdtempSync(
+    path.join(fs.realpathSync(os.tmpdir()), "buildchain-publication-recovery-"),
+  );
+  try {
+    // Historical validation proves the original document's integrity only.
+    // Fresh execution authorization is a separate admission before any effect.
+    const evaluatedAt = retained.qualified.qualification.issuedAt;
+    verifyPipelineQualification({
+      plan,
+      materialization,
+      qualified: retained.qualified,
+      evaluatedAt,
+    });
+    await restorePipelineProducts(
+      archive,
+      retained.qualified,
+      retained.sealed,
+      path.join(directory, "products"),
+    );
+    if (!retained.signing)
+      return { sealed: retained.sealed.root, signing: null };
+    const bundlePath = writeImmutablePublicationFile(
+      path.join(directory, "attestation.json"),
+      await archive.read(retained.bundle),
+    );
+    const signing = verifySigning({
+      plan,
+      materialization,
+      qualified: retained.qualified,
+      bundlePath,
+      directory: path.join(directory, "signing"),
+      token: host.token,
+      evaluatedAt,
+    });
+    const documents = pipelineReleaseDocuments({
+      plan,
+      materialization,
+      qualified: retained.qualified,
+      capsules: retained.capsules,
+      signing,
+      evaluatedAt,
+    });
+    if (
+      recordDigest(signing) !== recordDigest(retained.signing) ||
+      recordDigest(documents) !== recordDigest(retained.documents)
+    )
+      throw new Error(
+        "Recovery signature or original release documents changed",
+      );
+    return {
+      sealed: retained.sealed.root,
+      signing: recordDigest(signing),
+      documents: recordDigest(documents),
+    };
+  } finally {
+    fs.rmSync(directory, { recursive: true, force: true });
+  }
+}
+
+export async function inspectRecoveryPublication(
+  session,
+  admitted,
+  host,
+  { verifySigning = verifyPipelineSigning } = {},
+) {
+  const integration = await qualifyRecoveryIntegration(
+    session,
+    admitted.admission,
+    host,
+  );
+  const materials = await readRecoveryPublicationMaterials(session, host);
+  const plan = recoveryPublicationMaterial(materials, "publication/plan/");
+  if (!plan) {
+    if (materials.some((item) => !item.id.startsWith("publication/worker/")))
+      throw new Error(
+        "Publication recovery has materials without their authoritative plan",
+      );
+    return {
+      schema: "buildchain.pipeline-recovery-publication/v1",
+      mode: "unprepared",
+      integration,
+      materials,
+    };
+  }
+  verifyPipelinePublicationPlan(plan);
+  const original = session.observed.history.find(
+    (item) => item.identity.id === plan.attempt,
+  );
+  if (
+    !original ||
+    plan.generation !== session.observed.generation ||
+    recordDigest(plan.intentSource) !==
+      recordDigest(original.generation.source) ||
+    plan.source.commit !== integration.integration.mergeCommit
+  )
+    throw new Error(
+      "Recovered publication plan changed its original source or protected integration",
+    );
+  const materialization = recoveryPublicationMaterial(
+    materials,
+    "publication/materialization/",
+  );
+  if (!materialization)
+    return {
+      schema: "buildchain.pipeline-recovery-publication/v1",
+      mode: "materialize",
+      integration,
+      materials,
+    };
+  const source = await host.source.source(
+    materialization.source.commit,
+    plan.source.configPath,
+  );
+  if (
+    recordDigest(source.identity) !== recordDigest(materialization.source) ||
+    recordDigest(source.plan) !== plan.contractRoot
+  )
+    throw new Error(
+      "Recovered publication materialization changed its exact source or contract",
+    );
+  const qualified = recoveryPublicationMaterial(
+    materials,
+    "publication/qualified/",
+  );
+  const prepared =
+    materials
+      .filter((item) =>
+        item.id.startsWith("publication/qualification-prepared/"),
+      )
+      .at(-1)?.value ||
+    recoveryPublicationMaterial(materials, "publication/predecessor-prepared/");
+  const retained = qualified || prepared;
+  if (
+    !qualified &&
+    materials.some((item) => item.id.startsWith("publication/effect/"))
+  )
+    throw new Error(
+      "Partial publication lost its original signed qualification; rebuilding cannot replace published bytes",
+    );
+  const originalPlan = retained
+    ? publicationLineageMaterial(
+        materials,
+        "buildchain.pipeline-publication-plan/v1",
+        (value) => value.root === retained.qualified.planRoot,
+      )
+    : plan;
+  const originalMaterialization = retained
+    ? publicationLineageMaterial(
+        materials,
+        "buildchain.pipeline-version-materialization/v1",
+        (value) =>
+          value.planRoot === originalPlan.root &&
+          recordDigest(value.source) ===
+            recordDigest(retained.qualified.source),
+      )
+    : materialization;
+  if (
+    originalPlan.version !== plan.version ||
+    originalPlan.contractRoot !== plan.contractRoot ||
+    recordDigest(originalPlan.outputs) !== recordDigest(plan.outputs)
+  )
+    throw new Error(
+      "Recovered publication lineage changed the sealed product contract",
+    );
+  const readback = retained
+    ? await inspectRetainedBytes(
+        {
+          plan: originalPlan,
+          materialization: originalMaterialization,
+          retained,
+          archive: host.productArchive(session),
+          host,
+        },
+        verifySigning,
+      )
+    : null;
+  return {
+    schema: "buildchain.pipeline-recovery-publication/v1",
+    mode: qualified ? "qualified" : prepared ? "prepared" : "build",
+    integration,
+    materials,
+    readback,
+    preparedRoot: prepared ? recordDigest(prepared) : null,
+    build: retained
+      ? null
+      : await planRecoveryPublicationBuild(
+          session,
+          plan,
+          materialization,
+          materials,
+          host,
+        ),
+  };
+}

package/packages/core/publication/pipeline/recovery-materials.js

@@ -0,0 +1,104 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { retainedRecoveryEvidence } from "../../workflow/pipeline/recovery-session.js";
+import {
+  PUBLICATION_IMPORT,
+  publicationImportedValues,
+} from "./imported-materials.js";
+
+export async function readRecoveryPublicationMaterials(session, host) {
+  const current = session.observed.history.at(-1);
+  const references = current.events
+    .flatMap((event) => event.payload.materials)
+    .filter((item) => item.id.startsWith("publication/"));
+  const unique = [
+    ...new Map(
+      references.map((reference) => [
+        `${reference.id}:${reference.digest}`,
+        reference,
+      ]),
+    ).values(),
+  ];
+  if (unique.length > 1000)
+    throw new Error(
+      "Publication recovery material inventory exceeds its bound",
+    );
+  const result = [];
+  for (const reference of unique) {
+    const value = await host.materialStore(session).read(reference);
+    if (value.schema === PUBLICATION_IMPORT)
+      result.push(
+        ...publicationImportedValues(value).map((item) => ({
+          id: item.id,
+          reference,
+          memberId: item.id,
+          value: item.value,
+        })),
+      );
+    else result.push({ id: reference.id, reference, value });
+  }
+  if (
+    !result.some((item) => item.id.startsWith("publication/plan/")) &&
+    current.identity.requestKey.startsWith("recover:")
+  ) {
+    const { evidence } = await retainedRecoveryEvidence(session, host);
+    for (const item of evidence.publication?.materials || []) {
+      const producer = session.observed.history.find(
+        (entry) => entry.identity.id === item.reference.producerAttempt,
+      );
+      if (!producer)
+        throw new Error(
+          "Publication recovery material has no canonical predecessor",
+        );
+      const retained = {
+        ...session,
+        observed: { ...session.observed, history: [producer] },
+      };
+      const container = await host.materialStore(retained).read(item.reference);
+      const value = item.memberId
+        ? publicationImportedValues(container).find(
+            (member) => member.id === item.memberId,
+          )?.value
+        : container;
+      if (recordDigest(value) !== recordDigest(item.value))
+        throw new Error("Retained predecessor publication material changed");
+      result.push({ ...item, value });
+    }
+  }
+  if (result.length > 1000)
+    throw new Error(
+      "Expanded publication recovery inventory exceeds its bound",
+    );
+  return result;
+}
+
+export function recoveryPublicationMaterial(
+  materials,
+  prefix,
+  required = false,
+) {
+  const values = [
+    ...new Map(
+      materials
+        .filter((item) => item.id.startsWith(prefix))
+        .map((item) => [recordDigest(item.value), item.value]),
+    ).values(),
+  ];
+  if (values.length > 1 || (required && values.length !== 1))
+    throw new Error(
+      `Publication recovery needs one exact retained material: ${prefix}`,
+    );
+  return values[0] || null;
+}
+
+export function publicationLineageMaterial(materials, schema, accepts) {
+  const values = [
+    ...new Map(
+      materials.map((item) => [recordDigest(item.value), item.value]),
+    ).values(),
+  ].filter((value) => value.schema === schema && accepts(value));
+  if (values.length !== 1)
+    throw new Error(
+      `Publication recovery has no exact original lineage material: ${schema}`,
+    );
+  return values[0];
+}

package/packages/core/publication/pipeline/recovery-plan.js

@@ -0,0 +1,46 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { verifyPipelinePublicationPlan } from "./plan.js";
+import { verifyRootedPublication } from "./documents.js";
+
+// Before signing or publishing, a new publisher may qualify the already sealed
+// bytes under an explicit derived plan. Original manifests and source stay exact.
+export function deriveRecoveryPublicationPlan(
+  original,
+  materialization,
+  execution,
+  recoveryPlanRoot,
+) {
+  verifyPipelinePublicationPlan(original);
+  const { root: previous, ...body } = original;
+  const value = {
+    ...body,
+    attempt: execution.attempt,
+    runtime: execution.runtime,
+    publisher: execution.publisher,
+    recovery: {
+      predecessorPlanRoot: previous,
+      admissionRoot: recoveryPlanRoot,
+    },
+  };
+  const plan = { ...value, root: recordDigest(value) };
+  if (!materialization) return { plan, materialization: null };
+  verifyRootedPublication(
+    materialization,
+    "buildchain.pipeline-version-materialization/v1",
+  );
+  if (
+    materialization.planRoot !== original.root ||
+    recordDigest(materialization.protectedSource) !==
+      recordDigest(original.source)
+  )
+    throw new Error(
+      "Recovery version materialization changed its original plan",
+    );
+  const { root, ...source } = materialization;
+  const derived = {
+    ...source,
+    planRoot: plan.root,
+    recovery: { predecessorRoot: root, predecessorPlanRoot: previous },
+  };
+  return { plan, materialization: { ...derived, root: recordDigest(derived) } };
+}

package/packages/core/publication/pipeline/recovery-prepare.js

@@ -0,0 +1,73 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { retainedRecoveryEvidence } from "../../workflow/pipeline/recovery-session.js";
+import { recoveryPublicationMaterial } from "./recovery-materials.js";
+
+import { importRecoveredPublication } from "./recovery-import.js";
+
+export async function prepareRecoveredPublication(
+  session,
+  publisherSha,
+  host,
+  journal,
+  phase,
+) {
+  if (
+    !session.observed.history.at(-1).identity.requestKey.startsWith("recover:")
+  )
+    return null;
+  const { plan: recoveryPlan, evidence } = await retainedRecoveryEvidence(
+    session,
+    host,
+  );
+  const publication = evidence.publication;
+  if (!publication || publication.mode === "unprepared") return null;
+  const plan = recoveryPublicationMaterial(
+    publication.materials,
+    "publication/plan/",
+    true,
+  );
+  if (recordDigest(host.runtime) !== recordDigest(recoveryPlan.runtime))
+    throw new Error(
+      "Recovered publication executor differs from the admitted repair",
+    );
+  const runtime = await host.request(
+    `/repos/${host.runtime.repository}/git/commits/${host.runtime.sha}`,
+  );
+  if (
+    runtime.sha !== host.runtime.sha ||
+    !/^[0-9a-f]{40}$/u.test(runtime.tree?.sha || "")
+  )
+    throw new Error("Recovery publisher runtime has no exact admitted tree");
+  const execution = {
+    attempt: session.observed.attempt,
+    runId: host.runId,
+    runAttempt: host.runAttempt,
+    publisher: {
+      repository: "kungfu-systems/buildchain",
+      workflow: ".github/workflows/.release-pipeline-products.yml",
+      workflowSha: publisherSha,
+      job: "apply",
+    },
+    runtime: {
+      repository: host.runtime.repository,
+      commit: runtime.sha,
+      tree: runtime.tree.sha,
+    },
+  };
+  await importRecoveredPublication(
+    publication,
+    execution,
+    recoveryPlan.root,
+    journal,
+    phase,
+  );
+  return {
+    planRoot: recoveryPlan.root,
+    predecessor: recoveryPlan.predecessor,
+    originalPlanRoot: plan.root,
+    mode: publication.mode,
+    build: publication.build || null,
+    preserveTransaction: publication.mode === "qualified",
+    execution,
+  };
+}

package/packages/core/publication/pipeline/recovery-qualification.js

@@ -0,0 +1,128 @@
+import path from "node:path";
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { createDomainPublicationQualificationReceipt } from "../publication-qualification.js";
+import { restorePipelineProducts } from "./sealed-products.js";
+import { inspectPipelinePublicationArtifacts } from "./qualification.js";
+import { verifyPipelineQualification } from "./documents.js";
+import { reobservePublicationBuild } from "./recovery-build-readback.js";
+import { requalifyPublicationCapsules } from "./recovery-capsules.js";
+import { readPipelineCaller } from "../../providers/github/pipeline-run-entry.js";
+
+function originalManifests(qualified) {
+  return [...new Set(qualified.artifacts.map((item) => item.manifestRoot))].map(
+    (root) => {
+      const artifacts = qualified.artifacts
+        .filter((item) => item.manifestRoot === root)
+        .map(({ manifestRoot, providerArtifactId, ...item }) => item);
+      const body = {
+        schema: "buildchain.pipeline-publication-products/v1",
+        planRoot: qualified.planRoot,
+        source: qualified.source,
+        platform: artifacts[0].platform,
+        artifacts,
+      };
+      if (recordDigest(body) !== root)
+        throw new Error(
+          "Recovered artifact descriptors cannot reconstruct their original manifest bytes",
+        );
+      return { ...body, root };
+    },
+  );
+}
+
+export async function requalifySealedPublication({
+  context,
+  prepared,
+  originalPlan,
+  originalMaterialization,
+  archive,
+  host,
+  directory,
+  now = new Date(),
+}) {
+  const previous = prepared.qualified;
+  verifyPipelineQualification({
+    plan: originalPlan,
+    materialization: originalMaterialization,
+    qualified: previous,
+    evaluatedAt: previous.qualification.issuedAt,
+  });
+  const { plan, materialization } = context;
+  if (
+    recordDigest(plan.outputs) !== recordDigest(originalPlan.outputs) ||
+    plan.version !== originalPlan.version ||
+    plan.contractRoot !== originalPlan.contractRoot ||
+    recordDigest(materialization.source) !== recordDigest(previous.source)
+  )
+    throw new Error(
+      "Signing recovery cannot change the already sealed product contract or source",
+    );
+  await reobservePublicationBuild(previous.build, previous.source, host);
+  const products = await restorePipelineProducts(
+    archive,
+    previous,
+    prepared.sealed,
+    path.join(directory, "requalified-products"),
+  );
+  for (const manifest of originalManifests(previous))
+    inspectPipelinePublicationArtifacts(products, manifest, plan);
+  const { run } = await host.runs.read(context.runId, context.runAttempt);
+  await readPipelineCaller(
+    run,
+    previous.source.configPath,
+    host.request,
+    host.repository,
+  );
+  const buildBody = {
+    schema: "buildchain.pipeline-publication-requalification/v1",
+    operation: "requalify-sealed-bytes",
+    outcome: "success",
+    planRoot: plan.root,
+    source: previous.source,
+    runId: context.runId,
+    runAttempt: context.runAttempt,
+    providerSource: run.head_sha,
+    predecessorBuild: previous.build,
+    sealedRoot: prepared.sealed.root,
+  };
+  const build = { ...buildBody, root: recordDigest(buildBody) };
+  const qualification = createDomainPublicationQualificationReceipt({
+    ...previous.qualification,
+    candidateRoot: recordDigest({
+      planRoot: plan.root,
+      source: previous.source,
+      artifacts: previous.artifacts,
+    }),
+    issuedAt: now.toISOString(),
+    expiresAt: new Date(now.getTime() + 3600000).toISOString(),
+  });
+  const body = {
+    schema: "buildchain.pipeline-publication-qualification/v1",
+    planRoot: plan.root,
+    source: previous.source,
+    artifacts: previous.artifacts,
+    build,
+    qualification,
+    predecessorRoot: previous.root,
+  };
+  const qualified = { ...body, root: recordDigest(body) };
+  const capsules = requalifyPublicationCapsules(
+    prepared.capsules,
+    qualified,
+    plan,
+  );
+  const { root, ...sealedBody } = prepared.sealed;
+  const sealedValue = {
+    ...sealedBody,
+    qualificationRoot: qualified.root,
+    predecessorRoot: root,
+  };
+  const sealed = { ...sealedValue, root: recordDigest(sealedValue) };
+  return {
+    schema: "buildchain.pipeline-qualification-preparation/v1",
+    contextRoot: recordDigest(context),
+    qualified,
+    capsules,
+    sealed,
+  };
+}

package/packages/core/publication/pipeline/recovery-readback.js

@@ -0,0 +1,56 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+
+export async function observeRecoveryPublicationEffects({
+  effects,
+  receipts,
+  retained,
+  provider,
+  fence,
+}) {
+  const transactionRoot = retained.documents.transaction.transactionRoot;
+  const observations = [];
+  for (const effect of effects) {
+    const { root, ...body } = effect;
+    if (root !== recordDigest(body))
+      throw new Error(
+        "Recovery publication effect changed its immutable declaration",
+      );
+    const prior = receipts.filter((receipt) => receipt.effectId === effect.id);
+    for (const receipt of prior) {
+      const { root: receiptRoot, ...value } = receipt;
+      if (
+        receiptRoot !== recordDigest(value) ||
+        value.schema !== "buildchain.pipeline-publication-effect/v1" ||
+        receipt.transactionRoot !== transactionRoot ||
+        receipt.effectRoot !== effect.root
+      )
+        throw new Error(
+          "Recovery cannot rewrite a prior publication receipt or transaction",
+        );
+    }
+    await fence();
+    const observed = await provider.observe(effect);
+    const matched = provider.matches(effect, observed);
+    if (
+      (!matched && prior.some((item) => item.state === "success")) ||
+      (observed.state !== "absent" &&
+        !matched &&
+        !provider.canApply?.(effect, observed))
+    )
+      throw new Error(
+        "Recovery publication destination conflicts with retained bytes",
+      );
+    observations.push({ effect, observed, matched });
+  }
+  const body = {
+    schema: "buildchain.pipeline-recovery-publication-readback/v1",
+    transactionRoot,
+    qualificationRoot: retained.qualified.root,
+    signingRoot: recordDigest(retained.signing),
+    sealedRoot: retained.sealed.root,
+    effectsRoot: recordDigest(effects),
+    receiptRoots: receipts.map((receipt) => receipt.root),
+    observations,
+  };
+  return { ...body, root: recordDigest(body) };
+}

package/packages/core/publication/pipeline/recovery-signing.js

@@ -0,0 +1,58 @@
+import path from "node:path";
+import { uniquePublicationMaterial } from "./context.js";
+import { requalifySealedPublication } from "./recovery-qualification.js";
+import { preparePipelineSigning } from "./signing.js";
+
+async function originalMaterial(journal, prefix, accepts) {
+  const matches = (await journal.materials(prefix)).filter(accepts);
+  if (matches.length !== 1)
+    throw new Error(
+      `Signing recovery has no unique original material: ${prefix}`,
+    );
+  return matches[0];
+}
+
+export async function prepareRecoveredSigning(
+  context,
+  host,
+  journal,
+  archive,
+  directory,
+) {
+  const prepared = await uniquePublicationMaterial(
+    journal,
+    "publication/predecessor-prepared/",
+  );
+  const originalPlan = await originalMaterial(
+    journal,
+    "publication/predecessor-plan/",
+    (plan) => plan.root === prepared.qualified.planRoot,
+  );
+  const originalMaterialization = await originalMaterial(
+    journal,
+    "publication/predecessor-materialization/",
+    (value) =>
+      value.planRoot === originalPlan.root &&
+      value.source.commit === prepared.qualified.source.commit,
+  );
+  const result = await requalifySealedPublication({
+    context,
+    prepared,
+    originalPlan,
+    originalMaterialization,
+    archive,
+    host,
+    directory,
+  });
+  await journal.fence();
+  await journal.record(
+    `publication/qualification-prepared/${context.runId}-${context.runAttempt}`,
+    result,
+  );
+  return preparePipelineSigning({
+    plan: context.plan,
+    materialization: context.materialization,
+    qualified: result.qualified,
+    directory: path.join(directory, "signing"),
+  });
+}