@kungfu-tech/buildchain 4.1.3-alpha.0 → 4.1.3-alpha.2

package/packages/core/publication/npm/pipeline-provider.js

@@ -0,0 +1,109 @@
+import fs from "node:fs";
+import path from "node:path";
+import { publishedDigest, runNpm } from "./registry.js";
+import { createNativeChildEnvironment } from "../../dev-delivery/native/execution.js";
+import {
+  publicationFile,
+  publicationPath,
+  writeImmutablePublicationFile,
+} from "../pipeline/files.js";
+
+export function pipelineNpmEnvironment(directory, environment = process.env) {
+  const cwd = path.resolve(directory);
+  const config = path.join(cwd, ".buildchain-publisher.npmrc");
+  fs.mkdirSync(cwd, { recursive: true });
+  writeImmutablePublicationFile(
+    config,
+    environment.NODE_AUTH_TOKEN
+      ? "//registry.npmjs.org/:_authToken=${NODE_AUTH_TOKEN}\n"
+      : "",
+  );
+  const env = {
+    ...createNativeChildEnvironment(environment),
+    NPM_CONFIG_USERCONFIG: config,
+    NPM_CONFIG_REGISTRY: "https://registry.npmjs.org/",
+    NPM_CONFIG_IGNORE_SCRIPTS: "true",
+  };
+  for (const key of [
+    "ACTIONS_ID_TOKEN_REQUEST_URL",
+    "ACTIONS_ID_TOKEN_REQUEST_TOKEN",
+    "NODE_AUTH_TOKEN",
+    "GITHUB_ACTIONS",
+  ])
+    if (environment[key]) env[key] = environment[key];
+  return env;
+}
+
+export function pipelineNpmProvider({
+  directory,
+  artifacts,
+  environment = process.env,
+  run = runNpm,
+  lookup = publishedDigest,
+}) {
+  const cwd = path.resolve(directory);
+  const env = pipelineNpmEnvironment(directory, environment);
+  const registry = "https://registry.npmjs.org/";
+  const args = { cwd, env, registry };
+  function artifactFor(effect) {
+    const found = artifacts.filter(
+      (artifact) => artifact.id === effect.product,
+    );
+    if (
+      found.length !== 1 ||
+      found[0].package?.name !== effect.name ||
+      found[0].package?.version !== effect.version ||
+      found[0].package?.integrity !== effect.integrity
+    )
+      throw new Error("npm publication changed qualified package identity");
+    const artifact = found[0];
+    const file = publicationPath(cwd, artifact.file);
+    const bytes = publicationFile(file);
+    if (
+      bytes.integrity !== effect.integrity ||
+      bytes.digest !== artifact.digest
+    )
+      throw new Error("npm publication bytes changed after qualification");
+    return file;
+  }
+  return {
+    async observe(effect) {
+      artifactFor(effect);
+      if (effect.access === "restricted" && !env.NODE_AUTH_TOKEN)
+        throw new Error(
+          "Restricted npm readback requires a configured package read credential",
+        );
+      const integrity = lookup({
+        ...args,
+        name: effect.name,
+        version: effect.version,
+      });
+      return integrity ? { state: "present", integrity } : { state: "absent" };
+    },
+    matches: (effect, observed) =>
+      observed.state === "present" && observed.integrity === effect.integrity,
+    async apply(effect) {
+      const file = artifactFor(effect);
+      const result = run({
+        cwd,
+        env,
+        args: [
+          "publish",
+          file,
+          "--ignore-scripts",
+          "--provenance=false",
+          "--access",
+          effect.access,
+          "--tag",
+          effect.tag,
+          `--registry=${registry}`,
+        ],
+        allowFailure: true,
+      });
+      if (result.status !== 0)
+        throw new Error(
+          `Sealed npm publication failed with exit ${result.status}`,
+        );
+    },
+  };
+}

package/packages/core/publication/npm/registry.js

@@ -19,17 +19,23 @@ export function runNpm({ cwd, args, env, allowFailure = false }) {
   return result;
 }
 
-function parseNpmView(stdout) {
+export function parseNpmView(stdout) {
   const raw = String(stdout || "").trim();
-  if (!raw) {
-    return undefined;
-  }
-  const parsed = JSON.parse(raw);
+  if (!raw) throw new Error("npm view omitted its exact package response");
+  const value = JSON.parse(raw);
+  if (Array.isArray(value) && value.length !== 1)
+    throw new Error("npm view must return exactly one package version");
+  const parsed = Array.isArray(value) ? value[0] : value;
+  if (typeof parsed === "string" && parsed)
+    return { integrity: parsed, shasum: "" };
   const dist = parsed?.dist || parsed;
-  return {
+  const result = {
     integrity: dist?.integrity || parsed?.["dist.integrity"] || "",
     shasum: dist?.shasum || parsed?.["dist.shasum"] || "",
   };
+  if (!result.integrity && !result.shasum)
+    throw new Error("npm view omitted the exact package integrity");
+  return result;
 }
 
 export function publishedDigest({ cwd, name, version, registry, env }) {
@@ -54,8 +60,5 @@ export function publishedDigest({ cwd, name, version, registry, env }) {
     throw new Error(`npm view ${name}@${version} failed\n${output}`.trim());
   }
   const view = parseNpmView(result.stdout);
-  if (!view) {
-    return undefined;
-  }
   return view.integrity || (view.shasum ? `sha1:${view.shasum}` : "");
 }

package/packages/core/publication/pipeline/actions.js

@@ -0,0 +1,100 @@
+import path from "node:path";
+import { pipelineHost } from "../../workflow/pipeline/host.js";
+import { buildPipelineSource } from "../../workflow/pipeline/build.js";
+import { githubPipelinePublicationArtifacts } from "../../providers/github/pipeline-publication-artifacts.js";
+import { preparePipelinePublication } from "./prepare.js";
+import { packPipelineProducts } from "./pack.js";
+import {
+  preparePipelineQualification,
+  sealPipelineQualification,
+} from "./qualify.js";
+import { applyPipelinePublication } from "./apply.js";
+import { settlePipelinePublication } from "./settle.js";
+import { githubJsonClient } from "../../providers/github/json-client.js";
+
+const contextInput = (core) =>
+  JSON.parse(core.getInput("context", { required: true }));
+const directory = (env) =>
+  path.join(env.GITHUB_WORKSPACE, ".buildchain/publication");
+
+export async function preparePipelinePublicationAction(core, env) {
+  const host = await pipelineHost(core, env, "Prepare publication");
+  const result = await preparePipelinePublication(
+    core.getInput("attempt", { required: true }),
+    core.getInput("definition-sha", { required: true }),
+    host,
+  );
+  core.setOutput("operation", result.operation);
+  core.setOutput("context", JSON.stringify(result.context || {}));
+  core.setOutput(
+    "matrix",
+    JSON.stringify({ include: result.context?.platforms || [] }),
+  );
+}
+
+export async function buildPipelinePublicationAction(core, env) {
+  const context = contextInput(core);
+  const platform = core.getInput("platform", { required: true });
+  const cwd = path.join(env.GITHUB_WORKSPACE, ".buildchain/product");
+  await buildPipelineSource({
+    cwd,
+    source: context.materialization.source,
+    platform,
+    environment: env,
+  });
+  const output = path.join(directory(env), "products");
+  const manifest = packPipelineProducts({
+    cwd,
+    output,
+    plan: context.plan,
+    platform,
+    source: context.materialization.source,
+    environment: env,
+  });
+  const artifacts = githubPipelinePublicationArtifacts({});
+  await artifacts.upload(context.plan, manifest, output);
+}
+
+export async function preparePipelineQualificationAction(core, env) {
+  const host = await pipelineHost(core, env, "Qualify and sign products");
+  const prepared = await preparePipelineQualification(
+    contextInput(core),
+    host,
+    directory(env),
+  );
+  core.setOutput("subject-path", prepared.subjectPath);
+  core.setOutput("predicate-path", prepared.predicatePath);
+  core.setOutput("predicate-type", prepared.predicateType);
+}
+
+export async function sealPipelineQualificationAction(core, env) {
+  const host = await pipelineHost(core, env, "Qualify and sign products");
+  await sealPipelineQualification(
+    contextInput(core),
+    host,
+    directory(env),
+    core.getInput("bundle-path", { required: true }),
+  );
+}
+
+export async function applyPipelinePublicationAction(core, env) {
+  const host = await pipelineHost(core, env, "Apply qualified publication");
+  await applyPipelinePublication(contextInput(core), host, directory(env), env);
+}
+
+export async function settlePipelinePublicationAction(core, env) {
+  const host = await pipelineHost(core, env, "Settle product follow-ups");
+  const prToken = core.getInput("pr-token");
+  if (prToken && prToken !== host.token)
+    host.pullRequests = githubJsonClient({
+      token: prToken,
+      userAgent: "buildchain-next-development",
+    });
+  const result = await settlePipelinePublication(
+    contextInput(core),
+    host,
+    directory(env),
+    env,
+  );
+  core.info(result.reason);
+}

package/packages/core/publication/pipeline/apply.js

@@ -0,0 +1,182 @@
+import path from "node:path";
+import fs from "node:fs";
+import { createHash } from "node:crypto";
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { canonicalJson } from "../../release/discussion/envelope.js";
+import {
+  createReleaseReceipt,
+  RELEASE_RECEIPT_CONTRACT,
+} from "../../release/release-invocation.js";
+import { githubPipelineProductRelease } from "../../providers/github/pipeline-product-release.js";
+import { pipelineNpmProvider } from "../npm/pipeline-provider.js";
+import { publicationContext, uniquePublicationMaterial } from "./context.js";
+import { restorePipelineProducts } from "./sealed-products.js";
+import { pipelineReleaseDocuments } from "./documents.js";
+import { writeImmutablePublicationFile } from "./files.js";
+import { verifyPipelineSigning } from "./signing.js";
+import { pipelinePublicationEffects, applyPipelineEffects } from "./effects.js";
+import { observeRecoveryPublicationEffects } from "./recovery-readback.js";
+import {
+  createRecoveryPublicationAdmission,
+  verifyRecoveryPublicationAdmission,
+} from "./recovery-admission.js";
+
+export async function applyPipelinePublication(
+  context,
+  host,
+  directory,
+  environment,
+  {
+    verifySigning = verifyPipelineSigning,
+    npmProvider = pipelineNpmProvider,
+  } = {},
+) {
+  const { journal, archive } = await publicationContext(context, host);
+  const retained = await uniquePublicationMaterial(
+    journal,
+    "publication/qualified/",
+  );
+  const { plan, materialization } = context;
+  const { qualified, capsules, sealed } = retained;
+  const evaluatedAt = context.recovery?.preserveTransaction
+    ? qualified.qualification.issuedAt
+    : undefined;
+  const products = await restorePipelineProducts(
+    archive,
+    qualified,
+    sealed,
+    path.join(directory, "sealed-products"),
+  );
+  const bundlePath = writeImmutablePublicationFile(
+    path.join(directory, "attestation.json"),
+    await archive.read(retained.bundle),
+  );
+  const signing = verifySigning({
+    plan,
+    materialization,
+    qualified,
+    bundlePath,
+    directory: path.join(directory, "signing-readback"),
+    token: host.token,
+    evaluatedAt,
+  });
+  const documents = pipelineReleaseDocuments({
+    plan,
+    materialization,
+    qualified,
+    capsules,
+    signing,
+    evaluatedAt,
+  });
+  if (
+    recordDigest(documents) !== recordDigest(retained.documents) ||
+    recordDigest(signing) !== recordDigest(retained.signing)
+  )
+    throw new Error(
+      "Retained release documents changed during publisher admission",
+    );
+  const npm = npmProvider({
+    directory: products,
+    artifacts: qualified.artifacts,
+    environment,
+  });
+  const evidence = Object.entries({
+    qualification: Buffer.from(`${canonicalJson(qualified)}\n`),
+    capsules: Buffer.from(`${canonicalJson(capsules)}\n`),
+    invocation: Buffer.from(
+      `${canonicalJson(documents.invocation.invocation)}\n`,
+    ),
+    attestation: fs.readFileSync(bundlePath),
+  }).map(([id, bytes]) => ({
+    id,
+    name: `buildchain.${id}.json`,
+    bytes,
+    size: bytes.length,
+    digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}`,
+  }));
+  const github = githubPipelineProductRelease({
+    ...host,
+    plan,
+    qualified,
+    documents,
+    directory: products,
+    evidence,
+  });
+  const providerFor = (effect) =>
+    effect.kind === "npm-package" ? npm : github;
+  const provider = {
+    observe: (effect) => providerFor(effect).observe(effect),
+    matches: (effect, value) => providerFor(effect).matches(effect, value),
+    apply: (effect) => providerFor(effect).apply(effect),
+  };
+  const effects = pipelinePublicationEffects({
+    plan,
+    qualified,
+    documents,
+    evidence: evidence.map(({ bytes, ...asset }) => asset),
+  });
+  const receipts = await journal.materials("publication/effect/");
+  let fence = journal.fence;
+  if (context.recovery?.preserveTransaction) {
+    const readback = await observeRecoveryPublicationEffects({
+      effects,
+      receipts,
+      retained,
+      provider,
+      fence,
+    });
+    const input = {
+      context,
+      retained,
+      execution: context.recovery.execution,
+      readback,
+    };
+    const admission = createRecoveryPublicationAdmission(input);
+    await journal.record("publication/recovery-authorization", {
+      admission,
+      readback,
+    });
+    fence = async () => {
+      await journal.fence();
+      verifyRecoveryPublicationAdmission(admission, input);
+    };
+  }
+  const results = await applyPipelineEffects({
+    effects,
+    transactionRoot: documents.transaction.transactionRoot,
+    receipts,
+    provider,
+    fence,
+    retain: (receipt) => journal.record("publication/effect", receipt),
+  });
+  const receipt = createReleaseReceipt({
+    schema: RELEASE_RECEIPT_CONTRACT,
+    transactionRoot: documents.transaction.transactionRoot,
+    outcome: "complete",
+    releasePassportRoot: documents.passport.passportRoot,
+    providerTransactionRoot: recordDigest(effects),
+    providerStateRoot: recordDigest(results),
+    providerReceiptRoots: [...new Set(results.map(({ root }) => root))].sort(),
+  });
+  // This is publication success only. Distribution and next-development remain
+  // separate mandatory nodes of the same business attempt.
+  let complete = {
+    schema: "buildchain.pipeline-publication-complete/v1",
+    release: receipt,
+    completedAt: new Date().toISOString(),
+  };
+  const originals = await journal.materials(
+    "publication/predecessor-complete/",
+  );
+  if (
+    originals.length > 1 ||
+    (originals.length &&
+      recordDigest(originals[0].release) !== recordDigest(receipt))
+  )
+    throw new Error(
+      "Recovery cannot replace the original completed publication receipt",
+    );
+  if (originals.length) complete = originals[0];
+  await journal.record("publication/complete", complete, { state: "success" });
+  return complete;
+}

package/packages/core/publication/pipeline/build-segments.js

@@ -0,0 +1,107 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { assertRecoveryProductContract } from "./recovery-build-plan.js";
+
+export const PUBLICATION_BUILD_AGGREGATE =
+  "buildchain.pipeline-publication-build-qualification/v1";
+
+export function qualifyPublicationBuildSegments(
+  context,
+  segments,
+  providerSource,
+) {
+  if (!/^[0-9a-f]{40}$/u.test(providerSource || ""))
+    throw new Error(
+      "Publication qualification requires the actual current signing execution source",
+    );
+  if (!segments.length || segments.length > 100)
+    throw new Error("Publication build segments exceed their exact bound");
+  const platforms = [],
+    artifactIds = [];
+  for (const segment of segments) {
+    const { build, plan, materialization } = segment;
+    assertRecoveryProductContract(
+      plan,
+      context.plan,
+      materialization.source,
+      context.materialization,
+    );
+    const { root, ...body } = build;
+    if (
+      root !== recordDigest(body) ||
+      build.schema !== "buildchain.pipeline-publication-build-readback/v1" ||
+      build.outcome !== "success" ||
+      build.planRoot !== plan.root ||
+      recordDigest(build.source) !== recordDigest(materialization.source) ||
+      build.platforms.length !== build.jobs.length ||
+      build.platforms.length !== build.artifactIds.length
+    )
+      throw new Error(
+        "Publication segment lacks exact independently observed build evidence",
+      );
+    for (const [index, platform] of build.platforms.entries()) {
+      const job = build.jobs[index],
+        name = `Build publication (${platform})`;
+      if (
+        job.run_id !== build.runId ||
+        job.run_attempt !== build.runAttempt ||
+        job.status !== "completed" ||
+        job.conclusion !== "success" ||
+        !(job.name === name || job.name.endsWith(` / ${name}`))
+      )
+        throw new Error("Publication segment producer job or platform changed");
+    }
+    platforms.push(...build.platforms);
+    artifactIds.push(...build.artifactIds);
+  }
+  const expected = [
+    ...new Set(context.plan.outputs.map(({ platform }) => platform)),
+  ].sort();
+  if (
+    recordDigest([...platforms].sort()) !== recordDigest(expected) ||
+    new Set(artifactIds).size !== artifactIds.length
+  )
+    throw new Error(
+      "Publication segments must cover each declared platform and artifact exactly once",
+    );
+  const body = {
+    schema: PUBLICATION_BUILD_AGGREGATE,
+    operation: "requalify-original-product-segments",
+    outcome: "success",
+    planRoot: context.plan.root,
+    source: context.materialization.source,
+    providerSource,
+    runId: context.runId,
+    runAttempt: context.runAttempt,
+    platforms: expected,
+    artifactIds: artifactIds.sort((a, b) => a - b),
+    segments,
+  };
+  return { ...body, root: recordDigest(body) };
+}
+
+export function publicationArtifactProducer(build, artifactId) {
+  if (build.schema !== PUBLICATION_BUILD_AGGREGATE)
+    return { build, plan: null };
+  const selected = build.segments.filter((segment) =>
+    segment.build.artifactIds.includes(artifactId),
+  );
+  if (selected.length !== 1)
+    throw new Error("Publication artifact has no unique original producer");
+  return selected[0];
+}
+
+export function verifyPublicationBuildAggregate(build, context) {
+  if (
+    recordDigest(build) !==
+    recordDigest(
+      qualifyPublicationBuildSegments(
+        context,
+        build.segments,
+        build.providerSource,
+      ),
+    )
+  )
+    throw new Error(
+      "Publication aggregate changed its exact source or original segments",
+    );
+}

package/packages/core/publication/pipeline/capsules.js

@@ -0,0 +1,89 @@
+import { publicationArtifactProducer } from "./build-segments.js";
+import { recordDigest } from "../../release/discussion/envelope.js";
+import {
+  STAGE_CAPSULE_CONTRACT,
+  STAGE_CAPSULE_IDENTITY_CONTRACT,
+  stageCapsuleIdentityRoot,
+  stageCapsuleRoot,
+  validateStageCapsule,
+} from "../../build/stage-capsule.js";
+import { verifyPipelineQualification } from "./documents.js";
+
+export function pipelineProductCapsules({
+  plan,
+  materialization,
+  qualified,
+  bundles,
+  evaluatedAt,
+}) {
+  verifyPipelineQualification({
+    plan,
+    materialization,
+    qualified,
+    evaluatedAt,
+  });
+  const capsules = qualified.artifacts.map((artifact) => {
+    const bundle = bundles.find(
+      ({ providerArtifact }) =>
+        providerArtifact.id === artifact.providerArtifactId,
+    );
+    if (
+      !bundle ||
+      !bundle.providerArtifact.expires_at ||
+      Date.parse(bundle.providerArtifact.expires_at) <=
+        Date.parse(qualified.qualification.issuedAt)
+    )
+      throw new Error(
+        "Product Capsule requires a live provider retention promise",
+      );
+    const producer = publicationArtifactProducer(
+      qualified.build,
+      artifact.providerArtifactId,
+    );
+    const identity = {
+      schema: STAGE_CAPSULE_IDENTITY_CONTRACT,
+      sourceRoot: recordDigest(qualified.source),
+      platform: artifact.platform,
+      platformRoot: recordDigest({ platform: artifact.platform }),
+      stage: "verify",
+      toolchainRoots: [],
+      runtimeRoot: recordDigest(producer.plan?.runtime || plan.runtime),
+      policyRoot: plan.contractRoot,
+      declaredInputs: [
+        { name: "publication-plan", root: producer.plan?.root || plan.root },
+      ],
+      transformationRoot: recordDigest({
+        source: qualified.source,
+        product: artifact.product,
+        artifact: artifact.id,
+        kind: artifact.kind,
+      }),
+      outputManifestRoot: artifact.manifestRoot,
+      qualificationRoot: qualified.qualification.receiptRoot,
+      observationRoots: [
+        { name: "provider-build", root: qualified.build.root },
+      ],
+    };
+    const capsule = {
+      schema: STAGE_CAPSULE_CONTRACT,
+      writerAuthority: "typescript-v3",
+      rustAuthority: "validation-only",
+      identity,
+      identityRoot: stageCapsuleIdentityRoot(identity),
+      retentionPromise: {
+        class: "github-artifact",
+        retainUntil: bundle.providerArtifact.expires_at,
+      },
+      capsuleRoot: `sha256:${"0".repeat(64)}`,
+    };
+    capsule.capsuleRoot = stageCapsuleRoot(capsule);
+    validateStageCapsule(capsule);
+    return { id: artifact.id, artifact, capsule };
+  });
+  const body = {
+    schema: "buildchain.pipeline-product-capsules/v1",
+    qualificationRoot: qualified.root,
+    capsules,
+  };
+  return { ...body, root: recordDigest(body) };
+}

package/packages/core/publication/pipeline/context.js

@@ -0,0 +1,39 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { resumePipelineSession } from "../../workflow/pipeline/session.js";
+import { pipelinePublicationJournal } from "./journal.js";
+
+export async function publicationContext(context, host) {
+  if (
+    context.schema !== "buildchain.pipeline-publication-context/v1" ||
+    context.runId !== host.runId ||
+    context.runAttempt !== host.runAttempt
+  )
+    throw new Error(
+      "Publication context is not from this exact provider execution",
+    );
+  const session = await resumePipelineSession(
+    { ...host, attempt: context.attempt },
+    host,
+  );
+  if (session.observed.generation !== context.generation)
+    throw new Error(
+      "Publication context belongs to a superseded source generation",
+    );
+  const journal = pipelinePublicationJournal(session, host);
+  await journal.fence();
+  const contexts = await journal.materials("publication/context/");
+  if (!contexts.some((value) => recordDigest(value) === recordDigest(context)))
+    throw new Error(
+      "Publication context is not retained in the authoritative attempt",
+    );
+  return { session, journal, archive: host.productArchive(session) };
+}
+
+export async function uniquePublicationMaterial(journal, prefix) {
+  const values = await journal.materials(prefix);
+  if (values.length !== 1)
+    throw new Error(
+      `Publication requires one exact retained material: ${prefix}`,
+    );
+  return values[0];
+}