@kungfu-tech/buildchain 4.1.3-alpha.0 → 4.1.3-alpha.10

package/packages/core/publication/pipeline/distribution.js

@@ -0,0 +1,130 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { pipelineNpmChannel } from "../npm/pipeline-channel.js";
+import { applyPipelineEffects, pipelinePublicationEffects } from "./effects.js";
+
+export async function distributePipelineProducts(
+  context,
+  host,
+  journal,
+  retained,
+  environment,
+) {
+  const { plan, materialization } = context;
+  const channel = `v${plan.version.split(".")[0]}${plan.channel === "alpha" ? "-alpha" : ""}`;
+  const refPath = `/repos/${host.repository}/git/ref/tags/${channel}`;
+  const npm = pipelineNpmChannel({ environment });
+  async function observe(effect) {
+    if (effect.kind === "git-channel") {
+      const value = await host.request(refPath, { allow404: true });
+      if (value && value.object?.type !== "commit")
+        throw new Error("Floating channel requires a lightweight commit ref");
+      return value
+        ? { state: "present", commit: value.object.sha }
+        : { state: "absent", commit: null };
+    }
+    // Only retained historical plans contain npm channel effects. New releases
+    // set their channel in npm publish and never enter a separate npm operation.
+    const value = await npm.observe(effect);
+    if (value.version !== effect.version)
+      throw new Error(
+        `Historical npm channel ${effect.name}@${effect.tag} requires a one-time authenticated correction to ${effect.version}; recovery never republishes or rewrites npm tags`,
+      );
+    return value;
+  }
+  const plans = await journal.materials("publication/distribution-plan/");
+  if (plans.length > 1)
+    throw new Error("Distribution has conflicting retained plans");
+  let effects = plans[0]?.effects;
+  if (!effects) {
+    const raw = [
+      {
+        id: "git-channel",
+        kind: "git-channel",
+        channel,
+        commit: materialization.source.commit,
+      },
+    ];
+    const receipts = await journal.materials("publication/effect/");
+    if (receipts.length) {
+      const published = pipelinePublicationEffects({
+        plan,
+        ...retained,
+        receipts,
+      });
+      for (const effect of published.filter(
+        (item) =>
+          item.kind === "npm-package" && item.tag.startsWith("buildchain-"),
+      ))
+        raw.push({
+          id: `npm-channel:${effect.product}`,
+          kind: "npm-channel",
+          name: effect.name,
+          access: effect.access,
+          tag: plan.channel === "alpha" ? "alpha" : "latest",
+          version: effect.version,
+        });
+    }
+    effects = [];
+    for (const effect of raw) {
+      const before = await observe(effect);
+      const body = { ...effect, expected: before };
+      effects.push({ ...body, root: recordDigest(body) });
+    }
+    await journal.record(
+      "publication/distribution-plan",
+      { planRoot: plan.root, effects },
+      { phase: "distribution" },
+    );
+  }
+  const matches = (effect, value) =>
+    value.state === "present" &&
+    (effect.kind === "git-channel"
+      ? value.commit === effect.commit
+      : value.version === effect.version);
+  const provider = {
+    observe,
+    matches,
+    canApply: (effect, value) =>
+      recordDigest(value) === recordDigest(effect.expected),
+    async apply(effect) {
+      const current = await observe(effect);
+      if (matches(effect, current)) return;
+      if (recordDigest(current) !== recordDigest(effect.expected))
+        throw new Error(
+          "Floating channel changed since retained distribution admission",
+        );
+      await journal.fence();
+      if (effect.kind === "git-channel") {
+        if (current.state === "absent")
+          return host.request(`/repos/${host.repository}/git/refs`, {
+            method: "POST",
+            body: { ref: `refs/tags/${channel}`, sha: effect.commit },
+          });
+        // Provider-enforced fast-forward preserves published history. A
+        // divergent channel requires a separately proved source reconciliation.
+        return host.request(
+          `/repos/${host.repository}/git/refs/tags/${channel}`,
+          { method: "PATCH", body: { sha: effect.commit, force: false } },
+        );
+      }
+      throw new Error("Distribution cannot write npm channels");
+    },
+  };
+  const receipts = await applyPipelineEffects({
+    effects,
+    transactionRoot: retained.documents.transaction.transactionRoot,
+    receipts: await journal.materials("publication/distribution-effect/"),
+    provider,
+    fence: journal.fence,
+    retain: (receipt) =>
+      journal.record("publication/distribution-effect", receipt, {
+        phase: "distribution",
+      }),
+  });
+  return {
+    schema: "buildchain.pipeline-distribution-complete/v1",
+    planRoot: plan.root,
+    effects,
+    receipts,
+  };
+}

package/packages/core/publication/pipeline/documents.js

@@ -0,0 +1,192 @@
+import { createHash } from "node:crypto";
+import {
+  canonicalJson,
+  recordDigest,
+} from "../../release/discussion/envelope.js";
+import { releaseTailRoot } from "../../release/release-tail-provider-plane.js";
+import {
+  RELEASE_INVOCATION_CONTRACT,
+  RELEASE_PROVIDER_CONTRACT,
+  createReleaseInvocation,
+  createDomainReleaseTransaction,
+} from "../../release/release-invocation.js";
+import {
+  validatePublicationQualificationReceipt,
+  domainPublicationQualificationRoot,
+} from "../publication-qualification.js";
+import { verifyPipelinePublicationPlan } from "./plan.js";
+
+export function pipelineReleaseEvidence({
+  plan,
+  qualified,
+  capsules,
+  documents,
+  bundle,
+}) {
+  verifyPipelinePublicationPlan(plan);
+  if (plan.evidenceVersion !== undefined && plan.evidenceVersion !== 1)
+    throw new Error("Unsupported retained publication evidence version");
+  // An older retained transaction keeps its original evidence inventory.
+  // Adding a disclosure during recovery would change its provider receipt.
+  const values = {
+    ...(plan.evidenceVersion === 1 ? { plan } : {}),
+    qualification: qualified,
+    capsules,
+    invocation: documents.invocation.invocation,
+  };
+  const bytes = Object.entries(values).map(([id, value]) => [
+    id,
+    Buffer.from(`${canonicalJson(value)}\n`),
+  ]);
+  bytes.push(["attestation", Buffer.from(bundle)]);
+  return bytes.map(([id, value]) => ({
+    id,
+    name: `buildchain.${id}.json`,
+    bytes: value,
+    size: value.length,
+    digest: `sha256:${createHash("sha256").update(value).digest("hex")}`,
+  }));
+}
+
+export function verifyRootedPublication(value, schema) {
+  const { root, ...body } = value || {};
+  if (body.schema !== schema || root !== recordDigest(body))
+    throw new Error(`Invalid retained ${schema} bytes`);
+  return value;
+}
+
+export function verifyPipelineQualification({
+  plan,
+  materialization,
+  qualified,
+  evaluatedAt,
+}) {
+  verifyPipelinePublicationPlan(plan);
+  verifyRootedPublication(
+    materialization,
+    "buildchain.pipeline-version-materialization/v1",
+  );
+  verifyRootedPublication(
+    qualified,
+    "buildchain.pipeline-publication-qualification/v1",
+  );
+  if (
+    qualified.planRoot !== plan.root ||
+    materialization.planRoot !== plan.root ||
+    recordDigest(materialization.protectedSource) !==
+      recordDigest(plan.source) ||
+    recordDigest(qualified.source) !== recordDigest(materialization.source) ||
+    materialization.material.version !== plan.version
+  )
+    throw new Error(
+      "Publication plan, materialized source and qualification differ",
+    );
+  const artifacts = qualified.artifacts
+    .map((artifact) => ({
+      role: `${artifact.product}/${artifact.artifact}`,
+      platform: artifact.platform,
+      artifactRoot: artifact.digest,
+      manifestRoot: artifact.manifestRoot,
+    }))
+    .sort((a, b) =>
+      `${a.role}\0${a.platform}`.localeCompare(`${b.role}\0${b.platform}`),
+    );
+  validatePublicationQualificationReceipt(qualified.qualification, {
+    repository: plan.source.repository,
+    sourceSha: qualified.source.commit,
+    sourceRoot: recordDigest(qualified.source),
+    candidateRoot: recordDigest({
+      planRoot: plan.root,
+      source: qualified.source,
+      artifacts: qualified.artifacts,
+    }),
+    artifactRoot: domainPublicationQualificationRoot(artifacts),
+    policyDigest: plan.contractRoot,
+    ...(evaluatedAt ? { evaluatedAt } : {}),
+  });
+  return qualified;
+}
+
+export function pipelineReleaseDocuments({
+  plan,
+  materialization,
+  qualified,
+  capsules,
+  signing,
+  evaluatedAt,
+}) {
+  verifyPipelineQualification({
+    plan,
+    materialization,
+    qualified,
+    evaluatedAt,
+  });
+  verifyRootedPublication(capsules, "buildchain.pipeline-product-capsules/v1");
+  if (
+    capsules.qualificationRoot !== qualified.root ||
+    signing.planRoot !== plan.root ||
+    signing.qualificationRoot !== qualified.root ||
+    signing.materializationRoot !== materialization.root ||
+    signing.verified !== true
+  )
+    throw new Error(
+      "Release requires independently verified signing and capsules for this qualification",
+    );
+  // The Rust domain still owns publisher admission and QUALIFY/APPLY/SETTLE.
+  const invocation = createReleaseInvocation({
+    schema: RELEASE_INVOCATION_CONTRACT,
+    publisher: plan.publisher,
+    runtime: plan.runtime,
+    candidate: {
+      repository: qualified.source.repository,
+      commit: qualified.source.commit,
+      tree: qualified.source.tree,
+      version: plan.version,
+    },
+    target: {
+      channel: plan.channel,
+      tag: plan.tag,
+      expectedOldSha: plan.expectedTagSha,
+    },
+    authority: {
+      policyRoot: plan.contractRoot,
+      qualificationRoot: qualified.qualification.receiptRoot,
+      warrantRoot: qualified.qualification.receiptRoot,
+    },
+    provider: {
+      adapter: "built-in-provider-plane",
+      contract: RELEASE_PROVIDER_CONTRACT,
+      repository: qualified.source.repository,
+    },
+    parent: { invocationRoot: null, transactionRoot: null, receiptRoot: null },
+  });
+  const transaction = createDomainReleaseTransaction({
+    invocationRoot: invocation.roots.invocationRoot,
+    publisherRoot: invocation.roots.publisherRoot,
+    runtimeRoot: invocation.roots.runtimeRoot,
+    providerRoot: invocation.roots.providerRoot,
+    parentRoot: invocation.roots.parentRoot,
+  });
+  const body = {
+    schema: "kungfu.buildchain.release-passport/v4",
+    repository: qualified.source.repository,
+    source: qualified.source,
+    // This binding explicitly retains the declared version overlay. It never
+    // claims tree equivalence with the protected commit.
+    versionMaterialization: materialization,
+    release: { version: plan.version, tag: plan.tag, channel: plan.channel },
+    candidateRoot: qualified.qualification.candidateRoot,
+    policyDigest: qualified.qualification.policyDigest,
+    artifactRoot: qualified.qualification.artifactRoot,
+    publicationQualificationRoot: qualified.qualification.receiptRoot,
+    stageCapsuleAggregateRoot: capsules.root,
+    stageCapsuleRoots: capsules.capsules
+      .map(({ capsule }) => capsule.capsuleRoot)
+      .sort(),
+    signingRoot: recordDigest(signing),
+    invocationRoot: invocation.roots.invocationRoot,
+    transactionRoot: transaction.transactionRoot,
+  };
+  const passport = { ...body, passportRoot: releaseTailRoot(body) };
+  return { invocation, transaction, passport };
+}

package/packages/core/publication/pipeline/effects.js

@@ -0,0 +1,192 @@
+import { setTimeout } from "node:timers/promises";
+import { recordDigest } from "../../release/discussion/envelope.js";
+
+const KINDS = new Set([
+  "exact-tag",
+  "npm-package",
+  "release-asset",
+  "release-visibility",
+  "npm-channel",
+  "git-channel",
+]);
+
+export function pipelinePublicationEffects({
+  plan,
+  qualified,
+  documents,
+  evidence = [],
+  receipts = [],
+}) {
+  const effects = [
+    {
+      id: "exact-tag",
+      kind: "exact-tag",
+      tag: plan.tag,
+      commit: qualified.source.commit,
+    },
+  ];
+  for (const artifact of qualified.artifacts) {
+    for (const target of artifact.targets) {
+      if (target.provider === "npm")
+        effects.push({
+          id: `npm:${artifact.id}`,
+          kind: "npm-package",
+          product: artifact.id,
+          name: artifact.package.name,
+          version: artifact.package.version,
+          integrity: artifact.package.integrity,
+          access: target.access || "public",
+          tag: plan.channel === "alpha" ? "alpha" : "latest",
+        });
+      if (target.provider === "github-release")
+        effects.push({
+          id: `asset:${artifact.id}`,
+          kind: "release-asset",
+          product: artifact.id,
+          tag: plan.tag,
+          name: artifact.file.split("/").at(-1),
+          digest: artifact.digest,
+          size: artifact.size,
+        });
+    }
+  }
+  for (const asset of evidence)
+    effects.push({
+      id: `evidence:${asset.id}`,
+      kind: "release-asset",
+      product: `evidence:${asset.id}`,
+      tag: plan.tag,
+      name: asset.name,
+      digest: asset.digest,
+      size: asset.size,
+    });
+  effects.push({
+    id: "release-passport",
+    kind: "release-asset",
+    product: "release-passport",
+    tag: plan.tag,
+    name: "buildchain.release.json",
+    documentRoot: documents.passport.passportRoot,
+  });
+  effects.push({
+    id: "release-visibility",
+    kind: "release-visibility",
+    tag: plan.tag,
+    prerelease: plan.channel === "alpha",
+  });
+  return effects.map((effect) => {
+    if (effect.kind === "npm-package") {
+      // Historical receipts keep their exact identity across runtime repairs.
+      const original = {
+        ...effect,
+        tag: `buildchain-${plan.root.slice(7, 23)}`,
+      };
+      if (
+        receipts.some(
+          (receipt) =>
+            receipt.effectId === effect.id &&
+            receipt.effectRoot === recordDigest(original),
+        )
+      )
+        effect = original;
+    }
+    return { ...effect, root: recordDigest(effect) };
+  });
+}
+
+// This executor owns only bounded provider operations. Journal admission/fencing
+// and provider implementations are required dependencies, never consumer hooks.
+// Each irreversible success is retained before a later operation is attempted.
+export async function applyPipelineEffects({
+  effects,
+  transactionRoot,
+  receipts,
+  provider,
+  fence,
+  retain,
+  wait = setTimeout,
+}) {
+  const results = [];
+  for (const effect of effects) {
+    const { root, ...body } = effect;
+    if (!KINDS.has(effect.kind) || root !== recordDigest(body))
+      throw new Error("Unrecognized or modified publication effect");
+    await fence();
+    const prior = receipts.filter((receipt) => receipt.effectId === effect.id);
+    if (
+      prior.some(
+        (receipt) =>
+          receipt.effectRoot !== root ||
+          receipt.transactionRoot !== transactionRoot,
+      )
+    )
+      throw new Error(
+        "Publication effect conflicts with immutable prior intent",
+      );
+    const completed = prior.find((receipt) => receipt.state === "success");
+    let observed = await provider.observe(effect);
+    if (completed) {
+      if (!provider.matches(effect, observed))
+        throw new Error(
+          "A completed publication no longer matches provider bytes",
+        );
+      results.push(completed);
+      continue;
+    }
+    if (
+      observed.state !== "absent" &&
+      !provider.matches(effect, observed) &&
+      !provider.canApply?.(effect, observed)
+    )
+      throw new Error(
+        "Publication destination already contains conflicting bytes",
+      );
+    if (!prior.length)
+      await retain(rootedReceipt(effect, transactionRoot, "pending", observed));
+    if (!provider.matches(effect, observed)) {
+      await fence();
+      // Preserve unknown outcomes. A lost response is reconciled from provider
+      // state before any retry; absence still propagates the original failure.
+      let failure;
+      try {
+        await provider.apply(effect);
+      } catch (error) {
+        failure = error;
+      }
+      // Accepted npm versions remain unavailable during registry scanning.
+      // Bound that wait to fifteen minutes without repeating the write.
+      const scanning = effect.kind === "npm-package" && !failure;
+      const maximumReads = scanning ? 31 : 3;
+      for (let attempt = 0; attempt < maximumReads; attempt++) {
+        if (attempt) await wait(scanning ? 30_000 : 1000);
+        await fence();
+        observed = await provider.observe(effect);
+        if (observed.state !== "absent" || provider.matches(effect, observed))
+          break;
+      }
+      if (!provider.matches(effect, observed)) {
+        if (failure) throw failure;
+        throw new Error(
+          `Publication write lacks exact successful provider readback (${effect.kind})`,
+        );
+      }
+    }
+    await fence();
+    const receipt = rootedReceipt(effect, transactionRoot, "success", observed);
+    await retain(receipt);
+    results.push(receipt);
+  }
+  return results;
+}
+
+function rootedReceipt(effect, transactionRoot, state, observed) {
+  const body = {
+    schema: "buildchain.pipeline-publication-effect/v1",
+    effectId: effect.id,
+    effectRoot: effect.root,
+    transactionRoot,
+    state,
+    observed,
+  };
+  return { ...body, root: recordDigest(body) };
+}

package/packages/core/publication/pipeline/files.js

@@ -0,0 +1,66 @@
+import fs from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+
+export function publicationPath(root, relative, kind = "file") {
+  const base = fs.realpathSync(root);
+  const target = path.resolve(base, relative);
+  if (target !== base && !target.startsWith(`${base}${path.sep}`))
+    throw new Error("Publication path escapes its owned directory");
+  let current = base;
+  for (const part of path
+    .relative(base, target)
+    .split(path.sep)
+    .filter(Boolean)) {
+    current = path.join(current, part);
+    if (fs.lstatSync(current).isSymbolicLink())
+      throw new Error("Publication paths cannot traverse symbolic links");
+  }
+  const stat = fs.statSync(target);
+  if (
+    (kind === "file" && !stat.isFile()) ||
+    (kind === "directory" && !stat.isDirectory())
+  )
+    throw new Error(`Publication expected a ${kind}`);
+  return target;
+}
+
+export function publicationFile(file) {
+  const size = fs.statSync(file).size;
+  if (!size || size > 256 * 1024 * 1024)
+    throw new Error("Publication artifact must contain 1 byte to 256 MiB");
+  const bytes = fs.readFileSync(file);
+  if (!bytes.length || bytes.length > 256 * 1024 * 1024)
+    throw new Error("Publication artifact must contain 1 byte to 256 MiB");
+  return {
+    bytes,
+    size: bytes.length,
+    digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}`,
+    integrity: `sha512-${createHash("sha512").update(bytes).digest("base64")}`,
+  };
+}
+
+export function writeImmutablePublicationFile(file, bytes) {
+  // Refuse parent symlinks before mkdir/write, including during recovery into
+  // an existing output directory.
+  let parent = path.dirname(path.resolve(file));
+  while (parent !== path.dirname(parent)) {
+    if (fs.existsSync(parent) && fs.lstatSync(parent).isSymbolicLink())
+      throw new Error("Publication output cannot traverse symbolic links");
+    parent = path.dirname(parent);
+  }
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  try {
+    fs.writeFileSync(file, bytes, { flag: "wx" });
+  } catch (error) {
+    if (
+      error.code !== "EEXIST" ||
+      fs.lstatSync(file).isSymbolicLink() ||
+      !fs.readFileSync(file).equals(Buffer.from(bytes))
+    )
+      throw new Error(
+        `Publication refuses conflicting retained bytes: ${path.basename(file)}`,
+      );
+  }
+  return file;
+}

package/packages/core/publication/pipeline/imported-materials.js

@@ -0,0 +1,32 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+
+export const PUBLICATION_IMPORT = "buildchain.pipeline-publication-import/v1";
+
+export function publicationImportedValues(value) {
+  const { root, ...body } = value;
+  if (
+    body.schema !== PUBLICATION_IMPORT ||
+    root !== recordDigest(body) ||
+    !Array.isArray(body.values) ||
+    !body.values.length ||
+    body.values.length > 1000
+  )
+    throw new Error(
+      "Publication import is not one bounded immutable material bundle",
+    );
+  const ids = new Set();
+  for (const item of body.values) {
+    if (
+      typeof item.id !== "string" ||
+      !item.id.startsWith("publication/") ||
+      /^publication\/(?:worker|context|recovery-import)\//u.test(item.id) ||
+      ids.has(item.id) ||
+      !item.id.endsWith(`/${recordDigest(item.value).slice(7)}`)
+    )
+      throw new Error(
+        "Publication import contains duplicate, recursive or unbound materials",
+      );
+    ids.add(item.id);
+  }
+  return body.values;
+}