@kungfu-tech/buildchain 4.1.3-alpha.0 → 4.1.3-alpha.2

package/packages/core/publication/pipeline/pack.js

@@ -0,0 +1,151 @@
+import fs from "node:fs";
+import path from "node:path";
+import { execFileSync } from "node:child_process";
+import { packNpmArtifact, sealedPackResult } from "../npm/package.js";
+import { readNpmPackageJsonFromTarball } from "../../release/candidate/payloads.js";
+import { createNativeChildEnvironment } from "../../dev-delivery/native/execution.js";
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { verifyPipelinePublicationPlan } from "./plan.js";
+import { assertPipelinePackagePolicy } from "./package-policy.js";
+import {
+  publicationPath,
+  publicationFile,
+  writeImmutablePublicationFile,
+} from "./files.js";
+
+function packageArtifact(directory, output, expected, version, environment) {
+  const product = publicationPath(directory, expected.path, "directory");
+  const pack = packNpmArtifact({
+    cwd: product,
+    env: createNativeChildEnvironment(environment),
+    outputDirectory: output,
+    ignoreScripts: true,
+    registry: "https://registry.npmjs.org/",
+  });
+  const sealed = sealedPackResult({
+    tarballPath: pack.tarballPath,
+    integrity: pack.integrity,
+  });
+  const pkg = readNpmPackageJsonFromTarball(pack.tarballPath);
+  assertPipelinePackagePolicy(pkg);
+  if (
+    pkg.name !== pack.name ||
+    pkg.version !== version ||
+    pack.version !== version ||
+    pkg.private === true
+  )
+    throw new Error(
+      "Sealed npm package does not match the planned public package version",
+    );
+  return {
+    file: pack.tarballPath,
+    suffix: ".tgz",
+    package: { name: pkg.name, version, integrity: sealed.integrity },
+  };
+}
+
+export function inspectPipelineFileArtifact(directory, expected) {
+  const file = publicationPath(directory, expected.path);
+  if (expected.kind === "pdf") {
+    const bytes = fs.readFileSync(file);
+    if (
+      !bytes.subarray(0, 8).toString().startsWith("%PDF-") ||
+      !bytes.subarray(-1024).toString().includes("%%EOF")
+    )
+      throw new Error("Declared Paper output is not a complete PDF document");
+    return { file, suffix: ".pdf" };
+  }
+  const suffix = [".tar.gz", ".tar.xz", ".tgz", ".zip", ".tar"].find((value) =>
+    file.endsWith(value),
+  );
+  if (!suffix)
+    throw new Error("Declared binary output must be a standard archive");
+  const names = execFileSync("tar", ["-tf", path.basename(file)], {
+    cwd: path.dirname(file),
+    encoding: "utf8",
+    maxBuffer: 16 * 1024 * 1024,
+  })
+    .trim()
+    .split(/\r?\n/u);
+  if (
+    !names[0] ||
+    names.some(
+      (name) =>
+        name.startsWith("/") ||
+        /^[A-Za-z]:/u.test(name) ||
+        name.replaceAll("\\", "/").split("/").includes(".."),
+    )
+  )
+    throw new Error("Binary archive contains unsafe or empty paths");
+  return { file, suffix };
+}
+
+export function packPipelineProducts({
+  cwd,
+  output,
+  plan,
+  platform,
+  source,
+  environment = process.env,
+}) {
+  verifyPipelinePublicationPlan(plan);
+  const expected = plan.outputs.filter((item) => item.platform === platform);
+  if (!expected.length)
+    throw new Error("Publication platform was not declared");
+  fs.mkdirSync(output, { recursive: true });
+  const artifacts = expected.map((item) => {
+    const directory = publicationPath(cwd, item.directory, "directory");
+    const staging = path.join(output, "npm-pack", item.product, item.artifact);
+    const packed =
+      item.kind === "npm-package"
+        ? packageArtifact(directory, staging, item, plan.version, environment)
+        : inspectPipelineFileArtifact(directory, item);
+    const observed = publicationFile(packed.file);
+    const name = `${item.product}-${platform}-${item.artifact}${packed.suffix}`;
+    writeImmutablePublicationFile(
+      path.join(output, "payloads", name),
+      observed.bytes,
+    );
+    return {
+      ...item,
+      file: `payloads/${name}`,
+      size: observed.size,
+      digest: observed.digest,
+      ...(packed.package ? { package: packed.package } : {}),
+    };
+  });
+  const body = {
+    schema: "buildchain.pipeline-publication-products/v1",
+    planRoot: plan.root,
+    source,
+    platform,
+    artifacts,
+  };
+  const manifest = { ...body, root: recordDigest(body) };
+  writeImmutablePublicationFile(
+    path.join(output, "manifest.json"),
+    `${JSON.stringify(manifest, null, 2)}\n`,
+  );
+  return manifest;
+}
+
+export function verifyPipelineProductFiles(directory, manifest) {
+  const { root, ...body } = manifest;
+  if (
+    body.schema !== "buildchain.pipeline-publication-products/v1" ||
+    root !== recordDigest(body)
+  )
+    throw new Error("Publication manifest root does not match retained bytes");
+  for (const artifact of manifest.artifacts) {
+    const observed = publicationFile(publicationPath(directory, artifact.file));
+    if (
+      observed.size !== artifact.size ||
+      observed.digest !== artifact.digest ||
+      (artifact.package && observed.integrity !== artifact.package.integrity)
+    )
+      throw new Error(
+        "Publication artifact changed after its manifest was sealed",
+      );
+  }
+  return manifest;
+}

package/packages/core/publication/pipeline/package-policy.js

@@ -0,0 +1,22 @@
+export function assertPipelinePackagePolicy(pkg) {
+  const config = pkg.publishConfig;
+  if (config === undefined) return;
+  if (
+    !config ||
+    Array.isArray(config) ||
+    typeof config !== "object" ||
+    Object.keys(config).some(
+      (key) => !["access", "registry", "tag", "provenance"].includes(key),
+    )
+  )
+    throw new Error(
+      "npm publishConfig cannot add provider, credential or execution configuration to the product contract",
+    );
+  if (
+    config.registry !== undefined &&
+    !["https://registry.npmjs.org", "https://registry.npmjs.org/"].includes(
+      config.registry,
+    )
+  )
+    throw new Error("npm publication requires the declared npmjs provider");
+}

package/packages/core/publication/pipeline/plan.js

@@ -0,0 +1,114 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { selectProductPublicationIntent } from "../../release/product-publication.js";
+
+export function pipelineExpectedProducts(plan) {
+  const outputs = plan.products
+    .flatMap((product) =>
+      product.platforms.flatMap((platform) =>
+        product.artifacts.map((artifact) => ({
+          id: `${product.id}/${platform}/${artifact.id}`,
+          product: product.id,
+          platform,
+          artifact: artifact.id,
+          kind: artifact.kind,
+          directory: product.directory || ".",
+          path: artifact.path,
+          targets: product.targets
+            .filter((target) => target.artifacts.includes(artifact.id))
+            .map(({ provider, access }) => ({
+              provider,
+              ...(access ? { access } : {}),
+            })),
+        })),
+      ),
+    )
+    .sort((left, right) => left.id.localeCompare(right.id));
+  if (
+    !outputs.length ||
+    outputs.length > 256 ||
+    new Set(outputs.map(({ id }) => id)).size !== outputs.length
+  )
+    throw new Error(
+      "Publication requires a bounded unique product/platform/artifact inventory",
+    );
+  return outputs;
+}
+
+export function planPipelinePublication({
+  attempt,
+  generation,
+  source,
+  intentSource = source,
+  runtime,
+  publisher,
+  contract,
+  route,
+  version,
+  sourceTimestamp,
+  expectedTagSha = null,
+  previousChannelCommit = null,
+}) {
+  if (route.operation === "develop")
+    throw new Error("Development delivery cannot publish products");
+  const developmentRoutes = contract.channels.filter(
+    (entry) =>
+      entry.operation === "alpha" &&
+      entry.to === (route.operation === "alpha" ? route.to : route.from),
+  );
+  if (developmentRoutes.length !== 1)
+    throw new Error(
+      "Publication requires one declared protected development channel",
+    );
+  const outputs = pipelineExpectedProducts(contract);
+  // The Rust-owned version/route decision operates on a declaration here.
+  // Actual sealed artifact roots are admitted independently after the build.
+  const selection = selectProductPublicationIntent({
+    channel: route.operation === "alpha" ? "alpha" : "stable",
+    targetRef: route.to,
+    sourceSha: source.commit,
+    sourceTimestamp,
+    repository: source.repository,
+    artifactKind: "custom",
+    requiredArtifactsRoot: recordDigest(outputs),
+    candidateVersion: version,
+    observedVersions: [],
+  });
+  if (contract.version.strategy === "anchored" && selection.version !== version)
+    throw new Error(
+      "Anchored publication requires an explicitly materialized version; inference is forbidden",
+    );
+  const body = {
+    schema: "buildchain.pipeline-publication-plan/v1",
+    attempt,
+    generation,
+    source,
+    intentSource,
+    runtime,
+    publisher,
+    contractRoot: recordDigest(contract),
+    version: selection.version,
+    candidateVersion: version,
+    tag: selection.exactTag,
+    channel: selection.channel,
+    route,
+    sourceTimestamp,
+    versionPolicy: contract.version,
+    developmentBranch: developmentRoutes[0].from,
+    expectedTagSha,
+    previousChannelCommit,
+    outputs,
+    versionSelection: selection,
+    phases: ["QUALIFY", "APPLY", "SETTLE"],
+  };
+  return { ...body, root: recordDigest(body) };
+}
+
+export function verifyPipelinePublicationPlan(plan) {
+  const { root, ...body } = plan || {};
+  if (
+    body.schema !== "buildchain.pipeline-publication-plan/v1" ||
+    root !== recordDigest(body)
+  )
+    throw new Error("Publication plan bytes do not match their retained root");
+  return plan;
+}

package/packages/core/publication/pipeline/prepare.js

@@ -0,0 +1,151 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { resumePipelineSession } from "../../workflow/pipeline/session.js";
+import { pipelinePlatforms } from "../../workflow/pipeline/platforms.js";
+import { githubPipelineVersion } from "../../providers/github/pipeline-version.js";
+import { createPipelinePublicationPlan } from "./source-plan.js";
+import { pipelinePublicationJournal } from "./journal.js";
+import { prepareRecoveredPublication } from "./recovery-prepare.js";
+
+function publicationNextOperation(phase, existing, recovery) {
+  if (phase !== "publish") return "settle";
+  if (existing.length) return "apply";
+  if (recovery?.mode === "prepared" || recovery?.build?.scheduled.length === 0)
+    return "qualify";
+  return "build";
+}
+
+export async function preparePipelinePublication(attempt, publisherSha, host) {
+  const { run: execution } = await host.runs.read(host.runId, host.runAttempt);
+  const definitions = (execution.referenced_workflows || []).filter((entry) =>
+    entry.path?.startsWith(
+      "kungfu-systems/buildchain/.github/workflows/.release-pipeline-products.yml@",
+    ),
+  );
+  if (
+    !/^[0-9a-f]{40}$/u.test(publisherSha || "") ||
+    definitions.length !== 1 ||
+    definitions[0].sha !== publisherSha
+  )
+    throw new Error(
+      "Publisher definition is not the exact hosted reusable workflow",
+    );
+  const session = await resumePipelineSession({ ...host, attempt }, host);
+  const observed = session.observed;
+  if (observed.phases.merge?.payload.state !== "success")
+    throw new Error(
+      "Product publication requires completed protected integration",
+    );
+  if (observed.status === "complete") return { operation: "wait" };
+  const phase = observed.missing[0];
+  if (!["publish", "distribution", "next-development"].includes(phase))
+    throw new Error("Attempt is not at product publication");
+  const prior = observed.history
+    .at(-1)
+    .events.filter((event) =>
+      event.payload.materials.some((material) =>
+        material.id.startsWith("publication/worker/"),
+      ),
+    )
+    .at(-1)?.payload.writer;
+  if (
+    prior &&
+    (prior.runId !== String(host.runId) ||
+      prior.runAttempt !== String(host.runAttempt))
+  ) {
+    const { run } = await host.runs.read(
+      Number(prior.runId),
+      Number(prior.runAttempt),
+    );
+    if (run.status !== "completed") return { operation: "wait" };
+  }
+  const journal = pipelinePublicationJournal(session, host);
+  const claim = {
+    schema: "buildchain.pipeline-publication-worker/v1",
+    attempt,
+    runId: host.runId,
+    runAttempt: host.runAttempt,
+  };
+  await journal.record("publication/worker", claim, {
+    phase,
+    expectedHead: observed.head,
+  });
+  await journal.fence();
+  const recovery = await prepareRecoveredPublication(
+    session,
+    publisherSha,
+    host,
+    journal,
+    phase,
+  );
+  const plans = await journal.materials("publication/plan/");
+  if (plans.length > 1)
+    throw new Error("Publication has conflicting retained plans");
+  const version = githubPipelineVersion(host.request, host.repository);
+  let plan = plans[0];
+  if (!plan) {
+    plan = await createPipelinePublicationPlan(
+      session,
+      publisherSha,
+      host,
+      version,
+    );
+    await journal.record("publication/plan", plan);
+  }
+  if (
+    !recovery &&
+    (plan.publisher.workflowSha !== publisherSha ||
+      plan.runtime.commit !== host.runtime.sha)
+  )
+    throw new Error(
+      "Publication resume requires its retained publisher and runtime; use typed recovery for upgrades",
+    );
+  const materializations = await journal.materials(
+    "publication/materialization/",
+  );
+  if (materializations.length > 1)
+    throw new Error("Publication materialized source is ambiguous");
+  let materialization = materializations[0];
+  if (!materialization) {
+    await journal.fence();
+    materialization = await version.materialize(plan);
+    const verified = await host.source.source(
+      materialization.source.commit,
+      plan.source.configPath,
+    );
+    if (recordDigest(verified.plan) !== plan.contractRoot)
+      throw new Error("Version materialization altered the product contract");
+    const { root, ...body } = materialization;
+    materialization = { ...body, source: verified.identity };
+    materialization.root = recordDigest(materialization);
+    await journal.record("publication/materialization", materialization);
+  }
+  const admitted = await host.source.source(
+    materialization.source.commit,
+    plan.source.configPath,
+  );
+  if (
+    recordDigest(admitted.identity) !== recordDigest(materialization.source) ||
+    recordDigest(admitted.plan) !== plan.contractRoot
+  )
+    throw new Error("Retained publication source or contract drifted");
+  const existing = await journal.materials("publication/qualified/");
+  const context = {
+    schema: "buildchain.pipeline-publication-context/v1",
+    attempt,
+    generation: observed.generation,
+    plan,
+    materialization,
+    platforms: pipelinePlatforms(admitted.plan).filter(
+      ({ platform }) =>
+        !recovery?.build || recovery.build.scheduled.includes(platform),
+    ),
+    runId: host.runId,
+    runAttempt: host.runAttempt,
+    ...(recovery ? { recovery } : {}),
+  };
+  await journal.record("publication/context", context, { phase });
+  return {
+    operation: publicationNextOperation(phase, existing, recovery),
+    context,
+  };
+}

package/packages/core/publication/pipeline/qualification.js

@@ -0,0 +1,170 @@
+import {
+  PUBLICATION_BUILD_AGGREGATE,
+  publicationArtifactProducer,
+  verifyPublicationBuildAggregate,
+} from "./build-segments.js";
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { createDomainPublicationQualificationReceipt } from "../publication-qualification.js";
+import { verifyPipelinePublicationPlan } from "./plan.js";
+import {
+  inspectPipelineFileArtifact,
+  verifyPipelineProductFiles,
+} from "./pack.js";
+import { publicationPath } from "./files.js";
+import { assertPipelinePackagePolicy } from "./package-policy.js";
+import { readNpmPackageJsonFromTarball } from "../../release/candidate/payloads.js";
+
+function productDescriptor(artifact) {
+  const { file, size, digest, package: pkg, ...descriptor } = artifact;
+  return descriptor;
+}
+
+export function inspectPipelinePublicationArtifacts(directory, manifest, plan) {
+  verifyPipelineProductFiles(directory, manifest);
+  for (const artifact of manifest.artifacts) {
+    if (artifact.kind === "npm-package") {
+      const pkg = readNpmPackageJsonFromTarball(
+        publicationPath(directory, artifact.file),
+      );
+      assertPipelinePackagePolicy(pkg);
+      if (
+        !artifact.package ||
+        pkg.name !== artifact.package.name ||
+        pkg.version !== plan.version ||
+        pkg.version !== artifact.package.version ||
+        pkg.private === true
+      )
+        throw new Error(
+          "Independent npm qualification rejected package identity or version",
+        );
+    } else
+      inspectPipelineFileArtifact(directory, {
+        path: artifact.file,
+        kind: artifact.kind,
+      });
+  }
+}
+
+export function qualifyPipelineProducts({
+  plan,
+  source,
+  bundles,
+  build,
+  policyRoot,
+  now = new Date(),
+}) {
+  verifyPipelinePublicationPlan(plan);
+  if (build.schema === PUBLICATION_BUILD_AGGREGATE)
+    verifyPublicationBuildAggregate(build, {
+      plan,
+      materialization: { source },
+      runId: build.runId,
+      runAttempt: build.runAttempt,
+    });
+  const { root: buildRoot, ...buildBody } = build;
+  if (
+    buildRoot !== recordDigest(buildBody) ||
+    !Number.isSafeInteger(build.runId) ||
+    build.runId < 1 ||
+    ![
+      "buildchain.pipeline-publication-build-readback/v1",
+      PUBLICATION_BUILD_AGGREGATE,
+    ].includes(build.schema) ||
+    build.outcome !== "success" ||
+    build.planRoot !== plan.root ||
+    recordDigest(build.source) !== recordDigest(source)
+  )
+    throw new Error(
+      "Publication requires exact independent completed build readback",
+    );
+  const platforms = [
+    ...new Set(plan.outputs.map(({ platform }) => platform)),
+  ].sort();
+  const observed = bundles.map(({ manifest }) => manifest.platform).sort();
+  if (
+    recordDigest(platforms) !== recordDigest(observed) ||
+    recordDigest(platforms) !== recordDigest(build.platforms)
+  )
+    throw new Error(
+      "Publication build must cover exactly every declared platform",
+    );
+  const artifactIds = bundles
+    .map(({ providerArtifact }) => providerArtifact.id)
+    .sort((left, right) => left - right);
+  if (
+    new Set(artifactIds).size !== platforms.length ||
+    recordDigest(artifactIds) !== recordDigest(build.artifactIds)
+  )
+    throw new Error(
+      "Publication readback must cover exactly the retained provider artifacts",
+    );
+  const descriptors = [],
+    artifacts = [];
+  for (const { directory, manifest, providerArtifact } of bundles) {
+    const producer = publicationArtifactProducer(build, providerArtifact.id);
+    if (
+      manifest.planRoot !== producer.build.planRoot ||
+      recordDigest(manifest.source) !== recordDigest(source) ||
+      !build.artifactIds.includes(providerArtifact.id) ||
+      providerArtifact.expired ||
+      providerArtifact.workflow_run?.id !== producer.build.runId
+    )
+      throw new Error(
+        "Publication artifact is not bound to the exact admitted source run",
+      );
+    inspectPipelinePublicationArtifacts(directory, manifest, plan);
+    descriptors.push(...manifest.artifacts.map(productDescriptor));
+    artifacts.push(
+      ...manifest.artifacts.map((artifact) => ({
+        ...artifact,
+        manifestRoot: manifest.root,
+        providerArtifactId: providerArtifact.id,
+      })),
+    );
+  }
+  descriptors.sort((left, right) => left.id.localeCompare(right.id));
+  artifacts.sort((left, right) => left.id.localeCompare(right.id));
+  if (recordDigest(descriptors) !== recordDigest(plan.outputs))
+    throw new Error(
+      "Publication artifacts do not exactly match declared outputs and targets",
+    );
+  if (new Set(artifacts.map(({ file }) => file)).size !== artifacts.length)
+    throw new Error("Publication asset filenames must be unique");
+  const packages = artifacts.filter((artifact) => artifact.package);
+  if (
+    new Set(packages.map(({ package: pkg }) => pkg.name)).size !==
+    packages.length
+  )
+    throw new Error(
+      "Multi-platform npm publication requires distinct declared package names",
+    );
+  const candidateRoot = recordDigest({
+    planRoot: plan.root,
+    source,
+    artifacts,
+  });
+  const qualification = createDomainPublicationQualificationReceipt({
+    repository: source.repository,
+    candidateRoot,
+    sourceSha: source.commit,
+    sourceRoot: recordDigest(source),
+    policyDigest: policyRoot,
+    artifacts: artifacts.map((artifact) => ({
+      role: `${artifact.product}/${artifact.artifact}`,
+      platform: artifact.platform,
+      artifactRoot: artifact.digest,
+      manifestRoot: artifact.manifestRoot,
+    })),
+    issuedAt: now.toISOString(),
+    expiresAt: new Date(now.getTime() + 60 * 60 * 1000).toISOString(),
+  });
+  const body = {
+    schema: "buildchain.pipeline-publication-qualification/v1",
+    planRoot: plan.root,
+    source,
+    build,
+    artifacts,
+    qualification,
+  };
+  return { ...body, root: recordDigest(body) };
+}