@kungfu-tech/buildchain 4.1.3-alpha.0 → 4.1.3-alpha.10

package/packages/core/publication/pipeline/journal.js

@@ -0,0 +1,122 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { assertPipelineExecution } from "../../workflow/pipeline/fence.js";
+import {
+  PUBLICATION_IMPORT,
+  publicationImportedValues,
+} from "./imported-materials.js";
+
+export function pipelinePublicationJournal(session, host) {
+  const initial = session.observed;
+  const expected = {
+    attempt: initial.attempt,
+    generation: initial.generation,
+    intent: session.intent.id,
+    sourceRoot: recordDigest(initial.history.at(-1).generation.source),
+  };
+  const store = host.materialStore(session);
+  async function fence() {
+    const observed = await session.journal.read();
+    assertPipelineExecution(observed, expected);
+    const writer = observed.history
+      .at(-1)
+      .events.filter((event) =>
+        event.payload.materials.some((material) =>
+          material.id.startsWith("publication/worker/"),
+        ),
+      )
+      .at(-1)?.payload.writer;
+    if (
+      writer &&
+      (writer.runId !== String(host.runId) ||
+        writer.runAttempt !== String(host.runAttempt))
+    )
+      throw new Error(
+        "Publication worker lost its exact provider execution reservation",
+      );
+    return observed;
+  }
+  async function materials(prefix) {
+    const observed = await session.journal.read();
+    if (observed.attempt !== initial.attempt)
+      throw new Error("Publication journal is no longer the current attempt");
+    const references = observed.history
+      .at(-1)
+      .events.flatMap((event) => event.payload.materials)
+      .filter(
+        (reference) =>
+          reference.id.startsWith(prefix) ||
+          reference.id.startsWith("publication/recovery-import/"),
+      );
+    const unique = [
+      ...new Map(
+        references.map((reference) => [reference.digest, reference]),
+      ).values(),
+    ];
+    const values = [];
+    for (const reference of unique) {
+      const value = await store.read(reference);
+      if (value.schema === PUBLICATION_IMPORT)
+        values.push(
+          ...publicationImportedValues(value)
+            .filter((item) => item.id.startsWith(prefix))
+            .map((item) => item.value),
+        );
+      else if (reference.id.startsWith(prefix)) values.push(value);
+    }
+    return [
+      ...new Map(values.map((value) => [recordDigest(value), value])).values(),
+    ];
+  }
+  async function record(
+    id,
+    value,
+    { phase = "publish", state = "running", expectedHead } = {},
+  ) {
+    const root = recordDigest(value);
+    const observed = await session.journal.read();
+    const eventKey = `publication:${id}:${root}`;
+    const prior = observed.history
+      .at(-1)
+      .events.find((event) => event.payload.eventKey === eventKey);
+    if (prior) {
+      if (
+        observed.attempt !== initial.attempt ||
+        prior.node !== phase ||
+        prior.payload.state !== state ||
+        prior.payload.materials.length !== 1 ||
+        recordDigest(await store.read(prior.payload.materials[0])) !== root
+      )
+        throw new Error(
+          "Publication retry changed its immutable recorded result",
+        );
+      return value;
+    }
+    const reference = await store.retain(
+      `${id}/${root.slice(7)}`,
+      value,
+      "receipt",
+    );
+    await session.progress.progress({
+      attempt: initial.attempt,
+      phase,
+      state,
+      eventKey,
+      materials: [reference],
+      ...(expectedHead !== undefined ? { expectedHead } : {}),
+    });
+    return value;
+  }
+  async function recordImport(value, phase) {
+    publicationImportedValues(value);
+    const observed = await fence();
+    const id = "publication/recovery-import";
+    const eventKey = `publication:${id}:${recordDigest(value)}`;
+    const prior = observed.history
+      .at(-1)
+      .events.find((event) => event.payload.eventKey === eventKey);
+    // Import is one atomic append, not a new result for each resumed phase.
+    // Replay still verifies the original event, state and retained bytes.
+    return record(id, value, { phase: prior?.node || phase });
+  }
+  return { fence, materials, record, recordImport };
+}

package/packages/core/publication/pipeline/next-development.js

@@ -0,0 +1,117 @@
+import { observeAdvancedPipelineDevelopment } from "./development-current.js";
+import { createPipelineDevelopmentTransition } from "./development-transition.js";
+import { githubPipelineVersion } from "../../providers/github/pipeline-version.js";
+import { uniquePublicationMaterial } from "./context.js";
+import { observePipelineVersionAnchor } from "./development-anchor.js";
+import { developmentTransitionReadback } from "./development-proof.js";
+import { prepareNextDevelopmentPullRequest } from "./development-pr.js";
+
+export async function nextPipelineDevelopment(context, host, journal) {
+  const { plan } = context;
+  const publication = await uniquePublicationMaterial(
+    journal,
+    "publication/complete/",
+  );
+  const transitions = await journal.materials(
+    "publication/next-development-plan/",
+  );
+  if (transitions.length > 1)
+    throw new Error("Next-development has conflicting retained declarations");
+  let transition = transitions[0];
+  if (!transition) {
+    transition = createPipelineDevelopmentTransition(context, publication);
+    await journal.record("publication/next-development-plan", transition, {
+      phase: "next-development",
+    });
+  }
+  if (transition.model.strategy === "anchored")
+    return observePipelineVersionAnchor(context, host, transition);
+  const retained = await journal.materials("publication/next-development-pr/");
+  if (retained.length > 1)
+    throw new Error("Next-development has conflicting retained PRs");
+  let selected = retained[0];
+  const adapter = githubPipelineVersion(host.request, host.repository);
+  if (!selected) {
+    const advanced = await observeAdvancedPipelineDevelopment(
+      context,
+      host,
+      transition,
+      adapter,
+    );
+    if (advanced) return advanced;
+    selected = await prepareNextDevelopmentPullRequest(
+      context,
+      host,
+      journal,
+      transition,
+      adapter,
+    );
+    if (selected.versionContext) return selected;
+  }
+  const pr = await host.request(
+    `/repos/${host.repository}/pulls/${selected.number}`,
+  );
+  if (
+    pr.head?.sha !== selected.source.commit ||
+    pr.head?.ref !== selected.branch ||
+    pr.base?.ref !== selected.base ||
+    pr.head?.repo?.full_name !== host.repository
+  )
+    throw new Error(
+      "Next-development PR changed its retained source or target",
+    );
+  const before = await adapter.inspect(
+    selected.materialization.protectedSource,
+    plan.versionPolicy,
+  );
+  const after = await adapter.inspect(selected.source, plan.versionPolicy);
+  const pending = developmentTransitionReadback(transition, {
+    before,
+    after,
+    evidence: selected,
+    createdAt: pr.created_at,
+  });
+  if (!pr.merged)
+    return {
+      state: "waiting",
+      reason:
+        pr.state === "open"
+          ? "next-development-protected-pr-pending"
+          : "next-development-pr-closed",
+      pullRequest: selected.number,
+      source: selected.source,
+      transition: pending,
+    };
+  const integration = await host.integration.observe({
+    intent: {
+      source: { pullRequest: selected.number, targetBranch: selected.base },
+    },
+    generation: {
+      source: { ...selected.source, configPath: plan.source.configPath },
+    },
+  });
+  const source = await host.source.source(
+    integration.mergeCommit,
+    plan.source.configPath,
+  );
+  const readback = await adapter.inspect(source.identity, plan.versionPolicy);
+  if (readback.version !== transition.target.version)
+    throw new Error(
+      "Protected next-development version readback differs from its declaration",
+    );
+  return {
+    state: "success",
+    reason: "protected-next-development-verified",
+    transitionRoot: transition.idempotencyKey,
+    integration,
+    version: readback.version,
+    publicationRoot: publication.release.receiptRoot,
+    transition: developmentTransitionReadback(transition, {
+      before,
+      after: readback,
+      evidence: integration,
+      createdAt: pr.created_at,
+      mergedAt: pr.merged_at,
+    }),
+  };
+}

package/packages/core/publication/pipeline/pack.js

@@ -0,0 +1,180 @@
+import fs from "node:fs";
+import path from "node:path";
+import { execFileSync } from "node:child_process";
+import { packNpmArtifact, sealedPackResult } from "../npm/package.js";
+import { readNpmPackageJsonFromTarball } from "../../release/candidate/payloads.js";
+import { createNativeChildEnvironment } from "../../dev-delivery/native/execution.js";
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { verifyPipelinePublicationPlan } from "./plan.js";
+import { assertPipelinePackagePolicy } from "./package-policy.js";
+import {
+  publicationPath,
+  publicationFile,
+  writeImmutablePublicationFile,
+} from "./files.js";
+
+function packageArtifact(directory, output, expected, version, environment) {
+  const product = publicationPath(directory, expected.path, "directory");
+  const pack = packNpmArtifact({
+    cwd: product,
+    env: createNativeChildEnvironment(environment),
+    outputDirectory: output,
+    ignoreScripts: true,
+    registry: "https://registry.npmjs.org/",
+  });
+  const sealed = sealedPackResult({
+    tarballPath: pack.tarballPath,
+    integrity: pack.integrity,
+  });
+  const pkg = readNpmPackageJsonFromTarball(pack.tarballPath);
+  assertPipelinePackagePolicy(pkg);
+  if (
+    pkg.name !== pack.name ||
+    pkg.version !== version ||
+    pack.version !== version ||
+    pkg.private === true
+  )
+    throw new Error(
+      "Sealed npm package does not match the planned public package version",
+    );
+  return {
+    file: pack.tarballPath,
+    suffix: ".tgz",
+    package: { name: pkg.name, version, integrity: sealed.integrity },
+  };
+}
+
+function archiveNames(file, suffix) {
+  const options = {
+    cwd: path.dirname(file),
+    encoding: "utf8",
+    maxBuffer: 16 * 1024 * 1024,
+    timeout: 30000,
+    stdio: ["ignore", "pipe", "pipe"],
+  };
+  if (suffix === ".zip") {
+    const script = [
+      "import json,sys,zipfile",
+      "with zipfile.ZipFile(sys.argv[1]) as archive:",
+      "    print(json.dumps(archive.namelist()))",
+    ].join("\n");
+    return JSON.parse(
+      execFileSync(
+        process.platform === "win32" ? "python" : "python3",
+        ["-c", script, path.basename(file)],
+        options,
+      ),
+    );
+  }
+  return execFileSync("tar", ["-tf", path.basename(file)], options)
+    .trim()
+    .split(/\r?\n/u);
+}
+
+export function inspectPipelineFileArtifact(directory, expected) {
+  const file = publicationPath(directory, expected.path);
+  if (expected.kind === "pdf") {
+    const bytes = fs.readFileSync(file);
+    if (
+      !bytes.subarray(0, 8).toString().startsWith("%PDF-") ||
+      !bytes.subarray(-1024).toString().includes("%%EOF")
+    )
+      throw new Error("Declared Paper output is not a complete PDF document");
+    return { file, suffix: ".pdf" };
+  }
+  const suffix = [".tar.gz", ".tar.xz", ".tgz", ".zip", ".tar"].find((value) =>
+    file.endsWith(value),
+  );
+  if (!suffix)
+    throw new Error("Declared binary output must be a standard archive");
+  const names = archiveNames(file, suffix);
+  if (
+    !names[0] ||
+    names.some(
+      (name) =>
+        !name.trim() ||
+        /[\u0000-\u001f\u007f]/u.test(name) ||
+        name.replaceAll("\\", "/").startsWith("/") ||
+        /^[A-Za-z]:/u.test(name) ||
+        name.replaceAll("\\", "/").split("/").includes(".."),
+    )
+  )
+    throw new Error("Binary archive contains unsafe or empty paths");
+  return { file, suffix };
+}
+
+export function packPipelineProducts({
+  cwd,
+  output,
+  plan,
+  platform,
+  source,
+  environment = process.env,
+}) {
+  verifyPipelinePublicationPlan(plan);
+  const expected = plan.outputs.filter((item) => item.platform === platform);
+  if (!expected.length)
+    throw new Error("Publication platform was not declared");
+  fs.mkdirSync(output, { recursive: true });
+  const artifacts = expected.map((item) => {
+    const directory = publicationPath(cwd, item.directory, "directory");
+    const staging = path.join(output, "npm-pack", item.product, item.artifact);
+    const packed =
+      item.kind === "npm-package"
+        ? packageArtifact(directory, staging, item, plan.version, environment)
+        : inspectPipelineFileArtifact(directory, item);
+    const observed = publicationFile(packed.file);
+    const name =
+      item.filename ||
+      `${item.product}-${platform}-${item.artifact}${packed.suffix}`;
+    if (!name.endsWith(packed.suffix))
+      throw new Error(
+        "Declared publication filename does not match the actual artifact format",
+      );
+    writeImmutablePublicationFile(
+      path.join(output, "payloads", name),
+      observed.bytes,
+    );
+    return {
+      ...item,
+      file: `payloads/${name}`,
+      size: observed.size,
+      digest: observed.digest,
+      ...(packed.package ? { package: packed.package } : {}),
+    };
+  });
+  const body = {
+    schema: "buildchain.pipeline-publication-products/v1",
+    planRoot: plan.root,
+    source,
+    platform,
+    artifacts,
+  };
+  const manifest = { ...body, root: recordDigest(body) };
+  writeImmutablePublicationFile(
+    path.join(output, "manifest.json"),
+    `${JSON.stringify(manifest, null, 2)}\n`,
+  );
+  return manifest;
+}
+
+export function verifyPipelineProductFiles(directory, manifest) {
+  const { root, ...body } = manifest;
+  if (
+    body.schema !== "buildchain.pipeline-publication-products/v1" ||
+    root !== recordDigest(body)
+  )
+    throw new Error("Publication manifest root does not match retained bytes");
+  for (const artifact of manifest.artifacts) {
+    const observed = publicationFile(publicationPath(directory, artifact.file));
+    if (
+      observed.size !== artifact.size ||
+      observed.digest !== artifact.digest ||
+      (artifact.package && observed.integrity !== artifact.package.integrity)
+    )
+      throw new Error(
+        "Publication artifact changed after its manifest was sealed",
+      );
+  }
+  return manifest;
+}

package/packages/core/publication/pipeline/package-policy.js

@@ -0,0 +1,22 @@
+export function assertPipelinePackagePolicy(pkg) {
+  const config = pkg.publishConfig;
+  if (config === undefined) return;
+  if (
+    !config ||
+    Array.isArray(config) ||
+    typeof config !== "object" ||
+    Object.keys(config).some(
+      (key) => !["access", "registry", "tag", "provenance"].includes(key),
+    )
+  )
+    throw new Error(
+      "npm publishConfig cannot add provider, credential or execution configuration to the product contract",
+    );
+  if (
+    config.registry !== undefined &&
+    !["https://registry.npmjs.org", "https://registry.npmjs.org/"].includes(
+      config.registry,
+    )
+  )
+    throw new Error("npm publication requires the declared npmjs provider");
+}

package/packages/core/publication/pipeline/plan.js

@@ -0,0 +1,136 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { selectProductPublicationIntent } from "../../release/product-publication.js";
+
+export function pipelineExpectedProducts(plan) {
+  const outputs = plan.products
+    .flatMap((product) =>
+      product.platforms.flatMap((platform) =>
+        product.artifacts.map((artifact) => ({
+          id: `${product.id}/${platform}/${artifact.id}`,
+          product: product.id,
+          platform,
+          artifact: artifact.id,
+          kind: artifact.kind,
+          directory: product.directory || ".",
+          path: artifact.path,
+          ...(artifact.filename ? { filename: artifact.filename } : {}),
+          targets: product.targets
+            .filter((target) => target.artifacts.includes(artifact.id))
+            .map(({ provider, access }) => ({
+              provider,
+              ...(access ? { access } : {}),
+            })),
+        })),
+      ),
+    )
+    .sort((left, right) => left.id.localeCompare(right.id));
+  if (
+    !outputs.length ||
+    outputs.length > 256 ||
+    new Set(outputs.map(({ id }) => id)).size !== outputs.length
+  )
+    throw new Error(
+      "Publication requires a bounded unique product/platform/artifact inventory",
+    );
+  const filenames = outputs.map(
+    (output) =>
+      output.filename ||
+      `${output.product}-${output.platform}-${output.artifact}${
+        output.kind === "npm-package"
+          ? ".tgz"
+          : output.kind === "pdf"
+            ? ".pdf"
+            : [".tar.gz", ".tar.xz", ".tgz", ".zip", ".tar"].find((suffix) =>
+                output.path.endsWith(suffix),
+              )
+      }`,
+  );
+  if (new Set(filenames).size !== filenames.length)
+    throw new Error(
+      "Publication artifact filenames must be unique across every declared platform",
+    );
+  return outputs;
+}
+
+export function planPipelinePublication({
+  attempt,
+  generation,
+  source,
+  intentSource = source,
+  runtime,
+  publisher,
+  contract,
+  route,
+  version,
+  sourceTimestamp,
+  expectedTagSha = null,
+  previousChannelCommit = null,
+}) {
+  if (route.operation === "develop")
+    throw new Error("Development delivery cannot publish products");
+  const developmentRoutes = contract.channels.filter(
+    (entry) =>
+      entry.operation === "alpha" &&
+      entry.to === (route.operation === "alpha" ? route.to : route.from),
+  );
+  if (developmentRoutes.length !== 1)
+    throw new Error(
+      "Publication requires one declared protected development channel",
+    );
+  const outputs = pipelineExpectedProducts(contract);
+  // The Rust-owned version/route decision operates on a declaration here.
+  // Actual sealed artifact roots are admitted independently after the build.
+  const selection = selectProductPublicationIntent({
+    channel: route.operation === "alpha" ? "alpha" : "stable",
+    targetRef: route.to,
+    sourceSha: source.commit,
+    sourceTimestamp,
+    repository: source.repository,
+    artifactKind: "custom",
+    requiredArtifactsRoot: recordDigest(outputs),
+    candidateVersion: version,
+    observedVersions: [],
+  });
+  if (contract.version.strategy === "anchored" && selection.version !== version)
+    throw new Error(
+      "Anchored publication requires an explicitly materialized version; inference is forbidden",
+    );
+  const body = {
+    schema: "buildchain.pipeline-publication-plan/v1",
+    evidenceVersion: 1,
+    attempt,
+    generation,
+    source,
+    intentSource,
+    runtime,
+    publisher,
+    contractRoot: recordDigest(contract),
+    version: selection.version,
+    candidateVersion: version,
+    tag: selection.exactTag,
+    channel: selection.channel,
+    route,
+    sourceTimestamp,
+    versionPolicy: contract.version,
+    ...(contract.stable
+      ? { stablePolicy: structuredClone(contract.stable) }
+      : {}),
+    developmentBranch: developmentRoutes[0].from,
+    expectedTagSha,
+    previousChannelCommit,
+    outputs,
+    versionSelection: selection,
+    phases: ["QUALIFY", "APPLY", "SETTLE"],
+  };
+  return { ...body, root: recordDigest(body) };
+}
+
+export function verifyPipelinePublicationPlan(plan) {
+  const { root, ...body } = plan || {};
+  if (
+    body.schema !== "buildchain.pipeline-publication-plan/v1" ||
+    root !== recordDigest(body)
+  )
+    throw new Error("Publication plan bytes do not match their retained root");
+  return plan;
+}