@kungfu-tech/buildchain 4.1.3-alpha.0 → 4.1.3-alpha.10

package/packages/core/publication/pipeline/version-regeneration.js

@@ -0,0 +1,162 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { object, relativePath } from "../../consumer/contract/shape.js";
+import { materializePipelineVersion } from "./version.js";
+
+export function pipelineVersionMaterialPaths(policy, configPath) {
+  const primary = [...new Set(policy.files.map(({ path }) => path))];
+  const derived = policy.derived_files || [];
+  const paths = [...primary, ...derived];
+  if (
+    !paths.length ||
+    paths.length > 128 ||
+    new Set(paths).size !== paths.length
+  )
+    throw new Error(
+      "Version material requires a bounded disjoint path inventory",
+    );
+  for (const file of paths) {
+    relativePath(file, "version material path");
+    if (
+      file.includes("*") ||
+      file === configPath ||
+      [
+        ".buildchain/contract-lock.json",
+        ".buildchain/alpha-contract-lock.json",
+      ].includes(file) ||
+      /^\.(?:git|github)\//u.test(file)
+    )
+      throw new Error(
+        "Version material cannot rewrite its source contract or workflow authority",
+      );
+  }
+  return paths.sort();
+}
+
+function checkFiles(files, paths) {
+  if (recordDigest(Object.keys(files).sort()) !== recordDigest(paths))
+    throw new Error("Version regeneration changed its declared file inventory");
+  let total = 0;
+  for (const file of paths) {
+    if (
+      typeof files[file] !== "string" ||
+      Buffer.byteLength(files[file]) > 4 * 1024 * 1024
+    )
+      throw new Error("Version regeneration requires bounded text files");
+    total += Buffer.byteLength(files[file]);
+  }
+  if (total > 64 * 1024 * 1024)
+    throw new Error(
+      "Version regeneration exceeds its complete material byte limit",
+    );
+}
+
+// This is an observation from a credentialless product job. A provider adapter
+// must qualify its exact producer execution before passing it to collection.
+export function pipelineVersionRegenerationResult(
+  preparation,
+  platform,
+  files,
+) {
+  const paths = pipelineVersionMaterialPaths(
+    preparation.versionPolicy,
+    preparation.source.configPath,
+  );
+  checkFiles(files, paths);
+  if (!preparation.platforms.includes(platform))
+    throw new Error("Version regeneration platform was not declared");
+  const body = {
+    schema: "buildchain.pipeline-version-regeneration/v1",
+    preparationRoot: preparation.root,
+    source: preparation.source,
+    platform,
+    version: preparation.version,
+    files,
+  };
+  return { ...body, root: recordDigest(body) };
+}
+
+function qualifiedFiles(preparation, result, paths) {
+  object(result, [
+    "schema",
+    "preparationRoot",
+    "source",
+    "platform",
+    "version",
+    "files",
+    "root",
+  ]);
+  const expected = pipelineVersionRegenerationResult(
+    preparation,
+    result.platform,
+    result.files,
+  );
+  if (recordDigest(result) !== recordDigest(expected))
+    throw new Error(
+      "Version regeneration changed its preparation, source, version or result root",
+    );
+  checkFiles(result.files, paths);
+  return result.files;
+}
+
+export function collectPipelineVersionMaterial(preparation, files, results) {
+  const policy = preparation.versionPolicy;
+  const paths = pipelineVersionMaterialPaths(
+    policy,
+    preparation.source.configPath,
+  );
+  checkFiles(files, paths);
+  const expectedPlatforms = [...new Set(preparation.platforms)].sort();
+  const actualPlatforms = results.map(({ platform }) => platform).sort();
+  if (
+    recordDigest(actualPlatforms) !== recordDigest(expectedPlatforms) ||
+    !actualPlatforms.length
+  )
+    throw new Error(
+      "Version regeneration requires every declared platform exactly once",
+    );
+  const material = materializePipelineVersion(
+    policy,
+    files,
+    preparation.version,
+  );
+  const primary = new Set(policy.files.map(({ path }) => path));
+  const expected = {
+    ...files,
+    ...Object.fromEntries(
+      material.changes.map(({ path, content }) => [path, content]),
+    ),
+  };
+  const derived = new Map();
+  for (const result of results) {
+    const observed = qualifiedFiles(preparation, result, paths);
+    for (const file of paths) {
+      if (primary.has(file)) {
+        if (observed[file] !== expected[file])
+          throw new Error(
+            "Product commands changed bytes outside the planned version fields",
+          );
+      } else if (observed[file] !== files[file]) {
+        if (derived.has(file) && derived.get(file) !== observed[file])
+          throw new Error(
+            "Platforms produced conflicting derived version bytes",
+          );
+        derived.set(file, observed[file]);
+      }
+    }
+  }
+  const changes = [...derived].map(([path, content]) => ({
+    path,
+    before: recordDigest(files[path]),
+    content,
+    after: recordDigest(content),
+  }));
+  return {
+    ...material,
+    changes: [...material.changes, ...changes].sort((left, right) =>
+      left.path.localeCompare(right.path),
+    ),
+    regenerationRoots: results
+      .toSorted((left, right) => left.platform.localeCompare(right.platform))
+      .map(({ root }) => root),
+  };
+}

package/packages/core/publication/pipeline/version-source-guard.js

@@ -0,0 +1,69 @@
+import fs from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { execFileSync } from "node:child_process";
+import { recordDigest } from "../../release/discussion/envelope.js";
+
+function git(cwd, ...args) {
+  return execFileSync("git", ["-C", cwd, ...args], {
+    encoding: "utf8",
+    maxBuffer: 16 * 1024 * 1024,
+  });
+}
+
+function fileState(cwd, file) {
+  const absolute = path.join(cwd, file);
+  if (fs.realpathSync(path.dirname(absolute)) !== path.dirname(absolute))
+    throw new Error(
+      "Version source parent directories cannot become symbolic links",
+    );
+  const stat = fs.lstatSync(absolute);
+  const bytes = stat.isSymbolicLink()
+    ? Buffer.from(fs.readlinkSync(absolute))
+    : stat.isFile()
+      ? fs.readFileSync(absolute)
+      : null;
+  if (!bytes)
+    throw new Error(
+      "Version preparation requires regular tracked source or symbolic links",
+    );
+  return {
+    kind: stat.isSymbolicLink() ? "link" : "file",
+    executable:
+      process.platform === "win32" ? null : Boolean(stat.mode & 0o111),
+    digest: createHash("sha256").update(bytes).digest("hex"),
+  };
+}
+
+export function guardPipelineVersionSource(cwd, source, allowedPaths) {
+  const entries = git(cwd, "ls-tree", "-r", "-z", source.commit)
+    .split("\0")
+    .filter(Boolean);
+  const files = entries.map((entry) => {
+    const match =
+      /^(?:100644|100755|120000) blob [0-9a-f]{40}\t([\s\S]+)$/u.exec(entry);
+    if (!match)
+      throw new Error(
+        "Version preparation cannot rewrite a source containing unresolved submodules",
+      );
+    return match[1];
+  });
+  const before = new Map(files.map((file) => [file, fileState(cwd, file)]));
+  return () => {
+    const current = git(cwd, "rev-parse", "HEAD", "HEAD^{tree}")
+      .trim()
+      .split(/\r?\n/u);
+    if (recordDigest(current) !== recordDigest([source.commit, source.tree]))
+      throw new Error(
+        "Version product commands changed their admitted Git HEAD or tree",
+      );
+    for (const [file, expected] of before) {
+      const current = fileState(cwd, file);
+      if (allowedPaths.includes(file)) current.digest = expected.digest;
+      if (recordDigest(current) !== recordDigest(expected))
+        throw new Error(
+          `Version product commands changed undeclared source bytes: ${file}`,
+        );
+    }
+  };
+}

package/packages/core/publication/pipeline/version.js

@@ -0,0 +1,81 @@
+import { parse, stringify } from "smol-toml";
+import { recordDigest } from "../../release/discussion/envelope.js";
+
+function document(file, bytes) {
+  const text = Buffer.isBuffer(bytes) ? bytes.toString("utf8") : bytes;
+  return file.format === "json" ? JSON.parse(text) : parse(text);
+}
+
+function field(value, key) {
+  const names = key.split(".");
+  let parent = value;
+  for (const name of names.slice(0, -1)) {
+    if (
+      ["__proto__", "prototype", "constructor"].includes(name) ||
+      !Object.hasOwn(parent, name) ||
+      !parent[name] ||
+      typeof parent[name] !== "object"
+    )
+      throw new Error(
+        "Version field does not resolve to an owned document key",
+      );
+    parent = parent[name];
+  }
+  const name = names.at(-1);
+  if (
+    ["__proto__", "prototype", "constructor"].includes(name) ||
+    !Object.hasOwn(parent, name) ||
+    typeof parent[name] !== "string"
+  )
+    throw new Error("Version field must be an existing string");
+  return { parent, name };
+}
+
+export function readPipelineVersion(policy, files) {
+  const versions = policy.files.map((file) => {
+    if (!Object.hasOwn(files, file.path))
+      throw new Error(`Missing declared version file: ${file.path}`);
+    const { parent, name } = field(document(file, files[file.path]), file.key);
+    return parent[name];
+  });
+  if (!versions.length || new Set(versions).size !== 1)
+    throw new Error(
+      "All declared version fields must identify the same candidate version",
+    );
+  return versions[0];
+}
+
+export function materializePipelineVersion(policy, files, version) {
+  const before = readPipelineVersion(policy, files);
+  if (policy.strategy === "anchored" && before !== version)
+    throw new Error("Buildchain cannot rewrite anchored version authority");
+  const documents = new Map();
+  for (const file of policy.files) {
+    if (!documents.has(file.path))
+      documents.set(file.path, {
+        file,
+        value: document(file, files[file.path]),
+      });
+    const entry = documents.get(file.path);
+    if (entry.file.format !== file.format)
+      throw new Error("One version document cannot have conflicting formats");
+    const { parent, name } = field(entry.value, file.key);
+    parent[name] = version;
+  }
+  const changes = [...documents].flatMap(([filePath, { file, value }]) => {
+    if (before === version) return [];
+    const content =
+      file.format === "json"
+        ? `${JSON.stringify(value, null, 2)}\n`
+        : stringify(value);
+    return [
+      {
+        path: filePath,
+        before: recordDigest(String(files[filePath])),
+        content,
+        after: recordDigest(content),
+      },
+    ];
+  });
+  return { before, version, changes };
+}

package/packages/core/publication/pipeline/worker.js

@@ -0,0 +1,72 @@
+import { resumePipelineSession } from "../../workflow/pipeline/session.js";
+import { pipelinePublicationJournal } from "./journal.js";
+import { retainedPipelineStableWait } from "./stable-wait.js";
+
+export async function claimPipelinePublicationWorker(
+  attempt,
+  publisherSha,
+  host,
+) {
+  const { run: execution } = await host.runs.read(host.runId, host.runAttempt);
+  const definitions = (execution.referenced_workflows || []).filter((entry) =>
+    entry.path?.startsWith(
+      "kungfu-systems/buildchain/.github/workflows/.release-pipeline-products.yml@",
+    ),
+  );
+  if (
+    !/^[0-9a-f]{40}$/u.test(publisherSha || "") ||
+    definitions.length !== 1 ||
+    definitions[0].sha !== publisherSha
+  )
+    throw new Error(
+      "Publisher definition is not the exact hosted reusable workflow",
+    );
+  const session = await resumePipelineSession({ ...host, attempt }, host);
+  const observed = session.observed;
+  if (observed.phases.merge?.payload.state !== "success")
+    throw new Error(
+      "Product publication requires completed protected integration",
+    );
+  if (observed.status === "complete") return null;
+  const phase = observed.missing[0];
+  if (!["publish", "distribution", "next-development"].includes(phase))
+    throw new Error("Attempt is not at product publication");
+  const prior = observed.history
+    .at(-1)
+    .events.filter((event) =>
+      event.payload.materials.some((material) =>
+        material.id.startsWith("publication/worker/"),
+      ),
+    )
+    .at(-1)?.payload.writer;
+  if (
+    prior &&
+    (prior.runId !== String(host.runId) ||
+      prior.runAttempt !== String(host.runAttempt))
+  ) {
+    const { run } = await host.runs.read(
+      Number(prior.runId),
+      Number(prior.runAttempt),
+    );
+    // A time-only wait releases publication ownership before its outer run ends.
+    // The next worker still claims the exact native head under the publication lock.
+    if (
+      run.status !== "completed" &&
+      !(await retainedPipelineStableWait(observed, host))
+    )
+      return null;
+  }
+  const journal = pipelinePublicationJournal(session, host);
+  const claim = {
+    schema: "buildchain.pipeline-publication-worker/v1",
+    attempt,
+    runId: host.runId,
+    runAttempt: host.runAttempt,
+  };
+  await journal.record("publication/worker", claim, {
+    phase,
+    expectedHead: observed.head,
+  });
+  await journal.fence();
+  return { session, journal, phase };
+}

package/packages/core/publication/publication-reproducibility.js

@@ -1,3 +1,4 @@
+import { readNpmPackResult } from "./npm/pack-result.js";
 import crypto from "node:crypto";
 import fs from "node:fs";
 import os from "node:os";
@@ -564,8 +565,7 @@ function packPublication({ buildRoot, packageDir, source, toolchain }) {
       label: "npm pack reproducibility candidate",
     },
   );
-  const parsed = JSON.parse(output);
-  const result = Array.isArray(parsed) ? parsed[0] : parsed;
+  const result = readNpmPackResult(output);
   if (!result?.filename || !result?.integrity || !result?.shasum) {
     throw new Error(
       "npm pack reproducibility candidate did not report filename, integrity, and shasum",

package/packages/core/release/discussion/binary.js

@@ -1,3 +1,4 @@
+import { releaseCheckpoints } from "./checkpoints.js";
 import { discussionTransport } from "../../providers/github/discussions/transport.js";
 import { releaseDiscussionStore } from "./store.js";
 import { createProgress, decodeRecord } from "./envelope.js";
@@ -46,6 +47,12 @@ export async function observeBinaryDistribution(
   const attemptRecord = state.records.find(
     (record) => record.attempt === state.attempt,
   );
+  Object.assign(session, {
+    attempt: state.attempt,
+    predecessor: attemptRecord.predecessor,
+    runtime,
+    writer,
+  });
   const sequence =
     Math.max(
       -1,
@@ -82,6 +89,10 @@ export async function observeBinaryDistribution(
       await record("failure", 1, {
         code: String(error.code || "execution-failed"),
       });
+      await releaseCheckpoints({ session, store, octokit }).diagnostics(
+        "binary-distribution",
+        error.code,
+      );
     } catch (recordError) {
       error.discussionRecordingError = recordError.message;
     }

package/packages/core/release/discussion/checkpoints.js

@@ -1,3 +1,8 @@
+import {
+  retainAttachment,
+  diagnosticReport,
+  diagnosticLog,
+} from "./evidence.js";
 import fs from "node:fs";
 import path from "node:path";
 import { createProgress, recordDigest } from "./envelope.js";
@@ -10,7 +15,11 @@ export function releaseCheckpoints({ session, store, octokit }) {
     repository: session.intent.repository,
     intentId: session.intent.id,
   });
-  async function checkpoint(node, value) {
+  async function checkpoint(
+    node,
+    value,
+    { attachments = [], label = value.schema || "Recovery checkpoint" } = {},
+  ) {
     const observed = await store.read(session);
     const records = observed.records.filter(
       (record) =>
@@ -21,20 +30,29 @@ export function releaseCheckpoints({ session, store, octokit }) {
     const root = recordDigest(value);
     const duplicate = records.find((record) => record.payload.root === root);
     if (duplicate) return duplicate;
-    const handle = await materials.put(Buffer.from(JSON.stringify(value)));
+    const sequence =
+      Math.max(-1, ...records.map((record) => record.sequence)) + 1;
+    const handle = await retainAttachment(materials, {
+      name: `${node}-checkpoint-${sequence}.json`,
+      mediaType: "application/json",
+      bytes: Buffer.from(JSON.stringify(value, null, 2) + "\n"),
+    });
     const record = createProgress({
       intent: session.intent,
       runtime: session.runtime,
       attempt: session.attempt,
+      writer: session.writer || session.attempt,
       predecessor: session.predecessor,
       kind: "checkpoint",
       node,
       status: "running",
-      sequence: Math.max(-1, ...records.map((record) => record.sequence)) + 1,
+      sequence,
       payload: {
         schema: "buildchain.release-checkpoint/v1",
         root,
         material: handle,
+        label,
+        attachments: [handle, ...attachments],
       },
     });
     await store.append(session, record);
@@ -69,7 +87,30 @@ export function releaseCheckpoints({ session, store, octokit }) {
       throw new Error("Unsupported retained publication reader manifest");
     return materials.read(manifest.reader);
   }
-  return { checkpoint, readCheckpoint, materials, publicationReader };
+  async function diagnostics(node, code) {
+    const report = diagnosticReport(
+      session,
+      await store.read(session),
+      node,
+      code,
+    );
+    const log = await retainAttachment(materials, {
+      name: `${node}-diagnostics.log`,
+      mediaType: "text/plain",
+      bytes: diagnosticLog(report),
+    });
+    return checkpoint(node, report, {
+      attachments: [log],
+      label: `Execution diagnostics: ${report.code}`,
+    });
+  }
+  return {
+    checkpoint,
+    readCheckpoint,
+    materials,
+    publicationReader,
+    diagnostics,
+  };
 }
 
 function assertNoSymlink(file, base) {

package/packages/core/release/discussion/envelope.js

@@ -94,6 +94,7 @@ export function createIntent({
     id: recordDigest(identity),
     ...identity,
     expectedNodes,
+    organization: "attempt-threads/v1",
     runtime,
     source,
   };

package/packages/core/release/discussion/evidence.js

@@ -0,0 +1,51 @@
+// Evidence is declared transaction data, never an implicit upload of workspace
+// files, environment variables, raw exceptions or authenticated HTTP responses.
+export async function retainAttachment(materials, { name, mediaType, bytes }) {
+  if (
+    !/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,159}$/u.test(name) ||
+    !["application/json", "text/plain", "application/octet-stream"].includes(
+      mediaType,
+    )
+  )
+    throw new Error("Invalid transaction attachment descriptor");
+  const handle = await materials.put(bytes, { name, mediaType });
+  if (!handle.downloadUrl)
+    throw new Error("Retained attachment is missing its provider download URL");
+  return { name, mediaType, ...handle };
+}
+
+export function diagnosticReport(session, state, node, code) {
+  return {
+    schema: "buildchain.release-diagnostics/v1",
+    intent: session.intent.id,
+    attempt: session.attempt,
+    node,
+    code: String(code || "execution-failed")
+      .replace(/[^a-zA-Z0-9._-]/gu, "-")
+      .slice(0, 160),
+    events: state.records
+      .filter((record) => record.attempt === session.attempt)
+      .map(({ id, node, kind, status, sequence, writer }) => ({
+        id,
+        node,
+        kind,
+        status,
+        sequence,
+        writer,
+      })),
+  };
+}
+
+export function diagnosticLog(report) {
+  return Buffer.from(
+    [
+      `Release intent ${report.intent}`,
+      `Attempt ${report.attempt}: ${report.node} ${report.code}`,
+      ...report.events.map(
+        (event) =>
+          `${event.node} ${event.kind} ${event.sequence} ${event.status} writer=${event.writer} record=${event.id}`,
+      ),
+      "",
+    ].join("\n"),
+  );
+}

package/packages/core/release/discussion/presentation.js

@@ -0,0 +1,92 @@
+import { isAttemptRoot } from "./threads.js";
+
+const text = (value) =>
+  String(value ?? "")
+    .replace(
+      /[&<>\[\]`*_\\|]/gu,
+      (character) => `&#${character.charCodeAt(0)};`,
+    )
+    .replace(/[\r\n]/gu, " ");
+const code = (value) => `\`${String(value).replace(/[`\r\n]/gu, " ")}\``;
+
+function runLink(repository, writer) {
+  const match = /^(\d+):(\d+)(?::.*)?$/u.exec(writer);
+  return match
+    ? `[${text(writer)}](https://github.com/${repository}/actions/runs/${match[1]}/attempts/${match[2]})`
+    : code(writer);
+}
+
+function runtimeLink(runtime) {
+  return `[${runtime.repository}@${runtime.sha.slice(0, 12)}](https://github.com/${runtime.repository}/commit/${runtime.sha})`;
+}
+
+export function renderIntent(intent) {
+  return [
+    `## ${intent.source?.qualification ? "Qualification" : "Release"}: ${text(intent.key)}`,
+    `Repository: ${code(intent.repository)}`,
+    "Each top-level transaction comment owns one attempt. Its replies contain node results, recovery checkpoints and downloadable evidence. Recovery attempts link to their predecessors.",
+    `Expected nodes: ${intent.expectedNodes.map(code).join(" → ")}`,
+    `Intent: ${code(intent.id)}`,
+  ].join("\n\n");
+}
+
+function attachmentRows(record, repository) {
+  return (record.payload?.attachments || []).map((attachment) => {
+    const url = new URL(attachment.downloadUrl);
+    if (
+      url.protocol !== "https:" ||
+      url.hostname !== "github.com" ||
+      !url.pathname.startsWith(`/${repository}/releases/download/`) ||
+      url.search ||
+      url.hash
+    )
+      throw new Error(
+        "Attachment download must belong to the consumer repository",
+      );
+    return `| [${text(attachment.name)}](${url.href}) | ${text(attachment.mediaType)} | ${attachment.size} | ${code(attachment.digest)} |`;
+  });
+}
+
+export function renderEvent(record, intent, state) {
+  if (isAttemptRoot(record)) {
+    const predecessor = state.roots.get(record.predecessor);
+    return [
+      `## Attempt ${text(record.attempt)}`,
+      `Execution: ${runLink(intent.repository, record.writer)}  \nRuntime: ${runtimeLink(record.runtime)}`,
+      record.predecessor
+        ? `Resumes: ${predecessor?.url ? `[${text(record.predecessor)}](${predecessor.url})` : code(record.predecessor)}`
+        : "Initial execution of this release intent.",
+      `Nodes: ${intent.expectedNodes.map(code).join(" → ")}`,
+      "Results and evidence are appended below. This root preserves the original execution context.",
+    ].join("\n\n");
+  }
+  const label =
+    record.kind === "checkpoint"
+      ? `checkpoint ${record.sequence}`
+      : record.status;
+  const lines = [
+    `### ${text(record.node)} · ${label}`,
+    `Attempt: ${code(record.attempt)} · Writer: ${runLink(intent.repository, record.writer)}`,
+    `Runtime: ${runtimeLink(record.runtime)}`,
+  ];
+  if (record.payload?.label) lines.push(text(record.payload.label));
+  if (record.payload?.code)
+    lines.push(`Failure code: ${code(record.payload.code)}`);
+  const attachments = attachmentRows(record, intent.repository);
+  if (attachments.length)
+    lines.push(
+      "| Attachment | Type | Bytes | SHA-256 |\n| --- | --- | ---: | --- |\n" +
+        attachments.join("\n"),
+      "Retained files use the transaction material archive. Access requires repository permission to read draft release assets.",
+    );
+  const { attachments: _attachments, ...details } = record.payload || {};
+  if (Object.keys(details).length) {
+    const json = JSON.stringify(details, null, 2);
+    // HTML escaping keeps payload strings from closing the presentation container.
+    lines.push(
+      `<details><summary>Record details</summary><pre>${json.slice(0, 6000).replace(/[&<>]/gu, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;" })[c])}${json.length > 6000 ? "\n… See the retained JSON attachment for the complete material." : ""}</pre></details>`,
+    );
+  }
+  lines.push(`Record: ${code(record.id)}`);
+  return lines.join("\n\n");
+}

package/packages/core/release/discussion/publication.js

@@ -40,8 +40,10 @@ export async function publishWithDiscussion(
     observe = () => {},
   },
 ) {
+  let retained;
   const journal = await openReleaseSession({
     graphql: octokit.graphql,
+    onFailure: (node, error) => retained.diagnostics(node, error.code),
     repository: request.repository,
     key: request.version,
     source: { version: request.version },
@@ -54,7 +56,7 @@ export async function publishWithDiscussion(
       request["publish-transaction-override"],
     ),
   });
-  const retained = releaseCheckpoints({
+  retained = releaseCheckpoints({
     session: journal.session,
     store: journal.store,
     octokit,