@kungfu-tech/buildchain 4.1.3-alpha.2 → 4.1.3-alpha.4

package/packages/core/publication/pipeline/version-build.js

@@ -0,0 +1,130 @@
+import fs from "node:fs";
+import path from "node:path";
+import { execFileSync } from "node:child_process";
+import { recordDigest } from "../../release/discussion/envelope.js";
+import {
+  inspectPipelineSource,
+  buildPipelineProducts,
+} from "../../workflow/pipeline/build.js";
+import { verifyPipelineVersionPreparation } from "./version-preparation.js";
+import { guardPipelineVersionSource } from "./version-source-guard.js";
+import { materializePipelineVersion } from "./version.js";
+import {
+  pipelineVersionMaterialPaths,
+  pipelineVersionRegenerationResult,
+  collectPipelineVersionMaterial,
+} from "./version-regeneration.js";
+
+function materialFiles(cwd, paths) {
+  return Object.fromEntries(
+    paths.map((file) => {
+      const absolute = path.resolve(cwd, file);
+      const stat = fs.lstatSync(absolute);
+      if (
+        !stat.isFile() ||
+        stat.size > 4 * 1024 * 1024 ||
+        fs.realpathSync(absolute) !== absolute
+      )
+        throw new Error(
+          "Version preparation requires bounded regular source files without symlinks",
+        );
+      const bytes = fs.readFileSync(absolute);
+      const content = bytes.toString("utf8");
+      if (!Buffer.from(content).equals(bytes))
+        throw new Error("Version preparation cannot rewrite non-UTF8 material");
+      return [file, content];
+    }),
+  );
+}
+
+function trackedModes(cwd, paths) {
+  const entries = execFileSync(
+    "git",
+    ["-C", cwd, "ls-files", "--stage", "-z", "--", ...paths],
+    { encoding: "utf8" },
+  )
+    .split("\0")
+    .filter(Boolean);
+  const modes = Object.fromEntries(
+    entries.map((entry) => {
+      const match = /^(100644|100755) [0-9a-f]{40} 0\t(.+)$/u.exec(entry);
+      if (!match)
+        throw new Error("Version preparation requires tracked regular files");
+      return [match[2], match[1]];
+    }),
+  );
+  if (recordDigest(Object.keys(modes).sort()) !== recordDigest(paths))
+    throw new Error(
+      "Version preparation material is absent from the source tree",
+    );
+  return modes;
+}
+
+function assertPreparedDiff(cwd, paths, modes) {
+  const changes = execFileSync(
+    "git",
+    ["-C", cwd, "diff", "HEAD", "--name-status", "-z"],
+    { encoding: "utf8" },
+  )
+    .split("\0")
+    .filter(Boolean);
+  for (let index = 0; index < changes.length; index += 2)
+    if (changes[index] !== "M" || !paths.includes(changes[index + 1]))
+      throw new Error(
+        "Version product commands changed undeclared source paths",
+      );
+  if (recordDigest(trackedModes(cwd, paths)) !== recordDigest(modes))
+    throw new Error("Version product commands changed tracked file modes");
+  const raw = execFileSync("git", ["-C", cwd, "diff", "HEAD", "--summary"], {
+    encoding: "utf8",
+  });
+  if (raw.trim())
+    throw new Error(
+      "Version product commands changed source file kinds or modes",
+    );
+}
+
+export async function buildPipelineVersionMaterial({
+  cwd,
+  preparation,
+  platform,
+  environment = process.env,
+}) {
+  cwd = fs.realpathSync(cwd);
+  const contract = inspectPipelineSource(cwd, preparation.source);
+  verifyPipelineVersionPreparation(preparation, contract);
+  if (environment.BUILDCHAIN_RUNTIME_SHA !== preparation.runtime.sha)
+    throw new Error(
+      "Version preparation is not using its admitted execution runtime",
+    );
+  const paths = pipelineVersionMaterialPaths(
+    preparation.versionPolicy,
+    preparation.source.configPath,
+  );
+  const modes = trackedModes(cwd, paths);
+  const guard = guardPipelineVersionSource(cwd, preparation.source, paths);
+  const before = materialFiles(cwd, paths);
+  const material = materializePipelineVersion(
+    preparation.versionPolicy,
+    before,
+    preparation.version,
+  );
+  // Product code runs only in this credentialless job. The provider writer
+  // independently observes job completion and verifies these exact bytes.
+  for (const { path: file, content } of material.changes)
+    fs.writeFileSync(path.join(cwd, file), content);
+  await buildPipelineProducts({ cwd, plan: contract, platform, environment });
+  guard();
+  assertPreparedDiff(cwd, paths, modes);
+  const result = pipelineVersionRegenerationResult(
+    preparation,
+    platform,
+    materialFiles(cwd, paths),
+  );
+  collectPipelineVersionMaterial(
+    { ...preparation, platforms: [platform] },
+    before,
+    [result],
+  );
+  return result;
+}

package/packages/core/publication/pipeline/version-context.js

@@ -0,0 +1,143 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { resumePipelineSession } from "../../workflow/pipeline/session.js";
+import { pipelinePlatforms } from "../../workflow/pipeline/platforms.js";
+import { githubPipelineVersion } from "../../providers/github/pipeline-version.js";
+import { githubPipelineVersionArtifacts } from "../../providers/github/pipeline-version-artifacts.js";
+import { pipelinePublicationJournal } from "./journal.js";
+import {
+  planPipelineVersionPreparation,
+  verifyPipelineVersionPreparation,
+} from "./version-preparation.js";
+import { collectPipelineVersionMaterial } from "./version-regeneration.js";
+
+const phaseOf = (purpose) =>
+  purpose === "publication" ? "publish" : "next-development";
+
+export async function preparePipelineVersionContext(
+  parent,
+  purpose,
+  publication,
+  host,
+  journal,
+  definitionSha,
+) {
+  const admitted = await host.source.source(
+    parent.source.commit,
+    parent.source.configPath,
+  );
+  if (
+    recordDigest(admitted.identity) !== recordDigest(parent.source) ||
+    recordDigest(admitted.plan) !== publication.plan.contractRoot
+  )
+    throw new Error(
+      "Version preparation source or contract differs from its retained parent",
+    );
+  const preparation = planPipelineVersionPreparation({
+    attempt: publication.attempt,
+    generation: publication.generation,
+    source: parent.source,
+    contract: admitted.plan,
+    version: parent.version,
+    runtime: host.runtime,
+    purpose,
+    parentRoot: parent.root,
+  });
+  const context = {
+    schema: "buildchain.pipeline-version-context/v1",
+    attempt: publication.attempt,
+    generation: publication.generation,
+    preparation,
+    definitionSha,
+    runId: host.runId,
+    runAttempt: host.runAttempt,
+    platforms: pipelinePlatforms(admitted.plan),
+    publication,
+  };
+  await journal.record(`publication/version-${purpose}-context`, context, {
+    phase: phaseOf(purpose),
+  });
+  return context;
+}
+
+export async function qualifyPipelineVersionContext(
+  context,
+  host,
+  directory,
+  artifactClient,
+) {
+  if (
+    context.schema !== "buildchain.pipeline-version-context/v1" ||
+    context.runId !== host.runId ||
+    context.runAttempt !== host.runAttempt
+  )
+    throw new Error(
+      "Version context belongs to a different provider execution",
+    );
+  const session = await resumePipelineSession(
+    { ...host, attempt: context.attempt },
+    host,
+  );
+  const journal = pipelinePublicationJournal(session, host);
+  await journal.fence();
+  const { preparation } = context;
+  if (
+    context.generation !== session.observed.generation ||
+    preparation.attempt !== context.attempt ||
+    preparation.generation !== context.generation ||
+    recordDigest(preparation.runtime) !== recordDigest(host.runtime)
+  )
+    throw new Error(
+      "Version context changed its admitted attempt, generation or runtime",
+    );
+  const retained = await journal.materials(
+    `publication/version-${preparation.purpose}-context/`,
+  );
+  if (!retained.some((value) => recordDigest(value) === recordDigest(context)))
+    throw new Error(
+      "Version context is not retained in the authoritative attempt",
+    );
+  const admitted = await host.source.source(
+    preparation.source.commit,
+    preparation.source.configPath,
+  );
+  if (recordDigest(admitted.identity) !== recordDigest(preparation.source))
+    throw new Error("Version context source changed before qualification");
+  verifyPipelineVersionPreparation(preparation, admitted.plan);
+  const artifacts = githubPipelineVersionArtifacts(host, artifactClient);
+  const observed = await artifacts.download(context, directory);
+  const adapter = githubPipelineVersion(host.request, host.repository);
+  const before = await adapter.inspect(
+    preparation.source,
+    preparation.versionPolicy,
+  );
+  collectPipelineVersionMaterial(preparation, before.files, observed.results);
+  const regeneration = { context, ...observed };
+  await journal.fence();
+  await journal.record(
+    `publication/version-${preparation.purpose}-qualified`,
+    regeneration,
+    { phase: phaseOf(preparation.purpose) },
+  );
+  return { session, journal, regeneration };
+}
+
+export async function retainedPipelineVersionRegeneration(
+  journal,
+  parent,
+  purpose,
+  runtime,
+) {
+  const values = (
+    await journal.materials(`publication/version-${purpose}-qualified/`)
+  ).filter(
+    ({ context }) =>
+      context.preparation.parentRoot === parent.root &&
+      recordDigest(context.preparation.runtime) === recordDigest(runtime),
+  );
+  if (
+    values.length > 1 &&
+    new Set(values.map(({ results }) => recordDigest(results))).size !== 1
+  )
+    throw new Error("Version regeneration has conflicting qualified bytes");
+  return values.at(-1);
+}

package/packages/core/publication/pipeline/version-material.js

@@ -0,0 +1,36 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { collectPipelineVersionMaterial } from "./version-regeneration.js";
+import { materializePipelineVersion } from "./version.js";
+
+// Provider execution is qualified and journaled by version-context before this
+// pure byte check. A self-reported digest never grants Git write authority.
+export function preparedPipelineVersionMaterial(plan, files, regeneration) {
+  if (!regeneration) {
+    if (plan.versionPolicy.derived_files?.length)
+      throw new Error(
+        "Derived version files require retained qualified regeneration",
+      );
+    return materializePipelineVersion(plan.versionPolicy, files, plan.version);
+  }
+  const { context, build, results } = regeneration;
+  const { preparation } = context;
+  const { root: buildRoot, ...buildBody } = build;
+  if (
+    preparation.parentRoot !== plan.root ||
+    preparation.version !== plan.version ||
+    recordDigest(preparation.source) !== recordDigest(plan.source) ||
+    recordDigest(preparation.versionPolicy) !==
+      recordDigest(plan.versionPolicy) ||
+    build.preparationRoot !== preparation.root ||
+    build.runId !== context.runId ||
+    build.runAttempt !== context.runAttempt ||
+    buildRoot !== recordDigest(buildBody)
+  )
+    throw new Error(
+      "Qualified regeneration does not belong to this exact version materialization",
+    );
+  return {
+    ...collectPipelineVersionMaterial(preparation, files, results),
+    regenerationBuildRoot: buildRoot,
+  };
+}

package/packages/core/publication/pipeline/version-preparation.js

@@ -0,0 +1,69 @@
+import {
+  recordDigest,
+  validateRuntime,
+} from "../../release/discussion/envelope.js";
+import { validateConsumerSource } from "../../workflow/attempt/identity.js";
+import { pipelinePlatforms } from "../../workflow/pipeline/platforms.js";
+import { choice, text } from "../../consumer/contract/shape.js";
+import { pipelineVersionMaterialPaths } from "./version-regeneration.js";
+
+export function planPipelineVersionPreparation({
+  attempt,
+  generation,
+  source,
+  contract,
+  version,
+  runtime,
+  purpose,
+  parentRoot,
+}) {
+  text(attempt, "version preparation attempt", /^attempt-[0-9a-f]{64}$/u);
+  text(generation, "version preparation generation", /^sha256:[0-9a-f]{64}$/u);
+  text(parentRoot, "version preparation parent", /^sha256:[0-9a-f]{64}$/u);
+  text(
+    source.repository,
+    "version preparation repository",
+    /^[\w.-]+\/[\w.-]+$/u,
+  );
+  validateConsumerSource(source, source.repository);
+  validateRuntime(runtime);
+  choice(
+    runtime.repository,
+    ["kungfu-systems/buildchain"],
+    "version preparation runtime",
+  );
+  choice(
+    purpose,
+    ["publication", "development"],
+    "version preparation purpose",
+  );
+  text(
+    version,
+    "version preparation target",
+    /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/u,
+  );
+  pipelineVersionMaterialPaths(contract.version, source.configPath);
+  const body = {
+    schema: "buildchain.pipeline-version-preparation/v1",
+    attempt,
+    generation,
+    source,
+    version,
+    runtime,
+    purpose,
+    parentRoot,
+    contractRoot: recordDigest(contract),
+    versionPolicy: contract.version,
+    platforms: pipelinePlatforms(contract).map(({ platform }) => platform),
+  };
+  return structuredClone({ ...body, root: recordDigest(body) });
+}
+
+export function verifyPipelineVersionPreparation(preparation, contract) {
+  const expected = planPipelineVersionPreparation({ ...preparation, contract });
+  if (recordDigest(preparation) !== recordDigest(expected))
+    throw new Error(
+      "Version preparation changed its source-bound product contract or root",
+    );
+  return preparation;
+}

package/packages/core/publication/pipeline/version-regeneration.js

@@ -0,0 +1,162 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { object, relativePath } from "../../consumer/contract/shape.js";
+import { materializePipelineVersion } from "./version.js";
+
+export function pipelineVersionMaterialPaths(policy, configPath) {
+  const primary = [...new Set(policy.files.map(({ path }) => path))];
+  const derived = policy.derived_files || [];
+  const paths = [...primary, ...derived];
+  if (
+    !paths.length ||
+    paths.length > 128 ||
+    new Set(paths).size !== paths.length
+  )
+    throw new Error(
+      "Version material requires a bounded disjoint path inventory",
+    );
+  for (const file of paths) {
+    relativePath(file, "version material path");
+    if (
+      file.includes("*") ||
+      file === configPath ||
+      [
+        ".buildchain/contract-lock.json",
+        ".buildchain/alpha-contract-lock.json",
+      ].includes(file) ||
+      /^\.(?:git|github)\//u.test(file)
+    )
+      throw new Error(
+        "Version material cannot rewrite its source contract or workflow authority",
+      );
+  }
+  return paths.sort();
+}
+
+function checkFiles(files, paths) {
+  if (recordDigest(Object.keys(files).sort()) !== recordDigest(paths))
+    throw new Error("Version regeneration changed its declared file inventory");
+  let total = 0;
+  for (const file of paths) {
+    if (
+      typeof files[file] !== "string" ||
+      Buffer.byteLength(files[file]) > 4 * 1024 * 1024
+    )
+      throw new Error("Version regeneration requires bounded text files");
+    total += Buffer.byteLength(files[file]);
+  }
+  if (total > 64 * 1024 * 1024)
+    throw new Error(
+      "Version regeneration exceeds its complete material byte limit",
+    );
+}
+
+// This is an observation from a credentialless product job. A provider adapter
+// must qualify its exact producer execution before passing it to collection.
+export function pipelineVersionRegenerationResult(
+  preparation,
+  platform,
+  files,
+) {
+  const paths = pipelineVersionMaterialPaths(
+    preparation.versionPolicy,
+    preparation.source.configPath,
+  );
+  checkFiles(files, paths);
+  if (!preparation.platforms.includes(platform))
+    throw new Error("Version regeneration platform was not declared");
+  const body = {
+    schema: "buildchain.pipeline-version-regeneration/v1",
+    preparationRoot: preparation.root,
+    source: preparation.source,
+    platform,
+    version: preparation.version,
+    files,
+  };
+  return { ...body, root: recordDigest(body) };
+}
+
+function qualifiedFiles(preparation, result, paths) {
+  object(result, [
+    "schema",
+    "preparationRoot",
+    "source",
+    "platform",
+    "version",
+    "files",
+    "root",
+  ]);
+  const expected = pipelineVersionRegenerationResult(
+    preparation,
+    result.platform,
+    result.files,
+  );
+  if (recordDigest(result) !== recordDigest(expected))
+    throw new Error(
+      "Version regeneration changed its preparation, source, version or result root",
+    );
+  checkFiles(result.files, paths);
+  return result.files;
+}
+
+export function collectPipelineVersionMaterial(preparation, files, results) {
+  const policy = preparation.versionPolicy;
+  const paths = pipelineVersionMaterialPaths(
+    policy,
+    preparation.source.configPath,
+  );
+  checkFiles(files, paths);
+  const expectedPlatforms = [...new Set(preparation.platforms)].sort();
+  const actualPlatforms = results.map(({ platform }) => platform).sort();
+  if (
+    recordDigest(actualPlatforms) !== recordDigest(expectedPlatforms) ||
+    !actualPlatforms.length
+  )
+    throw new Error(
+      "Version regeneration requires every declared platform exactly once",
+    );
+  const material = materializePipelineVersion(
+    policy,
+    files,
+    preparation.version,
+  );
+  const primary = new Set(policy.files.map(({ path }) => path));
+  const expected = {
+    ...files,
+    ...Object.fromEntries(
+      material.changes.map(({ path, content }) => [path, content]),
+    ),
+  };
+  const derived = new Map();
+  for (const result of results) {
+    const observed = qualifiedFiles(preparation, result, paths);
+    for (const file of paths) {
+      if (primary.has(file)) {
+        if (observed[file] !== expected[file])
+          throw new Error(
+            "Product commands changed bytes outside the planned version fields",
+          );
+      } else if (observed[file] !== files[file]) {
+        if (derived.has(file) && derived.get(file) !== observed[file])
+          throw new Error(
+            "Platforms produced conflicting derived version bytes",
+          );
+        derived.set(file, observed[file]);
+      }
+    }
+  }
+  const changes = [...derived].map(([path, content]) => ({
+    path,
+    before: recordDigest(files[path]),
+    content,
+    after: recordDigest(content),
+  }));
+  return {
+    ...material,
+    changes: [...material.changes, ...changes].sort((left, right) =>
+      left.path.localeCompare(right.path),
+    ),
+    regenerationRoots: results
+      .toSorted((left, right) => left.platform.localeCompare(right.platform))
+      .map(({ root }) => root),
+  };
+}

package/packages/core/publication/pipeline/version-source-guard.js

@@ -0,0 +1,69 @@
+import fs from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { execFileSync } from "node:child_process";
+import { recordDigest } from "../../release/discussion/envelope.js";
+
+function git(cwd, ...args) {
+  return execFileSync("git", ["-C", cwd, ...args], {
+    encoding: "utf8",
+    maxBuffer: 16 * 1024 * 1024,
+  });
+}
+
+function fileState(cwd, file) {
+  const absolute = path.join(cwd, file);
+  if (fs.realpathSync(path.dirname(absolute)) !== path.dirname(absolute))
+    throw new Error(
+      "Version source parent directories cannot become symbolic links",
+    );
+  const stat = fs.lstatSync(absolute);
+  const bytes = stat.isSymbolicLink()
+    ? Buffer.from(fs.readlinkSync(absolute))
+    : stat.isFile()
+      ? fs.readFileSync(absolute)
+      : null;
+  if (!bytes)
+    throw new Error(
+      "Version preparation requires regular tracked source or symbolic links",
+    );
+  return {
+    kind: stat.isSymbolicLink() ? "link" : "file",
+    executable:
+      process.platform === "win32" ? null : Boolean(stat.mode & 0o111),
+    digest: createHash("sha256").update(bytes).digest("hex"),
+  };
+}
+
+export function guardPipelineVersionSource(cwd, source, allowedPaths) {
+  const entries = git(cwd, "ls-tree", "-r", "-z", source.commit)
+    .split("\0")
+    .filter(Boolean);
+  const files = entries.map((entry) => {
+    const match =
+      /^(?:100644|100755|120000) blob [0-9a-f]{40}\t([\s\S]+)$/u.exec(entry);
+    if (!match)
+      throw new Error(
+        "Version preparation cannot rewrite a source containing unresolved submodules",
+      );
+    return match[1];
+  });
+  const before = new Map(files.map((file) => [file, fileState(cwd, file)]));
+  return () => {
+    const current = git(cwd, "rev-parse", "HEAD", "HEAD^{tree}")
+      .trim()
+      .split(/\r?\n/u);
+    if (recordDigest(current) !== recordDigest([source.commit, source.tree]))
+      throw new Error(
+        "Version product commands changed their admitted Git HEAD or tree",
+      );
+    for (const [file, expected] of before) {
+      const current = fileState(cwd, file);
+      if (allowedPaths.includes(file)) current.digest = expected.digest;
+      if (recordDigest(current) !== recordDigest(expected))
+        throw new Error(
+          `Version product commands changed undeclared source bytes: ${file}`,
+        );
+    }
+  };
+}

package/packages/core/publication/pipeline/worker.js

@@ -0,0 +1,72 @@
+import { resumePipelineSession } from "../../workflow/pipeline/session.js";
+import { pipelinePublicationJournal } from "./journal.js";
+import { retainedPipelineStableWait } from "./stable-wait.js";
+
+export async function claimPipelinePublicationWorker(
+  attempt,
+  publisherSha,
+  host,
+) {
+  const { run: execution } = await host.runs.read(host.runId, host.runAttempt);
+  const definitions = (execution.referenced_workflows || []).filter((entry) =>
+    entry.path?.startsWith(
+      "kungfu-systems/buildchain/.github/workflows/.release-pipeline-products.yml@",
+    ),
+  );
+  if (
+    !/^[0-9a-f]{40}$/u.test(publisherSha || "") ||
+    definitions.length !== 1 ||
+    definitions[0].sha !== publisherSha
+  )
+    throw new Error(
+      "Publisher definition is not the exact hosted reusable workflow",
+    );
+  const session = await resumePipelineSession({ ...host, attempt }, host);
+  const observed = session.observed;
+  if (observed.phases.merge?.payload.state !== "success")
+    throw new Error(
+      "Product publication requires completed protected integration",
+    );
+  if (observed.status === "complete") return null;
+  const phase = observed.missing[0];
+  if (!["publish", "distribution", "next-development"].includes(phase))
+    throw new Error("Attempt is not at product publication");
+  const prior = observed.history
+    .at(-1)
+    .events.filter((event) =>
+      event.payload.materials.some((material) =>
+        material.id.startsWith("publication/worker/"),
+      ),
+    )
+    .at(-1)?.payload.writer;
+  if (
+    prior &&
+    (prior.runId !== String(host.runId) ||
+      prior.runAttempt !== String(host.runAttempt))
+  ) {
+    const { run } = await host.runs.read(
+      Number(prior.runId),
+      Number(prior.runAttempt),
+    );
+    // A time-only wait releases publication ownership before its outer run ends.
+    // The next worker still claims the exact native head under the publication lock.
+    if (
+      run.status !== "completed" &&
+      !(await retainedPipelineStableWait(observed, host))
+    )
+      return null;
+  }
+  const journal = pipelinePublicationJournal(session, host);
+  const claim = {
+    schema: "buildchain.pipeline-publication-worker/v1",
+    attempt,
+    runId: host.runId,
+    runAttempt: host.runAttempt,
+  };
+  await journal.record("publication/worker", claim, {
+    phase,
+    expectedHead: observed.head,
+  });
+  await journal.fence();
+  return { session, journal, phase };
+}