@kungfu-tech/buildchain 4.1.3-alpha.2 → 4.1.3-alpha.3

package/packages/core/providers/github/pipeline-stable-source.js

@@ -0,0 +1,354 @@
+import { createHash } from "node:crypto";
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { verifyPipelinePublicationPlan } from "../../publication/pipeline/plan.js";
+import { compareDevelopmentVersions } from "../../publication/publication-development.js";
+import { githubPipelineSource } from "./pipeline-source.js";
+import { githubPipelineVersion } from "./pipeline-version.js";
+import { normalInputs } from "../../consumer/contract/entries.js";
+
+const SHA = /^[0-9a-f]{40}$/u;
+const STABLE = /^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/u;
+
+function publishedRelease(value, repository, tag, prerelease) {
+  if (
+    !Number.isSafeInteger(value?.id) ||
+    value.id < 1 ||
+    value.tag_name !== tag ||
+    value.draft !== false ||
+    value.prerelease !== prerelease ||
+    value.html_url !== `https://github.com/${repository}/releases/tag/${tag}` ||
+    typeof value.published_at !== "string" ||
+    !Number.isFinite(Date.parse(value.published_at))
+  )
+    throw new Error(
+      "Stable eligibility requires the exact public release identity",
+    );
+  return {
+    id: value.id,
+    tag,
+    publishedAt: new Date(value.published_at).toISOString(),
+  };
+}
+
+async function exactTag(get, tag) {
+  const pointer = await get(`/git/ref/tags/${encodeURIComponent(tag)}`);
+  if (
+    pointer.ref !== `refs/tags/${tag}` ||
+    !SHA.test(pointer.object?.sha || "")
+  )
+    throw new Error("Stable release tag identity changed");
+  let object = pointer.object;
+  for (let depth = 0; object.type === "tag" && depth < 4; depth++) {
+    const annotated = await get(`/git/tags/${object.sha}`);
+    if (annotated.sha !== object.sha || !SHA.test(annotated.object?.sha || ""))
+      throw new Error("Stable annotated tag identity changed");
+    object = annotated.object;
+  }
+  if (object.type !== "commit" || !SHA.test(object.sha))
+    throw new Error(
+      "Stable release tag must resolve to a bounded exact commit",
+    );
+  return { object: pointer.object.sha, commit: object.sha };
+}
+
+async function releases(get) {
+  const result = [],
+    ids = new Set();
+  for (let page = 1; page <= 20; page++) {
+    const values = await get(`/releases?per_page=100&page=${page}`);
+    if (!Array.isArray(values) || values.length > 100)
+      throw new Error("Stable release inventory is incomplete");
+    for (const value of values) {
+      if (!Number.isSafeInteger(value.id) || value.id < 1 || ids.has(value.id))
+        throw new Error(
+          "Stable release inventory has duplicate or invalid identities",
+        );
+      ids.add(value.id);
+      result.push(value);
+    }
+    if (values.length < 100) return result;
+  }
+  throw new Error("Stable release inventory exceeds its complete-read bound");
+}
+
+function predecessors(values, plan) {
+  const older = [],
+    tags = new Set(),
+    major = plan.version.split(".")[0];
+  for (const value of values) {
+    const match = STABLE.exec(value.tag_name || "");
+    if (!match || match[1] !== major || value.draft === true) continue;
+    const release = publishedRelease(
+      value,
+      plan.source.repository,
+      value.tag_name,
+      false,
+    );
+    if (tags.has(release.tag))
+      throw new Error("Stable release inventory repeats an exact tag");
+    tags.add(release.tag);
+    const order = compareDevelopmentVersions(
+      release.tag.slice(1),
+      plan.version,
+    );
+    if (order > 0)
+      throw new Error("A newer stable product has already been published");
+    // The current version can already exist during recovery of the same
+    // transaction. The effect journal separately verifies its exact bytes.
+    if (order < 0) older.push(release);
+  }
+  older.sort((a, b) =>
+    compareDevelopmentVersions(b.tag.slice(1), a.tag.slice(1)),
+  );
+  const comparison = older[0] || null;
+  const latest =
+    [...older].sort((a, b) => b.publishedAt.localeCompare(a.publishedAt))[0] ||
+    null;
+  return { comparison, latest };
+}
+
+async function treeFiles(get, commitSha) {
+  const commit = await get(`/git/commits/${commitSha}`);
+  if (commit.sha !== commitSha || !SHA.test(commit.tree?.sha || ""))
+    throw new Error("Stable source commit or tree identity changed");
+  const tree = await get(`/git/trees/${commit.tree.sha}?recursive=1`);
+  if (
+    tree.sha !== commit.tree.sha ||
+    tree.truncated !== false ||
+    !Array.isArray(tree.tree) ||
+    tree.tree.length > 100_000
+  )
+    throw new Error("Stable product difference requires a complete Git tree");
+  const files = new Map(),
+    paths = new Set();
+  for (const entry of tree.tree) {
+    if (
+      typeof entry.path !== "string" ||
+      !entry.path ||
+      entry.path
+        .split("/")
+        .some((part) => !part || part === "." || part === "..") ||
+      paths.has(entry.path) ||
+      !SHA.test(entry.sha || "")
+    )
+      throw new Error("Stable product tree has an ambiguous or invalid path");
+    paths.add(entry.path);
+    const modes = {
+      tree: ["040000"],
+      blob: ["100644", "100755", "120000"],
+      commit: ["160000"],
+    };
+    if (!modes[entry.type]?.includes(entry.mode))
+      throw new Error(
+        "Stable product tree has an unsupported entry type or mode",
+      );
+    if (entry.type !== "tree")
+      files.set(entry.path, {
+        sha: entry.sha,
+        mode: entry.mode,
+        type: entry.type,
+      });
+  }
+  return { tree: commit.tree.sha, files };
+}
+
+async function impactDocument(get, files, pathname, version) {
+  const entry = files.get(pathname);
+  if (entry?.type !== "blob" || !["100644", "100755"].includes(entry.mode))
+    throw new Error(
+      "Stable impact must be a regular file in the exact Alpha tree",
+    );
+  const blob = await get(`/git/blobs/${entry.sha}`);
+  if (
+    blob.sha !== entry.sha ||
+    blob.encoding !== "base64" ||
+    !Number.isSafeInteger(blob.size) ||
+    blob.size < 1 ||
+    blob.size > 1024 * 1024 ||
+    typeof blob.content !== "string" ||
+    blob.content.length > 2 * 1024 * 1024
+  )
+    throw new Error("Stable impact exceeds its exact Git blob boundary");
+  const bytes = Buffer.from(blob.content, "base64");
+  const sha = createHash("sha1")
+    .update(`blob ${bytes.length}\0`)
+    .update(bytes)
+    .digest("hex");
+  if (bytes.length !== blob.size || sha !== entry.sha)
+    throw new Error("Stable impact Git blob bytes changed");
+  const value = JSON.parse(
+    new TextDecoder("utf-8", { fatal: true }).decode(bytes),
+  );
+  if (value?.release?.version !== version)
+    throw new Error("Stable impact does not belong to the exact Alpha version");
+  return {
+    path: pathname,
+    blob: sha,
+    digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}`,
+    value,
+  };
+}
+
+function changedPaths(before, after) {
+  return [...new Set([...before.keys(), ...after.keys()])]
+    .filter(
+      (pathname) =>
+        recordDigest(before.get(pathname) || null) !==
+        recordDigest(after.get(pathname) || null),
+    )
+    .sort();
+}
+
+function stableSourceReader(plan, host) {
+  verifyPipelinePublicationPlan(plan);
+  const repository = plan.source.repository;
+  if (
+    host.repository !== repository ||
+    !/^[\w.-]+\/[\w.-]+$/u.test(repository) ||
+    plan.channel !== "stable" ||
+    !plan.stablePolicy ||
+    !/^\d+\.\d+\.\d+-alpha\.\d+$/u.test(plan.candidateVersion) ||
+    plan.candidateVersion.split("-alpha.")[0] !== plan.version
+  )
+    throw new Error(
+      "Stable source collection requires its exact repository and Alpha candidate plan",
+    );
+  const base = `/repos/${repository}`;
+  normalInputs({ "config-path": plan.source.configPath });
+  const request = (endpoint, options = {}) => {
+    if (
+      !endpoint.startsWith(`${base}/`) ||
+      (options.method && options.method !== "GET")
+    )
+      throw new Error(
+        "Stable source collection cannot mutate or cross repositories",
+      );
+    return host.request(endpoint, options);
+  };
+  const get = (pathname) => request(`${base}${pathname}`);
+  return { repository, request, get };
+}
+
+async function recheckStableSources(
+  get,
+  { plan, pointer, release, selected, prior, latest },
+) {
+  const repository = plan.source.repository,
+    tag = `v${plan.candidateVersion}`;
+  const repeatedRelease = publishedRelease(
+    await get(`/releases/tags/${tag}`),
+    repository,
+    tag,
+    true,
+  );
+  if (
+    recordDigest(await exactTag(get, tag)) !== recordDigest(pointer) ||
+    recordDigest(repeatedRelease) !== recordDigest(release) ||
+    recordDigest(predecessors(await releases(get), plan)) !==
+      recordDigest(selected) ||
+    (prior &&
+      recordDigest(await exactTag(get, selected.comparison.tag)) !==
+        recordDigest(prior)) ||
+    (latest &&
+      recordDigest(await exactTag(get, selected.latest.tag)) !==
+        recordDigest(latest))
+  )
+    throw new Error("Stable source facts changed during provider readback");
+}
+
+// Read-only provider facts. This result intentionally contains no successful
+// canary or release authorization; those require independent qualification.
+export async function readPipelineStableSource(plan, host) {
+  const { repository, request, get } = stableSourceReader(plan, host);
+  const tag = `v${plan.candidateVersion}`;
+  const pointer = await exactTag(get, tag);
+  if (pointer.commit !== plan.intentSource.commit)
+    throw new Error("Published Alpha differs from the exact channel PR source");
+  const release = publishedRelease(
+    await get(`/releases/tags/${tag}`),
+    repository,
+    tag,
+    true,
+  );
+  const candidate = await githubPipelineSource(request, repository).source(
+    pointer.commit,
+    plan.source.configPath,
+  );
+  if (
+    recordDigest(candidate.identity) !== recordDigest(plan.intentSource) ||
+    recordDigest(candidate.plan) !== plan.contractRoot ||
+    recordDigest(candidate.plan.stable) !== recordDigest(plan.stablePolicy) ||
+    recordDigest(candidate.plan.version) !== recordDigest(plan.versionPolicy)
+  )
+    throw new Error(
+      "Published Alpha source or product policy differs from the admitted plan",
+    );
+  const version = await githubPipelineVersion(request, repository).inspect(
+    candidate.identity,
+    candidate.plan.version,
+  );
+  if (version.version !== plan.candidateVersion)
+    throw new Error(
+      "Published Alpha version documents disagree with its exact tag",
+    );
+  const selected = predecessors(await releases(get), plan);
+  const prior = selected.comparison
+    ? await exactTag(get, selected.comparison.tag)
+    : null;
+  if ((prior?.commit || null) !== plan.previousChannelCommit)
+    throw new Error(
+      "Stable comparison differs from the retained published channel",
+    );
+  const current = await treeFiles(get, pointer.commit);
+  if (current.tree !== candidate.identity.tree)
+    throw new Error(
+      "Alpha tree changed between independent source observations",
+    );
+  const config = current.files.get(candidate.identity.configPath);
+  if (
+    config?.type !== "blob" ||
+    !["100644", "100755"].includes(config.mode) ||
+    config.sha !== candidate.identity.configBlob
+  )
+    throw new Error(
+      "Alpha configuration is not the exact blob in its complete tree",
+    );
+  const previous = prior
+    ? await treeFiles(get, prior.commit)
+    : { files: new Map() };
+  const impact = await impactDocument(
+    get,
+    current.files,
+    plan.stablePolicy.impact_file,
+    plan.candidateVersion,
+  );
+  const latest =
+    selected.latest &&
+    (selected.latest.tag === selected.comparison.tag
+      ? prior
+      : await exactTag(get, selected.latest.tag));
+  await recheckStableSources(get, {
+    plan,
+    pointer,
+    release,
+    selected,
+    prior,
+    latest,
+  });
+  const body = {
+    schema: "buildchain.pipeline-stable-source/v1",
+    planRoot: plan.root,
+    source: candidate.identity,
+    contractRoot: plan.contractRoot,
+    candidate: { ...release, sha: pointer.commit, tree: current.tree },
+    previousStable: selected.latest
+      ? { ...selected.latest, sha: latest.commit }
+      : null,
+    comparisonStable: selected.comparison
+      ? { ...selected.comparison, sha: prior.commit, tree: previous.tree }
+      : null,
+    impact,
+    changedPaths: changedPaths(previous.files, current.files),
+  };
+  return { ...body, root: recordDigest(body) };
+}

package/packages/core/providers/github/pipeline-version-artifacts.js

@@ -0,0 +1,91 @@
+import fs from "node:fs";
+import path from "node:path";
+import artifact from "@actions/artifact";
+import { recordDigest } from "../../release/discussion/envelope.js";
+import {
+  publicationPath,
+  writeImmutablePublicationFile,
+} from "../../publication/pipeline/files.js";
+import { pipelineVersionRegenerationResult } from "../../publication/pipeline/version-regeneration.js";
+import {
+  pipelineVersionArtifactName,
+  readPipelineVersionBuild,
+} from "./pipeline-version-readback.js";
+
+function verifyResult(context, result, platform) {
+  const expected = pipelineVersionRegenerationResult(
+    context.preparation,
+    platform,
+    result.files,
+  );
+  if (recordDigest(result) !== recordDigest(expected))
+    throw new Error(
+      "Version artifact changed its exact preparation, platform or bytes",
+    );
+  return result;
+}
+
+export function githubPipelineVersionArtifacts(host, client = artifact) {
+  async function upload(context, result, directory) {
+    verifyResult(context, result, result.platform);
+    const file = writeImmutablePublicationFile(
+      path.join(directory, "version.json"),
+      JSON.stringify(result),
+    );
+    const uploaded = await client.uploadArtifact(
+      pipelineVersionArtifactName(context, result.platform),
+      [file],
+      directory,
+      {
+        retentionDays: 30,
+        compressionLevel: 0,
+      },
+    );
+    if (!uploaded.id || !uploaded.digest)
+      throw new Error("Version upload omitted immutable provider coordinates");
+    return uploaded;
+  }
+  async function download(context, directory) {
+    const { build, assets } = await readPipelineVersionBuild(context, host);
+    const [repositoryOwner, repositoryName] = host.repository.split("/");
+    const results = [];
+    for (const [index, asset] of assets.entries()) {
+      const target = path.join(directory, String(asset.id));
+      fs.mkdirSync(target, { recursive: true });
+      const downloaded = await client.downloadArtifact(asset.id, {
+        path: target,
+        expectedHash: asset.digest,
+        findBy: {
+          repositoryOwner,
+          repositoryName,
+          workflowRunId: context.runId,
+          token: host.token,
+        },
+      });
+      if (downloaded.digestMismatch)
+        throw new Error(
+          "Version download digest differs from its provider archive",
+        );
+      if (
+        recordDigest(fs.readdirSync(target)) !== recordDigest(["version.json"])
+      )
+        throw new Error("Version artifact contains undeclared files");
+      const file = publicationPath(target, "version.json");
+      // JSON escaping can expand the bounded UTF-8 document inventory sixfold.
+      if (fs.statSync(file).size > 384 * 1024 * 1024)
+        throw new Error("Version artifact exceeds its encoded material limit");
+      const bytes = fs.readFileSync(file);
+      if (!Buffer.from(bytes.toString("utf8")).equals(bytes))
+        throw new Error("Version artifact is not canonical UTF-8 text");
+      results.push(
+        verifyResult(
+          context,
+          JSON.parse(bytes.toString("utf8")),
+          context.preparation.platforms[index],
+        ),
+      );
+    }
+    return { build, results };
+  }
+  return { upload, download };
+}

package/packages/core/providers/github/pipeline-version-readback.js

@@ -0,0 +1,117 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { readPipelineCaller } from "./pipeline-run-entry.js";
+
+export const pipelineVersionArtifactName = (context, platform) =>
+  `buildchain-version-${context.preparation.root.slice(7)}-${context.runId}-${context.runAttempt}-${platform}`;
+
+export const pipelineVersionJobName = (preparation, platform) =>
+  `Prepare version (${preparation.purpose}, ${platform})`;
+
+export async function readPipelineVersionBuild(context, host) {
+  const { preparation, runId, runAttempt, definitionSha } = context;
+  const { repository, request, runs } = host;
+  if (preparation.source.repository !== repository)
+    throw new Error("Version preparation crossed its repository boundary");
+  const { run, jobs } = await runs.read(runId, runAttempt);
+  await readPipelineCaller(
+    run,
+    preparation.source.configPath,
+    request,
+    repository,
+  );
+  const definitions = (run.referenced_workflows || []).filter((entry) =>
+    entry.path?.startsWith(
+      "kungfu-systems/buildchain/.github/workflows/.release-pipeline-products.yml@",
+    ),
+  );
+  const versionDefinitions = (run.referenced_workflows || []).filter((entry) =>
+    entry.path?.startsWith(
+      "kungfu-systems/buildchain/.github/workflows/.release-pipeline-version.yml@",
+    ),
+  );
+  if (
+    definitions.length !== 1 ||
+    definitions[0].sha !== definitionSha ||
+    versionDefinitions.length !== 1 ||
+    versionDefinitions[0].sha !== definitionSha ||
+    !/^[0-9a-f]{40}$/u.test(definitionSha || "")
+  )
+    throw new Error("Version preparation changed its exact hosted definition");
+  const selected = preparation.platforms.map((platform) => {
+    const name = pipelineVersionJobName(preparation, platform);
+    const matches = jobs.filter(
+      (job) => job.name === name || job.name.endsWith(` / ${name}`),
+    );
+    if (
+      matches.length !== 1 ||
+      matches[0].run_id !== runId ||
+      matches[0].run_attempt !== runAttempt ||
+      matches[0].status !== "completed" ||
+      matches[0].conclusion !== "success"
+    )
+      throw new Error(
+        "Version preparation requires every exact successful product job",
+      );
+    return matches[0];
+  });
+  if (
+    !selected.length ||
+    new Set(preparation.platforms).size !== selected.length
+  )
+    throw new Error(
+      "Version preparation platform inventory is empty or duplicated",
+    );
+  const inventory = await request(
+    `/repos/${repository}/actions/runs/${runId}/artifacts?per_page=100`,
+  );
+  if (
+    !Array.isArray(inventory.artifacts) ||
+    inventory.total_count !== inventory.artifacts.length ||
+    inventory.total_count > 100
+  )
+    throw new Error("Version preparation artifact inventory is incomplete");
+  const assets = preparation.platforms.map((platform) => {
+    const matches = inventory.artifacts.filter(
+      (asset) => asset.name === pipelineVersionArtifactName(context, platform),
+    );
+    if (
+      matches.length !== 1 ||
+      !Number.isSafeInteger(matches[0].id) ||
+      matches[0].id < 1 ||
+      matches[0].expired ||
+      matches[0].workflow_run?.id !== runId ||
+      matches[0].workflow_run?.head_sha !== run.head_sha ||
+      !/^sha256:[0-9a-f]{64}$/u.test(matches[0].digest || "")
+    )
+      throw new Error(
+        "Version preparation has no unique immutable platform artifact",
+      );
+    return matches[0];
+  });
+  const again = await runs.read(runId, runAttempt);
+  if (
+    again.run.head_sha !== run.head_sha ||
+    recordDigest(again.run.referenced_workflows) !==
+      recordDigest(run.referenced_workflows) ||
+    selected.some(
+      (job) =>
+        !again.jobs.some((value) => recordDigest(value) === recordDigest(job)),
+    )
+  )
+    throw new Error(
+      "Version preparation producer changed during independent readback",
+    );
+  const body = {
+    schema: "buildchain.pipeline-version-build-readback/v1",
+    preparationRoot: preparation.root,
+    source: preparation.source,
+    runtime: preparation.runtime,
+    runId,
+    runAttempt,
+    definitionSha,
+    providerSource: run.head_sha,
+    jobs: selected,
+    artifacts: assets.map(({ id, digest }) => ({ id, digest })),
+  };
+  return { build: { ...body, root: recordDigest(body) }, assets };
+}

package/packages/core/providers/github/pipeline-version.js

@@ -1,9 +1,7 @@
 import { createHash } from "node:crypto";
 import { recordDigest } from "../../release/discussion/envelope.js";
-import {
-  materializePipelineVersion,
-  readPipelineVersion,
-} from "../../publication/pipeline/version.js";
+import { readPipelineVersion } from "../../publication/pipeline/version.js";
+import { preparedPipelineVersionMaterial } from "../../publication/pipeline/version-material.js";
 import { verifyPipelinePublicationPlan } from "../../publication/pipeline/plan.js";
 
 function blobBytes(value, sha) {
@@ -64,15 +62,15 @@ export function githubPipelineVersion(request, repository) {
       sourceTimestamp: commit.committer.date,
     };
   }
-  async function materialize(plan) {
+  async function materialize(plan, regeneration) {
     verifyPipelinePublicationPlan(plan);
     const input = await inspect(plan.source, plan.versionPolicy);
     if (input.version !== plan.candidateVersion)
       throw new Error("Version material changed after planning");
-    const material = materializePipelineVersion(
-      plan.versionPolicy,
+    const material = preparedPipelineVersionMaterial(
+      plan,
       input.files,
-      plan.version,
+      regeneration,
     );
     let source = plan.source;
     if (material.changes.length)
@@ -86,12 +84,12 @@ export function githubPipelineVersion(request, repository) {
     };
     return { ...body, root: recordDigest(body) };
   }
-  async function materializeDevelopment(plan) {
+  async function materializeDevelopment(plan, regeneration) {
     const input = await inspect(plan.source, plan.versionPolicy);
-    const material = materializePipelineVersion(
-      plan.versionPolicy,
+    const material = preparedPipelineVersionMaterial(
+      plan,
       input.files,
-      plan.version,
+      regeneration,
     );
     const branch = `feature/buildchain-next/${plan.root.slice(7)}`;
     const source = material.changes.length

package/packages/core/publication/pipeline/actions.js

@@ -25,6 +25,7 @@ export async function preparePipelinePublicationAction(core, env) {
     host,
   );
   core.setOutput("operation", result.operation);
+  core.setOutput("stable-wait", result.wait ? JSON.stringify(result.wait) : "");
   core.setOutput("context", JSON.stringify(result.context || {}));
   core.setOutput(
     "matrix",
@@ -79,7 +80,13 @@ export async function sealPipelineQualificationAction(core, env) {
 
 export async function applyPipelinePublicationAction(core, env) {
   const host = await pipelineHost(core, env, "Apply qualified publication");
-  await applyPipelinePublication(contextInput(core), host, directory(env), env);
+  const result = await applyPipelinePublication(
+    contextInput(core),
+    host,
+    directory(env),
+    env,
+  );
+  core.setOutput("stable-wait", result.wait ? JSON.stringify(result.wait) : "");
 }
 
 export async function settlePipelinePublicationAction(core, env) {
@@ -97,4 +104,13 @@ export async function settlePipelinePublicationAction(core, env) {
     env,
   );
   core.info(result.reason);
+  core.setOutput(
+    "version-context",
+    JSON.stringify(result.versionContext || {}),
+  );
+  core.setOutput("operation", result.versionContext ? "regenerate" : "wait");
+  core.setOutput(
+    "matrix",
+    JSON.stringify({ include: result.versionContext?.platforms || [] }),
+  );
 }