@kungfu-tech/buildchain 4.1.3-alpha.0 → 4.1.3-alpha.2

package/packages/core/providers/github/pipeline-product-payload.js

@@ -0,0 +1,50 @@
+import { createHash } from "node:crypto";
+import { canonicalJson } from "../../release/discussion/envelope.js";
+import {
+  publicationFile,
+  publicationPath,
+} from "../../publication/pipeline/files.js";
+const sha256 = (bytes) =>
+  `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
+
+export function pipelineProductPayload({
+  qualified,
+  documents,
+  directory,
+  evidence,
+}) {
+  const payload = (effect) => {
+    if (effect.product.startsWith("evidence:")) {
+      const asset = evidence.find(
+        (entry) => `evidence:${entry.id}` === effect.product,
+      );
+      if (
+        !asset ||
+        sha256(asset.bytes) !== effect.digest ||
+        asset.bytes.length !== effect.size
+      )
+        throw new Error(
+          "Release evidence differs from admitted immutable bytes",
+        );
+      return asset.bytes;
+    }
+    if (effect.product === "release-passport")
+      return Buffer.from(`${canonicalJson(documents.passport)}\n`);
+    const artifact = qualified.artifacts.find(
+      (entry) => entry.id === effect.product,
+    );
+    if (
+      !artifact ||
+      artifact.digest !== effect.digest ||
+      artifact.size !== effect.size
+    )
+      throw new Error(
+        "GitHub Release payload does not match qualified product",
+      );
+    const file = publicationFile(publicationPath(directory, artifact.file));
+    if (file.digest !== artifact.digest || file.size !== artifact.size)
+      throw new Error("GitHub Release payload bytes changed");
+    return file.bytes;
+  };
+  return payload;
+}

package/packages/core/providers/github/pipeline-product-release.js

@@ -0,0 +1,191 @@
+import { createHash } from "node:crypto";
+import { pipelineProductPayload } from "./pipeline-product-payload.js";
+
+const sha256 = (bytes) =>
+  `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
+
+export function githubPipelineProductRelease({
+  github,
+  request,
+  repository,
+  plan,
+  qualified,
+  documents,
+  directory,
+  evidence = [],
+}) {
+  const [owner, repo] = repository.split("/");
+  const prefix = `/repos/${repository}`;
+  const marker = `Buildchain release transaction ${documents.transaction.transactionRoot}`;
+  const payload = pipelineProductPayload({
+    qualified,
+    documents,
+    directory,
+    evidence,
+  });
+  async function tag() {
+    const value = await request(
+      `${prefix}/git/ref/tags/${encodeURIComponent(plan.tag)}`,
+      { allow404: true },
+    );
+    if (!value) return { state: "absent" };
+    let object = value.object;
+    for (let depth = 0; object.type === "tag" && depth < 4; depth++)
+      object = (await request(`${prefix}/git/tags/${object.sha}`)).object;
+    if (object.type !== "commit")
+      throw new Error("Exact release tag does not resolve to a commit");
+    return {
+      state: "present",
+      commit: object.sha,
+      ref: value.ref,
+      objectSha: value.object.sha,
+    };
+  }
+  async function release(create = false) {
+    const observedTag = await tag();
+    if (observedTag.state === "absent" && !create) return null;
+    if (observedTag.commit !== qualified.source.commit)
+      throw new Error("Release requires the exact immutable publication tag");
+    let value = await findRelease();
+    if (!value && create) {
+      try {
+        await github.rest.repos.createRelease({
+          owner,
+          repo,
+          tag_name: plan.tag,
+          target_commitish: qualified.source.commit,
+          name: plan.tag,
+          body: marker,
+          draft: true,
+          prerelease: plan.channel === "alpha",
+          make_latest: "false",
+        });
+      } catch (error) {
+        value = await findRelease();
+        if (!value) throw error;
+      }
+      value = await findRelease();
+      if (!value)
+        throw new Error("Release creation lacks exact provider readback");
+    }
+    if (
+      value &&
+      (value.tag_name !== plan.tag ||
+        value.body !== marker ||
+        value.prerelease !== (plan.channel === "alpha"))
+    )
+      throw new Error(
+        "Existing GitHub Release belongs to a different immutable transaction",
+      );
+    return value;
+  }
+  const findRelease = () => findPipelineRelease(github, owner, repo, plan.tag);
+  async function asset(effect) {
+    const published = await release();
+    if (!published) return { state: "absent" };
+    const assets = await github.paginate(github.rest.repos.listReleaseAssets, {
+      owner,
+      repo,
+      release_id: published.id,
+      per_page: 100,
+    });
+    const matches = assets.filter((entry) => entry.name === effect.name);
+    if (!matches.length) return { state: "absent" };
+    if (matches.length !== 1 || matches[0].size > 256 * 1024 * 1024)
+      throw new Error(
+        "Release asset identity is ambiguous or exceeds its bound",
+      );
+    const bytes = Buffer.from(
+      (
+        await github.rest.repos.getReleaseAsset({
+          owner,
+          repo,
+          asset_id: matches[0].id,
+          headers: { accept: "application/octet-stream" },
+        })
+      ).data,
+    );
+    return {
+      state: "present",
+      id: matches[0].id,
+      releaseId: published.id,
+      size: bytes.length,
+      digest: sha256(bytes),
+    };
+  }
+  return {
+    async observe(effect) {
+      if (effect.kind === "exact-tag") return tag();
+      if (effect.kind === "release-asset") return asset(effect);
+      if (effect.kind === "release-visibility") {
+        const value = await release();
+        return value && !value.draft
+          ? {
+              state: "present",
+              id: value.id,
+              tag: value.tag_name,
+              prerelease: value.prerelease,
+            }
+          : { state: "absent" };
+      }
+      throw new Error("Unsupported GitHub product publication operation");
+    },
+    matches(effect, observed) {
+      if (observed.state !== "present") return false;
+      if (effect.kind === "exact-tag") return observed.commit === effect.commit;
+      if (effect.kind === "release-asset") {
+        const bytes = payload(effect);
+        return (
+          observed.digest === sha256(bytes) && observed.size === bytes.length
+        );
+      }
+      return (
+        effect.kind === "release-visibility" &&
+        observed.tag === effect.tag &&
+        observed.prerelease === effect.prerelease
+      );
+    },
+    async apply(effect) {
+      if (effect.kind === "exact-tag")
+        return request(`${prefix}/git/refs`, {
+          method: "POST",
+          body: { ref: `refs/tags/${effect.tag}`, sha: effect.commit },
+        });
+      const value = await release(true);
+      if (effect.kind === "release-asset")
+        return github.rest.repos.uploadReleaseAsset({
+          owner,
+          repo,
+          release_id: value.id,
+          name: effect.name,
+          data: payload(effect),
+          headers: { "content-type": "application/octet-stream" },
+        });
+      if (effect.kind === "release-visibility")
+        return github.rest.repos.updateRelease({
+          owner,
+          repo,
+          release_id: value.id,
+          draft: false,
+          make_latest: "false",
+        });
+      throw new Error("Unsupported GitHub product publication operation");
+    },
+  };
+}
+
+async function findPipelineRelease(github, owner, repo, tag) {
+  let pages = 0;
+  const values = await github.paginate(
+    github.rest.repos.listReleases,
+    { owner, repo, per_page: 100 },
+    (response) => {
+      if (++pages > 100)
+        throw new Error("Release inventory exceeds its bounded readback");
+      return response.data.filter((entry) => entry.tag_name === tag);
+    },
+  );
+  if (values.length > 1)
+    throw new Error("Exact release tag has multiple provider releases");
+  return values[0];
+}

package/packages/core/providers/github/pipeline-publication-artifacts.js

@@ -0,0 +1,153 @@
+import fs from "node:fs";
+import path from "node:path";
+import artifact from "@actions/artifact";
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { readPipelineCaller } from "./pipeline-run-entry.js";
+import { publicationPath } from "../../publication/pipeline/files.js";
+
+export const pipelinePublicationArtifactName = (plan, platform) =>
+  `buildchain-products-${plan.root.slice(7)}-${platform}`;
+
+export function githubPipelinePublicationArtifacts({
+  request,
+  runs,
+  repository,
+  token,
+  client = artifact,
+}) {
+  const prefix = `/repos/${repository}/actions`;
+  async function upload(plan, manifest, directory) {
+    const paths = [
+      publicationPath(directory, "manifest.json"),
+      ...manifest.artifacts.map(({ file }) => publicationPath(directory, file)),
+    ];
+    const result = await client.uploadArtifact(
+      pipelinePublicationArtifactName(plan, manifest.platform),
+      paths,
+      directory,
+      { retentionDays: 30, compressionLevel: 0 },
+    );
+    if (!result.id || !result.digest)
+      throw new Error(
+        "Product upload omitted its immutable provider coordinates",
+      );
+    return result;
+  }
+  async function buildReadback(context, selectedPlatforms) {
+    const { plan, materialization, runId, runAttempt } = context;
+    const { run, jobs } = await runs.read(runId, runAttempt);
+    await readPipelineCaller(
+      run,
+      materialization.source.configPath,
+      request,
+      repository,
+    );
+    const declared = [
+      ...new Set(plan.outputs.map(({ platform }) => platform)),
+    ].sort();
+    const platforms = selectedPlatforms || declared;
+    if (
+      !platforms.length ||
+      new Set(platforms).size !== platforms.length ||
+      platforms.some((platform) => !declared.includes(platform))
+    )
+      throw new Error("Publication subset must name unique declared platforms");
+    const selected = platforms.map((platform) => {
+      const name = `Build publication (${platform})`;
+      const matches = jobs.filter(
+        (job) => job.name === name || job.name.endsWith(` / ${name}`),
+      );
+      if (
+        matches.length !== 1 ||
+        matches[0].run_id !== runId ||
+        matches[0].run_attempt !== runAttempt ||
+        matches[0].status !== "completed" ||
+        matches[0].conclusion !== "success"
+      )
+        throw new Error(
+          "Publication needs every exact credentialless build job to succeed",
+        );
+      return matches[0];
+    });
+    const response = await request(
+      `${prefix}/runs/${runId}/artifacts?per_page=100`,
+    );
+    if (
+      !Array.isArray(response.artifacts) ||
+      response.total_count !== response.artifacts.length ||
+      response.total_count > 100
+    )
+      throw new Error("Publication artifact inventory is incomplete");
+    const assets = platforms.map((platform) => {
+      const matches = response.artifacts.filter(
+        (asset) =>
+          asset.name === pipelinePublicationArtifactName(plan, platform),
+      );
+      if (
+        matches.length !== 1 ||
+        !Number.isSafeInteger(matches[0].id) ||
+        matches[0].id < 1 ||
+        matches[0].expired ||
+        matches[0].workflow_run?.id !== runId ||
+        matches[0].workflow_run?.head_sha !== run.head_sha ||
+        !/^sha256:[0-9a-f]{64}$/u.test(matches[0].digest || "")
+      )
+        throw new Error(
+          "Publication requires one retained artifact per exact platform",
+        );
+      return matches[0];
+    });
+    const again = await runs.read(runId, runAttempt);
+    for (const job of selected)
+      if (
+        !again.jobs.some((entry) => recordDigest(entry) === recordDigest(job))
+      )
+        throw new Error(
+          "Publication build changed during independent readback",
+        );
+    if (again.run.head_sha !== run.head_sha)
+      throw new Error("Publication provider source changed during readback");
+    const body = {
+      schema: "buildchain.pipeline-publication-build-readback/v1",
+      outcome: "success",
+      planRoot: plan.root,
+      source: materialization.source,
+      runId,
+      runAttempt,
+      providerSource: run.head_sha,
+      platforms,
+      jobs: selected,
+      artifactIds: assets.map(({ id }) => id).sort((a, b) => a - b),
+    };
+    return { build: { ...body, root: recordDigest(body) }, assets };
+  }
+  async function download(context, workspace, selectedPlatforms) {
+    const { build, assets } = await buildReadback(context, selectedPlatforms);
+    const [repositoryOwner, repositoryName] = repository.split("/");
+    const bundles = [];
+    for (const asset of assets) {
+      const directory = path.join(workspace, String(asset.id));
+      fs.mkdirSync(directory, { recursive: true });
+      const result = await client.downloadArtifact(asset.id, {
+        path: directory,
+        expectedHash: asset.digest,
+        findBy: {
+          repositoryOwner,
+          repositoryName,
+          workflowRunId: context.runId,
+          token,
+        },
+      });
+      if (result.digestMismatch)
+        throw new Error(
+          "Product download digest does not match the provider archive",
+        );
+      const manifest = JSON.parse(
+        fs.readFileSync(publicationPath(directory, "manifest.json"), "utf8"),
+      );
+      bundles.push({ directory, manifest, providerArtifact: asset });
+    }
+    return { build, bundles };
+  }
+  return { upload, download, buildReadback };
+}

package/packages/core/providers/github/pipeline-run-entry.js

@@ -0,0 +1,86 @@
+import { createHash } from "node:crypto";
+import {
+  PIPELINE_ENTRY,
+  RECOVERY_ENTRY,
+  consumerWorkflows,
+} from "../../consumer/contract/entries.js";
+
+export function pipelineRunEntry(run) {
+  const definitions = (run.referenced_workflows || []).filter((item) =>
+    [PIPELINE_ENTRY, RECOVERY_ENTRY].some((entry) =>
+      item.path?.startsWith(`kungfu-systems/buildchain/${entry}@`),
+    ),
+  );
+  if (
+    definitions.length !== 1 ||
+    !/^[0-9a-f]{40}$/u.test(definitions[0].sha || "")
+  )
+    throw new Error(
+      "Build provider run has no unique exact pipeline or recovery entry",
+    );
+  const definition = definitions[0];
+  const recovery = definition.path.startsWith(
+    `kungfu-systems/buildchain/${RECOVERY_ENTRY}@`,
+  );
+  const caller = `.github/workflows/${recovery ? "buildchain-recover" : "buildchain"}.yml`;
+  const channel = definition.path
+    .split("@")
+    .at(-1)
+    .replace(/^refs\/tags\//u, "");
+  if (
+    !["v4", "v4-alpha"].includes(channel) ||
+    run.path?.split("@")[0] !== caller ||
+    (recovery
+      ? run.event !== "workflow_dispatch"
+      : ![
+          "pull_request",
+          "repository_dispatch",
+          "pull_request_review",
+          "merge_group",
+        ].includes(run.event))
+  )
+    throw new Error(
+      "Build run did not enter through its trusted published consumer workflow",
+    );
+  return { definition, channel, caller, recovery };
+}
+
+// Entry presence and a matching job name alone are insufficient: an altered
+// caller could add a lookalike job beside a failed reusable-workflow call.
+export async function readPipelineCaller(run, configPath, request, repository) {
+  const entry = pipelineRunEntry(run);
+  if (
+    run.repository?.full_name !== repository ||
+    run.head_repository?.full_name !== repository ||
+    !/^[0-9a-f]{40}$/u.test(run.head_sha || "")
+  )
+    throw new Error("Pipeline execution crossed a repository or fork boundary");
+  const file = await request(
+    `/repos/${repository}/contents/${entry.caller}?ref=${run.head_sha}`,
+  );
+  if (
+    file.type !== "file" ||
+    file.encoding !== "base64" ||
+    !Number.isSafeInteger(file.size) ||
+    file.size < 1 ||
+    file.size > 64000 ||
+    typeof file.content !== "string" ||
+    file.content.length > 128000
+  )
+    throw new Error("Pipeline caller workflow is absent or exceeds its bound");
+  const bytes = Buffer.from(file.content, "base64");
+  const blob = createHash("sha1")
+    .update(`blob ${bytes.length}\0`)
+    .update(bytes)
+    .digest("hex");
+  if (
+    bytes.length !== file.size ||
+    blob !== file.sha ||
+    bytes.toString("utf8") !==
+      consumerWorkflows(entry.channel, configPath)[entry.caller]
+  )
+    throw new Error(
+      "Pipeline caller differs from the trusted minimal workflow contract",
+    );
+  return { commit: run.head_sha, path: entry.caller, blob };
+}

package/packages/core/providers/github/pipeline-runs.js

@@ -0,0 +1,132 @@
+import { providerWriter } from "../../workflow/attempt/identity.js";
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { pipelineRunEntry, readPipelineCaller } from "./pipeline-run-entry.js";
+
+function uniqueJob(jobs, name) {
+  const matches = jobs.filter(
+    (job) => job.name === name || job.name.endsWith(` / ${name}`),
+  );
+  if (matches.length !== 1)
+    throw new Error(`Pipeline needs one exact provider job: ${name}`);
+  return matches[0];
+}
+
+export function githubPipelineRuns(request, repository) {
+  if (!/^[\w.-]+\/[\w.-]+$/u.test(repository || ""))
+    throw new Error("Invalid pipeline run repository");
+  const base = `/repos/${repository}`;
+  async function read(runId, runAttempt) {
+    for (const value of [runId, runAttempt])
+      if (!Number.isSafeInteger(value) || value < 1)
+        throw new Error("Invalid pipeline run coordinates");
+    const run = await request(`${base}/actions/runs/${runId}`);
+    if (
+      run.id !== runId ||
+      run.run_attempt !== runAttempt ||
+      run.repository?.full_name !== repository
+    )
+      throw new Error("Pipeline provider run attempt changed");
+    const jobs = [];
+    for (let page = 1; page <= 100; page++) {
+      const response = await request(
+        `${base}/actions/runs/${runId}/attempts/${runAttempt}/jobs?per_page=100&page=${page}`,
+      );
+      if (!Array.isArray(response.jobs))
+        throw new Error("Missing pipeline provider jobs");
+      jobs.push(...response.jobs);
+      if (response.jobs.length < 100) return { run, jobs };
+    }
+    throw new Error("Pipeline job inventory exceeds its bound");
+  }
+  async function writer(runId, runAttempt, jobName) {
+    const { jobs } = await read(runId, runAttempt);
+    const job = uniqueJob(jobs, jobName);
+    if (
+      job.run_id !== runId ||
+      job.run_attempt !== runAttempt ||
+      job.status !== "in_progress"
+    )
+      throw new Error("Pipeline writer is not the exact active provider job");
+    const value = {
+      repository,
+      runId: String(runId),
+      runAttempt: String(runAttempt),
+      jobId: String(job.id),
+    };
+    providerWriter(value, repository);
+    return value;
+  }
+  async function build(runId, runAttempt, source, platforms) {
+    const { run, jobs } = await read(runId, runAttempt);
+    if (
+      ![
+        "pull_request",
+        "repository_dispatch",
+        "pull_request_review",
+        "merge_group",
+        "workflow_dispatch",
+      ].includes(run.event) ||
+      (["pull_request", "merge_group"].includes(run.event) &&
+        run.head_sha !== source.commit) ||
+      source.repository !== repository
+    )
+      throw new Error("Build provider run does not match admitted PR source");
+    const entry = pipelineRunEntry(run);
+    await readPipelineCaller(run, source.configPath, request, repository);
+    if (
+      !Array.isArray(platforms) ||
+      !platforms.length ||
+      new Set(platforms).size !== platforms.length
+    )
+      throw new Error(
+        "Build provider receipt requires declared unique platforms",
+      );
+    const selected = platforms.map((platform) =>
+      uniqueJob(jobs, `Build product (${platform})`),
+    );
+    if (
+      selected.some(
+        (job) =>
+          job.run_id !== runId ||
+          job.run_attempt !== runAttempt ||
+          job.status !== "completed" ||
+          !job.conclusion,
+      )
+    )
+      throw new Error("Product build jobs are not provider-terminal");
+    const again = await request(`${base}/actions/runs/${runId}`);
+    if (
+      again.id !== run.id ||
+      again.run_attempt !== runAttempt ||
+      again.head_sha !== run.head_sha
+    )
+      throw new Error("Product build run changed during readback");
+    const body = {
+      schema: "buildchain.pipeline-build-readback/v1",
+      source,
+      runId,
+      runAttempt,
+      entry: entry.definition,
+      jobs: selected,
+      outcome: selected.every((job) => job.conclusion === "success")
+        ? "success"
+        : "failure",
+    };
+    return { ...body, root: recordDigest(body) };
+  }
+  async function completed(
+    runId,
+    runAttempt,
+    { sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) } = {},
+  ) {
+    for (let attempt = 0; attempt < 20; attempt++) {
+      const { run } = await read(runId, runAttempt);
+      if (run.status === "completed") return run;
+      await sleep(3000);
+    }
+    throw new Error(
+      "Predecessor workflow did not become terminal within the bounded provider readback window",
+    );
+  }
+  return { read, writer, build, completed };
+}