@kungfu-tech/buildchain 4.1.3-alpha.1 → 4.1.3-alpha.3

package/packages/core/workflow/pipeline/recovery-admission.js

@@ -0,0 +1,122 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { readPipelineCaller } from "../../providers/github/pipeline-run-entry.js";
+import { RECOVERY_ENTRY } from "../../consumer/contract/entries.js";
+
+export async function recoveryExecution(host, definitionSha) {
+  const { run } = await host.runs.read(host.runId, host.runAttempt);
+  const definitions = (run.referenced_workflows || []).filter((item) =>
+    item.path?.startsWith(`kungfu-systems/buildchain/${RECOVERY_ENTRY}@`),
+  );
+  if (
+    run.event !== "workflow_dispatch" ||
+    run.status !== "in_progress" ||
+    run.repository?.full_name !== host.repository ||
+    run.head_repository?.full_name !== host.repository ||
+    run.path?.split("@")[0] !== ".github/workflows/buildchain-recover.yml" ||
+    definitions.length !== 1 ||
+    definitions[0].sha !== definitionSha ||
+    !/^[0-9a-f]{40}$/u.test(definitionSha || "")
+  )
+    throw new Error(
+      "Recovery must execute the exact canonical entry from the same consumer repository",
+    );
+  const channel = definitions[0].path
+    .split("@")
+    .at(-1)
+    .replace(/^refs\/tags\//u, "");
+  if (!["v4", "v4-alpha"].includes(channel))
+    throw new Error("Recovery entry must be a published floating channel");
+  return {
+    run,
+    channel,
+    entry: {
+      repository: "kungfu-systems/buildchain",
+      workflow: RECOVERY_ENTRY,
+      sha: definitionSha,
+    },
+  };
+}
+
+export async function admitRecoverySource(session, execution, host) {
+  const current = session.observed.history.at(-1);
+  const original = current.generation.source;
+  const source = await host.source.source(original.commit, original.configPath);
+  if (
+    recordDigest(source.identity) !== recordDigest(original) ||
+    host.selection.source?.sha !== original.commit ||
+    host.selection.source?.repository !== host.repository
+  )
+    throw new Error(
+      "Recovery source, config or selected runtime source changed",
+    );
+  const caller = await readPipelineCaller(
+    execution.run,
+    original.configPath,
+    host.request,
+    host.repository,
+  );
+  const admission = await host.source.observeIntent(
+    session.intent,
+    current.generation,
+    { "config-path": original.configPath },
+  );
+  if (
+    !admission.live.source ||
+    recordDigest(admission.live.source) !== recordDigest(original) ||
+    admission.live.targetBranch !== session.intent.source.targetBranch ||
+    (!admission.live.merged &&
+      (admission.live.state !== "open" ||
+        admission.live.routeEnabled === false ||
+        admission.live.baseCommit !== current.generation.baseCommit))
+  )
+    throw new Error(
+      "Selected source was closed, superseded or rebased; use its current admitted attempt",
+    );
+  return {
+    source,
+    admission,
+    caller,
+  };
+}
+
+export async function recoveryPredecessorRuns(session, host) {
+  const current = session.observed.history.at(-1);
+  const keys = [
+    ...new Map(
+      current.runs.map((writer) => [
+        `${writer.runId}:${writer.runAttempt}`,
+        writer,
+      ]),
+    ).values(),
+  ];
+  if (keys.length > 100)
+    throw new Error("Recovery predecessor run inventory exceeds its bound");
+  const observations = [];
+  for (const writer of keys) {
+    if (
+      Number(writer.runId) === host.runId &&
+      Number(writer.runAttempt) === host.runAttempt
+    )
+      throw new Error("Recovery cannot replace its own live execution");
+    const { run } = await host.runs.read(
+      Number(writer.runId),
+      Number(writer.runAttempt),
+    );
+    if (
+      run.repository?.full_name !== host.repository ||
+      run.head_repository?.full_name !== host.repository
+    )
+      throw new Error(
+        "Recovery predecessor execution crossed a repository or fork boundary",
+      );
+    observations.push(run);
+  }
+  return {
+    schema: "buildchain.pipeline-recovery-executions/v1",
+    predecessor: current.identity.id,
+    runs: observations,
+    terminal: observations.every(
+      (run) => run.status === "completed" && run.conclusion,
+    ),
+  };
+}

package/packages/core/workflow/pipeline/recovery-build-control.js

@@ -0,0 +1,95 @@
+import { object } from "../../consumer/contract/shape.js";
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { resumePipelineSession } from "./session.js";
+import { recoveryBuildEvidence } from "./recovery-build-evidence.js";
+import { qualifyPipelineBuild } from "./build-qualification.js";
+import { retainPipelineBuildResult } from "./build-result.js";
+
+export const RECOVERY_BUILD_CONTEXT =
+  "buildchain.pipeline-recovery-build-context/v1";
+
+export async function beginRecoveryBuild(session, host) {
+  session.observed = await session.journal.read();
+  const { plan, build } = await recoveryBuildEvidence(session, host);
+  const observed = session.observed;
+  const previous = observed.phases.build;
+  if (
+    previous &&
+    (previous.payload.writer.runId !== String(host.runId) ||
+      previous.payload.writer.runAttempt !== String(host.runAttempt))
+  )
+    return null;
+  await session.progress.progress({
+    attempt: observed.attempt,
+    phase: "build",
+    state: "running",
+    eventKey: `build-start:${host.runId}:${host.runAttempt}`,
+    expectedHead: observed.head,
+    reason: build.reason,
+  });
+  return {
+    schema: RECOVERY_BUILD_CONTEXT,
+    attempt: observed.attempt,
+    generation: observed.generation,
+    source: build.source,
+    runId: host.runId,
+    runAttempt: host.runAttempt,
+    platforms: build.scheduled,
+    recoveryPlanRoot: plan.root,
+  };
+}
+
+export async function recordRecoveryBuild(context, host) {
+  object(context, [
+    "schema",
+    "attempt",
+    "generation",
+    "source",
+    "runId",
+    "runAttempt",
+    "platforms",
+    "recoveryPlanRoot",
+  ]);
+  if (
+    context.schema !== RECOVERY_BUILD_CONTEXT ||
+    context.runId !== host.runId ||
+    context.runAttempt !== host.runAttempt
+  )
+    throw new Error("Recovery build callback belongs to another execution");
+  const session = await resumePipelineSession(
+    { ...host, attempt: context.attempt },
+    host,
+  );
+  const { plan, build } = await recoveryBuildEvidence(session, host);
+  const writer = session.observed.phases.build?.payload.writer;
+  if (
+    context.generation !== session.observed.generation ||
+    context.recoveryPlanRoot !== plan.root ||
+    recordDigest(context.source) !== recordDigest(build.source) ||
+    recordDigest(context.platforms) !== recordDigest(build.scheduled) ||
+    writer?.runId !== String(host.runId) ||
+    writer?.runAttempt !== String(host.runAttempt)
+  )
+    throw new Error(
+      "Recovery build callback changed its reserved work or admission",
+    );
+  const segments = structuredClone(build.segments);
+  if (build.scheduled.length) {
+    const platforms = build.scheduled.map((item) => item.platform);
+    const readback = await host.runs.build(
+      host.runId,
+      host.runAttempt,
+      build.source,
+      platforms,
+    );
+    segments.push({ readback, platforms, runtime: host.runtime });
+  }
+  const result = qualifyPipelineBuild({
+    source: build.source,
+    platforms: build.platforms.map((item) => item.platform),
+    segments,
+    runId: host.runId,
+    runAttempt: host.runAttempt,
+  });
+  return retainPipelineBuildResult(context, result, session, host);
+}

package/packages/core/workflow/pipeline/recovery-build-evidence.js

@@ -0,0 +1,71 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { retainedRecoveryEvidence } from "./recovery-session.js";
+import { pipelinePlatforms } from "./platforms.js";
+import { buildReadbackPlatforms } from "./build-qualification.js";
+import { qualifyRecoveryRuntime } from "./recovery-runtime.js";
+
+export async function recoveryBuildEvidence(session, host) {
+  const { plan, evidence } = await retainedRecoveryEvidence(session, host);
+  const build = evidence.build;
+  if (!build) throw new Error("Recovery admission has no qualified build plan");
+  const { root, ...body } = build;
+  const current = session.observed.history.at(-1);
+  const source = await host.source.source(
+    current.generation.source.commit,
+    current.generation.source.configPath,
+  );
+  if (
+    root !== recordDigest(body) ||
+    body.schema !== "buildchain.pipeline-recovery-build/v1" ||
+    body.predecessor !== plan.predecessor ||
+    recordDigest(body.source) !== recordDigest(current.generation.source) ||
+    recordDigest(source.identity) !== recordDigest(body.source) ||
+    recordDigest(pipelinePlatforms(source.plan)) !==
+      recordDigest(body.platforms) ||
+    recordDigest(host.runtime) !== recordDigest(plan.runtime)
+  )
+    throw new Error(
+      "Recovery build plan changed its source, platform inventory or executor",
+    );
+  const selected = new Set();
+  for (const segment of build.segments) {
+    const old = segment.readback;
+    const inventory = buildReadbackPlatforms(old);
+    const fresh = await host.runs.build(
+      old.runId,
+      old.runAttempt,
+      build.source,
+      inventory,
+    );
+    const runtime = await qualifyRecoveryRuntime(
+      segment.runtime,
+      host.runtime,
+      "build",
+      host.request,
+    );
+    if (recordDigest(fresh) !== recordDigest(old) || !runtime.compatible)
+      throw new Error(
+        "Recovered platform changed during execution requalification",
+      );
+    for (const platform of segment.platforms) {
+      const job = old.jobs[inventory.indexOf(platform)];
+      if (
+        selected.has(platform) ||
+        job?.status !== "completed" ||
+        job.conclusion !== "success"
+      )
+        throw new Error(
+          "Recovery cannot reuse duplicate or unsuccessful platform results",
+        );
+      selected.add(platform);
+    }
+  }
+  if (
+    recordDigest(build.scheduled) !==
+    recordDigest(build.platforms.filter((item) => !selected.has(item.platform)))
+  )
+    throw new Error(
+      "Recovery scheduling differs from the qualified remaining platform set",
+    );
+  return { plan, build };
+}

package/packages/core/workflow/pipeline/recovery-build.js

@@ -0,0 +1,121 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { pipelinePlatforms } from "./platforms.js";
+import { qualifyRecoveryRuntime } from "./recovery-runtime.js";
+import { unrecordedRecoveryBuild } from "./recovery-unrecorded-build.js";
+import {
+  PIPELINE_BUILD_QUALIFICATION,
+  buildReadbackPlatforms,
+  verifyPipelineBuildQualification,
+  qualifyPipelineBuild,
+} from "./build-qualification.js";
+
+async function retainedSegments(session, source, platforms, host) {
+  const phase = session.observed.history.at(-1).phases.build;
+  const references = (phase?.payload.materials || []).filter(
+    (item) =>
+      item.id.startsWith("build/provider-") ||
+      item.id.startsWith("build/qualified-"),
+  );
+  if (!references.length && phase?.payload.state !== "success")
+    return unrecordedRecoveryBuild(session, source, platforms, host);
+  if (references.length !== 1)
+    throw new Error(
+      "Recovery cannot qualify the unique retained build receipt",
+    );
+  const value = await host.materialStore(session).read(references[0]);
+  if (value.schema === PIPELINE_BUILD_QUALIFICATION) {
+    await verifyPipelineBuildQualification(value, source, platforms, host.runs);
+    return value.segments;
+  }
+  const inventory = buildReadbackPlatforms(value);
+  if (
+    recordDigest(value.source) !== recordDigest(source) ||
+    recordDigest([...inventory].sort()) !== recordDigest([...platforms].sort())
+  )
+    throw new Error(
+      "Recovery build receipt changed its original platform inventory",
+    );
+  const fresh = await host.runs.build(
+    value.runId,
+    value.runAttempt,
+    source,
+    inventory,
+  );
+  if (recordDigest(fresh) !== recordDigest(value))
+    throw new Error("Recovery build receipt changed during provider readback");
+  const segments = [{ readback: value, platforms, runtime: phase.runtime }];
+  qualifyPipelineBuild({
+    source,
+    platforms,
+    segments,
+    runId: value.runId,
+    runAttempt: value.runAttempt,
+  });
+  return segments;
+}
+
+// Every retained segment is independently observed, even when its old overall
+// workflow failed elsewhere. Only successful jobs with an unchanged execution
+// implementation qualify for reuse. Missing evidence is not invented.
+export async function qualifyRecoveryBuild(session, admission, host) {
+  const source = session.observed.history.at(-1).generation.source;
+  if (recordDigest(admission.identity) !== recordDigest(source))
+    throw new Error("Recovery build admission changed its exact source");
+  const platforms = pipelinePlatforms(admission.plan);
+  const prior = await retainedSegments(
+    session,
+    source,
+    platforms.map((item) => item.platform),
+    host,
+  );
+  const segments = [],
+    runtimes = [],
+    selected = new Set();
+  for (const segment of prior) {
+    const runtime = await qualifyRecoveryRuntime(
+      segment.runtime,
+      host.runtime,
+      "build",
+      host.request,
+    );
+    runtimes.push(runtime);
+    const inventory = buildReadbackPlatforms(segment.readback);
+    const reusable = segment.platforms.filter((platform) => {
+      const job = segment.readback.jobs[inventory.indexOf(platform)];
+      return (
+        runtime.compatible &&
+        job.status === "completed" &&
+        job.conclusion === "success"
+      );
+    });
+    if (reusable.length) {
+      for (const platform of reusable) {
+        if (
+          selected.has(platform) ||
+          !platforms.some((item) => item.platform === platform)
+        )
+          throw new Error(
+            "Recovery build has duplicate or undeclared retained platforms",
+          );
+        selected.add(platform);
+      }
+      segments.push({ ...segment, platforms: reusable });
+    }
+  }
+  const body = {
+    schema: "buildchain.pipeline-recovery-build/v1",
+    predecessor: session.observed.attempt,
+    predecessorHead: session.observed.head,
+    source,
+    platforms,
+    scheduled: platforms.filter((item) => !selected.has(item.platform)),
+    segments,
+    runtimes,
+    reason: !prior.length
+      ? "No retained product result; execute declared platforms"
+      : runtimes.some((item) => !item.compatible)
+        ? "Changed stage implementation requires affected platform execution"
+        : "Reuse independently verified successful platforms; execute only missing results",
+  };
+  return { ...body, root: recordDigest(body) };
+}

package/packages/core/workflow/pipeline/recovery-controller.js

@@ -0,0 +1,91 @@
+import {
+  recoveryExecution,
+  admitRecoverySource,
+  recoveryPredecessorRuns,
+} from "./recovery-admission.js";
+import {
+  selectRecoveryAttempt,
+  openRecoveryAttempt,
+  retainedRecoveryPlan,
+} from "./recovery-session.js";
+import { qualifyRecoveryBuild } from "./recovery-build.js";
+import { observeRecoveryOwnership } from "./recovery-ownership.js";
+import {
+  terminateRecoveryPredecessor,
+  recoveryNodePlan,
+  replayRecoveryAdmission,
+} from "./recovery-transition.js";
+import { qualifyRecoveryIntegration } from "./recovery-integration.js";
+import { beginRecoveryBuild } from "./recovery-build-control.js";
+
+async function duplicateRecovery(session, host) {
+  await retainedRecoveryPlan(session, host);
+  // Root CAS can commit before the immutable lookup index acknowledgement.
+  await host.index.retain(session.intent, session.observed.attempt);
+  await host.project(session);
+  return {
+    operation: "wait",
+    attempt: session.observed.attempt,
+    reason: `Recovery already selected this successor (${session.observed.status}); any further recovery must select its exact attempt`,
+  };
+}
+
+export async function controlPipelineRecovery(attempt, definitionSha, host) {
+  const execution = await recoveryExecution(host, definitionSha);
+  let session = await selectRecoveryAttempt(attempt, host, execution.entry);
+  if (session.duplicate) return duplicateRecovery(session, host);
+  const admitted = await admitRecoverySource(session, execution, host);
+  const executions = await recoveryPredecessorRuns(session, host);
+  if (!executions.terminal) {
+    await host.project(session);
+    return {
+      operation: "wait",
+      attempt,
+      reason:
+        "Predecessor provider executions are still active; wait for completion or cancel their actual runs before recovery",
+    };
+  }
+  const build = await qualifyRecoveryBuild(session, admitted.source, host);
+  // Post-merge recovery must qualify the retained publication transaction before
+  // opening a successor. Its adapter preserves signed bytes and completed effects.
+  const integration = admitted.admission.live.merged
+    ? await qualifyRecoveryIntegration(session, admitted.admission, host)
+    : null;
+  const publication =
+    admitted.admission.live.merged &&
+    session.intent.expectedNodes.includes("publish")
+      ? await host.recoverPublication(session, admitted, execution)
+      : null;
+  const ownership = await observeRecoveryOwnership(session, executions, host);
+  session = await terminateRecoveryPredecessor(
+    session,
+    executions,
+    ownership,
+    host,
+  );
+  const evidence = {
+    admitted,
+    executions,
+    ownership,
+    build,
+    publication,
+    integration,
+  };
+  const plan = recoveryNodePlan(session, evidence, execution.entry, host);
+  session = await openRecoveryAttempt(session, plan, host, evidence);
+  await replayRecoveryAdmission(session, host);
+  const context = await beginRecoveryBuild(session, host);
+  await host.project(session);
+  if (!context)
+    return {
+      operation: "wait",
+      attempt: session.observed.attempt,
+      reason: "Recovery build already has an execution owner",
+    };
+  return {
+    operation: context.platforms.length ? "build" : "qualify-build",
+    context,
+    attempt: context.attempt,
+    reason: build.reason,
+  };
+}

package/packages/core/workflow/pipeline/recovery-integration.js

@@ -0,0 +1,66 @@
+import { recordDigest } from "../../release/discussion/envelope.js";
+import { settleRecoveryOwnership } from "./recovery-ownership-settlement.js";
+import { retainedRecoveryEvidence } from "./recovery-session.js";
+
+export async function qualifyRecoveryIntegration(session, admission, host) {
+  const current = {
+    ...session.observed.history.at(-1),
+    intent: session.intent,
+  };
+  const integration = await host.integration.observe(current);
+  const review = await host.policy.observeMerged(
+    current,
+    admission.protectedPlan.review,
+    integration,
+  );
+  if (!review.review || !review.checksPassing)
+    throw new Error(
+      "Recovery of protected integration requires current exact review and required checks",
+    );
+  return { integration, review };
+}
+
+async function replayPhase(session, phase, value, host) {
+  const observed = await session.journal.read();
+  const old = observed.phases[phase];
+  if (old?.payload.state === "success") return;
+  const root = recordDigest(value);
+  const reference = await host
+    .materialStore(session)
+    .retain(`recovery/${phase}-${root.slice(7)}`, value, "provider-readback");
+  await session.progress.progress({
+    attempt: observed.attempt,
+    phase,
+    state: "success",
+    eventKey: `recovery-${phase}:${root}`,
+    materials: [reference],
+    expectedHead: observed.head,
+  });
+}
+
+export async function replayRecoveryIntegration(session, host) {
+  session.observed = await session.journal.read();
+  const { evidence } = await retainedRecoveryEvidence(session, host);
+  if (!evidence.admitted.admission.live.merged) return;
+  const qualified = await qualifyRecoveryIntegration(
+    session,
+    evidence.admitted.admission,
+    host,
+  );
+  await replayPhase(session, "review", qualified.review, host);
+  if (session.intent.expectedNodes.includes("warrant")) {
+    const ownership = await settleRecoveryOwnership(session, qualified, host);
+    await replayPhase(
+      session,
+      "warrant",
+      {
+        schema: "buildchain.pipeline-recovery-merged-warrant/v1",
+        candidate: ownership.candidate,
+        proof: ownership.proof,
+        predecessor: evidence.ownership.ownerAttempt,
+      },
+      host,
+    );
+  }
+  await replayPhase(session, "merge", qualified.integration, host);
+}

package/packages/core/workflow/pipeline/recovery-merge-proof.js

@@ -0,0 +1,90 @@
+import {
+  createIntegrationDeliveryProof,
+  verifyIntegrationDeliveryProof,
+} from "../../dev-delivery/dev-delivery-warrant.js";
+import { recoveryOwnerSession } from "./recovery-ownership.js";
+
+export function verifyRecoveryMergeProof(
+  proof,
+  candidate,
+  integration,
+  session,
+) {
+  if (
+    !verifyIntegrationDeliveryProof(proof, {
+      repository: session.intent.repository,
+      protectedBase: session.intent.source.targetBranch,
+      sourceProofRoot: candidate.sourceProofRoot,
+      warrantCandidateId: candidate.candidateId,
+      currentBase: integration.mergeCommit,
+      replayTree: integration.mergeTree,
+      mergeGroupHead: integration.mergeCommit,
+      mergeGroupTree: integration.mergeTree,
+    }).ok
+  )
+    throw new Error(
+      "Recovery merged Warrant has no exact original integration proof",
+    );
+  return proof;
+}
+
+export async function retainedRecoveryMergeProof(
+  session,
+  candidate,
+  integration,
+  host,
+) {
+  let count = 0;
+  for (const owner of [...session.observed.history].reverse()) {
+    const store = host.materialStore(recoveryOwnerSession(session, owner));
+    for (const reference of owner.events
+      .flatMap((event) => event.payload.materials)
+      .filter(
+        (item) =>
+          item.id ===
+            `merge/proof-${candidate.terminal.evidenceRoot.slice(7)}` ||
+          item.id.startsWith("recovery/ownership-request/"),
+      )) {
+      if (++count > 100)
+        throw new Error(
+          "Recovery integration proof inventory exceeds its bound",
+        );
+      const value = await store.read(reference);
+      const proof = reference.id.startsWith("merge/proof-")
+        ? value
+        : value.proof;
+      if (proof?.proofRoot === candidate.terminal.evidenceRoot)
+        return verifyRecoveryMergeProof(proof, candidate, integration, session);
+    }
+  }
+  throw new Error(
+    "Merged predecessor lost its retained pre-effect integration proof",
+  );
+}
+
+export function createRecoveryMergeProof(
+  session,
+  queue,
+  candidate,
+  integration,
+) {
+  if (
+    queue.activeWarrant?.candidateId !== candidate.candidateId ||
+    queue.activeWarrant.phase !== "qualified"
+  )
+    throw new Error(
+      "Merged recovery requires the original qualified active Warrant",
+    );
+  return createIntegrationDeliveryProof({
+    repository: session.intent.repository,
+    protectedBase: session.intent.source.targetBranch,
+    sourceProofRoot: candidate.sourceProofRoot,
+    warrant: queue.activeWarrant,
+    currentBase: integration.mergeCommit,
+    replayTree: integration.mergeTree,
+    mergeGroupHead: integration.mergeCommit,
+    mergeGroupTree: integration.mergeTree,
+    requiredContextRoots: [integration.build.root, integration.root],
+    verifiedAt: new Date().toISOString(),
+  });
+}