@kungfu-tech/buildchain 4.0.2-alpha.2 → 4.0.2-alpha.20

package/scripts/check-v4-universal-workflow-bootstrap.mjs

@@ -0,0 +1,265 @@
+import assert from "node:assert/strict";
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { execFileSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const contract = JSON.parse(
+  fs.readFileSync(
+    path.join(root, "architecture/v4-universal-workflow-bootstrap.json"),
+    "utf8",
+  ),
+);
+const admissionPolicy = JSON.parse(
+  fs.readFileSync(
+    path.join(root, "architecture/v4-universal-workflow-train-admission.json"),
+    "utf8",
+  ),
+);
+const fileRoot = (relative) =>
+  `sha256:${crypto
+    .createHash("sha256")
+    .update(fs.readFileSync(path.join(root, relative)))
+    .digest("hex")}`;
+
+const workflowRoot = path.join(root, ".github/workflows");
+const discovered = fs
+  .readdirSync(workflowRoot)
+  .filter((name) => /\.ya?ml$/u.test(name))
+  .map((name) => `.github/workflows/${name}`)
+  .filter((relative) =>
+    /(?:^|\n)\s*workflow_call:\s*(?:\n|$)/u.test(
+      fs.readFileSync(path.join(root, relative), "utf8"),
+    ),
+  )
+  .sort();
+
+assert.equal(
+  contract.schema,
+  "kungfu-buildchain-v4-universal-workflow-bootstrap/v1",
+);
+assert.deepEqual(
+  contract.inventoryWorkflows,
+  [...new Set(contract.inventoryWorkflows)].sort(),
+  "inventoried public workflows must be sorted and duplicate-free",
+);
+assert.deepEqual(
+  contract.inventoryWorkflows,
+  discovered,
+  "every active public workflow_call surface must be inventoried",
+);
+for (const relative of contract.inventoryWorkflows)
+  assert.ok(
+    fs.statSync(path.join(root, relative)).isFile(),
+    `inventoried workflow is unavailable: ${relative}`,
+  );
+assert.deepEqual(
+  contract.bootstrapGovernedWorkflows,
+  [...new Set(contract.bootstrapGovernedWorkflows)].sort(),
+  "Bootstrap-governed workflows must be sorted and duplicate-free",
+);
+assert.deepEqual(
+  contract.retiredWorkflowSurfaces,
+  [...new Set(contract.retiredWorkflowSurfaces)].sort(),
+  "retired workflow surfaces must be sorted and duplicate-free",
+);
+for (const relative of contract.retiredWorkflowSurfaces) {
+  assert.ok(
+    contract.inventoryWorkflows.includes(relative),
+    `retired workflow is outside the inventory: ${relative}`,
+  );
+  assert.match(
+    fs.readFileSync(path.join(root, relative), "utf8"),
+    /retired/u,
+    `retired workflow does not fail closed explicitly: ${relative}`,
+  );
+}
+const activeWorkflows = contract.inventoryWorkflows.filter(
+  (relative) =>
+    !contract.retiredWorkflowSurfaces.includes(relative) &&
+    relative !== contract.bootstrap.consumerRecoveryWorkflow,
+);
+assert.deepEqual(
+  contract.bootstrapGovernedWorkflows,
+  activeWorkflows,
+  "every active public reusable workflow must be Bootstrap-governed",
+);
+for (const relative of contract.bootstrapGovernedWorkflows) {
+  assert.ok(
+    contract.inventoryWorkflows.includes(relative),
+    `Bootstrap-governed workflow is outside the inventory: ${relative}`,
+  );
+  const source = fs.readFileSync(path.join(root, relative), "utf8");
+  assert.ok(
+    relative === contract.bootstrap.publicWorkflow ||
+      /uses:\s+(?:\.\/)?\.github\/workflows\/bootstrap\.yml/u.test(source) ||
+      /uses:\s+kungfu-systems\/buildchain\/\.github\/workflows\/bootstrap\.yml@/u.test(
+        source,
+      ),
+    `workflow is declared governed without a Bootstrap edge: ${relative}`,
+  );
+}
+
+assert.deepEqual(contract.complexityBudget, {
+  bootstrapAbis: 1,
+  requestEnvelopes: 1,
+  candidateResolvers: 1,
+  candidateEngines: 1,
+  terminalReceiptAuthorities: 1,
+  perWorkflowBootstrapImplementations: 0,
+  buildchainOnlySelfReleasePaths: 0,
+  perIncidentConsumerEdits: 0,
+});
+assert.equal(contract.bootstrap.discoveryRefExecutionAuthority, false);
+assert.equal(contract.bootstrap.exactShaExecutionAuthority, true);
+assert.equal(contract.bootstrap.consumerSourceMutationPerIncident, false);
+assert.equal(contract.bootstrap.buildchainReleasePromotionPerIncident, false);
+assert.equal(contract.bootstrap.selfDogfoodUsesPublicContract, true);
+assert.equal(contract.bootstrap.recoveryDependsOnPublishedBuildchain, false);
+assert.equal(
+  contract.bootstrap.reviewRuntimeBinding,
+  "reviewed-head-or-protected-alpha-merge-to-exact-runtime",
+);
+assert.deepEqual(
+  contract.capabilityAdapters.map(({ id }) => id).sort(),
+  [...admissionPolicy.allowedCapabilities].sort(),
+  "every admitted capability must have one real candidate adapter",
+);
+for (const adapter of contract.capabilityAdapters) {
+  assert.match(adapter.execution, /^exact-candidate-/u);
+  assert.ok(adapter.payloadSchema.endsWith("/v1"));
+}
+assert.ok(!admissionPolicy.allowedCapabilities.includes("workflow-contract"));
+assert.deepEqual(
+  admissionPolicy.contractRoots,
+  [
+    fileRoot("architecture/v4-universal-workflow-bootstrap.json"),
+    fileRoot(contract.bootstrap.faultCampaign),
+    fileRoot("packages/core/v4-universal-workflow-bootstrap.js"),
+    fileRoot("scripts/v4-universal-workflow-backflow.mjs"),
+    fileRoot("scripts/v4-universal-workflow-engine.mjs"),
+    fileRoot("scripts/v4-universal-workflow-self-dogfood.mjs"),
+    fileRoot(contract.bootstrap.publicWorkflow),
+  ].sort(),
+);
+assert.doesNotMatch(
+  fs.readFileSync(
+    path.join(root, contract.bootstrap.candidateEnginePath),
+    "utf8",
+  ),
+  /request\.capability\.id\s*===\s*["']workflow-contract["']/u,
+  "contract-only validation must not be admitted as successful execution",
+);
+const bootstrapSource = fs.readFileSync(
+  path.join(root, contract.bootstrap.publicWorkflow),
+  "utf8",
+);
+assert.doesNotMatch(
+  bootstrapSource,
+  /admission-policy-json:/u,
+  "callers must not supply their own admission authority",
+);
+execFileSync(
+  process.execPath,
+  [path.join(root, contract.migration.compatibilityFacadeGenerator), "--check"],
+  { stdio: "inherit" },
+);
+assert.doesNotMatch(bootstrapSource, /^    permissions:/mu);
+assert.match(bootstrapSource, /settle:[\s\S]*needs: \[admit, execute\]/u);
+assert.match(
+  bootstrapSource,
+  /ascii_downcase\) == "kungfu-origin"[\s\S]*\.name == "check"[\s\S]*map\(select\(\.status == "completed" and \(\.conclusion \/\/ ""\) != ""\)\)[\s\S]*candidate\/architecture\/v4-universal-workflow-train-admission\.json/u,
+);
+assert.match(
+  bootstrapSource,
+  /CANDIDATE_MODE:[\s\S]*\.merged == true[\s\S]*\.merge_commit_sha == \$expected[\s\S]*\.base\.ref == "alpha\/v4\/v4\.0"[\s\S]*commitSha: \$expected[\s\S]*runtimeBinding:/u,
+);
+const consumerTemplate = fs.readFileSync(
+  path.join(root, contract.bootstrap.consumerTemplate),
+  "utf8",
+);
+const consumerRecoveryTemplate = fs.readFileSync(
+  path.join(root, contract.bootstrap.consumerRecoveryTemplate),
+  "utf8",
+);
+const consumerRecoveryWorkflow = fs.readFileSync(
+  path.join(root, contract.bootstrap.consumerRecoveryWorkflow),
+  "utf8",
+);
+const selfDogfood = fs.readFileSync(
+  path.join(root, contract.bootstrap.selfDogfoodWorkflow),
+  "utf8",
+);
+assert.match(
+  consumerTemplate,
+  /uses:\s+kungfu-systems\/buildchain\/\.github\/workflows\/bootstrap\.yml@v4/u,
+);
+assert.match(
+  consumerRecoveryTemplate,
+  /recovery-admit:[\s\S]*recovery-execute:/u,
+);
+assert.match(
+  consumerRecoveryTemplate,
+  /Parse exact recovery coordinates without Buildchain code[\s\S]*Prove exact independent review before candidate code runs[\s\S]*ascii_downcase\) == "kungfu-origin"[\s\S]*\.name == "check"[\s\S]*Admit candidate policy and exact-head checks[\s\S]*map\(select\(\.status == "completed" and \(\.conclusion \/\/ ""\) != ""\)\)/u,
+);
+assert.match(
+  consumerRecoveryTemplate,
+  /recovery-execute:[\s\S]*needs: recovery-admit[\s\S]*contents: write/u,
+);
+assert.doesNotMatch(
+  consumerRecoveryTemplate,
+  /uses:\s+kungfu-systems\/buildchain\/\.github\/workflows\//u,
+  "consumer recovery must not parse any published Buildchain workflow",
+);
+assert.equal(
+  consumerRecoveryWorkflow,
+  consumerRecoveryTemplate,
+  "the Buildchain self-consumer must exercise the exact distributable recovery shell",
+);
+assert.match(
+  selfDogfood,
+  /uses:\s+kungfu-systems\/buildchain\/\.github\/workflows\/bootstrap\.yml@train\/v4\/v4\.0\/universal-reusable-workflow-bootstrap/u,
+);
+assert.match(
+  selfDogfood,
+  /uses:\s+\.\/\.github\/workflows\/universal-bootstrap-recovery\.yml/u,
+);
+assert.doesNotMatch(
+  selfDogfood,
+  /\.github\/workflows\/bootstrap\.yml@(?:v4|v4-alpha)(?:\s|$)/u,
+  "Train self-dogfood must not depend on an alpha or stable Bootstrap tag",
+);
+for (const channel of ["conformance", "alpha", "stable"]) {
+  assert.match(selfDogfood, new RegExp(`primary-${channel}:`, "u"));
+  assert.match(selfDogfood, new RegExp(`recovery-${channel}:`, "u"));
+}
+assert.match(
+  selfDogfood,
+  /\.state == "APPROVED"[\s\S]*ascii_downcase\) == "kungfu-origin"[\s\S]*\.name == "check"[\s\S]*\.conclusion == "success"/u,
+);
+assert.match(
+  consumerRecoveryTemplate,
+  /result-json:[\s\S]*value:\s*\$\{\{ jobs\.recovery-execute\.outputs\.result-json \}\}/u,
+);
+
+console.log(
+  JSON.stringify({
+    ok: true,
+    schema: contract.schema,
+    publicWorkflowCount: discovered.length,
+    activePublicWorkflowCount: activeWorkflows.length,
+    retiredWorkflowCount: contract.retiredWorkflowSurfaces.length,
+    inventoriedWorkflowCount: contract.inventoryWorkflows.length,
+    inventoryCoveragePercent: 100,
+    governedWorkflowCount: contract.bootstrapGovernedWorkflows.length,
+    governedCoveragePercent: Number(
+      (
+        (contract.bootstrapGovernedWorkflows.length / activeWorkflows.length) *
+        100
+      ).toFixed(2),
+    ),
+    migration: contract.migration,
+  }),
+);

package/scripts/dev-delivery-warrant-store.mjs

@@ -7,6 +7,7 @@ import {
 import { normalizeDevDeliveryAuthorityState } from "../packages/core/dev-delivery-authority-state.js";
 
 const STATE_PATH = "queue.json";
+const READ_ATTEMPTS = 3;
 
 function text(value = "") {
   return String(value ?? "").trim();
@@ -20,6 +21,14 @@ function exactSha(value, label) {
   return normalized;
 }
 
+function positiveInteger(value, label) {
+  const parsed = Number(value);
+  if (!Number.isInteger(parsed) || parsed < 1) {
+    throw new Error(`${label} must be a positive integer`);
+  }
+  return parsed;
+}
+
 function normalizeRepository(value) {
   const normalized = text(value);
   const match = normalized.match(/^([^/\s]+)\/([^/\s]+)$/);
@@ -35,11 +44,10 @@ function encodeRef(ref) {
   return ref.split("/").map(encodeURIComponent).join("/");
 }
 
-function decodeBlob(blob) {
-  if (blob.encoding !== "base64")
-    throw new Error(
-      `unsupported Git blob encoding ${blob.encoding || "<empty>"}`,
-    );
+function decodeGitHubBlobEnvelope(raw) {
+  if (!/"encoding"\s*:\s*"base64"/.test(raw)) return raw;
+  const blob = JSON.parse(raw);
+  if (blob?.encoding !== "base64") return raw;
   return Buffer.from(
     String(blob.content || "").replace(/\s+/g, ""),
     "base64",
@@ -71,6 +79,7 @@ export class GitHubDevDeliveryStore {
     apiUrl = "https://api.github.com",
     fetchImpl = globalThis.fetch,
     createInitialState = createDevDeliveryQueue,
+    readAttempts = READ_ATTEMPTS,
   } = {}) {
     this.repository = normalizeRepository(repository);
     if (!fetchImpl) throw new Error("fetch is required");
@@ -82,6 +91,22 @@ export class GitHubDevDeliveryStore {
     this.apiUrl = apiUrl.replace(/\/+$/, "");
     this.fetch = fetchImpl;
     this.createInitialState = createInitialState;
+    this.readAttempts = positiveInteger(readAttempts, "readAttempts");
+  }
+
+  async retryParseableRead(operation) {
+    let failure;
+    for (let attempt = 1; attempt <= this.readAttempts; attempt += 1) {
+      try {
+        return await operation();
+      } catch (error) {
+        failure = error;
+        if (!(error instanceof SyntaxError) || attempt === this.readAttempts) {
+          throw error;
+        }
+      }
+    }
+    throw failure;
   }
 
   async request(method, requestPath, body) {
@@ -108,7 +133,42 @@ export class GitHubDevDeliveryStore {
     return data;
   }
 
+  async requestRaw(method, requestPath) {
+    const response = await this.fetch(`${this.apiUrl}${requestPath}`, {
+      method,
+      headers: {
+        accept: "application/vnd.github.raw+json",
+        authorization: `Bearer ${this.token}`,
+        "x-github-api-version": "2022-11-28",
+      },
+    });
+    const raw = await response.text();
+    if (!response.ok) {
+      let message = raw || `${method} ${requestPath} failed`;
+      try {
+        message = JSON.parse(raw)?.message || message;
+      } catch {
+        // Raw blob responses are not required to be JSON.
+      }
+      const error = new Error(message);
+      error.status = response.status;
+      throw error;
+    }
+    return raw;
+  }
+
   async read({ stateRef, protectedBase, now, allowLegacyV3Readback = false }) {
+    return this.retryParseableRead(async () =>
+      this.readOnce({ stateRef, protectedBase, now, allowLegacyV3Readback }),
+    );
+  }
+
+  async readOnce({
+    stateRef,
+    protectedBase,
+    now,
+    allowLegacyV3Readback = false,
+  }) {
     let ref;
     try {
       ref = await this.request(
@@ -135,6 +195,12 @@ export class GitHubDevDeliveryStore {
   }
 
   async readCommit(commitShaInput, { allowLegacyV3Readback = false } = {}) {
+    return this.retryParseableRead(async () =>
+      this.readCommitOnce(commitShaInput, { allowLegacyV3Readback }),
+    );
+  }
+
+  async readCommitOnce(commitShaInput, { allowLegacyV3Readback = false } = {}) {
     const commitSha = exactSha(commitShaInput, "state commit");
     const commit = await this.request(
       "GET",
@@ -149,11 +215,11 @@ export class GitHubDevDeliveryStore {
     );
     if (!entry?.sha)
       throw new Error(`${commitSha} does not contain ${STATE_PATH}`);
-    const blob = await this.request(
+    const raw = await this.requestRaw(
       "GET",
       `/repos/${this.repository.fullName}/git/blobs/${entry.sha}`,
     );
-    const bytes = decodeBlob(blob);
+    const bytes = decodeGitHubBlobEnvelope(raw);
     const queue = validateStoredState(JSON.parse(bytes), {
       allowLegacyV3Readback,
     });

package/scripts/dev-delivery-warrant-transition.mjs

@@ -0,0 +1,109 @@
+import { recoverExpiredDevDeliveryWarrant } from "../packages/core/dev-delivery-warrant.js";
+
+async function recoverExpiredLeaseBeforeReselection({
+  store,
+  options,
+  loaded,
+  initialLoaded,
+}) {
+  const recovery = recoverExpiredDevDeliveryWarrant(loaded.queue, {
+    now: options.now,
+  });
+  if (recovery.receipt.expectedOldStateRoot !== loaded.queue.stateRoot) {
+    throw new Error(
+      "expired Warrant recovery receipt expected-old root does not match the loaded authority",
+    );
+  }
+  await store.write({
+    stateRef: options.stateRef,
+    queue: recovery.queue,
+    expectedCommitSha: loaded.commitSha,
+    expectedStateRoot: loaded.queue.stateRoot,
+    receiptRoot: recovery.receiptRoot,
+  });
+  const reloaded = await store.read({
+    stateRef: options.stateRef,
+    protectedBase: options.branch,
+    now: options.now,
+  });
+  return {
+    loaded: reloaded,
+    concurrencyRecovery: {
+      schema: "kungfu.buildchain.dev-delivery-concurrency-recovery/v1",
+      action: "expired-lease-recovered-before-reselection",
+      initialCommitSha: initialLoaded.commitSha,
+      recoveredStateRoot: recovery.queue.stateRoot,
+      observedCommitSha: reloaded.commitSha,
+      observedStateRoot: reloaded.queue.stateRoot,
+    },
+  };
+}
+
+export async function persistDevDeliveryTransition({
+  store,
+  options,
+  loaded,
+  initialLoaded,
+  changed,
+  transitionFor,
+}) {
+  let concurrencyRecovery = null;
+  if (
+    options.execute &&
+    options.command === "select" &&
+    changed.recoveryReceipt?.action === "recovered-expired-lease"
+  ) {
+    const recovery = await recoverExpiredLeaseBeforeReselection({
+      store,
+      options,
+      loaded,
+      initialLoaded,
+    });
+    loaded = recovery.loaded;
+    changed = transitionFor(options.command, loaded.queue, options);
+    concurrencyRecovery = recovery.concurrencyRecovery;
+  }
+  if (!options.execute || changed.queue.stateRoot === loaded.queue.stateRoot)
+    return { loaded, changed, write: null, concurrencyRecovery };
+  if (changed.receipt.expectedOldStateRoot !== loaded.queue.stateRoot) {
+    throw new Error(
+      "transition receipt expected-old root does not match the loaded authority",
+    );
+  }
+  try {
+    const write = await store.write({
+      stateRef: options.stateRef,
+      queue: changed.queue,
+      expectedCommitSha: loaded.commitSha,
+      expectedStateRoot: loaded.queue.stateRoot,
+      receiptRoot: changed.receiptRoot,
+    });
+    return { loaded, changed, write, concurrencyRecovery };
+  } catch (error) {
+    if (options.command !== "settle" || options.expectedOldStateRoot)
+      throw error;
+    const latest = await store.read({
+      stateRef: options.stateRef,
+      protectedBase: options.branch,
+      now: options.now,
+    });
+    const reconciled = transitionFor(options.command, latest.queue, options);
+    if (
+      reconciled.queue.stateRoot !== latest.queue.stateRoot ||
+      reconciled.receipt.action !== "duplicate-terminal-event-noop"
+    )
+      throw error;
+    return {
+      loaded: latest,
+      changed: reconciled,
+      write: null,
+      concurrencyRecovery: {
+        schema: "kungfu.buildchain.dev-delivery-concurrency-recovery/v1",
+        action: "terminal-settlement-race-noop",
+        initialCommitSha: initialLoaded.commitSha,
+        observedCommitSha: latest.commitSha,
+        observedStateRoot: latest.queue.stateRoot,
+      },
+    };
+  }
+}