@theagilemonkeys/facility 0.3.1

package/templates/watchtower/canary.mjs

@@ -0,0 +1,216 @@
+#!/usr/bin/env node
+// Generated by facility — https://github.com/theam/facility
+//
+// Weekly synthetic canary. Flies the REAL pipeline: posts the canonical
+// /architect probe on the pinned canary issue, then asserts the crew run
+// appears, succeeds, and replies. Any broken link turns this workflow red,
+// and the health monitor (which watches facility-canary) raises it in the
+// incident issue the next morning.
+//
+// Authorization is by MESSAGE, not by sender: the crew workflow admits the
+// canary bot only for a comment that is byte-identical (SHA-256, CR-stripped)
+// to CANARY_PROBE_BODY below, on an agent-canary-labeled issue, resolving to
+// /architect. Within the crew trigger that blocks attacker-chosen
+// instructions, but the hash is an instruction-content gate, not a replay or
+// cost boundary: the token can post the fixed probe repeatedly, and it can
+// exercise any other GitHub permissions it carries. The generated workflow
+// narrows its minted token to Issues: write. Keep the backing App dedicated to
+// canary repositories with Issues: read and write as its only requested
+// repository permission, because its private key can mint tokens up to the
+// App's installed permissions. The pinned hash in facility-crew.yml is kept in
+// sync by the watchtower-locked guard.
+//
+// The probe must be posted with a GitHub App or PAT token (CANARY_COMMENT_TOKEN):
+// comments posted with the workflow's own GITHUB_TOKEN trigger no workflows
+// at all (GitHub's recursion guard) — the canary would test nothing.
+// Env: GH_TOKEN, GITHUB_REPOSITORY, CANARY_COMMENT_TOKEN (App/PAT).
+import { execFileSync } from "node:child_process";
+import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { verifyReceipt } from "../receipts/collect.mjs";
+
+export const CANARY_PROBE_BODY = [
+  "/architect",
+  "",
+  "Facility canary probe. This is a scheduled synthetic check of the agent",
+  "pipeline, authorized by message hash. Reply with a one-paragraph health",
+  "summary of this repository's facility setup (crew workflows present,",
+  "STANDARD.md readable, guards runnable) and stop. Do not plan features, do",
+  "not read beyond what the summary needs, and do not modify anything.",
+].join("\n");
+
+const CREW_WORKFLOW = "facility-crew";
+const POLL_SECONDS = 30;
+const TIMEOUT_MINUTES = 45;
+
+async function main() {
+  const repo = process.env.GITHUB_REPOSITORY;
+  if (!repo) throw new Error("GITHUB_REPOSITORY is required");
+  const token = process.env.CANARY_COMMENT_TOKEN;
+  if (!token) {
+    console.log(
+      "::notice::CANARY_COMMENT_TOKEN not configured — canary skipped. Mint it from a GitHub App so the probe comment triggers workflows.",
+    );
+    return;
+  }
+
+  const gh = (args) => execFileSync("gh", args, { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 });
+  const api = async (path, init = {}) => {
+    const response = await fetch(`https://api.github.com/repos/${repo}/${path}`, {
+      ...init,
+      headers: {
+        authorization: `Bearer ${token}`,
+        accept: "application/vnd.github+json",
+        ...(init.headers ?? {}),
+      },
+    });
+    if (!response.ok) throw new Error(`${path} → ${response.status}: ${await response.text()}`);
+    return response.json();
+  };
+
+  // 1. Find (or create) the pinned canary fixture issue.
+  try {
+    gh([
+      "label",
+      "create",
+      "agent-canary",
+      "--force",
+      "--color",
+      "FFD923",
+      "--description",
+      "facility canary fixture",
+    ]);
+  } catch {}
+  const issues = JSON.parse(
+    gh([
+      "issue",
+      "list",
+      "--label",
+      "agent-canary",
+      "--state",
+      "open",
+      "--json",
+      "number",
+      "--limit",
+      "1",
+    ]),
+  );
+  let issueNumber = issues[0]?.number;
+  if (!issueNumber) {
+    const createdUrl = gh([
+      "issue",
+      "create",
+      "--title",
+      "Facility canary",
+      "--label",
+      "agent-canary",
+      "--body",
+      "Pinned fixture issue for the weekly synthetic canary. The probe comments below are posted by the canary workflow; do not comment agent commands here yourself.",
+    ]).trim();
+    issueNumber = Number(createdUrl.split("/").pop());
+    if (!issueNumber) throw new Error(`could not parse issue number from: ${createdUrl}`);
+  }
+
+  // 2. Post the canonical probe with the App token (so it triggers workflows).
+  const startedAt = new Date().toISOString();
+  await api(`issues/${issueNumber}/comments`, {
+    method: "POST",
+    body: JSON.stringify({ body: CANARY_PROBE_BODY }),
+  });
+  console.log(`probe posted on #${issueNumber} at ${startedAt}`);
+
+  // 3. Assert the crew run appears and succeeds.
+  const deadline = Date.now() + TIMEOUT_MINUTES * 60_000;
+  let run = null;
+  while (Date.now() < deadline) {
+    const { workflow_runs: runs } = JSON.parse(
+      gh([
+        "api",
+        `repos/${repo}/actions/runs?event=issue_comment&created=>${startedAt}&per_page=20`,
+      ]),
+    );
+    run = runs.find((r) => r.name === CREW_WORKFLOW) ?? null;
+    if (run && run.status === "completed") break;
+    await new Promise((resolve) => setTimeout(resolve, POLL_SECONDS * 1000));
+  }
+  if (!run)
+    throw new Error(
+      `no ${CREW_WORKFLOW} run appeared within ${TIMEOUT_MINUTES} minutes — the trigger chain is broken`,
+    );
+  if (run.conclusion !== "success")
+    throw new Error(`crew run concluded '${run.conclusion}': ${run.html_url}`);
+
+  // 4. Download and verify the receipt produced by THIS exact crew run. A
+  // green workflow conclusion without its signed evidence is not a green
+  // agent flight.
+  const artifactName = `facility-run-receipt-${run.id}-crew`;
+  const receiptDir = mkdtempSync(join(tmpdir(), "facility-canary-receipt-"));
+  let receipt;
+  let receiptPath;
+  try {
+    gh([
+      "run",
+      "download",
+      String(run.id),
+      "--repo",
+      repo,
+      "--name",
+      artifactName,
+      "--dir",
+      receiptDir,
+    ]);
+    receiptPath = findFile(receiptDir, "facility-run.json");
+    if (!receiptPath) throw new Error(`${artifactName} did not contain facility-run.json`);
+    receipt = JSON.parse(readFileSync(receiptPath, "utf8"));
+    if (!verifyReceipt(receipt))
+      throw new Error(`${artifactName} failed its SHA-256 integrity check`);
+    if (
+      receipt.provider !== "claude_code" ||
+      receipt.mode !== "architect" ||
+      receipt.result !== "succeeded" ||
+      receipt.github?.issue !== issueNumber ||
+      !String(receipt.run_id ?? "").startsWith(`${run.id}:`)
+    ) {
+      throw new Error(`${artifactName} does not describe this architect canary run`);
+    }
+    gh(["attestation", "verify", receiptPath, "--repo", repo]);
+  } finally {
+    rmSync(receiptDir, { recursive: true, force: true });
+  }
+
+  // 5. Assert a reply landed on the issue after the probe.
+  const comments = JSON.parse(
+    gh(["api", `repos/${repo}/issues/${issueNumber}/comments?per_page=100`]),
+  );
+  const reply = comments.find((c) => c.created_at > startedAt && c.body !== CANARY_PROBE_BODY);
+  if (!reply)
+    throw new Error(
+      `crew run succeeded but no reply landed on #${issueNumber} — the reply chain is broken`,
+    );
+
+  console.log(
+    `canary green: run ${run.html_url} · receipt ${receipt.integrity.payload_sha256} (attested) · reply ${reply.html_url}`,
+  );
+}
+
+function findFile(dir, name) {
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    const path = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      const nested = findFile(path, name);
+      if (nested) return nested;
+    } else if (entry.name === name) {
+      return path;
+    }
+  }
+  return null;
+}
+
+// Import-safe: the watchtower-locked guard imports CANARY_PROBE_BODY without running the probe.
+if (process.argv[1] && import.meta.url.endsWith(process.argv[1].split("/").pop())) {
+  main().catch((error) => {
+    console.error(`::error::${error.message}`);
+    process.exit(1);
+  });
+}

package/templates/watchtower/health.mjs

@@ -0,0 +1,148 @@
+#!/usr/bin/env node
+// Generated by facility — https://github.com/theam/facility
+//
+// Daily health monitor. Watches the watchmen: pulls the last 24h of facility
+// workflow runs straight from the GitHub API (deliberately NOT from any
+// telemetry pipeline the facility itself writes — the monitor must not
+// depend on what it monitors), checks failure streaks and run budgets, and
+// maintains a single incident issue. The monitor run itself goes RED when
+// the facility is unhealthy, so the Actions tab is the at-a-glance signal.
+//
+// This workflow is intentionally NOT in its own watchlist (no self-recursion).
+// Budgets live next to this file in budgets.json — a reviewed file, not a
+// dashboard setting.
+// Env: GH_TOKEN (rw issues), GITHUB_REPOSITORY.
+import { execFileSync } from "node:child_process";
+import { readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const repo = process.env.GITHUB_REPOSITORY;
+if (!repo) throw new Error("GITHUB_REPOSITORY is required");
+const here = dirname(fileURLToPath(import.meta.url));
+
+const WATCHLIST = [
+  "facility-crew",
+  "facility-codex",
+  "facility-review",
+  "facility-address-review",
+  "facility-doctor",
+  "facility-security-sweep",
+  "facility-canary",
+];
+
+let budgets = { maxDailyFailures: { "*": 3 }, maxWeeklyRuns: {} };
+try {
+  budgets = { ...budgets, ...JSON.parse(readFileSync(join(here, "budgets.json"), "utf8")) };
+} catch {}
+const budgetFor = (map, name, fallback) => map?.[name] ?? map?.["*"] ?? fallback;
+
+const gh = (args) => execFileSync("gh", args, { encoding: "utf8", maxBuffer: 32 * 1024 * 1024 });
+const iso = (hoursAgo) => new Date(Date.now() - hoursAgo * 3600_000).toISOString();
+const runsSince = (sinceIso) =>
+  JSON.parse(
+    gh(["api", `repos/${repo}/actions/runs?created=>${sinceIso}&per_page=100`]),
+  ).workflow_runs.filter((r) => WATCHLIST.includes(r.name));
+
+const day = runsSince(iso(24));
+const week = runsSince(iso(24 * 7));
+
+const problems = [];
+const advisories = [];
+const rows = [];
+for (const name of WATCHLIST) {
+  const daily = day.filter((r) => r.name === name);
+  const failures = daily.filter((r) =>
+    ["failure", "startup_failure", "timed_out"].includes(r.conclusion ?? ""),
+  );
+  const weeklyCount = week.filter((r) => r.name === name).length;
+  const maxFail = budgetFor(budgets.maxDailyFailures, name, 3);
+  const maxWeekly = budgetFor(budgets.maxWeeklyRuns, name, Infinity);
+  const latest = daily[0] ?? week.filter((r) => r.name === name)[0];
+  const consecutiveFailures = (
+    daily.length ? daily : week.filter((r) => r.name === name)
+  ).findIndex((run) => !["failure", "startup_failure", "timed_out"].includes(run.conclusion ?? ""));
+  const failureStreak =
+    consecutiveFailures === -1
+      ? (daily.length ? daily : week.filter((r) => r.name === name)).length
+      : consecutiveFailures;
+  const classification =
+    weeklyCount === 0
+      ? "unknown"
+      : failures.length >= maxFail || weeklyCount > maxWeekly || failureStreak >= 2
+        ? "unhealthy"
+        : failures.length > 0 ||
+            ["failure", "startup_failure", "timed_out"].includes(latest?.conclusion ?? "")
+          ? "degraded"
+          : "healthy";
+  if (failures.length >= maxFail || failureStreak >= 2) {
+    problems.push(
+      `**${name}**: ${failures.length} failures in 24h (budget ${maxFail}), streak ${failureStreak} — latest: ${failures[0]?.html_url ?? latest?.html_url ?? "unknown"}`,
+    );
+  } else if (classification === "degraded") {
+    advisories.push(`**${name}**: a recent run failed, below the incident threshold.`);
+  }
+  if (weeklyCount > maxWeekly) {
+    problems.push(
+      `**${name}**: ${weeklyCount} runs this week (budget ${maxWeekly}) — runaway trigger or runaway spend?`,
+    );
+  }
+  rows.push(
+    `| ${name} | ${classification} | ${daily.length} | ${failures.length} | ${weeklyCount} |`,
+  );
+}
+
+const MARKER = "facility-health";
+try {
+  gh([
+    "label",
+    "create",
+    MARKER,
+    "--force",
+    "--color",
+    "D1242F",
+    "--description",
+    "facility health incident",
+  ]);
+} catch {}
+const openIncidents = JSON.parse(
+  gh(["issue", "list", "--label", MARKER, "--state", "open", "--json", "number", "--limit", "1"]),
+);
+
+const report = [
+  `Health check ${new Date().toISOString()} — last 24h / 7d, from the GitHub API.`,
+  "",
+  "| workflow | health | runs 24h | failures 24h | runs 7d |",
+  "|---|---|---|---|---|",
+  ...rows,
+  "",
+  problems.length
+    ? `### Problems\n${problems.map((p) => `- ${p}`).join("\n")}`
+    : "All watched workflows within budget.",
+  advisories.length ? `### Degraded\n${advisories.map((p) => `- ${p}`).join("\n")}` : "",
+].join("\n");
+
+if (problems.length) {
+  if (openIncidents.length === 0) {
+    gh([
+      "issue",
+      "create",
+      "--title",
+      "Facility health incident",
+      "--label",
+      MARKER,
+      "--body",
+      report,
+    ]);
+  } else {
+    gh(["issue", "comment", String(openIncidents[0].number), "--body", report]);
+  }
+  console.error(report);
+  process.exit(1); // red run = the at-a-glance signal
+}
+
+if (openIncidents.length > 0) {
+  gh(["issue", "comment", String(openIncidents[0].number), "--body", `Recovered.\n\n${report}`]);
+  gh(["issue", "close", String(openIncidents[0].number)]);
+}
+console.log(report);

package/templates/watchtower/outcomes.mjs

@@ -0,0 +1,188 @@
+#!/usr/bin/env node
+// Generated by facility — https://github.com/theam/facility
+//
+// Nightly outcome collector. Receipts measure what a run consumed; outcomes
+// measure whether the work was ACCEPTED. This joins every agent PR that
+// reached a terminal state in the lookback window with acceptance evidence:
+// human merger + enforced squash-only merge policy, linked issue lead time,
+// review rounds, and human fixup commits. Missing evidence is reported as
+// unassessed rather than guessed.
+//
+// Privacy: numbers, enums, and PR numbers only. Never titles, bodies, diffs.
+// Env: GH_TOKEN (read-only), GITHUB_REPOSITORY, WINDOW_HOURS (26),
+//      PR_LIMIT (100), OUTPUT_DIR (optional artifact dir),
+//      WATCHTOWER_WEBHOOK_URL (optional JSON sink).
+import { execFileSync } from "node:child_process";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+const repo = process.env.GITHUB_REPOSITORY;
+if (!repo) throw new Error("GITHUB_REPOSITORY is required");
+const windowHours = Number(process.env.WINDOW_HOURS || 26);
+const prLimit = Number(process.env.PR_LIMIT || 100);
+const since = Date.now() - windowHours * 3600_000;
+
+const gh = (args) => execFileSync("gh", args, { encoding: "utf8", maxBuffer: 32 * 1024 * 1024 });
+const api = (path) => JSON.parse(gh(["api", path]));
+const graphql = (query, fields) =>
+  JSON.parse(
+    gh([
+      "api",
+      "graphql",
+      "-f",
+      `query=${query}`,
+      ...Object.entries(fields).flatMap(([key, value]) => [
+        typeof value === "number" ? "-F" : "-f",
+        `${key}=${value}`,
+      ]),
+    ]),
+  ).data;
+const isBot = (login) => typeof login === "string" && login.endsWith("[bot]");
+const AGENT_BRANCH_PREFIXES = ["claude/", "codex/", "copilot/"];
+
+const closed = api(
+  `repos/${repo}/pulls?state=closed&sort=updated&direction=desc&per_page=${Math.min(prLimit, 100)}`,
+);
+const terminal = closed.filter((pr) => Date.parse(pr.closed_at) >= since);
+const [owner, name] = repo.split("/");
+
+const EVIDENCE_QUERY = `query FacilityOutcomeEvidence($owner: String!, $name: String!, $number: Int!) {
+  repository(owner: $owner, name: $name) {
+    mergeCommitAllowed
+    rebaseMergeAllowed
+    squashMergeAllowed
+    pullRequest(number: $number) {
+      mergedBy { login __typename }
+      mergeCommit { parents(first: 2) { totalCount } }
+      closingIssuesReferences(
+        first: 1
+        orderBy: { field: CREATED_AT, direction: ASC }
+      ) { nodes { number createdAt } }
+    }
+  }
+}`;
+
+function acceptanceEvidence(pr) {
+  if (!pr.merged_at) return { accepted: false, mergeMethod: null, issue: null };
+  try {
+    const repository = graphql(EVIDENCE_QUERY, { owner, name, number: pr.number }).repository;
+    const parents = repository.pullRequest?.mergeCommit?.parents.totalCount ?? null;
+    let mergeMethod = "unverified";
+    if (parents >= 2) {
+      mergeMethod = "merge";
+    } else if (parents === 1) {
+      if (repository.squashMergeAllowed && !repository.rebaseMergeAllowed) mergeMethod = "squash";
+      if (repository.rebaseMergeAllowed && !repository.squashMergeAllowed) mergeMethod = "rebase";
+    } else {
+      const allowed = [
+        repository.mergeCommitAllowed ? "merge" : null,
+        repository.rebaseMergeAllowed ? "rebase" : null,
+        repository.squashMergeAllowed ? "squash" : null,
+      ].filter(Boolean);
+      mergeMethod = allowed.length === 1 ? allowed[0] : "unverified";
+    }
+    const mergedBy = repository.pullRequest?.mergedBy ?? null;
+    const accepted =
+      mergeMethod === "unverified" || !mergedBy
+        ? null
+        : mergeMethod === "squash" && mergedBy.__typename === "User";
+    const issue = repository.pullRequest?.closingIssuesReferences.nodes?.[0] ?? null;
+    return { accepted, mergeMethod, mergedBy, issue };
+  } catch {
+    return {
+      accepted: null,
+      mergeMethod: "unverified",
+      mergedBy: null,
+      issue: null,
+      evidenceError: true,
+    };
+  }
+}
+
+function isAgentPr(pr, commits) {
+  if (isBot(pr.user?.login)) return true;
+  if (AGENT_BRANCH_PREFIXES.some((p) => pr.head?.ref?.startsWith(p))) return true;
+  // A bot-built branch where a human opened the PR: first commit is agent-authored.
+  return commits.length > 0 && isBot(commits[0]?.author?.login);
+}
+
+const outcomes = [];
+for (const pr of terminal) {
+  const commits = api(`repos/${repo}/pulls/${pr.number}/commits?per_page=100`);
+  if (!isAgentPr(pr, commits)) continue;
+  const reviews = api(`repos/${repo}/pulls/${pr.number}/reviews?per_page=100`);
+  const firstBotIndex = commits.findIndex((c) => isBot(c.author?.login));
+  const humanFixups =
+    firstBotIndex === -1
+      ? 0
+      : commits.slice(firstBotIndex + 1).filter((c) => c.author?.login && !isBot(c.author.login))
+          .length;
+  const reviewRounds = reviews.filter((r) => r.state === "CHANGES_REQUESTED").length;
+  const merged = Boolean(pr.merged_at);
+  const evidence = acceptanceEvidence(pr);
+  outcomes.push({
+    pr: pr.number,
+    lane: pr.head?.ref?.split("/")[0] ?? "unknown",
+    merged,
+    accepted: evidence.accepted,
+    assessed: evidence.accepted !== null,
+    mergeMethod: evidence.mergeMethod,
+    mergedBy: evidence.mergedBy?.login ?? null,
+    mergerType: evidence.mergedBy?.__typename ?? null,
+    evidenceError: evidence.evidenceError ?? null,
+    issue: evidence.issue?.number ?? null,
+    hoursIssueToMerge:
+      evidence.issue &&
+      pr.merged_at &&
+      Date.parse(pr.merged_at) >= Date.parse(evidence.issue.createdAt)
+        ? Number(
+            ((Date.parse(pr.merged_at) - Date.parse(evidence.issue.createdAt)) / 3600_000).toFixed(
+              2,
+            ),
+          )
+        : null,
+    hoursToTerminal: Math.round((Date.parse(pr.closed_at) - Date.parse(pr.created_at)) / 3600_000),
+    reviewRounds,
+    humanFixups,
+    oneShot: Boolean(pr.merged_at) && reviewRounds === 0 && humanFixups === 0,
+  });
+}
+
+const mergedCount = outcomes.filter((o) => o.merged).length;
+const assessedCount = outcomes.filter((o) => o.assessed).length;
+const acceptedCount = outcomes.filter((o) => o.accepted).length;
+const summary = {
+  schema: "facility.watchtower.outcomes.v2",
+  collectedAt: new Date().toISOString(),
+  windowHours,
+  agentPrs: outcomes.length,
+  merged: mergedCount,
+  rejected: outcomes.length - mergedCount,
+  assessed: assessedCount,
+  accepted: acceptedCount,
+  notAccepted: assessedCount - acceptedCount,
+  unassessed: outcomes.length - assessedCount,
+  acceptance: assessedCount ? Math.round((100 * acceptedCount) / assessedCount) : null,
+  oneShot: outcomes.filter((o) => o.oneShot).length,
+  outcomes,
+};
+
+if (process.env.OUTPUT_DIR) {
+  mkdirSync(process.env.OUTPUT_DIR, { recursive: true });
+  writeFileSync(join(process.env.OUTPUT_DIR, "outcomes.json"), JSON.stringify(summary, null, 2));
+}
+if (process.env.WATCHTOWER_WEBHOOK_URL) {
+  await fetch(process.env.WATCHTOWER_WEBHOOK_URL, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify(summary),
+  }).catch((error) => console.error(`webhook sink failed (non-fatal): ${error.message}`));
+}
+
+console.log(
+  JSON.stringify({
+    agentPrs: summary.agentPrs,
+    acceptance: summary.acceptance,
+    oneShot: summary.oneShot,
+  }),
+);