@mmerterden/multi-agent-pipeline 12.6.0 → 12.7.0

package/pipeline/scripts/scorecard.mjs

@@ -0,0 +1,258 @@
+#!/usr/bin/env node
+// scorecard.mjs - the measurable part of the review score, as a gate.
+//
+// A refactor review scores ten categories out of ten. Six of those numbers can
+// be measured; four cannot, and writing a number for those is how a scorecard
+// turns into self-congratulation. So this reports exactly two things:
+//
+//   MEASURED    a metric with a threshold, pass or fail
+//   UNMEASURED  a category no script can score, named as such
+//
+// Nothing here re-implements a check. Each metric either reads a committed
+// threshold file or shells out to the gate that already owns the question, so
+// there is one source of truth per question and the scorecard cannot drift from
+// the gates it summarises.
+//
+// Thresholds are locked at the values measured when the gate was written. They
+// are a ratchet: lowering one is a deliberate, reviewable edit, and a metric
+// that improves should have its floor raised.
+//
+// Runs on a clean clone: no $HOME lookups, no credentials, no network. The one
+// metric that needs local configuration (derived-skill drift) reports
+// NOT CONFIGURED rather than passing by default.
+//
+// Usage:  node pipeline/scripts/scorecard.mjs [--json]
+// Exit:   0 every measured metric within threshold, 1 otherwise
+
+import { execFileSync } from "node:child_process";
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
+const asJson = process.argv.includes("--json");
+
+// Locked floors. Raise when a metric improves; lowering one is the reviewable act.
+const FLOORS = {
+  coverageLines: 72,
+  coverageBranches: 68,
+  coverageFunctions: 85,
+  contextBytes: 60000,
+};
+
+const results = [];
+
+function measured(category, metric, ok, detail) {
+  results.push({ kind: "MEASURED", category, metric, ok, detail });
+}
+function unmeasured(category, why) {
+  results.push({ kind: "UNMEASURED", category, why });
+}
+
+// Run a gate and report whether it passed. The gate owns the assertion; this
+// only records the verdict, so the two can never disagree.
+function gate(category, metric, argv) {
+  try {
+    execFileSync(argv[0], argv.slice(1), { cwd: ROOT, stdio: "pipe" });
+    return measured(category, metric, true, "gate passed");
+  } catch (e) {
+    const out = `${e.stdout || ""}${e.stderr || ""}`.trim().split("\n").slice(-3).join(" | ");
+    return measured(category, metric, false, out || `exit ${e.status}`);
+  }
+}
+
+// --- Tests: the coverage floors must not be lowered ------------------------
+// c8 enforces the numbers on every run; what nothing enforced was the numbers
+// themselves. .c8rc.json reported 83% while the real figure over its own include
+// set was 33%, because `all` was missing - so the floors are worth pinning.
+{
+  const c8 = JSON.parse(readFileSync(join(ROOT, ".c8rc.json"), "utf-8"));
+  const checks = [
+    ["lines", c8.lines, FLOORS.coverageLines],
+    ["branches", c8.branches, FLOORS.coverageBranches],
+    ["functions", c8.functions, FLOORS.coverageFunctions],
+  ];
+  const low = checks.filter(([, actual, floor]) => !(actual >= floor));
+  measured(
+    "Tests",
+    "coverage floors held",
+    c8.all === true && low.length === 0,
+    c8.all !== true
+      ? "`all` is not set, so the floors only cover files the tests already load"
+      : low.length
+        ? low.map(([k, a, f]) => `${k} ${a} below the locked ${f}`).join(", ")
+        : checks.map(([k, a]) => `${k} ${a}`).join(", "),
+  );
+}
+
+// --- Dependency management: no high or critical advisories ------------------
+{
+  let vulns;
+  try {
+    const raw = execFileSync("npm", ["audit", "--json"], { cwd: ROOT, encoding: "utf-8" });
+    vulns = JSON.parse(raw).metadata.vulnerabilities;
+  } catch (e) {
+    // npm audit exits non-zero when it finds something; the JSON is still on stdout.
+    try {
+      vulns = JSON.parse(e.stdout).metadata.vulnerabilities;
+    } catch {
+      vulns = null;
+    }
+  }
+  if (!vulns) {
+    measured(
+      "Dependency management",
+      "no high or critical advisories",
+      false,
+      "npm audit produced no parseable report",
+    );
+  } else {
+    const bad = vulns.high + vulns.critical;
+    measured(
+      "Dependency management",
+      "no high or critical advisories",
+      bad === 0,
+      `critical ${vulns.critical}, high ${vulns.high}, moderate ${vulns.moderate}`,
+    );
+  }
+}
+
+// --- Performance: the bytes every run pays before it starts ----------------
+gate("Performance", "fixed per-run context within budget", [
+  "bash",
+  "pipeline/scripts/smoke-context-budget.sh",
+]);
+
+// --- CI/CD: the workflow audit runs, and finds nothing ---------------------
+// Strict by default, so a runner without the linters fails here rather than
+// reporting a clean audit it never performed.
+gate("CI/CD", "workflow audit clean (zizmor + actionlint)", [
+  "bash",
+  "pipeline/scripts/smoke-workflow-audit.sh",
+]);
+
+// --- Code quality: our own punctuation rule, both encodings ---------------
+gate("Code quality", "no banned punctuation in shipped files", [
+  "bash",
+  "pipeline/scripts/smoke-own-punctuation.sh",
+]);
+
+// --- Documentation: every skill description routes ------------------------
+gate("Documentation", "every skill description has a routing clause", [
+  "node",
+  "pipeline/scripts/lint-skills.mjs",
+]);
+
+// --- Documentation: the licence and changelog a consumer expects ----------
+{
+  const missing = ["LICENSE", "CHANGELOG.md", "README.md", "SECURITY.md"].filter(
+    (f) => !existsSync(join(ROOT, f)),
+  );
+  measured(
+    "Documentation",
+    "licence, changelog, readme, security present",
+    missing.length === 0,
+    missing.length ? `missing: ${missing.join(", ")}` : "all four present",
+  );
+}
+
+// --- Tests: no suite reports success without asserting anything ------------
+// The runner owns this check; asking it for the counts would re-run 120 suites,
+// so the scorecard asserts the capability is wired rather than re-measuring it.
+{
+  const runner = readFileSync(join(ROOT, "pipeline", "scripts", "run-smokes.mjs"), "utf-8");
+  const wired = runner.includes("countAssertions") && /asserted nothing/.test(runner);
+  measured(
+    "Tests",
+    "runner rejects a suite that asserted nothing",
+    wired,
+    wired
+      ? "zero-assertion check present in run-smokes.mjs"
+      : "run-smokes.mjs no longer counts assertions",
+  );
+}
+
+// --- Drift: needs local configuration, so it reports honestly -------------
+{
+  // The exit code alone cannot answer this. check-derived-drift exits 0 both
+  // when every derivation is current AND when nothing is configured at all, and
+  // those are different facts: one is a pass, the other is an absence of
+  // evidence. Reading `configured` off the JSON is what tells them apart. The
+  // first version of this block trusted exit 0 and reported an unconfigured
+  // machine as a passing metric, which is the exact failure this whole gate set
+  // exists to prevent.
+  let state;
+  let ok = false;
+  let parsed;
+  try {
+    parsed = JSON.parse(
+      execFileSync("node", ["pipeline/scripts/check-derived-drift.mjs", "--json"], {
+        cwd: ROOT,
+        encoding: "utf-8",
+        stdio: ["ignore", "pipe", "ignore"],
+      }),
+    );
+  } catch (e) {
+    // Non-zero exit still writes the report to stdout; an unparseable one means
+    // the check itself could not run.
+    try {
+      parsed = JSON.parse(e.stdout);
+    } catch {
+      parsed = undefined;
+    }
+  }
+  if (parsed && parsed.configured > 0) {
+    const bad = parsed.drifted + parsed.unverified + parsed.unreachable;
+    ok = bad === 0;
+    state = ok
+      ? `${parsed.configured} derivation(s) current with upstream`
+      : `${parsed.drifted} drifted, ${parsed.unverified} unverified, ${parsed.unreachable} unreachable`;
+  } else if (parsed) {
+    state = "NOT CONFIGURED - prefs carry no derivedSkillSources on this machine";
+  } else {
+    state = "NOT CONFIGURED - the drift check produced no parseable report";
+  }
+  if (state.startsWith("NOT CONFIGURED")) {
+    unmeasured("Derived-skill drift", state);
+  } else {
+    measured("Architecture", "derived skills current with upstream", ok, state);
+  }
+}
+
+// --- What no script can score --------------------------------------------
+unmeasured("Architecture", "layer separation and dependency direction are judgement, not a number");
+unmeasured("DevEx", "onboarding time and error-message quality need a person");
+unmeasured(
+  "Accessibility",
+  "nothing in this repo renders UI; the audits it ships are scored per target app",
+);
+unmeasured(
+  "Code quality",
+  "naming, cohesion and altitude are judgement; only the mechanical rules above are gated",
+);
+
+const failed = results.filter((r) => r.kind === "MEASURED" && !r.ok);
+const passed = results.filter((r) => r.kind === "MEASURED" && r.ok);
+
+if (asJson) {
+  console.log(JSON.stringify({ passed: passed.length, failed: failed.length, results }, null, 2));
+} else {
+  console.log("MEASURED");
+  for (const r of results.filter((x) => x.kind === "MEASURED")) {
+    console.log(`  ${r.ok ? "✓" : "✗"} ${r.category}: ${r.metric}`);
+    console.log(`      ${r.detail}`);
+  }
+  console.log("");
+  console.log("UNMEASURED (reported, never scored)");
+  for (const r of results.filter((x) => x.kind === "UNMEASURED")) {
+    console.log(`  - ${r.category}: ${r.why}`);
+  }
+  console.log("");
+  console.log(`══ scorecard: ${passed.length} measured metrics passed, ${failed.length} failed ══`);
+  if (failed.length === 0) {
+    console.log("Every mechanically checkable claim holds. The four unmeasured");
+    console.log("categories above are not covered by that statement.");
+  }
+}
+
+process.exit(failed.length === 0 ? 0 : 1);

package/pipeline/scripts/smoke-context-budget.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+#
+# smoke-context-budget.sh  -  the bytes every pipeline run pays before it starts.
+#
+# core/multi-agent/SKILL.md plus multi-agent-refs/rules.md load on every run of
+# every mode, whatever the task. They were 67150 bytes together, roughly 17k
+# tokens spent before the first phase, because the orchestrator skill had grown
+# its own inline copies of things that already existed as loadable references -
+# the log-file template (whose copy had gone stale and named a path the code no
+# longer uses), the help text the multi-agent-help skill owns, and a component
+# generation guide that only applies to design tasks.
+#
+# A budget nobody measures only moves one way, so this pins it. The ceiling is
+# the measured value plus a small margin: enough to add a paragraph, not enough
+# to move a chapter back in. Lowering it is always fine.
+#
+# Exit codes: 0=within budget, 1=over, 2=a tracked file is missing
+
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+
+# The files loaded unconditionally at run start.
+TRACKED="
+pipeline/skills/shared/core/multi-agent/SKILL.md
+pipeline/multi-agent-refs/rules.md
+"
+
+# Measured 57289 after the split. 60000 leaves ~2.7 kB of headroom.
+CEILING=${CONTEXT_BUDGET_CEILING:-60000}
+
+PASS=0
+FAIL=0
+pass() { PASS=$((PASS + 1)); echo "  ✓ $1"; }
+fail() { FAIL=$((FAIL + 1)); echo "  ✗ $1"; }
+
+echo "→ 1. every tracked file exists"
+TOTAL=0
+for rel in $TRACKED; do
+  f="$REPO_ROOT/$rel"
+  if [ ! -f "$f" ]; then
+    echo "  ✗ missing: $rel"
+    echo "    a tracked file that vanished makes the budget meaningless"
+    exit 2
+  fi
+  n=$(wc -c < "$f" | tr -d ' ')
+  TOTAL=$((TOTAL + n))
+  pass "$rel ($n bytes)"
+done
+
+echo "→ 2. fixed per-run load is within budget"
+if [ "$TOTAL" -le "$CEILING" ]; then
+  pass "fixed load $TOTAL bytes, ceiling $CEILING"
+else
+  fail "fixed load $TOTAL bytes exceeds the $CEILING ceiling by $((TOTAL - CEILING))"
+  echo "    Move detail into multi-agent-refs/ and point at it, rather than"
+  echo "    raising the ceiling. Every byte here is paid by every run."
+fi
+
+echo "→ 3. the orchestrator points at the references instead of copying them"
+SKILL="$REPO_ROOT/pipeline/skills/shared/core/multi-agent/SKILL.md"
+REFS=$(grep -c "multi-agent-refs" "$SKILL" || true)
+if [ "$REFS" -ge 3 ]; then
+  pass "orchestrator cites multi-agent-refs $REFS times"
+else
+  fail "orchestrator cites multi-agent-refs only $REFS time(s)"
+  echo "    It carried zero citations while duplicating three of those files."
+fi
+
+echo ""
+echo "══ smoke-context-budget: $PASS passed, $FAIL failed ══"
+[ "$FAIL" -gt 0 ] && exit 1 || exit 0

package/pipeline/scripts/smoke-model-fallback.sh

@@ -70,7 +70,7 @@ if [ "$FABLE_COUNT" -ge 4 ]; then
 else
   fail "expected >=4 fable personas (architects + Reviewer-1), found $FABLE_COUNT"
 fi
-# Opus stays a valid tier — the first fallback below fable, and the top tier for security personas.
+# Opus stays a valid tier - the first fallback below fable, and the top tier for security personas.
 OPUS_COUNT=$(grep -l "^preferredModel: opus" "$REPO_ROOT"/pipeline/agents/*.md 2>/dev/null | wc -l | tr -d ' ')
 if [ "$OPUS_COUNT" -ge 1 ]; then
   ok "opus personas intact ($OPUS_COUNT files)"

package/pipeline/scripts/smoke-no-mcp-in-dev-phases.sh

@@ -7,20 +7,30 @@
 # Phase 1; any entry in state.telemetry.mcpCalls[] with phase >= 2 is a
 # violation.
 #
-# Inputs
-#   --state-file <path>   Optional override (default: ~/.claude/state/run/latest.json)
+# Two modes.
 #
-# Outcomes
-#   - State file missing      -> WARN, exit 0 (no run to check)
+# CHECK MODE (--state-file <path>): audit that one state file.
 #   - telemetry.mcpCalls field absent -> PASS, exit 0 (older run, no telemetry)
 #   - All entries phase < 2   -> PASS, exit 0
 #   - One or more violations  -> FAIL summary on stdout, exit 1
+#   - Missing / corrupt file  -> exit 2
+#
+# SELF-TEST MODE (no arguments): drive check mode over fixtures that cover
+# every verdict, then audit the live state file if one happens to exist.
+#
+# Why: with no --state-file this script used to print "no run to check" and
+# exit 0. On a clean clone, on CI, and on any machine that had not just
+# finished a pipeline run - which is nearly always - the gate for Locked
+# decision 30 therefore asserted nothing while reporting success. A gate whose
+# green tick means "I had nothing to look at" is indistinguishable from one
+# that passed, and this suite had three of those.
 #
 # stdout: human-readable summary only. stderr: empty in normal operation.
 
 set -euo pipefail
 
 STATE_FILE="${HOME}/.claude/state/run/latest.json"
+STATE_FILE_EXPLICIT=0
 
 while [ "$#" -gt 0 ]; do
   case "$1" in
@@ -31,16 +41,19 @@ while [ "$#" -gt 0 ]; do
         exit 2
       fi
       STATE_FILE="$1"
+      STATE_FILE_EXPLICIT=1
       shift
       ;;
     --state-file=*)
       STATE_FILE="${1#--state-file=}"
+      STATE_FILE_EXPLICIT=1
       shift
       ;;
     -h|--help)
       echo "Usage: $0 [--state-file <path>]"
       echo ""
-      echo "Default state file: ~/.claude/state/run/latest.json"
+      echo "No arguments: run the fixture self-test suite."
+      echo "Default state file when checking live: ~/.claude/state/run/latest.json"
       exit 0
       ;;
     *)
@@ -55,9 +68,75 @@ if ! command -v jq >/dev/null 2>&1; then
   exit 2
 fi
 
+# ── self-test mode ─────────────────────────────────────────────────────────
+if [ "$STATE_FILE_EXPLICIT" -eq 0 ]; then
+  SELF="$0"
+  TMP=$(mktemp -d)
+  trap 'rm -rf "$TMP"' EXIT
+  PASS=0
+  FAIL=0
+
+  expect() {
+    local name="$1" want="$2" file="$3"
+    local got=0
+    bash "$SELF" --state-file "$file" >/dev/null 2>&1 || got=$?
+    if [ "$got" -eq "$want" ]; then
+      PASS=$((PASS + 1)); echo "  ✓ $name (exit $got)"
+    else
+      FAIL=$((FAIL + 1)); echo "  ✗ $name (exit $got, expected $want)"
+    fi
+  }
+
+  echo "→ Locked decision 30 gate, fixture self-test"
+
+  printf '%s' '{"telemetry":{"mcpCalls":[{"tool":"mcp__figma__get_design_context","phase":1}]}}' \
+    > "$TMP/phase1-only.json"
+  expect "MCP call in Phase 1 passes" 0 "$TMP/phase1-only.json"
+
+  printf '%s' '{"telemetry":{"mcpCalls":[{"tool":"mcp__figma__get_design_context","phase":3}]}}' \
+    > "$TMP/phase3.json"
+  expect "MCP call in Phase 3 is a violation" 1 "$TMP/phase3.json"
+
+  printf '%s' '{"telemetry":{"mcpCalls":[{"tool":"a","phase":1},{"tool":"b","phase":2}]}}' \
+    > "$TMP/mixed.json"
+  expect "one violation among clean entries still fails" 1 "$TMP/mixed.json"
+
+  printf '%s' '{"telemetry":{"mcpCalls":[]}}' > "$TMP/empty.json"
+  expect "empty call list passes" 0 "$TMP/empty.json"
+
+  printf '%s' '{"telemetry":{}}' > "$TMP/no-field.json"
+  expect "absent telemetry.mcpCalls passes (older state format)" 0 "$TMP/no-field.json"
+
+  printf '%s' '{"telemetry":{"mcpCalls":[{"tool":"a"}]}}' > "$TMP/no-phase.json"
+  expect "entry with no phase defaults to Phase 0 and passes" 0 "$TMP/no-phase.json"
+
+  printf '%s' 'not json at all' > "$TMP/corrupt.json"
+  expect "corrupt state is an error, never a silent pass" 2 "$TMP/corrupt.json"
+
+  expect "missing state file is an error in check mode" 2 "$TMP/does-not-exist.json"
+
+  # The live run, when there is one. Absence is not a failure here - the
+  # fixtures above already proved the gate works.
+  if [ -f "$STATE_FILE" ]; then
+    echo "→ live state file"
+    if bash "$SELF" --state-file "$STATE_FILE"; then
+      PASS=$((PASS + 1)); echo "  ✓ live run clean"
+    else
+      FAIL=$((FAIL + 1)); echo "  ✗ live run has violations"
+    fi
+  else
+    echo "→ live state file: none at $STATE_FILE (fixtures already exercised every path)"
+  fi
+
+  echo ""
+  echo "══ smoke-no-mcp-in-dev-phases: $PASS passed, $FAIL failed ══"
+  [ "$FAIL" -gt 0 ] && exit 1 || exit 0
+fi
+
+# ── check mode ─────────────────────────────────────────────────────────────
 if [ ! -f "$STATE_FILE" ]; then
-  echo "WARN: state file not found at $STATE_FILE - no run to check"
-  exit 0
+  echo "FAIL: state file not found at $STATE_FILE"
+  exit 2
 fi
 
 # Validate JSON parse first; corrupt state cannot be silently passed.

package/pipeline/scripts/smoke-own-punctuation.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+#
+# smoke-own-punctuation.sh  -  the punctuation rule applies to this repo's own
+# shipped files, not only to the documents it generates.
+#
+# validate-analysis-doc.mjs has enforced the locked policy (no em-dash, en-dash,
+# ellipsis or section sign) on generated analysis documents for a long time,
+# and smoke-generate-issue.sh / smoke-review-readiness.sh enforce it on issue
+# and review payloads. Nothing applied it to the pipeline's own tree, so 77
+# shipped files carried 719 em-dashes, 10 en-dashes and 24 ellipses while the
+# rule was described as enforced.
+#
+# ALLOWED holds the files that carry these characters as data rather than
+# prose: the detector's own table and the grep patterns that look for them.
+# Stripping those would disable the very checks this rule depends on. Every
+# entry needs a reason, and the list is expected to stay this short.
+#
+# Exit codes: 0=clean, 1=violations
+
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+cd "$REPO_ROOT"
+
+# file path -> why the characters are legitimately present
+ALLOWED_PATHS="
+pipeline/scripts/validate-analysis-doc.mjs|the detector's own table of banned characters
+pipeline/scripts/smoke-generate-issue.sh|greps for the banned characters
+pipeline/scripts/smoke-review-readiness.sh|greps for the banned characters
+pipeline/scripts/smoke-md2confluence.sh|dirty-input fixture plus the forbidden list
+pipeline/scripts/smoke-own-punctuation.sh|this file names the characters it bans
+pipeline/lib/md2confluence-v3.py|the converter's codepoint-to-ASCII mapping table
+"
+
+# The same characters written as escapes. prefs.schema.json carried 50 of them
+# in its descriptions and was invisible to a search for the literal character,
+# so the first version of this gate passed it. A rule that only sees one
+# encoding of what it bans is not enforcing the rule.
+ESCAPED='\\u2014|\\u2013|\\u2026|\\u00a7'
+
+is_allowed() {
+  printf '%s\n' "$ALLOWED_PATHS" | grep -q "^$1|"
+}
+
+PASS=0
+FAIL=0
+pass() { PASS=$((PASS + 1)); echo "  ✓ $1"; }
+fail() { FAIL=$((FAIL + 1)); echo "  ✗ $1"; }
+
+echo "→ 1. no em-dash, en-dash, ellipsis or section sign in shipped files"
+
+# CHANGELOG history is left as written; rewriting released notes to satisfy a
+# later style rule would falsify the record.
+VIOLATIONS=""
+while IFS= read -r f; do
+  case "$f" in CHANGELOG*.md) continue ;; esac
+  is_allowed "$f" && continue
+  VIOLATIONS="${VIOLATIONS}${f}\n"
+done < <(git grep -lI -e $'—' -e $'–' -e $'…' -e $'§' -- . 2>/dev/null || true)
+
+if [ -z "$VIOLATIONS" ]; then
+  pass "no banned punctuation outside the allowlist"
+else
+  COUNT=$(printf '%b' "$VIOLATIONS" | grep -c . || true)
+  fail "$COUNT file(s) carry banned punctuation"
+  printf '%b' "$VIOLATIONS" | sed 's/^/    /'
+  echo "    fix: plain ASCII - or : or parentheses; 'section 4' rather than the sign"
+fi
+
+echo "→ 2. same characters written as unicode escapes"
+ESC_VIOLATIONS=""
+while IFS= read -r f; do
+  case "$f" in CHANGELOG*.md) continue ;; esac
+  is_allowed "$f" && continue
+  ESC_VIOLATIONS="${ESC_VIOLATIONS}${f}\n"
+done < <(git grep -lI -E "$ESCAPED" -- . 2>/dev/null || true)
+
+if [ -z "$ESC_VIOLATIONS" ]; then
+  pass "no escaped forms outside the allowlist"
+else
+  COUNT=$(printf '%b' "$ESC_VIOLATIONS" | grep -c . || true)
+  fail "$COUNT file(s) carry the banned characters as unicode escapes"
+  printf '%b' "$ESC_VIOLATIONS" | sed 's/^/    /'
+fi
+
+echo "→ 3. every allowlist entry still exists and still needs the exemption"
+while IFS='|' read -r path reason; do
+  [ -z "$path" ] && continue
+  if [ ! -f "$path" ]; then
+    fail "allowlisted path no longer exists: $path"
+  elif ! grep -qI -e $'—' -e $'–' -e $'…' -e $'§' "$path" 2>/dev/null \
+    && ! grep -qIE "$ESCAPED" "$path" 2>/dev/null; then
+    # Either encoding justifies the entry: md2confluence-v3.py holds them as
+    # escapes, the detector table holds them literally.
+    fail "$path is allowlisted ($reason) but is now clean - drop it from the list"
+  else
+    pass "allowlist justified: ${path##*/} ($reason)"
+  fi
+done < <(printf '%s\n' "$ALLOWED_PATHS" | grep '|')
+
+echo ""
+echo "══ smoke-own-punctuation: $PASS passed, $FAIL failed ══"
+[ "$FAIL" -gt 0 ] && exit 1 || exit 0

package/pipeline/scripts/smoke-workflow-audit.sh

@@ -4,20 +4,38 @@
 # `.github/workflows/` with the two dedicated external linters:
 #
 #   - zizmor      GitHub Actions security auditor (template injection,
-#                 credential persistence, unpinned actions, ...). Runs in
-#                 offline mode so the smoke never touches the network.
+#                 credential persistence, cache poisoning, unpinned actions,
+#                 excessive permissions, ...). Runs offline, no network.
 #   - actionlint  workflow correctness linter (expression syntax, matrix
 #                 typos, shellcheck of run: blocks, ...).
 #
-# Both tools are optional: each is skipped with a SKIP line when its binary
-# is not on PATH (billing-friendly CI runners install neither; run locally
-# via `brew install zizmor actionlint` for full coverage). Structural
-# baseline checks that need no external binary live in
-# smoke-ci-workflows.sh - this smoke is the deeper, tool-backed layer.
-#
 # Any finding from either tool fails the smoke.
 #
-# Exit codes: 0=pass or all tools skipped, 1=findings, 2=workflows missing
+# WHY THIS IS STRICT BY DEFAULT
+#
+# Until now a missing binary produced a SKIP and the smoke exited 0, so it
+# reported "0 passed, 0 failed, 2 skipped" on the dev machine AND on the CI
+# runner - neither had the tools. ci-lite.yml justified not installing them by
+# saying to run the smoke locally "where the tools exist", but they did not
+# exist there either. The audit had therefore never run anywhere, while
+# appearing as a green step. When it was finally run it found nine real issues,
+# one of them high severity (a dependency cache restored into the job that
+# signs and publishes the release tarball).
+#
+# So the failure mode is inverted: absent tooling is a failure, and skipping is
+# something you opt into deliberately and locally.
+#
+#   brew install zizmor actionlint            # macOS
+#   pipx install zizmor && go install github.com/rhysd/actionlint/cmd/actionlint@latest
+#
+# WORKFLOW_AUDIT_ALLOW_MISSING=1 downgrades a missing binary back to a SKIP for
+# a local run. It is ignored when CI is set: CI installs both, pinned, and a
+# runner that cannot must fail loudly rather than pass quietly.
+#
+# Structural baseline checks that need no external binary live in
+# smoke-ci-workflows.sh - this smoke is the deeper, tool-backed layer.
+#
+# Exit codes: 0=pass, 1=findings or missing tooling, 2=workflows missing
 
 set -uo pipefail
 
@@ -26,6 +44,10 @@ WF_DIR="$REPO_ROOT/.github/workflows"
 
 [ -d "$WF_DIR" ] || { echo "FAIL: .github/workflows/ missing"; exit 2; }
 
+# In CI the escape hatch does not apply, whatever the environment says.
+ALLOW_MISSING="${WORKFLOW_AUDIT_ALLOW_MISSING:-0}"
+[ -n "${CI:-}" ] && ALLOW_MISSING=0
+
 PASS=0
 FAIL=0
 SKIPPED=0
@@ -33,6 +55,16 @@ pass() { PASS=$((PASS + 1)); echo "  ✓ $1"; }
 fail() { FAIL=$((FAIL + 1)); echo "  ✗ $1"; }
 skip() { SKIPPED=$((SKIPPED + 1)); echo "  ↷ SKIP: $1"; }
 
+# A tool that is not installed is a failure unless a local run opted out.
+missing_tool() {
+  local tool="$1" install="$2"
+  if [ "$ALLOW_MISSING" = "1" ]; then
+    skip "$tool not on PATH, WORKFLOW_AUDIT_ALLOW_MISSING=1 ($install)"
+  else
+    fail "$tool not on PATH - the audit cannot run ($install)"
+  fi
+}
+
 # ──────────────────────────────────────────────────────────────────────────
 echo "→ 1. zizmor (security audit, offline)"
 if command -v zizmor >/dev/null 2>&1; then
@@ -48,7 +80,7 @@ if command -v zizmor >/dev/null 2>&1; then
     echo "$OUT" | sed 's/^/    /'
   fi
 else
-  skip "zizmor not on PATH (brew install zizmor)"
+  missing_tool zizmor "brew install zizmor"
 fi
 
 # ──────────────────────────────────────────────────────────────────────────
@@ -61,7 +93,7 @@ if command -v actionlint >/dev/null 2>&1; then
     echo "$OUT" | sed 's/^/    /'
   fi
 else
-  skip "actionlint not on PATH (brew install actionlint)"
+  missing_tool actionlint "brew install actionlint"
 fi
 
 echo ""