@mmerterden/multi-agent-pipeline 12.7.0 → 12.8.0

package/pipeline/scripts/scorecard.mjs

@@ -1,258 +0,0 @@
-#!/usr/bin/env node
-// scorecard.mjs - the measurable part of the review score, as a gate.
-//
-// A refactor review scores ten categories out of ten. Six of those numbers can
-// be measured; four cannot, and writing a number for those is how a scorecard
-// turns into self-congratulation. So this reports exactly two things:
-//
-//   MEASURED    a metric with a threshold, pass or fail
-//   UNMEASURED  a category no script can score, named as such
-//
-// Nothing here re-implements a check. Each metric either reads a committed
-// threshold file or shells out to the gate that already owns the question, so
-// there is one source of truth per question and the scorecard cannot drift from
-// the gates it summarises.
-//
-// Thresholds are locked at the values measured when the gate was written. They
-// are a ratchet: lowering one is a deliberate, reviewable edit, and a metric
-// that improves should have its floor raised.
-//
-// Runs on a clean clone: no $HOME lookups, no credentials, no network. The one
-// metric that needs local configuration (derived-skill drift) reports
-// NOT CONFIGURED rather than passing by default.
-//
-// Usage:  node pipeline/scripts/scorecard.mjs [--json]
-// Exit:   0 every measured metric within threshold, 1 otherwise
-
-import { execFileSync } from "node:child_process";
-import { existsSync, readFileSync } from "node:fs";
-import { dirname, join } from "node:path";
-import { fileURLToPath } from "node:url";
-
-const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
-const asJson = process.argv.includes("--json");
-
-// Locked floors. Raise when a metric improves; lowering one is the reviewable act.
-const FLOORS = {
-  coverageLines: 72,
-  coverageBranches: 68,
-  coverageFunctions: 85,
-  contextBytes: 60000,
-};
-
-const results = [];
-
-function measured(category, metric, ok, detail) {
-  results.push({ kind: "MEASURED", category, metric, ok, detail });
-}
-function unmeasured(category, why) {
-  results.push({ kind: "UNMEASURED", category, why });
-}
-
-// Run a gate and report whether it passed. The gate owns the assertion; this
-// only records the verdict, so the two can never disagree.
-function gate(category, metric, argv) {
-  try {
-    execFileSync(argv[0], argv.slice(1), { cwd: ROOT, stdio: "pipe" });
-    return measured(category, metric, true, "gate passed");
-  } catch (e) {
-    const out = `${e.stdout || ""}${e.stderr || ""}`.trim().split("\n").slice(-3).join(" | ");
-    return measured(category, metric, false, out || `exit ${e.status}`);
-  }
-}
-
-// --- Tests: the coverage floors must not be lowered ------------------------
-// c8 enforces the numbers on every run; what nothing enforced was the numbers
-// themselves. .c8rc.json reported 83% while the real figure over its own include
-// set was 33%, because `all` was missing - so the floors are worth pinning.
-{
-  const c8 = JSON.parse(readFileSync(join(ROOT, ".c8rc.json"), "utf-8"));
-  const checks = [
-    ["lines", c8.lines, FLOORS.coverageLines],
-    ["branches", c8.branches, FLOORS.coverageBranches],
-    ["functions", c8.functions, FLOORS.coverageFunctions],
-  ];
-  const low = checks.filter(([, actual, floor]) => !(actual >= floor));
-  measured(
-    "Tests",
-    "coverage floors held",
-    c8.all === true && low.length === 0,
-    c8.all !== true
-      ? "`all` is not set, so the floors only cover files the tests already load"
-      : low.length
-        ? low.map(([k, a, f]) => `${k} ${a} below the locked ${f}`).join(", ")
-        : checks.map(([k, a]) => `${k} ${a}`).join(", "),
-  );
-}
-
-// --- Dependency management: no high or critical advisories ------------------
-{
-  let vulns;
-  try {
-    const raw = execFileSync("npm", ["audit", "--json"], { cwd: ROOT, encoding: "utf-8" });
-    vulns = JSON.parse(raw).metadata.vulnerabilities;
-  } catch (e) {
-    // npm audit exits non-zero when it finds something; the JSON is still on stdout.
-    try {
-      vulns = JSON.parse(e.stdout).metadata.vulnerabilities;
-    } catch {
-      vulns = null;
-    }
-  }
-  if (!vulns) {
-    measured(
-      "Dependency management",
-      "no high or critical advisories",
-      false,
-      "npm audit produced no parseable report",
-    );
-  } else {
-    const bad = vulns.high + vulns.critical;
-    measured(
-      "Dependency management",
-      "no high or critical advisories",
-      bad === 0,
-      `critical ${vulns.critical}, high ${vulns.high}, moderate ${vulns.moderate}`,
-    );
-  }
-}
-
-// --- Performance: the bytes every run pays before it starts ----------------
-gate("Performance", "fixed per-run context within budget", [
-  "bash",
-  "pipeline/scripts/smoke-context-budget.sh",
-]);
-
-// --- CI/CD: the workflow audit runs, and finds nothing ---------------------
-// Strict by default, so a runner without the linters fails here rather than
-// reporting a clean audit it never performed.
-gate("CI/CD", "workflow audit clean (zizmor + actionlint)", [
-  "bash",
-  "pipeline/scripts/smoke-workflow-audit.sh",
-]);
-
-// --- Code quality: our own punctuation rule, both encodings ---------------
-gate("Code quality", "no banned punctuation in shipped files", [
-  "bash",
-  "pipeline/scripts/smoke-own-punctuation.sh",
-]);
-
-// --- Documentation: every skill description routes ------------------------
-gate("Documentation", "every skill description has a routing clause", [
-  "node",
-  "pipeline/scripts/lint-skills.mjs",
-]);
-
-// --- Documentation: the licence and changelog a consumer expects ----------
-{
-  const missing = ["LICENSE", "CHANGELOG.md", "README.md", "SECURITY.md"].filter(
-    (f) => !existsSync(join(ROOT, f)),
-  );
-  measured(
-    "Documentation",
-    "licence, changelog, readme, security present",
-    missing.length === 0,
-    missing.length ? `missing: ${missing.join(", ")}` : "all four present",
-  );
-}
-
-// --- Tests: no suite reports success without asserting anything ------------
-// The runner owns this check; asking it for the counts would re-run 120 suites,
-// so the scorecard asserts the capability is wired rather than re-measuring it.
-{
-  const runner = readFileSync(join(ROOT, "pipeline", "scripts", "run-smokes.mjs"), "utf-8");
-  const wired = runner.includes("countAssertions") && /asserted nothing/.test(runner);
-  measured(
-    "Tests",
-    "runner rejects a suite that asserted nothing",
-    wired,
-    wired
-      ? "zero-assertion check present in run-smokes.mjs"
-      : "run-smokes.mjs no longer counts assertions",
-  );
-}
-
-// --- Drift: needs local configuration, so it reports honestly -------------
-{
-  // The exit code alone cannot answer this. check-derived-drift exits 0 both
-  // when every derivation is current AND when nothing is configured at all, and
-  // those are different facts: one is a pass, the other is an absence of
-  // evidence. Reading `configured` off the JSON is what tells them apart. The
-  // first version of this block trusted exit 0 and reported an unconfigured
-  // machine as a passing metric, which is the exact failure this whole gate set
-  // exists to prevent.
-  let state;
-  let ok = false;
-  let parsed;
-  try {
-    parsed = JSON.parse(
-      execFileSync("node", ["pipeline/scripts/check-derived-drift.mjs", "--json"], {
-        cwd: ROOT,
-        encoding: "utf-8",
-        stdio: ["ignore", "pipe", "ignore"],
-      }),
-    );
-  } catch (e) {
-    // Non-zero exit still writes the report to stdout; an unparseable one means
-    // the check itself could not run.
-    try {
-      parsed = JSON.parse(e.stdout);
-    } catch {
-      parsed = undefined;
-    }
-  }
-  if (parsed && parsed.configured > 0) {
-    const bad = parsed.drifted + parsed.unverified + parsed.unreachable;
-    ok = bad === 0;
-    state = ok
-      ? `${parsed.configured} derivation(s) current with upstream`
-      : `${parsed.drifted} drifted, ${parsed.unverified} unverified, ${parsed.unreachable} unreachable`;
-  } else if (parsed) {
-    state = "NOT CONFIGURED - prefs carry no derivedSkillSources on this machine";
-  } else {
-    state = "NOT CONFIGURED - the drift check produced no parseable report";
-  }
-  if (state.startsWith("NOT CONFIGURED")) {
-    unmeasured("Derived-skill drift", state);
-  } else {
-    measured("Architecture", "derived skills current with upstream", ok, state);
-  }
-}
-
-// --- What no script can score --------------------------------------------
-unmeasured("Architecture", "layer separation and dependency direction are judgement, not a number");
-unmeasured("DevEx", "onboarding time and error-message quality need a person");
-unmeasured(
-  "Accessibility",
-  "nothing in this repo renders UI; the audits it ships are scored per target app",
-);
-unmeasured(
-  "Code quality",
-  "naming, cohesion and altitude are judgement; only the mechanical rules above are gated",
-);
-
-const failed = results.filter((r) => r.kind === "MEASURED" && !r.ok);
-const passed = results.filter((r) => r.kind === "MEASURED" && r.ok);
-
-if (asJson) {
-  console.log(JSON.stringify({ passed: passed.length, failed: failed.length, results }, null, 2));
-} else {
-  console.log("MEASURED");
-  for (const r of results.filter((x) => x.kind === "MEASURED")) {
-    console.log(`  ${r.ok ? "✓" : "✗"} ${r.category}: ${r.metric}`);
-    console.log(`      ${r.detail}`);
-  }
-  console.log("");
-  console.log("UNMEASURED (reported, never scored)");
-  for (const r of results.filter((x) => x.kind === "UNMEASURED")) {
-    console.log(`  - ${r.category}: ${r.why}`);
-  }
-  console.log("");
-  console.log(`══ scorecard: ${passed.length} measured metrics passed, ${failed.length} failed ══`);
-  if (failed.length === 0) {
-    console.log("Every mechanically checkable claim holds. The four unmeasured");
-    console.log("categories above are not covered by that statement.");
-  }
-}
-
-process.exit(failed.length === 0 ? 0 : 1);

package/pipeline/scripts/smoke-add-detail.sh

@@ -1,137 +0,0 @@
-#!/usr/bin/env bash
-# smoke-add-detail.sh  -  contract check for the `add-detail` command.
-#
-# Does NOT hit real Bitbucket / Jira / GitHub. Instead:
-#   1. Generates a fake PR diff and body fixture.
-#   2. Runs the body-file + jq --rawfile + curl --data-binary pipeline
-#      against a local `jq` / `python -c` simulator that checks the
-#      resulting JSON payload round-trips with:
-#        - real newlines (no literal "\n")
-#        - no HTML entities (& / < / " / ')
-#        - required Bitbucket PUT fields present: reviewers, fromRef, toRef, draft
-#   3. Returns non-zero if any contract is violated.
-#
-# Intended as a CI guard so regressions like the "23 reviewers wiped" incident
-# cannot land unnoticed.
-
-set -euo pipefail
-
-TMP="$(mktemp -d -t multi-agent-smoke-XXXX)" || { echo "FATAL: mktemp failed" >&2; exit 1; }
-trap 'rm -rf "$TMP"' EXIT
-
-PASS=0
-FAIL=0
-
-pass() { PASS=$((PASS+1)); echo "  ✓ $1"; }
-fail() { FAIL=$((FAIL+1)); echo "  ✗ $1"; }
-
-need() {
-  command -v "$1" >/dev/null 2>&1 || {
-    echo "error: $1 is required" >&2
-    exit 2
-  }
-}
-need jq
-need python3
-
-echo "→ Writing fixture body with newlines, '&' and Turkish chars"
-cat > "$TMP/body.md" <<'EOF'
-## Özet
-
-- Kök neden: UILabel'e `adjustsFontSizeToFitWidth` eklenmemiş
-- Çözüm: `minimumScaleFactor = 0.7` ile ölçek desteği
-- Etki: İspanyolca & Almanca'da taşma düzeliyor
-
-## Test Senaryoları
-
-| # | Ön koşul | Adımlar | Beklenen |
-|---|----------|---------|----------|
-| 1 | Dil = ES | Check-in Summary aç | Şehir isimleri sığar |
-EOF
-
-echo "→ Building payload with jq --rawfile"
-jq -n \
-  --rawfile body "$TMP/body.md" \
-  --arg title 'fix(checkin): Spanish overflow [PROJ-12345]' \
-  '{
-    title: $title,
-    description: $body,
-    fromRef: { id: "refs/heads/bugfix/PROJ-12345", repository: { slug: "my-app", project: { key: "APP" } } },
-    toRef:   { id: "refs/heads/develop",           repository: { slug: "my-app", project: { key: "APP" } } },
-    reviewers: [{ user: { name: "alice" } }, { user: { name: "bob" } }],
-    draft: false,
-    state: "OPEN", open: true, closed: false, locked: false
-  }' > "$TMP/payload.json"
-
-echo "→ Contract checks"
-
-# Contract 1: payload is valid JSON.
-if jq -e . "$TMP/payload.json" >/dev/null 2>&1; then
-  pass "payload is valid JSON"
-else
-  fail "payload is not valid JSON"
-fi
-
-# Contract 2: description field round-trips with real newlines (not literal \n).
-DESC="$(jq -r .description "$TMP/payload.json")"
-if [[ "$DESC" == *$'\n'* ]]; then
-  pass "description contains real newlines"
-else
-  fail "description is single-line  -  newlines were escaped"
-fi
-
-# Contract 3: no literal backslash-n sequence survived.
-if grep -q '\\n' <<< "$DESC"; then
-  fail "literal \\n escaped into description"
-else
-  pass "no literal \\n in description"
-fi
-
-# Contract 4: no HTML entities in title or description.
-for field in title description; do
-  VALUE="$(jq -r --arg f "$field" '.[$f]' "$TMP/payload.json")"
-  if grep -Eq '&(amp|lt|gt|quot|#39);' <<< "$VALUE"; then
-    fail "$field contains HTML entities"
-  else
-    pass "$field has no HTML entities"
-  fi
-done
-
-# Contract 5: UTF-8 survives end-to-end (checks 'Ö', 'İ', 'ç').
-for glyph in 'Ö' 'İ' 'ç'; do
-  if grep -q "$glyph" <<< "$DESC"; then
-    pass "UTF-8 glyph '$glyph' preserved"
-  else
-    fail "UTF-8 glyph '$glyph' missing from description"
-  fi
-done
-
-# Contract 6: Bitbucket PUT contract  -  all of reviewers/fromRef/toRef/draft present.
-for field in reviewers fromRef toRef draft; do
-  if jq -e "has(\"$field\")" "$TMP/payload.json" >/dev/null; then
-    pass "payload has .$field"
-  else
-    fail "payload missing .$field (Bitbucket will wipe it)"
-  fi
-done
-
-# Contract 7: reviewers is a non-empty array for this fixture.
-LEN="$(jq '.reviewers | length' "$TMP/payload.json")"
-if [ "$LEN" -gt 0 ]; then
-  pass "reviewers array preserved ($LEN entries)"
-else
-  fail "reviewers array is empty  -  would wipe defaults"
-fi
-
-# Contract 8: fromRef / toRef are distinct (PR to self is rejected by Bitbucket).
-FROM="$(jq -r .fromRef.id "$TMP/payload.json")"
-TO="$(jq -r .toRef.id   "$TMP/payload.json")"
-if [ "$FROM" != "$TO" ]; then
-  pass "fromRef ($FROM) ≠ toRef ($TO)"
-else
-  fail "fromRef == toRef  -  invalid PR"
-fi
-
-echo
-echo "══ add-detail smoke: $PASS passed, $FAIL failed ══"
-[ "$FAIL" -eq 0 ]

package/pipeline/scripts/smoke-agent-guard.sh

@@ -1,74 +0,0 @@
-#!/usr/bin/env bash
-# smoke-agent-guard.sh  -  behavioral contract for the agent-guard PreToolUse hook.
-#
-# Verifies the guard blocks (exit 2) AI-attribution commits and force-push to a
-# protected branch, allows (exit 0) everything else, fails open on bad input, and
-# NEVER executes the inspected command (injection safety).
-#
-# Runs the guard from the repo copy in a throwaway git repo whose branch we set,
-# so the "bare push on a protected branch" path is exercised deterministically.
-#
-# Exit 0 = all pass, 1 = any failure.
-
-set -uo pipefail
-
-ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
-GUARD="$ROOT/pipeline/scripts/agent-guard.sh"
-command -v python3 >/dev/null 2>&1 || { echo "error: python3 required" >&2; exit 127; }
-[ -f "$GUARD" ] || { echo "FAIL: agent-guard.sh missing at $GUARD" >&2; exit 1; }
-[ -f "$ROOT/pipeline/scripts/agent-guard.py" ] || { echo "FAIL: agent-guard.py missing" >&2; exit 1; }
-
-PASS=0; FAIL=0
-pass() { PASS=$((PASS+1)); echo "  ✓ $1"; }
-fail() { FAIL=$((FAIL+1)); echo "  ✗ $1"; }
-
-# throwaway repo on a protected branch so `git rev-parse` inside the guard = main
-WORK="$(mktemp -d)"
-trap 'rm -rf "$WORK"' EXIT
-# init on main WITH a commit so `git rev-parse --abbrev-ref HEAD` resolves to main
-# (an unborn branch reports HEAD, which would not exercise the bare-push path)
-( cd "$WORK" \
-  && git init -q \
-  && git symbolic-ref HEAD refs/heads/main \
-  && git -c user.email=t@t -c user.name=t commit -q --allow-empty -m init ) 2>/dev/null
-
-J() { python3 -c 'import json,sys; print(json.dumps({"tool_input":{"command":sys.argv[1]}}))' "$1"; }
-run() { # label  expected-exit  command-string
-  local got
-  ( cd "$WORK" && printf '%s' "$(J "$3")" | bash "$GUARD" >/dev/null 2>&1 )
-  got=$?
-  if [ "$got" = "$2" ]; then pass "[$got] $1"; else fail "expected $2 got $got: $1"; fi
-}
-
-echo "→ blocks (exit 2)"
-run "commit Co-Authored-By: Claude" 2 'git commit -m "x
-
-Co-Authored-By: Claude <noreply@anthropic.com>"'
-run "commit robot-emoji"            2 'git commit -m "x
-
-🤖 Generated with Claude Code"'
-run "push --force origin main"      2 'git push --force origin main'
-run "push -f origin develop"        2 'git push -f origin develop'
-run "push --force-with-lease master" 2 'git push --force-with-lease origin master'
-run "bare push -f on main"          2 'git push -f'
-
-echo "→ allows (exit 0)"
-run "normal commit"                 0 'git commit -m "feat: normal"'
-run "push no force"                 0 'git push origin main'
-run "force push feature branch"     0 'git push --force origin feature/DEMO-1'
-run "force push tag"                0 'git push -f origin v1.2.3'
-run "non-git command"               0 'ls -la /tmp'
-run "empty command"                 0 ''
-
-echo "→ fail-open on bad input"
-( cd "$WORK" && printf 'not-json{' | bash "$GUARD" >/dev/null 2>&1 ); [ $? -eq 0 ] && pass "malformed json -> allow" || fail "malformed json blocked"
-( cd "$WORK" && printf '' | bash "$GUARD" >/dev/null 2>&1 ); [ $? -eq 0 ] && pass "empty stdin -> allow" || fail "empty stdin blocked"
-
-echo "→ injection safety (inspected command must NOT execute)"
-SENT="$WORK/sentinel"; : > "$SENT"
-( cd "$WORK" && printf '%s' "$(J "git commit -m \"\$(rm -f '$SENT')\"")" | bash "$GUARD" >/dev/null 2>&1 )
-[ -f "$SENT" ] && pass "sentinel survived (no execution)" || fail "sentinel deleted - command executed"
-
-echo ""
-echo "══ agent-guard smoke: $PASS passed, $FAIL failed ══"
-[ "$FAIL" -eq 0 ]