@event4u/agent-config 8.9.0 → 8.11.0

package/src/scripts/bench_humanizer_eval.ts

@@ -0,0 +1,240 @@
+#!/usr/bin/env node
+/**
+ * bench_humanizer_eval — paired eval for the humanizer feature.
+ *
+ * Two metrics over the before/after corpus in `tests/fixtures/ai-tells/`:
+ *
+ *   1. OBJECTIVE (free, deterministic): AI-tell reduction per pair via
+ *      `detect_ai_tells.ts` — hard hits, cluster score/500w, dash density/500w.
+ *   2. BLIND PREFERENCE (--judge, billable): an LLM judge sees each pair in
+ *      randomized A/B order (deterministic LCG seed per pair) and picks which
+ *      text reads more like a skilled human writer. LENGTH-CONTROLLED: pairs
+ *      whose word counts differ by more than ±25% are excluded from the
+ *      preference aggregate (the token program's verbosity-bias finding makes
+ *      uncontrolled pairs unusable) — they still count for the objective metric.
+ *
+ * Reports follow docs/benchmarks.md naming:
+ *   internal/bench/reports/humanizer-v1.{json,md}          (canonical pointer)
+ *   internal/bench/reports/<ISO>-humanizer-v1.{json,md}    (immutable trail)
+ *
+ * Claims discipline: this eval backs `claim:humanizer-tell-reduction` in
+ * docs/CLAIMS.md. It measures OUR pattern counts and a blind preference —
+ * never third-party "AI detector" outcomes; that claim class stays banned.
+ *
+ * Usage:
+ *   npx tsx src/scripts/bench_humanizer_eval.ts                          # objective only (free)
+ *   npx tsx src/scripts/bench_humanizer_eval.ts --judge --confirm-spend  # + blind preference (billable)
+ *
+ * `--judge` without `--confirm-spend` estimates the billable call count and
+ * halts (exit 2) — the spend is opt-in, never fired implicitly.
+ */
+
+import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { analyzeText, type TellReport } from "./detect_ai_tells.js";
+
+const ROOT = process.cwd();
+const FIXTURES = join(ROOT, "tests", "fixtures", "ai-tells");
+const REPORT_DIR = join(ROOT, "internal", "bench", "reports");
+const LENGTH_TOLERANCE = 0.25;
+
+interface PairResult {
+  name: string;
+  language: "en" | "de";
+  words_before: number;
+  words_after: number;
+  length_ratio: number;
+  length_controlled: boolean;
+  before: Pick<TellReport, "hard_total" | "cluster_score_per_500" | "dash_density_per_500">;
+  after: Pick<TellReport, "hard_total" | "cluster_score_per_500" | "dash_density_per_500">;
+  judge?: { preferred: "after" | "before"; order: "after-first" | "before-first" };
+}
+
+function slim(r: TellReport) {
+  return {
+    hard_total: r.hard_total,
+    cluster_score_per_500: r.cluster_score_per_500,
+    dash_density_per_500: r.dash_density_per_500,
+  };
+}
+
+/** Deterministic per-pair coin flip (LCG) so re-runs keep the same A/B order. */
+function coin(seed: number): boolean {
+  return ((seed * 1103515245 + 12345) >>> 16) % 2 === 0;
+}
+
+function loadPairs(): Array<{ name: string; language: "en" | "de"; before: string; after: string }> {
+  const out: Array<{ name: string; language: "en" | "de"; before: string; after: string }> = [];
+  for (const language of ["en", "de"] as const) {
+    const dir = join(FIXTURES, language);
+    let names: string[] = [];
+    try {
+      names = readdirSync(dir)
+        .filter((f) => f.endsWith(".before.md"))
+        .map((f) => f.replace(/\.before\.md$/, ""))
+        .sort();
+    } catch {
+      continue;
+    }
+    for (const name of names) {
+      out.push({
+        name,
+        language,
+        before: readFileSync(join(dir, `${name}.before.md`), "utf8"),
+        after: readFileSync(join(dir, `${name}.after.md`), "utf8"),
+      });
+    }
+  }
+  return out;
+}
+
+async function judgePair(
+  pair: { name: string; before: string; after: string },
+  index: number,
+): Promise<{ preferred: "after" | "before"; order: "after-first" | "before-first" }> {
+  const { AnthropicClient, load_anthropic_key } = await import("./ai_council/clients.js");
+  const client = new AnthropicClient({ api_key: load_anthropic_key() });
+  const afterFirst = coin(index + 1);
+  const [a, b] = afterFirst ? [pair.after, pair.before] : [pair.before, pair.after];
+  const system =
+    "You judge writing quality blind. You will see two texts, A and B, covering the same content. " +
+    "Pick the one that reads more like it was written by a skilled human writer for a real audience. " +
+    "Judge prose quality only — not length, not topic. Reply with exactly one character: A or B.";
+  const user = `Text A:\n\n${a}\n\n---\n\nText B:\n\n${b}\n\nWhich reads more like a skilled human writer? Reply A or B.`;
+  const resp = client.ask(system, user, 8);
+  const text = (resp.text ?? "").trim().toUpperCase();
+  const pickedA = text.startsWith("A");
+  const preferred = pickedA === afterFirst ? "after" : "before";
+  return { preferred, order: afterFirst ? "after-first" : "before-first" };
+}
+
+async function main(): Promise<void> {
+  const runJudge = process.argv.includes("--judge");
+  const confirmSpend = process.argv.includes("--confirm-spend");
+  const pairs = loadPairs();
+  if (pairs.length < 20) {
+    console.error(`corpus too small: ${pairs.length} pairs (< 20)`);
+    process.exit(2);
+  }
+
+  // Spend gate: --judge fires one billable API call per length-controlled
+  // pair (via the council client's curl transport). Estimate + halt unless
+  // --confirm-spend is passed — mirrors council:run's explicit-confirm model,
+  // never an interactive prompt. The objective-only default path is free.
+  if (runJudge && !confirmSpend) {
+    const controlledCount = pairs.filter((p) => {
+      const b = analyzeText(p.before, p.language).words;
+      const a = analyzeText(p.after, p.language).words;
+      return Math.abs(1 - a / b) <= LENGTH_TOLERANCE;
+    }).length;
+    process.stderr.write(
+      `--judge makes ${controlledCount} billable API call(s) (one per length-controlled pair). ` +
+        `Re-run with --confirm-spend to authorize the spend. ` +
+        `The objective-only run (no --judge) is free.\n`,
+    );
+    process.exit(2);
+  }
+
+  const results: PairResult[] = [];
+  for (let i = 0; i < pairs.length; i++) {
+    const p = pairs[i]!;
+    const before = analyzeText(p.before, p.language);
+    const after = analyzeText(p.after, p.language);
+    const ratio = after.words / before.words;
+    const controlled = Math.abs(1 - ratio) <= LENGTH_TOLERANCE;
+    const row: PairResult = {
+      name: p.name,
+      language: p.language,
+      words_before: before.words,
+      words_after: after.words,
+      length_ratio: Math.round(ratio * 100) / 100,
+      length_controlled: controlled,
+      before: slim(before),
+      after: slim(after),
+    };
+    if (runJudge && controlled) {
+      row.judge = await judgePair(p, i);
+      process.stdout.write(`  judged ${p.name}: prefers ${row.judge.preferred} (${row.judge.order})\n`);
+    }
+    results.push(row);
+  }
+
+  const mean = (xs: number[]) =>
+    Math.round((xs.reduce((s, x) => s + x, 0) / Math.max(xs.length, 1)) * 100) / 100;
+  const agg = {
+    pairs: results.length,
+    length_controlled_pairs: results.filter((r) => r.length_controlled).length,
+    mean_hard_before: mean(results.map((r) => r.before.hard_total)),
+    mean_hard_after: mean(results.map((r) => r.after.hard_total)),
+    mean_cluster_before: mean(results.map((r) => r.before.cluster_score_per_500)),
+    mean_cluster_after: mean(results.map((r) => r.after.cluster_score_per_500)),
+    mean_dash_before: mean(results.map((r) => r.before.dash_density_per_500)),
+    mean_dash_after: mean(results.map((r) => r.after.dash_density_per_500)),
+    judge_model: undefined as string | undefined,
+    judged_pairs: 0,
+    prefers_after: 0,
+    prefers_before: 0,
+  };
+  const judged = results.filter((r) => r.judge);
+  agg.judged_pairs = judged.length;
+  agg.prefers_after = judged.filter((r) => r.judge!.preferred === "after").length;
+  agg.prefers_before = judged.filter((r) => r.judge!.preferred === "before").length;
+  if (runJudge) {
+    const { DEFAULT_ANTHROPIC_MODEL } = await import("./ai_council/clients.js");
+    agg.judge_model = DEFAULT_ANTHROPIC_MODEL as string;
+  }
+
+  const iso = new Date().toISOString().replace(/\.\d+Z$/, "Z").replace(/:/g, "-");
+  const payload = { generated: iso, corpus: "tests/fixtures/ai-tells", aggregate: agg, results };
+
+  const md = [
+    "# Humanizer paired eval — v1",
+    "",
+    `> Generated ${iso} · corpus \`tests/fixtures/ai-tells\` (${agg.pairs} before/after pairs, ` +
+      `${agg.length_controlled_pairs} length-controlled ±${LENGTH_TOLERANCE * 100}%).`,
+    "",
+    "## Objective — AI-tell reduction (deterministic)",
+    "",
+    "| Metric (mean) | Before | After |",
+    "|---|---|---|",
+    `| Hard hits | ${agg.mean_hard_before} | ${agg.mean_hard_after} |`,
+    `| Cluster score /500w | ${agg.mean_cluster_before} | ${agg.mean_cluster_after} |`,
+    `| Dash density /500w | ${agg.mean_dash_before} | ${agg.mean_dash_after} |`,
+    "",
+    "## Blind preference (length-controlled)",
+    "",
+    agg.judged_pairs > 0
+      ? `Judge ${agg.judge_model}: prefers the humanized text in **${agg.prefers_after}/${agg.judged_pairs}** ` +
+        `pairs (randomized A/B order, deterministic seed). An honest null here keeps the detector as a ` +
+        `hygiene gate; the claim ledger only carries what this table shows.`
+      : "_Not run (objective-only invocation)._",
+    "",
+    "## Scope note",
+    "",
+    "This eval measures the package's own pattern counts and a blind prose-quality preference.",
+    'It never measures third-party "AI detector" outcomes — that claim class is banned',
+    "(unfalsifiable from our side; see roadmap non-goals).",
+    "",
+    "## Open question — real-draft lift is unmeasured",
+    "",
+    "The `before` fixtures were **deliberately tell-seeded**, so a perfect score",
+    "measures seeded-tell removal on a self-constructed corpus — NOT that real",
+    "ghostwriter drafts get better. Real-world lift stays **unmeasured** until",
+    "write-engine step 4b has processed real `/ghostwriter:write` drafts and those",
+    "have been paired-evaluated (the `road-to-humanizer-hardening` live-usage",
+    "blocker). The claim ledger is scoped to \"on the fixture corpus\" accordingly.",
+    "",
+  ].join("\n");
+
+  mkdirSync(REPORT_DIR, { recursive: true });
+  writeFileSync(join(REPORT_DIR, "humanizer-v1.json"), JSON.stringify(payload, null, 2));
+  writeFileSync(join(REPORT_DIR, "humanizer-v1.md"), md);
+  writeFileSync(join(REPORT_DIR, `${iso}-humanizer-v1.json`), JSON.stringify(payload, null, 2));
+  writeFileSync(join(REPORT_DIR, `${iso}-humanizer-v1.md`), md);
+  process.stdout.write(md + "\n");
+}
+
+main().catch((err) => {
+  console.error(String(err));
+  process.exit(2);
+});

package/src/scripts/build_discovery_manifest.ts

@@ -419,6 +419,12 @@ function* _iter_artefacts(): Generator<[string, string]> {
     // `_classify` subagent branch derives the manifest payload (default-off,
     // packs:[]) — static projection only, never runtime-dispatched.
     for (const p of _collect('subagents', '*.md')) {
+        // `_`-prefixed files are injected partials (e.g. _prompt-defense.md),
+        // not projectable agents — never discovery artefacts. Mirrors the
+        // same filter in condense.generate_claude_subagents.
+        if (path.basename(p).startsWith('_')) {
+            continue;
+        }
         yield [p, 'subagent'];
     }
 }

package/src/scripts/build_proof.ts

@@ -27,6 +27,7 @@ import { REPO, LEDGER_REL, load_ledger, pointer_unresolved } from './check_claim
 import { collectSkillGaps } from './check_skill_gaps.js';
 import { loadRows as loadComparisonRows } from './check_comparison.js';
 import { computeStatus as domainSoundnessStatus } from './domain_soundness_status.js';
+import { computeCoverage } from './skill_eval_coverage.js';
 
 const _FILE = fileURLToPath(import.meta.url);
 const OUT_REL = 'docs/proof.md';
@@ -108,16 +109,27 @@ function render(): string {
         L.push('Benchmark results are published under `docs/` including measured nulls.');
     }
     L.push('');
+    const cov = computeCoverage();
+    const prio = cov.tiers.priority;
+    const other = cov.tiers.other;
+    const prioLine =
+        prio.covered === prio.total
+            ? `\`rich\` + routers) are **fully covered (${prio.covered} of ${prio.total})**, the long tail`
+            : `\`rich\` + routers, **${prio.covered} of ${prio.total}**) are partially covered; the long tail`;
     L.push('**Behavioural-eval coverage — the honest baseline.** Skill *quality* is only');
-    L.push('as good as its measurement. Today **2 of 264** skills carry a behavioural');
-    L.push('`evals.json`, and the highest-traffic / highest-cost tiers (default-surface +');
-    L.push('`rich` + routers, **0 of 35**) are not yet covered. We publish that gap rather');
-    L.push('than imply "264 evaluated skills": coverage is measured per tier');
-    L.push('(`./scripts-run src/scripts/skill_eval_coverage`) and **CI-ratcheted so it can');
-    L.push('only rise** — a merged change can never lower it. Authoring the priority-tier');
-    L.push('evals is gated on per-case human ratification (a generated assertion that');
-    L.push('checks the wrong property is worse than none), so the number grows');
-    L.push('deliberately, not overnight.');
+    L.push(`as good as its measurement. Today **${cov.overall.covered} of ${cov.overall.total}** skills carry a behavioural`);
+    L.push('`evals.json`; the highest-traffic / highest-cost tiers (default-surface +');
+    L.push(prioLine);
+    L.push(`(${other.covered} of ${other.total}) is not. We publish that gap rather than imply`);
+    L.push(`"${cov.overall.total} evaluated skills": coverage is measured per tier`);
+    L.push('(`./scripts-run src/scripts/skill_eval_coverage`), **CI-ratcheted so it can');
+    L.push('only rise**, and the priority tiers carry a hard **tier floor**: every');
+    L.push('rich / default-surface / router skill MUST have a behavioural eval or an');
+    L.push('explicit exemption-with-reason in `internal/evals/tier-floor-exemptions.json`');
+    L.push('— no silent exemptions (the bright line deliberately replaces a');
+    L.push('weighted-coverage score). Authoring long-tail evals is gated on per-case');
+    L.push('human ratification (a generated assertion that checks the wrong property is');
+    L.push('worse than none), so the number grows deliberately, not overnight.');
     L.push('');
     const ds = domainSoundnessStatus();
     L.push('**Non-coding domain soundness — scoped, not proven.** The `finance` /');

package/src/scripts/check_claims.ts

@@ -42,6 +42,13 @@ const SURFACE_ROOTS = ['README.md', 'docs'];
 const CLAIM_MARKER = /<!--\s*claim:([A-Za-z0-9._-]+)\s*-->/g;
 const URL_DATED = /^https?:\/\/\S+\(\d{4}-\d{2}-\d{2}\)\s*$/;
 
+/** Witness sweep (P1.2) — marketing-class surfaces where a bare number must
+ *  not appear as a capability claim. docs/proof.md and docs/comparison.yaml
+ *  are structurally witnessed (pointer columns + their own linters). */
+const WITNESS_SURFACES = ['README.md', 'CAPABILITIES.yaml'];
+/** A percentage or multiplier reading as a measured capability figure. */
+const QUANTIFIED_CLAIM = /\b\d+(?:\.\d+)?\s*(?:%|[x×](?![A-Za-z0-9]))/;
+
 class ExitCode extends Error {
     code: number;
     constructor(code: number) {
@@ -224,6 +231,39 @@ function main(argv: string[] = process.argv.slice(2)): number {
         if (un) findings.push({ id: entry.id, file: LEDGER_REL, reason: `backed entry has dangling evidence — ${un}` });
     }
 
+    // 3. Witness sweep (road-to-opt-subagent-harvest P1.2): a QUANTIFIED
+    //    capability claim on a marketing-class surface must be witnessed — a
+    //    `<!-- claim:ID -->` marker on the same line, or an explicit
+    //    `unverified` annotation. Unmarkered numbers are exactly how an
+    //    "84.8%"-style figure spreads across surfaces with no methodology
+    //    (the anti-lesson this check encodes). Fenced code blocks are skipped.
+    for (const rel of WITNESS_SURFACES) {
+        const abs = path.join(REPO, rel);
+        let text: string;
+        try {
+            text = fs.readFileSync(abs, 'utf-8');
+        } catch {
+            continue;
+        }
+        let inFence = false;
+        const lines = text.split('\n');
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i] as string;
+            if (/^\s*(```|~~~)/.test(line)) {
+                inFence = !inFence;
+                continue;
+            }
+            if (inFence) continue;
+            if (!QUANTIFIED_CLAIM.test(line)) continue;
+            if (/<!--\s*claim:/.test(line) || /unverified/i.test(line)) continue;
+            findings.push({
+                id: '(unmarkered)',
+                file: `${rel}:${i + 1}`,
+                reason: `quantified claim without a claim marker or 'unverified' annotation — ${line.trim().slice(0, 70)}`,
+            });
+        }
+    }
+
     if (findings.length > 0) {
         process.stderr.write(`❌  check_claims: ${findings.length} unbacked/dangling claim(s):\n`);
         for (const f of findings) {

package/src/scripts/check_release_adjacent_health.ts

@@ -0,0 +1,121 @@
+#!/usr/bin/env tsx
+/**
+ * Release-adjacent workflow health tripwire — road-to-proof-under-real-conditions
+ * Phase 1. The "red for five releases without anyone noticing" detector:
+ * the MCP worker deploy failed on every release from one minor line to the
+ * next and nothing surfaced it, because post-tag workflows are nobody's PR.
+ *
+ * For each watched workflow, look at its most recent COMPLETED run on the
+ * default branch (any trigger). If that run's conclusion is not `success`
+ * and it finished more than --max-red-hours ago (default 48) with no newer
+ * successful run, the workflow counts as durably red → exit 1 and print a
+ * GitHub error annotation per offender. The calling workflow turns that
+ * into a visible failure on main.
+ *
+ * Auth: GITHUB_TOKEN (or GH_TOKEN) env. Repo: GITHUB_REPOSITORY env or
+ * --repo owner/name.
+ *
+ * Exit codes: 0 all green/absent · 1 durably-red workflow(s) · 2 usage/env.
+ */
+
+import { argv, env, exit } from 'node:process';
+
+const WATCHED = [
+    'publish-npm.yml',
+    'deploy-mcp-worker.yml',
+    'deploy-site.yml',
+    'site.yml',
+    'consumer-matrix.yml',
+    'release.yml',
+] as const;
+
+interface RunInfo {
+    conclusion: string | null;
+    updated_at: string;
+    html_url: string;
+}
+
+async function latestCompletedRun(
+    repo: string,
+    token: string,
+    workflow: string,
+    branch: string,
+): Promise<RunInfo | null> {
+    const url =
+        `https://api.github.com/repos/${repo}/actions/workflows/${workflow}/runs` +
+        `?status=completed&per_page=1${branch ? `&branch=${branch}` : ''}`;
+    const res = await fetch(url, {
+        headers: {
+            Authorization: `Bearer ${token}`,
+            Accept: 'application/vnd.github+json',
+            'X-GitHub-Api-Version': '2022-11-28',
+        },
+    });
+    if (res.status === 404) return null; // workflow not present (yet)
+    if (!res.ok) throw new Error(`GitHub API ${res.status} for ${workflow}: ${await res.text()}`);
+    const body = (await res.json()) as { workflow_runs?: RunInfo[] };
+    return body.workflow_runs?.[0] ?? null;
+}
+
+async function main(): Promise<number> {
+    const args = argv.slice(2);
+    const getOpt = (flag: string): string | undefined => {
+        const i = args.indexOf(flag);
+        return i >= 0 ? args[i + 1] : undefined;
+    };
+    const token = env.GITHUB_TOKEN ?? env.GH_TOKEN ?? '';
+    const repo = getOpt('--repo') ?? env.GITHUB_REPOSITORY ?? '';
+    const maxRedHours = Number(getOpt('--max-red-hours') ?? '48');
+    if (!token || !repo) {
+        process.stderr.write('❌  release-adjacent-health: GITHUB_TOKEN and GITHUB_REPOSITORY (or --repo) required.\n');
+        return 2;
+    }
+
+    // Tag/release-triggered workflows have no branch association — query
+    // without the branch filter for those; branch-scoped for PR/push ones.
+    const branchless = new Set(['publish-npm.yml', 'deploy-mcp-worker.yml', 'release.yml']);
+    let red = 0;
+    for (const wf of WATCHED) {
+        let run: RunInfo | null;
+        try {
+            run = await latestCompletedRun(repo, token, wf, branchless.has(wf) ? '' : 'main');
+        } catch (err) {
+            process.stderr.write(`⚠️  ${wf}: ${(err as Error).message}\n`);
+            continue; // transient API failure must not fake a red
+        }
+        if (!run) {
+            process.stderr.write(`·   ${wf}: no completed runs (or workflow absent) — skipping\n`);
+            continue;
+        }
+        if (run.conclusion === 'success') {
+            process.stderr.write(`✅  ${wf}: last completed run green\n`);
+            continue;
+        }
+        const ageHours = (Date.now() - Date.parse(run.updated_at)) / 3_600_000;
+        if (ageHours >= maxRedHours) {
+            process.stdout.write(
+                `::error::release-adjacent workflow durably red: ${wf} — last completed run ` +
+                    `'${run.conclusion}' ${ageHours.toFixed(0)}h ago with no newer success. ${run.html_url}\n`,
+            );
+            red += 1;
+        } else {
+            process.stderr.write(
+                `⚠️  ${wf}: last run '${run.conclusion}' ${ageHours.toFixed(0)}h ago (< ${maxRedHours}h window)\n`,
+            );
+        }
+    }
+    if (red > 0) {
+        process.stderr.write(`❌  ${red} release-adjacent workflow(s) durably red.\n`);
+        return 1;
+    }
+    process.stderr.write('✅  release-adjacent workflows healthy.\n');
+    return 0;
+}
+
+main().then(
+    (code) => exit(code),
+    (err) => {
+        process.stderr.write(`❌  release-adjacent-health crashed: ${(err as Error).stack ?? err}\n`);
+        exit(2);
+    },
+);