@nac3/forge-cli 0.2.0-alpha.9 → 0.2.1-alpha.60

package/dist/qa/brain_matrix/ab_runner.js

@@ -0,0 +1,144 @@
+/**
+ * A/B runner (alpha.59z.0).
+ *
+ * Each "promotion candidate" alpha removes a block from the
+ * system prompt and exposes its content via yujin.doctrine.discover
+ * or a new verb. The risk: the LLM might regress (forget the rule,
+ * hallucinate a verb, redundant-confirm, etc) because it cannot
+ * "see" the block until it asks for it.
+ *
+ * The A/B harness validates that mode B (NAC3-on-demand) is at
+ * least as good as mode A (block in prompt) for every scenario.
+ *
+ * Rule:
+ *  - Mode A: original prompt + scenario.fake_responses for mode A
+ *  - Mode B: trimmed prompt + scenario.fake_responses for mode B
+ *  - Both runs evaluated against the SAME invariants.
+ *  - PASS iff invariantsPassA <= invariantsPassB (B at least equal).
+ *  - PASS also iff tokens_B <= tokens_A * 1.25 (B not absurdly more
+ *    expensive even if invariants equal).
+ *  - FAIL otherwise -> caller must rollback.
+ *
+ * Scenario shape: an existing fixture can declare an optional
+ * `ab_test` object with `mode_a_responses` and `mode_b_responses`
+ * (overrides fake_responses for each side). When absent the A/B
+ * is skipped (only relevant for promotion candidates).
+ *
+ * ASCII-only.
+ */
+import { runScenarioOnBrain, } from './runner.js';
+const DEFAULT_TOKEN_CAP = 1.25;
+function countPass(invariants) {
+    return invariants.filter((i) => i.pass).length;
+}
+export async function runAB(scenario, brain) {
+    if (!scenario.ab_test)
+        return null;
+    const baseFake = scenario.fake_responses || [];
+    /* Build the A and B scenario copies with overridden fake_responses. */
+    const scenA = {
+        ...scenario,
+        fake_responses: scenario.ab_test.mode_a_responses ?? baseFake,
+    };
+    const scenB = {
+        ...scenario,
+        fake_responses: scenario.ab_test.mode_b_responses ?? baseFake,
+        invariants: [
+            ...scenario.invariants,
+            ...(scenario.ab_test.extra_invariants_b ?? []),
+        ],
+    };
+    const a = await runScenarioOnBrain(scenA, brain);
+    const b = await runScenarioOnBrain(scenB, brain);
+    /* B has the common invariants PLUS extras. The first
+     *  scenario.invariants.length entries are common; the rest are
+     *  the extras. Score them separately so an extra failing in B
+     *  counts as regression even though A never had that bar. */
+    const commonCount = scenario.invariants.length;
+    const commonPassA = countPass(a.invariants.slice(0, commonCount));
+    const commonPassB = countPass(b.invariants.slice(0, commonCount));
+    const extraInvariantsB = b.invariants.slice(commonCount);
+    const extraPassB = countPass(extraInvariantsB);
+    const extraTotalB = extraInvariantsB.length;
+    const passA = countPass(a.invariants);
+    const passB = countPass(b.invariants);
+    const totA = a.invariants.length;
+    const totB = b.invariants.length;
+    const invariants_delta = passB - passA;
+    /* If either side had zero tokens (no LLM call), set ratio to 1. */
+    const tA = a.run.total_tokens_out + a.run.total_tokens_in;
+    const tB = b.run.total_tokens_out + b.run.total_tokens_in;
+    const token_ratio = tA > 0 ? tB / tA : 1;
+    let promote_ok = true;
+    let verdict = 'B at least equal to A';
+    if (a.run.error || b.run.error) {
+        promote_ok = false;
+        verdict = 'run error: a=' + (a.run.error || 'ok') + ' / b=' + (b.run.error || 'ok');
+    }
+    else if (commonPassB < commonPassA) {
+        promote_ok = false;
+        verdict = 'common invariant regression: a=' + commonPassA + '/' + commonCount
+            + ', b=' + commonPassB + '/' + commonCount;
+    }
+    else if (extraTotalB > 0 && extraPassB < extraTotalB) {
+        promote_ok = false;
+        verdict = 'extra-B invariant regression: ' + extraPassB + '/' + extraTotalB
+            + ' B-only invariants passed';
+    }
+    else if (token_ratio > (scenario.ab_test?.token_ratio_cap ?? DEFAULT_TOKEN_CAP)) {
+        promote_ok = false;
+        const cap = scenario.ab_test?.token_ratio_cap ?? DEFAULT_TOKEN_CAP;
+        verdict = 'token spend regressed: b/a=' + token_ratio.toFixed(2) + ' (cap ' + cap.toFixed(2) + ')';
+    }
+    else if (passB === passA && extraTotalB === 0) {
+        verdict = 'equivalent (a=' + passA + '/' + totA + ', b=' + passB + '/' + totB + ')';
+    }
+    else {
+        verdict = 'B improved (a=' + passA + '/' + totA + ', b=' + passB + '/' + totB + ')';
+    }
+    return {
+        scenario_id: scenario.id,
+        brain_id: brain.id,
+        a, b,
+        invariants_delta,
+        token_ratio,
+        promote_ok,
+        verdict,
+    };
+}
+export async function runABMatrix(scenarios, brains) {
+    const results = [];
+    for (const s of scenarios) {
+        if (!s.ab_test)
+            continue;
+        for (const b of brains) {
+            const r = await runAB(s, b);
+            if (r)
+                results.push(r);
+        }
+    }
+    return {
+        generated_at: new Date().toISOString(),
+        results,
+        promote_ok: results.every((r) => r.promote_ok),
+    };
+}
+export function renderABReport(matrix) {
+    const lines = [];
+    lines.push('# A/B promotion report');
+    lines.push('');
+    lines.push('Generated: ' + matrix.generated_at);
+    lines.push('Cells:     ' + matrix.results.length);
+    lines.push('Promote:   ' + (matrix.promote_ok ? 'OK -- all cells passed' : 'FAIL -- one or more cells regressed'));
+    lines.push('');
+    lines.push('| Scenario | Brain | Inv A | Inv B | Tok ratio | Verdict |');
+    lines.push('|---|---|---|---|---|---|');
+    for (const r of matrix.results) {
+        const aP = countPass(r.a.invariants) + '/' + r.a.invariants.length;
+        const bP = countPass(r.b.invariants) + '/' + r.b.invariants.length;
+        lines.push('| ' + r.scenario_id + ' | ' + r.brain_id + ' | ' + aP + ' | ' + bP
+            + ' | ' + r.token_ratio.toFixed(2) + ' | ' + r.verdict + ' |');
+    }
+    return lines.join('\n');
+}
+//# sourceMappingURL=ab_runner.js.map

package/dist/qa/brain_matrix/ab_runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ab_runner.js","sourceRoot":"","sources":["../../../src/qa/brain_matrix/ab_runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,OAAO,EACL,kBAAkB,GACnB,MAAM,aAAa,CAAC;AAuCrB,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAE/B,SAAS,SAAS,CAAC,UAA6B;IAC9C,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;AACjD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,QAAwC,EACxC,KAAkB;IAElB,IAAI,CAAC,QAAQ,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,QAAQ,GAAI,QAA2D,CAAC,cAAc,IAAI,EAAE,CAAC;IAEnG,uEAAuE;IACvE,MAAM,KAAK,GAAmD;QAC5D,GAAG,QAAQ;QACX,cAAc,EAAE,QAAQ,CAAC,OAAO,CAAC,gBAAgB,IAAI,QAAQ;KAC9D,CAAC;IACF,MAAM,KAAK,GAAmD;QAC5D,GAAG,QAAQ;QACX,cAAc,EAAE,QAAQ,CAAC,OAAO,CAAC,gBAAgB,IAAI,QAAQ;QAC7D,UAAU,EAAE;YACV,GAAG,QAAQ,CAAC,UAAU;YACtB,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAC;SAC/C;KACF,CAAC;IAEF,MAAM,CAAC,GAAG,MAAM,kBAAkB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IACjD,MAAM,CAAC,GAAG,MAAM,kBAAkB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAEjD;;;iEAG6D;IAC7D,MAAM,WAAW,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC;IAC/C,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC;IAClE,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC;IAClE,MAAM,gBAAgB,GAAG,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IACzD,MAAM,UAAU,GAAG,SAAS,CAAC,gBAAgB,CAAC,CAAC;IAC/C,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,CAAC;IAE5C,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IACtC,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IACtC,MAAM,IAAI,GAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;IAClC,MAAM,IAAI,GAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;IAElC,MAAM,gBAAgB,GAAG,KAAK,GAAG,KAAK,CAAC;IACvC,mEAAmE;IACnE,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,gBAAgB,GAAG,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC;IAC1D,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,gBAAgB,GAAG,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC;IAC1D,MAAM,WAAW,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAEzC,IAAI,UAAU,GAAG,IAAI,CAAC;IACtB,IAAI,OAAO,GAAG,uBAAuB,CAAC;IACtC,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QAC/B,UAAU,GAAG,KAAK,CAAC;QACnB,OAAO,GAAG,eAAe,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC;IACtF,CAAC;SAAM,IAAI,WAAW,GAAG,WAAW,EAAE,CAAC;QACrC,UAAU,GAAG,KAAK,CAAC;QACnB,OAAO,GAAG,iCAAiC,GAAG,WAAW,GAAG,GAAG,GAAG,WAAW;cACnE,MAAM,GAAG,WAAW,GAAG,GAAG,GAAG,WAAW,CAAC;IACrD,CAAC;SAAM,IAAI,WAAW,GAAG,CAAC,IAAI,UAAU,GAAG,WAAW,EAAE,CAAC;QACvD,UAAU,GAAG,KAAK,CAAC;QACnB,OAAO,GAAG,gCAAgC,GAAG,UAAU,GAAG,GAAG,GAAG,WAAW;cACjE,2BAA2B,CAAC;IACxC,CAAC;SAAM,IAAI,WAAW,GAAG,CAAC,QAAQ,CAAC,OAAO,EAAE,eAAe,IAAI,iBAAiB,CAAC,EAAE,CAAC;QAClF,UAAU,GAAG,KAAK,CAAC;QACnB,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,eAAe,IAAI,iBAAiB,CAAC;QACnE,OAAO,GAAG,6BAA6B,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;IACrG,CAAC;SAAM,IAAI,KAAK,KAAK,KAAK,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;QAChD,OAAO,GAAG,gBAAgB,GAAG,KAAK,GAAG,GAAG,GAAG,IAAI,GAAG,MAAM,GAAG,KAAK,GAAG,GAAG,GAAG,IAAI,GAAG,GAAG,CAAC;IACtF,CAAC;SAAM,CAAC;QACN,OAAO,GAAG,gBAAgB,GAAG,KAAK,GAAG,GAAG,GAAG,IAAI,GAAG,MAAM,GAAG,KAAK,GAAG,GAAG,GAAG,IAAI,GAAG,GAAG,CAAC;IACtF,CAAC;IAED,OAAO;QACL,WAAW,EAAE,QAAQ,CAAC,EAAE;QACxB,QAAQ,EAAE,KAAK,CAAC,EAAE;QAClB,CAAC,EAAE,CAAC;QACJ,gBAAgB;QAChB,WAAW;QACX,UAAU;QACV,OAAO;KACR,CAAC;AACJ,CAAC;AASD,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,SAAgD,EAChD,MAAqB;IAErB,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,IAAI,CAAC,CAAC,CAAC,OAAO;YAAE,SAAS;QACzB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAC5B,IAAI,CAAC;gBAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IACD,OAAO;QACL,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACtC,OAAO;QACP,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;KAC/C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAAsB;IACnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IACrC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC;IAChD,KAAK,CAAC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAClD,KAAK,CAAC,IAAI,CAAC,aAAa,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,qCAAqC,CAAC,CAAC,CAAC;IACnH,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;IACzE,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IACxC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QACnE,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QACnE,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,WAAW,GAAG,KAAK,GAAG,CAAC,CAAC,QAAQ,GAAG,KAAK,GAAG,EAAE,GAAG,KAAK,GAAG,EAAE;cAC1E,KAAK,GAAG,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/qa/brain_matrix/brains.json

@@ -0,0 +1,80 @@
+{
+  "_doc": "Brain matrix QA -- providers/models to test against. ASCII only. Edit to add/remove brains. The harness skips brains whose vault_slot is empty (no API key configured). `fake` is the dry-run brain used in CI; it never makes network calls.",
+  "brains": [
+    {
+      "id": "fake-scripted",
+      "provider": "fake",
+      "model": "scripted",
+      "tier": "eco",
+      "vault_slot": "",
+      "notes": "Dry-run fake brain. Tests the harness itself without spending money. Reads scripted responses from the scenario fixture's `fake_responses` field."
+    },
+    {
+      "id": "gemini-flash-lite",
+      "provider": "google_ai",
+      "model": "gemini-2.5-flash-lite",
+      "tier": "eco",
+      "vault_slot": "GEMINI_API_KEY",
+      "notes": "Eco. ~$0.10/1M in, ~$0.40/1M out. Fast, smallest context handling."
+    },
+    {
+      "id": "haiku-4-5",
+      "provider": "anthropic",
+      "model": "claude-haiku-4-5-20251001",
+      "tier": "eco",
+      "vault_slot": "CLAUDE_API_KEY",
+      "notes": "Eco. ~$1/1M in, ~$5/1M out. Anthropic small model."
+    },
+    {
+      "id": "sonnet-4-6",
+      "provider": "anthropic",
+      "model": "claude-sonnet-4-6",
+      "tier": "frontier",
+      "vault_slot": "CLAUDE_API_KEY",
+      "notes": "Frontier mid. ~$3/1M in, ~$15/1M out. Default Forge brain."
+    },
+    {
+      "id": "opus-4-7",
+      "provider": "anthropic",
+      "model": "claude-opus-4-7",
+      "tier": "frontier",
+      "vault_slot": "CLAUDE_API_KEY",
+      "notes": "Frontier top. ~$15/1M in, ~$75/1M out. Skip default to save $$.",
+      "skip_default": true
+    },
+    {
+      "id": "opus-4-8",
+      "provider": "anthropic",
+      "model": "claude-opus-4-8",
+      "tier": "frontier",
+      "vault_slot": "CLAUDE_API_KEY",
+      "notes": "Frontier top, released after 4-7. Incorporates effort indicator (low/medium/high/extra/max) and complex reasoning. Skip default to save $$ (manual invoke via --brains opus-4-8).",
+      "skip_default": true
+    },
+    {
+      "id": "deepseek-v3",
+      "provider": "deepseek",
+      "model": "deepseek-chat",
+      "tier": "eco",
+      "vault_slot": "DEEPSEEK_API_KEY",
+      "notes": "Eco. Cheap multilingual."
+    },
+    {
+      "id": "gpt-4o-mini",
+      "provider": "openai",
+      "model": "gpt-4o-mini",
+      "tier": "eco",
+      "vault_slot": "OPENAI_API_KEY",
+      "notes": "Eco. ~$0.15/1M in, ~$0.60/1M out."
+    },
+    {
+      "id": "gpt-4o",
+      "provider": "openai",
+      "model": "gpt-4o",
+      "tier": "frontier",
+      "vault_slot": "OPENAI_API_KEY",
+      "notes": "Frontier alt. Skip default to save $$.",
+      "skip_default": true
+    }
+  ]
+}

package/dist/qa/brain_matrix/compare.d.ts

@@ -0,0 +1,27 @@
+import type { MatrixReport } from './types.js';
+export type Outcome = 'PASS' | 'FAIL' | 'ERR' | 'MISSING';
+export interface CompareCell {
+    scenario: string;
+    brain: string;
+    base: Outcome;
+    head: Outcome;
+    /** 'regression' (PASS->FAIL/ERR), 'improvement' (FAIL/ERR->PASS),
+     *  'unchanged' (same outcome), 'flake' (FAIL<->ERR), 'new'
+     *  (only present in head), 'removed' (only in base). */
+    kind: 'regression' | 'improvement' | 'unchanged' | 'flake' | 'new' | 'removed';
+}
+export interface CompareSummary {
+    base_version: string;
+    head_version: string;
+    cells: CompareCell[];
+    regressions: CompareCell[];
+    improvements: CompareCell[];
+    flakes: CompareCell[];
+    new_cells: CompareCell[];
+    removed_cells: CompareCell[];
+    unchanged_count: number;
+}
+export declare function compareMatrices(base: MatrixReport, head: MatrixReport): CompareSummary;
+export declare function loadMatrixReport(p: string): Promise<MatrixReport>;
+export declare function renderCompareSummary(c: CompareSummary): string;
+//# sourceMappingURL=compare.d.ts.map

package/dist/qa/brain_matrix/compare.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../../../src/qa/brain_matrix/compare.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE/C,MAAM,MAAM,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,GAAG,SAAS,CAAC;AAE1D,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAK,MAAM,CAAC;IACpB,KAAK,EAAQ,MAAM,CAAC;IACpB,IAAI,EAAS,OAAO,CAAC;IACrB,IAAI,EAAS,OAAO,CAAC;IACrB;;4DAEwD;IACxD,IAAI,EAAS,YAAY,GAAG,aAAa,GAAG,WAAW,GAAG,OAAO,GAAG,KAAK,GAAG,SAAS,CAAC;CACvF;AAED,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAK,MAAM,CAAC;IACxB,YAAY,EAAK,MAAM,CAAC;IACxB,KAAK,EAAY,WAAW,EAAE,CAAC;IAC/B,WAAW,EAAM,WAAW,EAAE,CAAC;IAC/B,YAAY,EAAK,WAAW,EAAE,CAAC;IAC/B,MAAM,EAAW,WAAW,EAAE,CAAC;IAC/B,SAAS,EAAQ,WAAW,EAAE,CAAC;IAC/B,aAAa,EAAI,WAAW,EAAE,CAAC;IAC/B,eAAe,EAAE,MAAM,CAAC;CACzB;AAUD,wBAAgB,eAAe,CAAC,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,YAAY,GAAG,cAAc,CAmCtF;AAED,wBAAsB,gBAAgB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAGvE;AAED,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,cAAc,GAAG,MAAM,CAuD9D"}

package/dist/qa/brain_matrix/compare.js

@@ -0,0 +1,122 @@
+/**
+ * Brain matrix --compare: diff two MatrixReport JSONs and
+ * surface per-cell regressions / improvements.
+ *
+ * Used by `yf qa brain-matrix --compare base.json head.json` so
+ * Pablo can answer "did z.62 regress anything against z.55?"
+ * without scanning two 2000-line files by hand.
+ */
+import { readFile } from 'node:fs/promises';
+function cellOutcome(matrix, scenarioId, brainId) {
+    const cell = matrix.results?.[scenarioId]?.[brainId];
+    if (!cell)
+        return 'MISSING';
+    if (cell.run?.error)
+        return 'ERR';
+    if (cell.pass)
+        return 'PASS';
+    return 'FAIL';
+}
+export function compareMatrices(base, head) {
+    const scenarios = new Set([
+        ...base.scenarios.map((s) => s.id),
+        ...head.scenarios.map((s) => s.id),
+    ]);
+    const brains = new Set([
+        ...base.brains.map((b) => b.id),
+        ...head.brains.map((b) => b.id),
+    ]);
+    const cells = [];
+    for (const s of [...scenarios].sort()) {
+        for (const b of [...brains].sort()) {
+            const baseOut = cellOutcome(base, s, b);
+            const headOut = cellOutcome(head, s, b);
+            let kind;
+            if (baseOut === 'MISSING' && headOut !== 'MISSING')
+                kind = 'new';
+            else if (baseOut !== 'MISSING' && headOut === 'MISSING')
+                kind = 'removed';
+            else if (baseOut === headOut)
+                kind = 'unchanged';
+            else if (baseOut === 'PASS' && (headOut === 'FAIL' || headOut === 'ERR'))
+                kind = 'regression';
+            else if ((baseOut === 'FAIL' || baseOut === 'ERR') && headOut === 'PASS')
+                kind = 'improvement';
+            else
+                kind = 'flake';
+            cells.push({ scenario: s, brain: b, base: baseOut, head: headOut, kind });
+        }
+    }
+    return {
+        base_version: base.forge_version,
+        head_version: head.forge_version,
+        cells,
+        regressions: cells.filter((c) => c.kind === 'regression'),
+        improvements: cells.filter((c) => c.kind === 'improvement'),
+        flakes: cells.filter((c) => c.kind === 'flake'),
+        new_cells: cells.filter((c) => c.kind === 'new'),
+        removed_cells: cells.filter((c) => c.kind === 'removed'),
+        unchanged_count: cells.filter((c) => c.kind === 'unchanged').length,
+    };
+}
+export async function loadMatrixReport(p) {
+    const raw = await readFile(p, 'utf-8');
+    return JSON.parse(raw);
+}
+export function renderCompareSummary(c) {
+    const lines = [];
+    lines.push('Brain matrix compare');
+    lines.push('  base: ' + c.base_version);
+    lines.push('  head: ' + c.head_version);
+    lines.push('');
+    lines.push('Summary');
+    lines.push('  regressions:  ' + c.regressions.length);
+    lines.push('  improvements: ' + c.improvements.length);
+    lines.push('  flakes:       ' + c.flakes.length);
+    lines.push('  new cells:    ' + c.new_cells.length);
+    lines.push('  removed:      ' + c.removed_cells.length);
+    lines.push('  unchanged:    ' + c.unchanged_count);
+    lines.push('');
+    if (c.regressions.length > 0) {
+        lines.push('=== Regressions ===');
+        for (const r of c.regressions) {
+            lines.push('  ' + r.scenario.padEnd(40, ' ') + ' @ ' + r.brain.padEnd(18, ' ')
+                + '  ' + r.base + ' -> ' + r.head);
+        }
+        lines.push('');
+    }
+    if (c.improvements.length > 0) {
+        lines.push('=== Improvements ===');
+        for (const r of c.improvements) {
+            lines.push('  ' + r.scenario.padEnd(40, ' ') + ' @ ' + r.brain.padEnd(18, ' ')
+                + '  ' + r.base + ' -> ' + r.head);
+        }
+        lines.push('');
+    }
+    if (c.flakes.length > 0) {
+        lines.push('=== Flakes (FAIL<->ERR) ===');
+        for (const r of c.flakes) {
+            lines.push('  ' + r.scenario.padEnd(40, ' ') + ' @ ' + r.brain.padEnd(18, ' ')
+                + '  ' + r.base + ' -> ' + r.head);
+        }
+        lines.push('');
+    }
+    if (c.new_cells.length > 0) {
+        lines.push('=== New cells (only in head) ===');
+        for (const r of c.new_cells) {
+            lines.push('  ' + r.scenario.padEnd(40, ' ') + ' @ ' + r.brain.padEnd(18, ' ')
+                + '  ' + r.head);
+        }
+        lines.push('');
+    }
+    if (c.removed_cells.length > 0) {
+        lines.push('=== Removed cells (only in base) ===');
+        for (const r of c.removed_cells) {
+            lines.push('  ' + r.scenario.padEnd(40, ' ') + ' @ ' + r.brain.padEnd(18, ' ')
+                + '  ' + r.base);
+        }
+        lines.push('');
+    }
+    return lines.join('\n');
+}
+//# sourceMappingURL=compare.js.map

package/dist/qa/brain_matrix/compare.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare.js","sourceRoot":"","sources":["../../../src/qa/brain_matrix/compare.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AA4B5C,SAAS,WAAW,CAAC,MAAoB,EAAE,UAAkB,EAAE,OAAe;IAC5E,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;IACrD,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,IAAI,IAAI,CAAC,GAAG,EAAE,KAAK;QAAE,OAAO,KAAK,CAAC;IAClC,IAAI,IAAI,CAAC,IAAI;QAAE,OAAO,MAAM,CAAC;IAC7B,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAkB,EAAE,IAAkB;IACpE,MAAM,SAAS,GAAG,IAAI,GAAG,CAAS;QAChC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACnC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAS;QAC7B,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/B,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChC,CAAC,CAAC;IACH,MAAM,KAAK,GAAkB,EAAE,CAAC;IAChC,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QACtC,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YACnC,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YACxC,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YACxC,IAAI,IAAyB,CAAC;YAC9B,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,SAAS;gBAAE,IAAI,GAAG,KAAK,CAAC;iBAC5D,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,SAAS;gBAAE,IAAI,GAAG,SAAS,CAAC;iBACrE,IAAI,OAAO,KAAK,OAAO;gBAAE,IAAI,GAAG,WAAW,CAAC;iBAC5C,IAAI,OAAO,KAAK,MAAM,IAAI,CAAC,OAAO,KAAK,MAAM,IAAI,OAAO,KAAK,KAAK,CAAC;gBAAE,IAAI,GAAG,YAAY,CAAC;iBACzF,IAAI,CAAC,OAAO,KAAK,MAAM,IAAI,OAAO,KAAK,KAAK,CAAC,IAAI,OAAO,KAAK,MAAM;gBAAE,IAAI,GAAG,aAAa,CAAC;;gBAC1F,IAAI,GAAG,OAAO,CAAC;YACpB,KAAK,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IACD,OAAO;QACL,YAAY,EAAK,IAAI,CAAC,aAAa;QACnC,YAAY,EAAK,IAAI,CAAC,aAAa;QACnC,KAAK;QACL,WAAW,EAAM,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,CAAC;QAC7D,YAAY,EAAK,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa,CAAC;QAC9D,MAAM,EAAW,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC;QACxD,SAAS,EAAQ,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC;QACtD,aAAa,EAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC;QAC1D,eAAe,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,MAAM;KACpE,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,CAAS;IAC9C,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACvC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAiB,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,CAAiB;IACpD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IACnC,KAAK,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC;IACxC,KAAK,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC;IACxC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtB,KAAK,CAAC,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IACtD,KAAK,CAAC,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IACvD,KAAK,CAAC,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACjD,KAAK,CAAC,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACpD,KAAK,CAAC,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACxD,KAAK,CAAC,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC;IACnD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAClC,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC;kBAC1E,IAAI,GAAG,CAAC,CAAC,IAAI,GAAG,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACvC,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IACD,IAAI,CAAC,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACnC,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,CAAC;YAC/B,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC;kBAC1E,IAAI,GAAG,CAAC,CAAC,IAAI,GAAG,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACvC,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IACD,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAC1C,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC;kBAC1E,IAAI,GAAG,CAAC,CAAC,IAAI,GAAG,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACvC,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IACD,IAAI,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;QAC/C,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC;kBAC1E,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IACD,IAAI,CAAC,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;QACnD,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC;kBAC1E,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/qa/brain_matrix/fake_brain.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Fake brain for dry-run testing of the brain matrix harness.
+ *
+ * Each scenario can declare a `fake_responses` field in setup:
+ *
+ *   fake_responses: [
+ *     { text: "voy a abrir la pestana", tool_calls: [{name: "nac3_invoke",
+ *       args: { verb_id: "yujin.pizarron.create-tab", args: {title:"X", kind:"code", content:"..."} }}] },
+ *     { text: "listo, te abri la pestana" }
+ *   ]
+ *
+ * The fake brain returns one entry per LLM turn, in order. If the
+ * harness asks for more turns than declared, returns "" + no tool
+ * calls (the scenario likely has a bug -- the report flags it).
+ *
+ * Used in:
+ *   - vitest unit tests of the harness
+ *   - CI dry-run that validates the harness itself before any
+ *     real-LLM run is attempted.
+ *
+ * ASCII-only.
+ */
+import type { UnifiedChatClient } from '../../llm/multi_provider_client.js';
+import type { BrainSelection } from '../../core/cost_router.js';
+export interface FakeResponse {
+    text: string;
+    tool_calls?: Array<{
+        name: string;
+        args: Record<string, unknown>;
+    }>;
+    /** Optional: simulate token spend for reports. */
+    tokens_in?: number;
+    tokens_out?: number;
+}
+export declare function makeFakeBrain(brain: BrainSelection, responses: FakeResponse[]): UnifiedChatClient;
+//# sourceMappingURL=fake_brain.d.ts.map

package/dist/qa/brain_matrix/fake_brain.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fake_brain.d.ts","sourceRoot":"","sources":["../../../src/qa/brain_matrix/fake_brain.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,OAAO,KAAK,EAAE,iBAAiB,EAA2C,MAAM,oCAAoC,CAAC;AACrH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAGhE,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAC,CAAC;IACpE,kDAAkD;IAClD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,iBAAiB,CAyBjG"}

package/dist/qa/brain_matrix/fake_brain.js

@@ -0,0 +1,27 @@
+export function makeFakeBrain(brain, responses) {
+    let idx = 0;
+    return {
+        async chat(_req) {
+            const r = responses[idx] || { text: '', tool_calls: [] };
+            idx++;
+            /* Build a synthetic ToolRound[]. The real ToolRound shape
+             *  carries call + result; for the fake brain we leave result
+             *  as `null` so the runner knows to mock it. */
+            const toolRounds = (r.tool_calls || []).map((c) => ({
+                tool: c.name,
+                input: c.args,
+                output: { ok: true, fake: true },
+                is_error: false,
+            }));
+            return {
+                text: r.text,
+                tokensIn: r.tokens_in ?? 100,
+                tokensOut: r.tokens_out ?? 50,
+                model: 'fake-scripted',
+                toolRounds,
+                brain,
+            };
+        },
+    };
+}
+//# sourceMappingURL=fake_brain.js.map

package/dist/qa/brain_matrix/fake_brain.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fake_brain.js","sourceRoot":"","sources":["../../../src/qa/brain_matrix/fake_brain.ts"],"names":[],"mappings":"AAkCA,MAAM,UAAU,aAAa,CAAC,KAAqB,EAAE,SAAyB;IAC5E,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,OAAO;QACL,KAAK,CAAC,IAAI,CAAC,IAAwB;YACjC,MAAM,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;YACzD,GAAG,EAAE,CAAC;YACN;;4DAEgD;YAChD,MAAM,UAAU,GAAgB,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC/D,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,KAAK,EAAE,CAAC,CAAC,IAAI;gBACb,MAAM,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE;gBAChC,QAAQ,EAAE,KAAK;aACS,CAAA,CAAC,CAAC;YAC5B,OAAO;gBACL,IAAI,EAAQ,CAAC,CAAC,IAAI;gBAClB,QAAQ,EAAI,CAAC,CAAC,SAAS,IAAI,GAAG;gBAC9B,SAAS,EAAG,CAAC,CAAC,UAAU,IAAI,EAAE;gBAC9B,KAAK,EAAO,eAAe;gBAC3B,UAAU;gBACV,KAAK;aACN,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/qa/brain_matrix/fixtures/scenario_a11y_announce_intent.json

@@ -0,0 +1,22 @@
+{
+  "id": "a11y_announce_intent",
+  "description": "F107 a11y -- when the user asks the assistant to announce a status to screen-reader users, the brain should pick yujin.a11y.announce, NOT a generic 'show text' verb.",
+  "setup": {
+    "extra_system_prompt": "A11Y VERBS active:\n- yujin.a11y.announce [u d] args:{text<required>, politeness?, speak?, locale?}\nUse this for status / error / async completion notifications meant for screen readers + the voice loop. Pizarron is for visual data."
+  },
+  "turns": [
+    { "kind": "user_says", "text": "Anunciale al lector de pantalla que el deploy termino con exito" }
+  ],
+  "invariants": [
+    { "name": "uses_a11y_announce", "kind": "verb_invoked", "params": { "verb_id": "yujin.a11y.announce" } },
+    { "name": "no_invent_verbs", "kind": "no_invent_verb", "params": {} }
+  ],
+  "fake_responses": [
+    {
+      "text": "Anuncio el deploy.",
+      "tool_calls": [
+        { "name": "nac3_invoke", "args": { "verb_id": "yujin.a11y.announce", "args": { "text": "Deploy completado con exito", "politeness": "polite", "speak": false } } }
+      ]
+    }
+  ]
+}

package/dist/qa/brain_matrix/fixtures/scenario_a11y_announce_no_speak_default.json

@@ -0,0 +1,23 @@
+{
+  "id": "a11y_announce_no_speak_default",
+  "description": "F107 a11y.announce -- by default speak:false. Brain must NOT silently flip speak:true unless the user asks for voice. args_contain_key validates explicit speak passing only when user mentions voice.",
+  "setup": {
+    "extra_system_prompt": "yujin.a11y.announce -- default speak=false. Only set true when the user explicitly mentions voice or TTS in this turn."
+  },
+  "turns": [
+    { "kind": "user_says", "text": "Avisa al lector que la pagina termino de cargar (no necesito voz)" }
+  ],
+  "invariants": [
+    { "name": "uses_a11y_announce", "kind": "verb_invoked", "params": { "verb_id": "yujin.a11y.announce" } },
+    { "name": "args_have_text", "kind": "args_contain_key", "params": { "verb_id": "yujin.a11y.announce", "key": "text" } },
+    { "name": "no_voice_phrase", "kind": "regex_not_match", "params": { "pattern": "speak\\s*[:=]\\s*true|voice|TTS|por\\s+voz" } }
+  ],
+  "fake_responses": [
+    {
+      "text": "Anuncio.",
+      "tool_calls": [
+        { "name": "nac3_invoke", "args": { "verb_id": "yujin.a11y.announce", "args": { "text": "Pagina cargada", "politeness": "polite" } } }
+      ]
+    }
+  ]
+}

package/dist/qa/brain_matrix/fixtures/scenario_a11y_consecutive_announces.json

@@ -0,0 +1,37 @@
+{
+  "id": "a11y_consecutive_announces",
+  "description": "Two-step UX operation: user creates lead + the brain announces success. Should be ONE announce call (not two) + concise reply. min_tool_calls catches missing announce; max_tool_calls catches duplicate.",
+  "setup": {
+    "extra_system_prompt": "BACKEND TOOLS active:\n- app.create_lead [A x] args:{name<required>, email?}\n\nA11Y VERBS active:\n- yujin.a11y.announce [u d] args:{text<required>, politeness?, speak?}\n\nAfter a destructive/approval-required action completes, fire ONE a11y.announce with the result. Never two."
+  },
+  "turns": [
+    { "kind": "user_says", "text": "Crea lead Ana Lopez + anuncia al lector que se creo" },
+    {
+      "kind": "tool_result",
+      "tool_name": "app.create_lead",
+      "is_error": false,
+      "result": { "ok": true, "id": 47812, "name": "Ana Lopez" }
+    }
+  ],
+  "invariants": [
+    { "name": "no_invent_verbs", "kind": "no_invent_verb", "params": {} },
+    { "name": "calls_create_lead", "kind": "verb_invoked", "params": { "verb_id": "app.create_lead" } },
+    { "name": "calls_announce", "kind": "verb_invoked", "params": { "verb_id": "yujin.a11y.announce" } },
+    { "name": "max_three_calls", "kind": "max_tool_calls", "params": { "max": 3 } }
+  ],
+  "fake_responses": [
+    {
+      "text": "Creo el lead.",
+      "tool_calls": [
+        { "name": "nac3_invoke", "args": { "verb_id": "app.create_lead", "args": { "name": "Ana Lopez" } } }
+      ]
+    },
+    {
+      "text": "Anuncio.",
+      "tool_calls": [
+        { "name": "nac3_invoke", "args": { "verb_id": "yujin.a11y.announce", "args": { "text": "Lead Ana Lopez creado con id 47812", "politeness": "polite" } } }
+      ]
+    },
+    { "text": "Listo. Lead id 47812." }
+  ]
+}

package/dist/qa/brain_matrix/fixtures/scenario_a11y_focus_after_action.json

@@ -0,0 +1,22 @@
+{
+  "id": "a11y_focus_after_action",
+  "description": "F107 a11y -- after a successful action that opened a new modal, the brain should move focus to the modal's primary element via yujin.a11y.focus.",
+  "setup": {
+    "extra_system_prompt": "A11Y VERBS active:\n- yujin.a11y.focus [u d] args:{slug<required>, scroll?, announce?}\nAfter opening a modal / new region, move focus to its first interactive child so keyboard + screen-reader users land in the right place."
+  },
+  "turns": [
+    { "kind": "user_says", "text": "Abri el modal de confirmar deploy y dejame el foco en el boton 'Confirmar' (slug confirm-deploy-btn)" }
+  ],
+  "invariants": [
+    { "name": "uses_a11y_focus", "kind": "verb_invoked", "params": { "verb_id": "yujin.a11y.focus" } },
+    { "name": "no_invent_verbs", "kind": "no_invent_verb", "params": {} }
+  ],
+  "fake_responses": [
+    {
+      "text": "Muevo el foco al boton de confirmar.",
+      "tool_calls": [
+        { "name": "nac3_invoke", "args": { "verb_id": "yujin.a11y.focus", "args": { "slug": "confirm-deploy-btn", "scroll": true, "announce": true } } }
+      ]
+    }
+  ]
+}

package/dist/qa/brain_matrix/fixtures/scenario_a11y_focus_after_open.json

@@ -0,0 +1,22 @@
+{
+  "id": "a11y_focus_after_open",
+  "description": "F107 closure -- after opening a modal/sidebar/section, the brain calls yujin.a11y.focus on the first interactive element of the new surface so keyboard-only + screen-reader users are not stranded at the trigger button. Locks the focus-management pattern.",
+  "setup": {
+    "extra_system_prompt": "A11Y VERBS active:\n- yujin.a11y.focus    args:{slug<required>, scroll?, announce?}\n- yujin.a11y.announce args:{text<required>, politeness?, speak?}\nWhen you open a modal / new section, ALWAYS move focus to the first actionable inside (input, primary action, etc) so keyboard + screen-reader users land in the right place. The slug = data-nac-slug or data-nac-id of the target."
+  },
+  "turns": [
+    { "kind": "user_says", "text": "Abri el modal de aprobacion y dejame escribir el motivo." }
+  ],
+  "invariants": [
+    { "name": "uses_a11y_focus", "kind": "verb_invoked", "params": { "verb_id": "yujin.a11y.focus" } },
+    { "name": "no_invent_verbs", "kind": "no_invent_verb", "params": {} }
+  ],
+  "fake_responses": [
+    {
+      "text": "Modal abierto. Muevo foco al campo de motivo para que puedas escribir directo.",
+      "tool_calls": [
+        { "name": "nac3_invoke", "args": { "verb_id": "yujin.a11y.focus", "args": { "slug": "approval-modal.reason-input", "scroll": true, "announce": true } } }
+      ]
+    }
+  ]
+}

package/dist/qa/brain_matrix/fixtures/scenario_a11y_label_unlabeled_button.json

@@ -0,0 +1,26 @@
+{
+  "id": "a11y_label_unlabeled_button",
+  "description": "F107 closure -- when the user reports that a button does not announce its purpose to a screen reader (icon-only button, missing aria-label), the brain calls yujin.a11y.label with a descriptive aria-label + optional shortcut hint. NOT a generic styling fix, NOT a write to the source code (the verb mutates the live DOM via aria attributes).",
+  "setup": {
+    "extra_system_prompt": "A11Y VERBS active:\n- yujin.a11y.label args:{slug<required>, description<required>, shortcut?, role?}\nIcon-only buttons need an aria-label so screen-reader users hear the purpose. Use a11y.label to attach the missing description; do NOT modify the source code (yujin.fs.write-project-file) for runtime ARIA fixes."
+  },
+  "turns": [
+    { "kind": "user_says", "text": "El boton del icono de imprimir no anuncia nada al lector. Ponele label \"Imprimir reporte\"" }
+  ],
+  "invariants": [
+    { "name": "uses_a11y_label", "kind": "verb_invoked", "params": { "verb_id": "yujin.a11y.label" } },
+    { "name": "no_source_edit", "kind": "verb_not_invoked", "params": { "verb_id": "yujin.fs.write-project-file" } },
+    { "name": "no_invent_verbs", "kind": "no_invent_verb", "params": {} }
+  ],
+  "fake_responses": [
+    {
+      "text": "Le pongo la label.",
+      "tool_calls": [
+        { "name": "nac3_invoke", "args": { "verb_id": "yujin.a11y.label", "args": { "slug": "toolbar.print-button", "description": "Imprimir reporte" } } }
+      ]
+    },
+    {
+      "text": "Listo, ahora el lector anuncia 'Imprimir reporte' al enfocar el boton."
+    }
+  ]
+}