@event4u/agent-config 9.8.0 → 9.10.0

package/src/scripts/_lib/anchor_eval.ts

@@ -0,0 +1,311 @@
+/**
+ * Constrained anchor evaluation with frozen verdicts (ADR-202, amended).
+ *
+ * ADR-202 originally called this "deterministic anchor-scoring against
+ * must_include / must_not" and claimed the scorer is "a pure function of
+ * (answer, anchors)". Measured against the corpus that claim is FALSE: 0 of 255
+ * `must_include` anchors carry a literal token or code span, and 17% are
+ * behavioural predicates ("mentions that UI redesign is outside scope"). A
+ * substring match over those measures nothing.
+ *
+ * What is deterministic is everything BELOW the verdict: aggregation, the
+ * per-rule floor, the non-inferiority arithmetic, κ, and the conservative
+ * disagreement resolution — all pure functions over frozen verdicts, all unit
+ * tested here without an API. The verdict itself is a constrained binary
+ * classification per (answer, anchor) by two independent evaluator models. That
+ * is far narrower than the pairwise preference judging which closed by
+ * diagnosis, and — the point — it restores a MEASURABLE κ, whose absence is the
+ * exact reason a single human judge was ruled inadmissible.
+ *
+ * Reproducibility comes from freezing, not from pinning: generate once, freeze
+ * the transcripts AND the verdicts as artefacts, and re-score over the frozen
+ * corpus. A re-generation is a new experiment, never a re-run of this one.
+ */
+
+export type AnchorKind = 'must_include' | 'must_not';
+export type Arm = 'thin' | 'eager';
+
+/** One evaluator's call on one anchor. `true` = satisfied (include) / violated (not). */
+export interface AnchorVerdict {
+    task_id: string;
+    arm: Arm;
+    kind: AnchorKind;
+    /** Index into the task's anchor list for that kind — stable across evaluators. */
+    index: number;
+    anchor: string;
+    hit: boolean;
+}
+
+/** Build the evaluator prompt for one (task, arm): every anchor in one call. */
+export function eval_prompt(
+    prompt: string,
+    answer: string,
+    must_include: readonly string[],
+    must_not: readonly string[],
+): string {
+    const lines: string[] = [
+        'You are grading ONE answer against a fixed checklist. For each checklist',
+        'item, decide only whether the answer satisfies it. Judge the answer on its',
+        'own — do not compare it to anything, do not reward or penalise length, and',
+        'do not grade style.',
+        '',
+        '## The request the answer responds to',
+        '',
+        prompt,
+        '',
+        '## The answer',
+        '',
+        answer,
+        '',
+        '## Checklist',
+        '',
+    ];
+    must_include.forEach((a, i) => lines.push(`I${i}. MUST BE PRESENT: ${a}`));
+    must_not.forEach((a, i) => lines.push(`N${i}. MUST BE ABSENT: ${a}`));
+    lines.push(
+        '',
+        '## Output format — structured, one object per checklist item',
+        '',
+        'Output a JSON array and NOTHING else. Exactly one object per checklist',
+        'item, in the order listed, using the item id as `anchor_id`:',
+        '',
+        '  [{"anchor_id":"I0","verdict":"yes"},{"anchor_id":"N0","verdict":"no"}]',
+        '',
+        '`verdict` is "yes" or "no".',
+        '  For I items: "yes" = the answer satisfies it.',
+        '  For N items: "yes" = the answer DOES the forbidden thing.',
+        '',
+        `Every one of the ${must_include.length + must_not.length} items must appear. No prose, no markdown fence.`,
+    );
+    return lines.join('\n');
+}
+
+/**
+ * Parse an evaluator reply into verdicts. Unparseable or missing items are
+ * returned as `null` so the caller resolves them conservatively rather than
+ * silently scoring them as passes.
+ */
+export function parse_eval(
+    reply: string,
+    must_include_len: number,
+    must_not_len: number,
+): { include: Array<boolean | null>; not: Array<boolean | null>; complete: boolean } {
+    const include: Array<boolean | null> = new Array(must_include_len).fill(null);
+    const not: Array<boolean | null> = new Array(must_not_len).fill(null);
+
+    const put = (tag: string, idx: number, val: boolean): void => {
+        if (tag === 'I') {
+            if (idx < include.length) include[idx] = val;
+        } else if (idx < not.length) not[idx] = val;
+    };
+
+    // Primary: structured objects. Tolerates a surrounding array, a fence, or
+    // one object per line — the shape matters, the packaging does not.
+    const obj = /\{\s*"anchor_id"\s*:\s*"([IN])(\d+)"\s*,\s*"verdict"\s*:\s*"(yes|no)"\s*\}/gi;
+    let m: RegExpExecArray | null;
+    let structured = 0;
+    while ((m = obj.exec(reply)) !== null) {
+        put(m[1]!.toUpperCase(), Number(m[2]), m[3]!.toLowerCase() === 'yes');
+        structured += 1;
+    }
+
+    // Fallback: the flat `I0=yes` form, so a model that ignores the JSON
+    // instruction is still read rather than silently scored as unfavourable.
+    if (structured === 0) {
+        const flat = /\b([IN])(\d+)\s*=\s*(yes|no)\b/gi;
+        while ((m = flat.exec(reply)) !== null) {
+            put(m[1]!.toUpperCase(), Number(m[2]), m[3]!.toLowerCase() === 'yes');
+        }
+    }
+
+    const complete = include.every((v) => v !== null) && not.every((v) => v !== null);
+    return { include, not, complete };
+}
+
+/**
+ * One evaluator call, with exactly one retry when the reply is incomplete or
+ * unparseable (attempt-2 mechanical change). A persistent failure is NOT retried
+ * further and NOT excluded — its `null`s flow into `resolve`, which reads them
+ * as the unfavourable verdict. A model that cannot answer must not be able to
+ * make a corpus look cleaner by staying silent.
+ */
+export function eval_with_retry(
+    ask: (user: string) => string,
+    prompt: string,
+    must_include_len: number,
+    must_not_len: number,
+): { include: Array<boolean | null>; not: Array<boolean | null>; complete: boolean; retried: boolean } {
+    const first = parse_eval(ask(prompt), must_include_len, must_not_len);
+    if (first.complete) return { ...first, retried: false };
+    const second = parse_eval(ask(prompt), must_include_len, must_not_len);
+    return { ...second, retried: true };
+}
+
+/**
+ * Resolve two evaluators conservatively (ADR-202 (c)): a `must_include` counts
+ * only when BOTH say satisfied; a `must_not` counts as violated when EITHER says
+ * violated. `null` (unparsed) is treated as the unfavourable reading, so a
+ * malformed reply can never become a pass.
+ */
+export function resolve(kind: AnchorKind, a: boolean | null, b: boolean | null): boolean {
+    if (kind === 'must_include') return a === true && b === true;
+    return a === true || b === true || a === null || b === null;
+}
+
+export interface TaskAnchors {
+    id: string;
+    rules: readonly string[];
+    must_include: readonly string[];
+    must_not: readonly string[];
+}
+
+export interface ScoreInput {
+    tasks: readonly TaskAnchors[];
+    /** Resolved verdicts, keyed `${task_id}|${arm}|${kind}|${index}` → hit. */
+    resolved: ReadonlyMap<string, boolean>;
+}
+
+export interface ArmScore {
+    include_pass: number;
+    include_total: number;
+    /** include_pass / include_total */
+    rate: number;
+    must_not_violations: Array<{ task_id: string; anchor: string }>;
+}
+
+export function key(task_id: string, arm: Arm, kind: AnchorKind, index: number): string {
+    return `${task_id}|${arm}|${kind}|${index}`;
+}
+
+export function score_arm(input: ScoreInput, arm: Arm): ArmScore {
+    let pass = 0;
+    let total = 0;
+    const violations: Array<{ task_id: string; anchor: string }> = [];
+    for (const t of input.tasks) {
+        t.must_include.forEach((_, i) => {
+            total += 1;
+            if (input.resolved.get(key(t.id, arm, 'must_include', i)) === true) pass += 1;
+        });
+        t.must_not.forEach((anchor, i) => {
+            if (input.resolved.get(key(t.id, arm, 'must_not', i)) === true) {
+                violations.push({ task_id: t.id, anchor });
+            }
+        });
+    }
+    return {
+        include_pass: pass,
+        include_total: total,
+        rate: total === 0 ? 0 : pass / total,
+        must_not_violations: violations,
+    };
+}
+
+/** Per-rule include-pass counts, for the registered floor. */
+export function per_rule_passes(input: ScoreInput, arm: Arm): Map<string, number> {
+    const out = new Map<string, number>();
+    for (const t of input.tasks) {
+        let n = 0;
+        t.must_include.forEach((_, i) => {
+            if (input.resolved.get(key(t.id, arm, 'must_include', i)) === true) n += 1;
+        });
+        for (const r of t.rules) out.set(r, (out.get(r) ?? 0) + n);
+    }
+    return out;
+}
+
+/**
+ * δ proposal from the observed spread: the standard deviation of the per-task
+ * `must_include` pass-rate difference (thin − eager), in percentage points.
+ * Derived from the FROZEN corpus and recorded before scoring — a property of the
+ * corpus, never of the verdict.
+ */
+export function delta_from_spread(input: ScoreInput): number {
+    const diffs: number[] = [];
+    for (const t of input.tasks) {
+        if (t.must_include.length === 0) continue;
+        let thin = 0;
+        let eager = 0;
+        t.must_include.forEach((_, i) => {
+            if (input.resolved.get(key(t.id, 'thin', 'must_include', i)) === true) thin += 1;
+            if (input.resolved.get(key(t.id, 'eager', 'must_include', i)) === true) eager += 1;
+        });
+        diffs.push((thin - eager) / t.must_include.length);
+    }
+    if (diffs.length === 0) return 0;
+    const mean = diffs.reduce((a, b) => a + b, 0) / diffs.length;
+    const varr = diffs.reduce((a, b) => a + (b - mean) ** 2, 0) / diffs.length;
+    return Math.sqrt(varr) * 100;
+}
+
+export interface Verdict {
+    kappa: number;
+    kappa_floor: number;
+    instrument_ok: boolean;
+    delta_pp: number;
+    delta_ceiling_pp: number;
+    delta_registered: boolean;
+    thin: ArmScore;
+    eager: ArmScore;
+    /** rate_thin − rate_eager, percentage points. */
+    rate_gap_pp: number;
+    must_not_ok: boolean;
+    non_inferiority_ok: boolean;
+    per_rule_floor_ok: boolean;
+    per_rule_floor_breaches: string[];
+    pass: boolean;
+}
+
+export interface VerdictOptions {
+    kappa: number;
+    kappa_floor?: number;
+    delta_ceiling_pp?: number;
+}
+
+/** Apply every registered threshold. Pure — the whole decision, no I/O. */
+export function verdict(input: ScoreInput, opts: VerdictOptions): Verdict {
+    const kappa_floor = opts.kappa_floor ?? 0.8;
+    const delta_ceiling_pp = opts.delta_ceiling_pp ?? 3;
+    const thin = score_arm(input, 'thin');
+    const eager = score_arm(input, 'eager');
+    const delta_pp = delta_from_spread(input);
+    const delta_registered = delta_pp <= delta_ceiling_pp;
+    const rate_gap_pp = (thin.rate - eager.rate) * 100;
+
+    // Registered: thin introduces NO must_not violation that eager avoids.
+    const eagerViolated = new Set(eager.must_not_violations.map((v) => `${v.task_id}|${v.anchor}`));
+    const introduced = thin.must_not_violations.filter((v) => !eagerViolated.has(`${v.task_id}|${v.anchor}`));
+    const must_not_ok = introduced.length === 0;
+
+    const effective_delta = Math.min(delta_pp, delta_ceiling_pp);
+    const non_inferiority_ok = rate_gap_pp >= -effective_delta;
+
+    const thinRule = per_rule_passes(input, 'thin');
+    const eagerRule = per_rule_passes(input, 'eager');
+    const breaches: string[] = [];
+    for (const [rule, e] of eagerRule) {
+        if (e >= 1 && (thinRule.get(rule) ?? 0) === 0) breaches.push(rule);
+    }
+
+    const instrument_ok = opts.kappa >= kappa_floor;
+    return {
+        kappa: opts.kappa,
+        kappa_floor,
+        instrument_ok,
+        delta_pp,
+        delta_ceiling_pp,
+        delta_registered,
+        thin,
+        eager,
+        rate_gap_pp,
+        must_not_ok,
+        non_inferiority_ok,
+        per_rule_floor_ok: breaches.length === 0,
+        per_rule_floor_breaches: breaches.sort(),
+        pass:
+            instrument_ok &&
+            delta_registered &&
+            must_not_ok &&
+            non_inferiority_ok &&
+            breaches.length === 0,
+    };
+}

package/src/scripts/_lib/auto_dispatch.corpus.test.ts

@@ -15,7 +15,13 @@
  */
 import { describe, expect, it } from 'vitest';
 
-import { classifyTask, type ActivationInputs, type TaskSignals } from './auto_dispatch.js';
+import {
+    classifyLookup,
+    classifyTask,
+    type ActivationInputs,
+    type LookupClass,
+    type TaskSignals,
+} from './auto_dispatch.js';
 
 const ON: ActivationInputs = { enabled: true, auto: 'on', subagent_spawn: true };
 
@@ -85,3 +91,60 @@ describe('parallelizable classifier — corpus recall', () => {
         }
     });
 });
+
+/**
+ * Lookup-class corpus (L0 — road-to-lean-agent-init Phase 1). The first four
+ * entries are the live-observed 2026-07-28 task shapes that burned ~1.21M
+ * subagent tokens; each must route to a primitive, never a spawn. The
+ * negative controls are non-lookup tasks that must escalate unchanged.
+ */
+const LOOKUP_CORPUS: Array<{ id: string; text: string; expect: LookupClass | null }> = [
+    // ── the four observed shapes (live evidence, 2026-07-28 session) ──
+    { id: 'lk-01', text: 'Confirming ExternalApiProviderEnum definition location', expect: 'definition' },
+    { id: 'lk-02', text: 'Confirming GWInput import call sites', expect: 'references' },
+    { id: 'lk-03', text: 'Running check_enforcement_coverage report', expect: 'report-run' },
+    { id: 'lk-04', text: 'Probing candidate strings with tsx', expect: 'string-existence' },
+    // ── generalizations of the same shapes ──
+    { id: 'lk-05', text: 'Where is classifyTask defined?', expect: 'definition' },
+    { id: 'lk-06', text: 'Who calls readOrchestrationMetrics across the repo?', expect: 'references' },
+    { id: 'lk-07', text: 'Does the string "lean-init-2026" exist anywhere in src/?', expect: 'string-existence' },
+    { id: 'lk-08', text: 'Run the roadmap:progress-check report and summarize', expect: 'report-run' },
+    { id: 'lk-09', text: 'Find all usages of InferredTier in tests', expect: 'references' },
+    { id: 'lk-10', text: 'Locate the enum that lists the dispatch actions', expect: 'definition' },
+    // ── negative controls — must escalate, never route to a primitive ──
+    { id: 'lk-n1', text: 'Refactor the auth middleware to use the new session store', expect: null },
+    { id: 'lk-n2', text: 'Fix the failing corpus test and explain the root cause', expect: null },
+    { id: 'lk-n3', text: 'Design a migration plan for the audit schema', expect: null },
+    { id: 'lk-n4', text: 'Review this diff for security issues', expect: null },
+];
+
+describe('lookup-class routing — corpus recall + FP=0 (lean-init L0)', () => {
+    for (const c of LOOKUP_CORPUS) {
+        it(`${c.id}: ${c.text}`, () => {
+            const r = classifyLookup(c.text);
+            expect(r.lookup_class).toBe(c.expect);
+            expect(r.route).toBe(c.expect === null ? 'escalate' : 'primitive');
+        });
+    }
+
+    it('recall on the four live-observed shapes is 4/4', () => {
+        const observed = LOOKUP_CORPUS.filter((c) => c.id.startsWith('lk-0') && Number(c.id.slice(3)) <= 4);
+        expect(observed).toHaveLength(4);
+        for (const c of observed) {
+            expect(classifyLookup(c.text).route).toBe('primitive');
+        }
+    });
+
+    it('false-positive rate is 0 — no negative control routes to a primitive', () => {
+        for (const c of LOOKUP_CORPUS.filter((x) => x.expect === null)) {
+            expect(classifyLookup(c.text).route).toBe('escalate');
+        }
+    });
+
+    it('unknown resolves to escalate with a non-degradation reason', () => {
+        const r = classifyLookup('Do something clever with the codebase');
+        expect(r.route).toBe('escalate');
+        expect(r.primitive).toBeNull();
+        expect(r.reason).toContain('never down-guessed');
+    });
+});

package/src/scripts/_lib/auto_dispatch.ts

@@ -94,6 +94,82 @@ export function classifyTask(signals: TaskSignals, activation: ActivationInputs)
     return { delegable: true, action, mode, reason };
 }
 
+// ── Lookup-class routing (L0 — road-to-lean-agent-init) ──
+
+/**
+ * Lookup-class task shapes that route to a deterministic primitive INSTEAD of
+ * a subagent spawn. Contract:
+ * `auto-dispatch-classification.md § Lookup-class rung`.
+ *
+ * Live evidence (2026-07-28): four `general-purpose` subagents burned ~1.21M
+ * tokens on tasks of exactly these shapes; the primitives answer each for <1k.
+ */
+export type LookupClass =
+    | 'definition' // "where is X defined" → code_graph query
+    | 'references' // "who calls / imports X" → code_graph query
+    | 'string-existence' // "does string Y exist" → FTS one-shot / capped grep
+    | 'report-run'; // "run report Z" → script-run with rtk wrap
+
+export type LookupPrimitive = 'code-graph-query' | 'fts-or-capped-grep' | 'script-run-rtk';
+
+export interface LookupRoute {
+    /** Matched lookup class, or null when the task is not lookup-shaped. */
+    lookup_class: LookupClass | null;
+    /** `primitive` = skip the spawn; `escalate` = normal classification path. */
+    route: 'primitive' | 'escalate';
+    primitive: LookupPrimitive | null;
+    reason: string;
+}
+
+const LOOKUP_PATTERNS: Array<{ cls: LookupClass; primitive: LookupPrimitive; re: RegExp }> = [
+    {
+        cls: 'definition',
+        primitive: 'code-graph-query',
+        re: /\b(where\s+is\s+\S+\s+defined|definition\s+(location|of|site)|find\s+the\s+definition|locate\s+the\s+(class|enum|function|type|interface|symbol)|confirm\w*\s+\S+\s+definition)\b/i,
+    },
+    {
+        cls: 'references',
+        primitive: 'code-graph-query',
+        re: /\b(who\s+(calls|imports|uses|references)|call\s*sites?|import\s+(call\s*)?sites?|reverse\s+references|where\s+is\s+\S+\s+(used|imported|referenced|called)|all\s+(callers|usages|references)\s+of)\b/i,
+    },
+    {
+        cls: 'string-existence',
+        primitive: 'fts-or-capped-grep',
+        re: /\b(does\s+(the\s+)?string\s+.*\bexist|is\s+(the\s+)?string\s+.*\b(present|used)\b|prob\w+\s+(candidate\s+)?strings?|search\s+for\s+the\s+(exact\s+)?string|(grep|check)\s+(for\s+)?(the\s+)?(literal|string)\b|does\s+\S+\s+appear\s+anywhere)/i,
+    },
+    {
+        cls: 'report-run',
+        primitive: 'script-run-rtk',
+        re: /\b(run(ning)?\s+(the\s+)?\S*(report|check|lint\w*|coverage)\b|run(ning)?\s+(check|lint|report)_\w+|regenerate\s+the\s+\S+\s+report)/i,
+    },
+];
+
+/**
+ * Route a task description to a lookup-class primitive or the regular
+ * escalation path. Deterministic regex layer, no LLM call (design lock:
+ * no LLM classifier fallback — cut C3). Non-matches escalate — never a
+ * silently degraded answer; an index-miss at execution time escalates the
+ * same way (runtime concern, documented in the classification context).
+ */
+export function classifyLookup(taskText: string): LookupRoute {
+    for (const p of LOOKUP_PATTERNS) {
+        if (p.re.test(taskText)) {
+            return {
+                lookup_class: p.cls,
+                route: 'primitive',
+                primitive: p.primitive,
+                reason: `lookup-class ${p.cls} — deterministic primitive, no spawn`,
+            };
+        }
+    }
+    return {
+        lookup_class: null,
+        route: 'escalate',
+        primitive: null,
+        reason: 'not lookup-shaped — regular classification path (never down-guessed)',
+    };
+}
+
 // ── Per-slice tier inference (v1.5 — road-to-cost-aware-model-routing) ──
 
 /**