agentic-qe 3.11.5 → 3.12.1

package/dist/learning/provenance-tier.js

@@ -0,0 +1,64 @@
+/**
+ * Provenance tiers for learning writes (ADR-121).
+ *
+ * Every learning write carries an explicit evidence tier. Authority tracks
+ * evidence strength: an execution-observed outcome is ground truth, an LLM
+ * judgment is a noisy opinion (ADR-119), a structural/co-occurrence inference
+ * is a weak guess. Only `oracle:test-exec` (or an explicitly-budgeted
+ * `judge:llm`) may change fleet behavior; `proxy:structural` is written and
+ * searchable but never auto-promoted.
+ *
+ * Mirrors ruflo ADR-174's in-code tier gate (not prose). Shared across every
+ * promotion path (pattern promotion, dream apply, flywheel accept) so the gate
+ * is defined once.
+ */
+/** Ordered strongest → weakest. Index is the rank (0 = strongest). */
+export const PROVENANCE_TIERS = [
+    'oracle:test-exec',
+    'judge:llm',
+    'proxy:structural',
+];
+/**
+ * Conservative default for undifferentiated / legacy / unknown writes.
+ * The backfill (ADR-121 §4) tags all pre-existing rows with this — it can
+ * never over-credit, only under-credit until a row is re-derived from
+ * execution evidence through the normal write path.
+ */
+export const DEFAULT_PROVENANCE_TIER = 'proxy:structural';
+/** Rank of a tier (0 = strongest). Unknown tiers rank as the weakest. */
+export function tierRank(tier) {
+    const idx = PROVENANCE_TIERS.indexOf(coerceTier(tier));
+    return idx < 0 ? PROVENANCE_TIERS.length - 1 : idx;
+}
+/**
+ * Normalize any raw value read from storage into a known tier.
+ * Null / undefined / unrecognized → the conservative default. Never throws —
+ * a bad tier must degrade to "weak", never crash the promotion path.
+ */
+export function coerceTier(raw) {
+    if (raw && PROVENANCE_TIERS.includes(raw)) {
+        return raw;
+    }
+    return DEFAULT_PROVENANCE_TIER;
+}
+/**
+ * The in-code promotion gate (ADR-121 §2). `oracle:test-exec` may always
+ * promote; `judge:llm` promotes only under an explicit budget flag;
+ * `proxy:structural` is NEVER auto-promoted.
+ *
+ * This is the single authority consulted by every promotion path. It does not
+ * decide whether the *metrics* clear threshold — only whether the *evidence
+ * tier* is strong enough to be allowed to change fleet behavior at all.
+ */
+export function tierAllowsPromotion(tier, options = {}) {
+    switch (coerceTier(tier)) {
+        case 'oracle:test-exec':
+            return true;
+        case 'judge:llm':
+            return options.allowJudgeTier === true;
+        case 'proxy:structural':
+        default:
+            return false;
+    }
+}
+//# sourceMappingURL=provenance-tier.js.map

package/dist/learning/qe-flywheel/corpus-scorer.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Real corpus scorer for the QE-policy flywheel (ADR-118) — the injected
+ * `PolicyScorer` seam that generation.ts DI's. Turns a `RetrievalPolicy` into
+ * its two decision signals over the live corpus:
+ *
+ *   1. heldOut  — self-supervised self-retrieval MRR over `qe_patterns`: for
+ *      each stored pattern, a query is built from its BODY tokens with the
+ *      SUBJECT (name) tokens WITHHELD; the correct label is the pattern's own
+ *      id. A pure LEXICAL retriever — parameterized entirely by the policy
+ *      weights (alpha blends body vs subject match, subject/bodyWeight scale the
+ *      term contributions, typePenaltyFactor penalizes pattern_type mismatch,
+ *      mmrLambda controls diversity in the ranked list) — ranks the corpus, and
+ *      Mean Reciprocal Rank of the true doc is the score. A policy whose weights
+ *      better match the corpus ranks the true doc higher, so heldOut is the
+ *      optimization signal. NO model, NO network — deterministic lexical only.
+ *
+ *   2. anchorMean — grade the ADR-117 frozen anchor via `loadAnchorSet` +
+ *      `evaluateOracle`, using a gold/reference test per item that kills its
+ *      mutants (mutationScore >= 0.8 on all items), then take the mean.
+ *
+ * HONEST SCOPE (ruflo ADR-176 "human anchor held flat"): in this retrieval-only
+ * wiring the retrieval policy does NOT feed test generation, so the anchor mean
+ * is a FLAT no-regression GUARD — constant across every policy. It is graded
+ * ONCE and cached. Coupling retrieval -> test-gen -> anchor (so the anchor moves
+ * with the policy) requires a model in the loop and is DEFERRED. Here the anchor
+ * is the flat floor the gate must not regress, while heldOut is the signal the
+ * flywheel actually optimizes.
+ */
+import type BetterSqlite3 from 'better-sqlite3';
+import type { PolicyScorer } from './generation.js';
+type SqliteDb = BetterSqlite3.Database;
+/** Default frozen anchor location (ADR-117), consistent with anchor-set tests. */
+export declare const DEFAULT_ANCHOR_PATH: string;
+export interface CorpusScorerOptions {
+    /** Read-only handle to the corpus DB (must expose a `qe_patterns` table). */
+    db: SqliteDb;
+    /** Frozen anchor path (ADR-117). Defaults to the pinned qe-anchor-v1. */
+    anchorPath?: string;
+    /**
+     * Fraction of patterns used as held-out queries (deterministic id-sorted
+     * prefix). Default 1 (query every pattern). Lower it to bound the O(n^2) cost
+     * on large corpora — the subject-withholding is the real held-out mechanism.
+     */
+    heldOutRatio?: number;
+}
+/**
+ * Build the real `PolicyScorer` for the flywheel. The corpus is read ONCE and
+ * the frozen anchor is graded ONCE (lazily, then cached) — the anchor mean is
+ * policy-independent (the honest flat guard), so caching makes the flat-guard
+ * property exact and avoids re-spawning the oracle per generation. Only heldOut
+ * varies with the policy.
+ */
+export declare function createCorpusScorer(opts: CorpusScorerOptions): PolicyScorer;
+export {};
+//# sourceMappingURL=corpus-scorer.d.ts.map

package/dist/learning/qe-flywheel/corpus-scorer.js

@@ -0,0 +1,293 @@
+/**
+ * Real corpus scorer for the QE-policy flywheel (ADR-118) — the injected
+ * `PolicyScorer` seam that generation.ts DI's. Turns a `RetrievalPolicy` into
+ * its two decision signals over the live corpus:
+ *
+ *   1. heldOut  — self-supervised self-retrieval MRR over `qe_patterns`: for
+ *      each stored pattern, a query is built from its BODY tokens with the
+ *      SUBJECT (name) tokens WITHHELD; the correct label is the pattern's own
+ *      id. A pure LEXICAL retriever — parameterized entirely by the policy
+ *      weights (alpha blends body vs subject match, subject/bodyWeight scale the
+ *      term contributions, typePenaltyFactor penalizes pattern_type mismatch,
+ *      mmrLambda controls diversity in the ranked list) — ranks the corpus, and
+ *      Mean Reciprocal Rank of the true doc is the score. A policy whose weights
+ *      better match the corpus ranks the true doc higher, so heldOut is the
+ *      optimization signal. NO model, NO network — deterministic lexical only.
+ *
+ *   2. anchorMean — grade the ADR-117 frozen anchor via `loadAnchorSet` +
+ *      `evaluateOracle`, using a gold/reference test per item that kills its
+ *      mutants (mutationScore >= 0.8 on all items), then take the mean.
+ *
+ * HONEST SCOPE (ruflo ADR-176 "human anchor held flat"): in this retrieval-only
+ * wiring the retrieval policy does NOT feed test generation, so the anchor mean
+ * is a FLAT no-regression GUARD — constant across every policy. It is graded
+ * ONCE and cached. Coupling retrieval -> test-gen -> anchor (so the anchor moves
+ * with the policy) requires a model in the loop and is DEFERRED. Here the anchor
+ * is the flat floor the gate must not regress, while heldOut is the signal the
+ * flywheel actually optimizes.
+ */
+import { join } from 'node:path';
+import { loadAnchorSet, anchorMean } from '../../validation/anchor-set.js';
+import { evaluateOracle } from '../../validation/oracle-eval.js';
+/** Default frozen anchor location (ADR-117), consistent with anchor-set tests. */
+export const DEFAULT_ANCHOR_PATH = join(process.cwd(), 'verification/anchors/qe-anchor-v1.json');
+/** Lowercase, split on non-alphanumerics, keep tokens of length >= 2, dedupe. */
+function tokenize(...parts) {
+    const out = new Set();
+    for (const part of parts) {
+        if (!part)
+            continue;
+        for (const tok of part.toLowerCase().split(/[^a-z0-9]+/)) {
+            if (tok.length >= 2)
+                out.add(tok);
+        }
+    }
+    return out;
+}
+function loadCorpus(db) {
+    const rows = db
+        .prepare(`SELECT id, pattern_type, name, description, template_json, context_json
+         FROM qe_patterns`)
+        .all();
+    return rows.map((r) => {
+        const subjectTokens = tokenize(r.name);
+        const bodyTokens = tokenize(r.description, r.template_json, r.context_json);
+        const allTokens = new Set([...subjectTokens, ...bodyTokens]);
+        return {
+            id: r.id,
+            patternType: r.pattern_type ?? '',
+            subjectTokens,
+            bodyTokens,
+            allTokens,
+        };
+    });
+}
+// ---------------------------------------------------------------------------
+// Held-out self-retrieval MRR (the optimization signal)
+// ---------------------------------------------------------------------------
+/** Count of query tokens present in the doc's token set (distinct overlap). */
+function overlap(query, docTokens) {
+    let n = 0;
+    for (const t of query)
+        if (docTokens.has(t))
+            n++;
+    return n;
+}
+/** Jaccard similarity of two token sets (for MMR diversity). */
+function jaccard(a, b) {
+    if (a.size === 0 && b.size === 0)
+        return 0;
+    let inter = 0;
+    for (const t of a)
+        if (b.has(t))
+            inter++;
+    const union = a.size + b.size - inter;
+    return union === 0 ? 0 : inter / union;
+}
+/**
+ * Policy-parameterized lexical relevance of `doc` to a body-only `query` drawn
+ * from held-out pattern `queried`. alpha blends the body and subject channels;
+ * subject/bodyWeight scale each channel; typePenaltyFactor discounts docs whose
+ * pattern_type differs from the queried pattern's.
+ */
+function relevance(query, doc, queried, policy) {
+    const bodyMatch = overlap(query, doc.bodyTokens);
+    const subjectMatch = overlap(query, doc.subjectTokens);
+    let rel = policy.alpha * policy.bodyWeight * bodyMatch +
+        (1 - policy.alpha) * policy.subjectWeight * subjectMatch;
+    if (doc.patternType !== queried.patternType) {
+        rel *= 1 - policy.typePenaltyFactor;
+    }
+    return rel;
+}
+/**
+ * Greedy MMR rerank: select next by lambda * normalized-relevance −
+ * (1 − lambda) * max-similarity-to-already-selected. lambda = 1 is pure
+ * relevance; lower lambda rewards diversity. Ties break by id for determinism.
+ */
+function rankMMR(scored, lambda) {
+    const remaining = [...scored].sort((a, b) => b.nrel - a.nrel || a.id.localeCompare(b.id));
+    const chosen = [];
+    while (remaining.length > 0) {
+        let bestIdx = 0;
+        let bestScore = -Infinity;
+        for (let i = 0; i < remaining.length; i++) {
+            const sim = chosen.length === 0
+                ? 0
+                : Math.max(...chosen.map((c) => jaccard(remaining[i].tokens, c.tokens)));
+            const mmr = lambda * remaining[i].nrel - (1 - lambda) * sim;
+            if (mmr > bestScore ||
+                (mmr === bestScore && remaining[i].id.localeCompare(remaining[bestIdx].id) < 0)) {
+                bestScore = mmr;
+                bestIdx = i;
+            }
+        }
+        chosen.push(remaining.splice(bestIdx, 1)[0]);
+    }
+    return chosen.map((c) => c.id);
+}
+/**
+ * MMR candidate-pool cap. The greedy MMR rerank is O(pool^2 * tokens) per query,
+ * so reranking the WHOLE corpus is cubic in corpus size and hangs on a real
+ * (hundreds of patterns) corpus. Standard practice is to rerank only the top-K
+ * by relevance — the true doc (queried by its own body) ranks into this pool, so
+ * MRR is unchanged, while cost stays bounded. Below this size the behavior is
+ * identical to reranking everything (so small-corpus tests are unaffected).
+ */
+const MMR_POOL_SIZE = 50;
+/** Deterministic held-out query split: id-sorted prefix of `ratio` of the corpus. */
+function selectHeldOut(docs, ratio) {
+    const clamped = Math.min(1, Math.max(0, ratio));
+    const sorted = [...docs].sort((a, b) => a.id.localeCompare(b.id));
+    const n = Math.max(1, Math.ceil(sorted.length * clamped));
+    return sorted.slice(0, n);
+}
+/**
+ * Mean Reciprocal Rank of each held-out pattern's own id when its BODY tokens
+ * are used as the query (subject withheld) and the whole corpus is the
+ * candidate pool. In [0, 1]; higher = the policy weights better recover the
+ * corpus from its own body signal.
+ */
+function scoreHeldOut(docs, policy, ratio) {
+    if (docs.length === 0)
+        return 0;
+    const queries = selectHeldOut(docs, ratio);
+    if (queries.length === 0)
+        return 0;
+    let sumReciprocalRank = 0;
+    for (const queried of queries) {
+        const query = queried.bodyTokens; // subject deliberately withheld
+        const rels = docs.map((d) => ({ doc: d, rel: relevance(query, d, queried, policy) }));
+        const maxRel = rels.reduce((m, r) => Math.max(m, r.rel), 0);
+        const scored = rels.map((r) => ({
+            id: r.doc.id,
+            nrel: maxRel > 0 ? r.rel / maxRel : 0,
+            tokens: r.doc.allTokens,
+        }));
+        // Rerank only the top-K by relevance (bounded MMR cost). Force-include the
+        // true doc so its rank is always measurable even if it fell just outside K.
+        const pool = scored
+            .slice()
+            .sort((a, b) => b.nrel - a.nrel || a.id.localeCompare(b.id))
+            .slice(0, MMR_POOL_SIZE);
+        if (!pool.some((s) => s.id === queried.id)) {
+            const trueDoc = scored.find((s) => s.id === queried.id);
+            if (trueDoc)
+                pool.push(trueDoc);
+        }
+        const ranked = rankMMR(pool, policy.mmrLambda);
+        const rank = ranked.indexOf(queried.id) + 1; // 1-based; 0 -> not found
+        sumReciprocalRank += rank > 0 ? 1 / rank : 0;
+    }
+    return sumReciprocalRank / queries.length;
+}
+// ---------------------------------------------------------------------------
+// Anchor flat guard (the no-regression floor)
+// ---------------------------------------------------------------------------
+/**
+ * Gold/reference tests for the frozen qe-anchor-v1 items — each is a minimal
+ * CORRECT test written to kill its item's operator mutants (verified >= 0.8 via
+ * evaluateOracle). Keyed by moduleName. These stand in for the "human anchor"
+ * that a policy-independent, model-free wiring cannot generate itself.
+ */
+const GOLD_TESTS = {
+    inRange: `import test from 'node:test';
+import assert from 'node:assert/strict';
+import { inRange } from '../src/inRange.mjs';
+test('inRange', () => {
+  assert.equal(inRange(2, 1, 3), true);
+  assert.equal(inRange(1, 1, 3), true);
+  assert.equal(inRange(3, 1, 3), true);
+  assert.equal(inRange(0, 1, 3), false);
+  assert.equal(inRange(5, 1, 3), false);
+});
+`,
+    letterGrade: `import test from 'node:test';
+import assert from 'node:assert/strict';
+import { letterGrade } from '../src/letterGrade.mjs';
+test('letterGrade', () => {
+  assert.equal(letterGrade(95), 'A');
+  assert.equal(letterGrade(90), 'A');
+  assert.equal(letterGrade(89), 'D');
+  assert.equal(letterGrade(60), 'D');
+  assert.equal(letterGrade(59), 'F');
+});
+`,
+    isBlank: `import test from 'node:test';
+import assert from 'node:assert/strict';
+import { isBlank } from '../src/isBlank.mjs';
+test('isBlank', () => {
+  assert.equal(isBlank(''), true);
+  assert.equal(isBlank('   '), true);
+  assert.equal(isBlank(null), true);
+  assert.equal(isBlank(undefined), true);
+  assert.equal(isBlank('abc'), false);
+});
+`,
+    countAbove: `import test from 'node:test';
+import assert from 'node:assert/strict';
+import { countAbove } from '../src/countAbove.mjs';
+test('countAbove', () => {
+  assert.equal(countAbove([1, 2, 3], 2), 1);
+  assert.equal(countAbove([2, 3], 2), 1);
+  assert.equal(countAbove([1, 1], 2), 0);
+  assert.equal(countAbove([], 2), 0);
+});
+`,
+    parseBool: `import test from 'node:test';
+import assert from 'node:assert/strict';
+import { parseBool } from '../src/parseBool.mjs';
+test('parseBool', () => {
+  assert.equal(parseBool('true'), true);
+  assert.equal(parseBool('TRUE'), true);
+  assert.equal(parseBool('false'), false);
+  assert.throws(() => parseBool('nope'));
+});
+`,
+};
+/** Grade one anchor item with its gold test; returns the oracle mutation score. */
+function gradeAnchorItem(item) {
+    const goldTest = GOLD_TESTS[item.moduleName];
+    if (!goldTest) {
+        throw new Error(`No gold reference test for anchor item "${item.id}" (module "${item.moduleName}"). ` +
+            `The corpus scorer's flat guard is wired for qe-anchor-v1; a re-frozen anchor ` +
+            `needs matching gold tests added here.`);
+    }
+    const result = evaluateOracle({
+        moduleName: item.moduleName,
+        referenceImpl: item.referenceImpl,
+        generatedTest: goldTest,
+        threshold: 0.8,
+    });
+    return result.mutationScore;
+}
+/** Grade the whole frozen anchor -> mean of per-item mutation scores (the flat floor). */
+function gradeAnchor(anchorPath) {
+    const anchor = loadAnchorSet(anchorPath); // throws on hash drift (ADR-117)
+    const scores = anchor.items.map(gradeAnchorItem);
+    return anchorMean(scores);
+}
+// ---------------------------------------------------------------------------
+// Factory
+// ---------------------------------------------------------------------------
+/**
+ * Build the real `PolicyScorer` for the flywheel. The corpus is read ONCE and
+ * the frozen anchor is graded ONCE (lazily, then cached) — the anchor mean is
+ * policy-independent (the honest flat guard), so caching makes the flat-guard
+ * property exact and avoids re-spawning the oracle per generation. Only heldOut
+ * varies with the policy.
+ */
+export function createCorpusScorer(opts) {
+    const anchorPath = opts.anchorPath ?? DEFAULT_ANCHOR_PATH;
+    const ratio = opts.heldOutRatio ?? 1;
+    const corpus = loadCorpus(opts.db);
+    let anchorMeanCached = null;
+    return (policy) => {
+        const heldOut = scoreHeldOut(corpus, policy, ratio);
+        if (anchorMeanCached === null) {
+            anchorMeanCached = gradeAnchor(anchorPath);
+        }
+        return { heldOut, anchorMean: anchorMeanCached };
+    };
+}
+//# sourceMappingURL=corpus-scorer.js.map

package/dist/learning/qe-flywheel/coupled-anchor.d.ts

@@ -0,0 +1,91 @@
+/**
+ * Coupled anchor scorer — retrieval → test-gen → anchor (ADR-118 / ADR-117).
+ *
+ * In the retrieval-only wiring the anchor was policy-INDEPENDENT (a flat guard):
+ * the retrieval policy didn't feed test generation, so it couldn't move the
+ * oracle score, and the no-regression guard was inert. This closes that loop:
+ *
+ *   retrieve(inputUnderTest, policy) → few-shot examples
+ *       → generate(inputUnderTest, examples) → a test
+ *           → evaluateOracle(referenceImpl, test) → mutation kill rate
+ *
+ * Now `anchorMean` VARIES with the retrieval policy — a policy that surfaces
+ * more useful examples yields better generated tests and a higher oracle score.
+ * The guard gets teeth: a candidate that games the cheap self-retrieval proxy
+ * but produces WORSE tests drops the anchor mean and is rejected (ADR-117
+ * no-regression), and the promotion is now genuinely oracle:test-exec-backed
+ * (ADR-121) rather than a framing stretch.
+ *
+ * `retrieve` and `generate` are injected seams: fake/deterministic in tests,
+ * a real policy-weighted DB retriever + an LLM test generator in production
+ * (the LLM step is the metered part — bind it deliberately).
+ */
+import type { Database as DatabaseType } from 'better-sqlite3';
+import type { RetrievalPolicy } from './policy.js';
+import type { PolicyScorer } from './generation.js';
+/** A pattern retrieved as a few-shot example for test generation. */
+export interface RetrievedExample {
+    id: string;
+    name: string;
+    body: string;
+}
+/** Retrieve few-shot examples for a query under a given retrieval policy. */
+export type RetrieveFn = (query: string, policy: RetrievalPolicy) => RetrievedExample[] | Promise<RetrievedExample[]>;
+/** Generate a test for `inputUnderTest`, optionally conditioned on examples. */
+export type TestGenerator = (inputUnderTest: string, examples: RetrievedExample[]) => string | Promise<string>;
+export interface CoupledAnchorOptions {
+    anchorPath?: string;
+    retrieve: RetrieveFn;
+    generate: TestGenerator;
+    /** Oracle mutation-kill threshold per item (default 0.8 — the ADR-117 High bar). */
+    threshold?: number;
+    /** Cap on mutants per item for speed (forwarded to evaluateOracle). */
+    maxMutants?: number;
+}
+/**
+ * Build the coupled anchorMean scorer: `(policy) => Promise<number>`.
+ * Grades each frozen anchor item by generating a test from policy-retrieved
+ * examples and scoring its mutation kill rate. The anchor now moves with the
+ * policy.
+ */
+export declare function createCoupledAnchorScorer(opts: CoupledAnchorOptions): (policy: RetrievalPolicy) => Promise<number>;
+export interface CoupledScorerOptions extends CoupledAnchorOptions {
+    /** DB handle for the self-retrieval held-out signal (read-only). */
+    db: DatabaseType;
+    /** Held-out query ratio for the proxy signal (bounds cost). */
+    heldOutRatio?: number;
+}
+/**
+ * A full PolicyScorer where `heldOut` is the cheap self-retrieval proxy (train
+ * signal) and `anchorMean` is the COUPLED oracle test-gen quality (the guard
+ * that now moves). The flywheel's accept() then requires a proxy gain WITHOUT an
+ * oracle-quality regression — the proxy-overfitting veto ADR-117 exists for.
+ */
+export declare function createCoupledScorer(opts: CoupledScorerOptions): PolicyScorer;
+/**
+ * A real policy-weighted DB retriever over `qe_patterns` — token overlap on the
+ * query, scaled by the policy's body/subject weights, top-K by score. Read-only.
+ * Injected as `retrieve` for live runs; tests use a deterministic fake instead.
+ */
+export declare function createDbRetriever(db: DatabaseType, topK?: number): RetrieveFn;
+export interface SemanticRetrieverOptions {
+    /** Read-only DB with `qe_patterns` + `qe_pattern_embeddings`. */
+    db: DatabaseType;
+    /** Embed a query into the pattern-embedding space. Default: all-MiniLM-L6-v2. */
+    embed?: (text: string) => Promise<number[]>;
+    topK?: number;
+}
+/**
+ * Policy-sensitive SEMANTIC retriever over the pre-computed `qe_pattern_embeddings`
+ * (all-MiniLM-L6-v2). Fixes the lexical retriever's honest-null: the anchor
+ * queries (toy functions) have ~0 lexical overlap with the corpus, so lexical
+ * scores were identical for every policy. Cosine similarity is a real, varying
+ * signal, and the policy re-shapes retrieval three ways:
+ *   - `alpha` blends semantic cosine vs lexical overlap,
+ *   - `subjectWeight`/`bodyWeight` scale the lexical component,
+ *   - `mmrLambda` trades relevance for diversity in a greedy MMR rerank
+ *     (this is the lever that reorders even when lexical overlap is 0).
+ * Query embeddings are cached (same query across policies embeds once).
+ */
+export declare function createSemanticRetriever(opts: SemanticRetrieverOptions): RetrieveFn;
+//# sourceMappingURL=coupled-anchor.d.ts.map