@wooojin/forgen 0.4.10 → 0.4.11

package/dist/engine/solution-matcher.d.ts

@@ -1,34 +1,30 @@
-import type { ScopeInfo } from '../core/types.js';
-import type { SolutionStatus, SolutionType } from './solution-format.js';
 /**
- * @deprecated Use `defaultNormalizer.normalizeTerms` from
- * `./term-normalizer.js` directly. Kept as a thin wrapper for the existing
- * `synonym-tfidf.test.ts` and any external consumers.
- */
-export declare function expandTagsWithSynonyms(tags: string[]): string[];
-/**
- * Compute the Dice coefficient between two strings using character bigrams.
- *
- * Dice = 2 * |intersection| / (|A| + |B|)
+ * Solution matcher — thin facade re-exporting from decomposed modules.
  *
- * Both strings are lowercased and whitespace-stripped before bigram generation.
- * Returns 0 for empty strings or single-character strings (no bigrams possible).
- * Returns 1.0 for identical non-trivial strings.
+ * All public exports are preserved for backward compatibility. Internal
+ * callers that import from './solution-matcher.js' continue to work.
  *
- * This is used as a lightweight fuzzy matching signal for borderline cases
- * where the TF-IDF tag intersection produces a low score but the query and
- * solution tags are character-similar (e.g., "database" vs "데이터베이스"
- * won't match, but "database" vs "databse" will get a high score).
+ * Module layout (post-decomposition):
+ *   scoring-algorithms.ts  — bigramSimilarity, bm25Score, tagWeight, COMMON_TAGS
+ *   relevance-scorer.ts    — calculateRelevance, CalculateRelevanceOptions
+ *   precision-guards.ts    — shouldRejectByR4T3Rules
+ *   ranking-pipeline.ts    — rankCandidates, RankableSolution, RankedCandidate
+ *   solution-matcher-eval.ts — EvalSolution/Query/Fixture/Result, ROUND3_BASELINE, evaluateQuery, evaluateSolutionMatcher
+ *   meta-learning/matcher-weight-loader.ts — loadTunedMatcherWeights
  */
-export declare function bigramSimilarity(a: string, b: string): number;
+import type { ScopeInfo } from '../core/types.js';
+import type { SolutionStatus, SolutionType } from './solution-format.js';
+export { bigramSimilarity, bm25Score, COMMON_TAGS, tagWeight } from './scoring-algorithms.js';
+export { calculateRelevance } from './relevance-scorer.js';
+export type { CalculateRelevanceOptions } from './relevance-scorer.js';
+export { shouldRejectByR4T3Rules } from './precision-guards.js';
+export { ROUND3_BASELINE, BASELINE_TOLERANCE, evaluateQuery, evaluateSolutionMatcher, } from './solution-matcher-eval.js';
+export type { EvalSolution, EvalQuery, EvalFixture, BucketMetrics, EvalResult, } from './solution-matcher-eval.js';
 /**
- * Simplified BM25 score for a single query-document pair.
- * Uses tag overlap with term frequency normalization.
- * k1=1.2, b=0.75 (standard BM25 parameters).
+ * @deprecated Use `defaultNormalizer.normalizeTerms` from
+ * `./term-normalizer.js` directly.
  */
-export declare function bm25Score(queryTags: string[], docTags: string[], avgDocLength: number): number;
-/** Apply IDF-like weight: common tags get reduced weight */
-export declare function tagWeight(tag: string): number;
+export declare function expandTagsWithSynonyms(tags: string[]): string[];
 export interface SolutionMatch {
     name: string;
     path: string;
@@ -41,361 +37,6 @@ export interface SolutionMatch {
     tags: string[];
     identifiers: string[];
     matchedTags: string[];
-    /**
-     * Identifier substrings (function/file names) that appeared literally in the
-     * prompt. Added 2026-04-21 so solution-injector can enforce a precision gate
-     * distinguishing "user typed a specific identifier" (strong signal, survives
-     * 1-tag overlap) from "only 1 tag happens to overlap" (often noise — common
-     * nouns like 'type', 'file', 'forgen' trigger rare-tag BM25 boost).
-     */
     matchedIdentifiers: string[];
 }
-/**
- * Optional hints for the v3 `calculateRelevance` path. Used by hot-path
- * callers (matchSolutions, searchSolutions) to avoid re-normalizing the
- * same query tags on every solution.
- */
-export interface CalculateRelevanceOptions {
-    /**
-     * Pre-normalized prompt tags (produced by `defaultNormalizer.normalizeTerms`).
-     * If provided, skips the per-call expansion. Callers loop-running against
-     * many solutions should compute this once outside the loop and pass it in.
-     */
-    normalizedPromptTags?: string[];
-    /**
-     * R4-T1: solution tags expanded with compound-split alternatives
-     * (`expandCompoundTags`). When supplied, the intersection/partial-match
-     * step uses this set INSTEAD of `solutionTags`, but the Jaccard union
-     * denominator still uses `solutionTags` (raw) so the score normalization
-     * stays semantically stable. Caller responsibility to pass the matching
-     * pair — `solutionTagsExpanded` MUST be a superset of `solutionTags`.
-     */
-    solutionTagsExpanded?: string[];
-    /** Average document (solution) tag count for BM25 normalization. Defaults to 6. */
-    avgDocLength?: number;
-    /** Meta-learning: dynamic ensemble weights (sum must equal 1.0). Defaults to {tfidf:0.5, bm25:0.3, bigram:0.2}. */
-    ensembleWeights?: {
-        tfidf: number;
-        bm25: number;
-        bigram: number;
-    };
-}
-export declare function calculateRelevance(promptTags: string[], solutionTags: string[], confidence: number, options?: CalculateRelevanceOptions): {
-    relevance: number;
-    matchedTags: string[];
-};
-/** @deprecated */
-export declare function calculateRelevance(prompt: string, keywords: string[]): number;
-export declare function shouldRejectByR4T3Rules(promptTags: readonly string[], matchedTags: readonly string[]): boolean;
-/**
- * In-memory solution shape for the bootstrap evaluator. Mirrors the index
- * entry fields that `matchSolutions` consumes (tags, identifiers, confidence)
- * but without any filesystem dependency — the evaluator is pure so CI can run
- * it without mounting a starter pack.
- */
-export interface EvalSolution {
-    name: string;
-    tags: string[];
-    identifiers?: string[];
-    confidence: number;
-}
-export interface EvalQuery {
-    query: string;
-    /** Names that should appear in the top-5. Empty array = expect no match (negative case). */
-    expectAnyOf: string[];
-}
-export interface EvalFixture {
-    solutions: EvalSolution[];
-    positive: EvalQuery[];
-    /** Bilingual or compound-word variants that exercise synonym expansion. */
-    paraphrase: EvalQuery[];
-    /** Unrelated queries that should not return a top-1 hit. */
-    negative: EvalQuery[];
-}
-/** Per-bucket metrics. Paraphrase and positive are reported separately so a
- *  bilingual regression (T2 synonym change) can't hide inside the aggregate. */
-export interface BucketMetrics {
-    /** |{q : ∃i≤5, ranked[i] ∈ q.expectAnyOf}| / |q| */
-    recallAt5: number;
-    /** Σ (1 / firstMatchRank) / |q|; rank > 5 contributes 0. */
-    mrrAt5: number;
-    /** |{q : ranked is empty}| / |q| */
-    noResultRate: number;
-    /** Number of queries in this bucket. */
-    total: number;
-}
-export interface EvalResult {
-    /** Combined (positive ∪ paraphrase) metrics — backwards-compatible headline numbers. */
-    recallAt5: number;
-    mrrAt5: number;
-    noResultRate: number;
-    /**
-     * Fraction of negative queries where the matcher returned ≥ 1 candidate
-     * (regardless of rank). Name is honest: this is the "any result" rate on
-     * the negative bucket, not a rank-1 precision metric. It's the correct
-     * baseline for "did synonym/stemming leak into unrelated queries?".
-     */
-    negativeAnyResultRate: number;
-    /** Per-bucket breakdown — use these to catch paraphrase-only regressions. */
-    byBucket: {
-        positive: BucketMetrics;
-        paraphrase: BucketMetrics;
-    };
-    total: {
-        positive: number;
-        paraphrase: number;
-        negative: number;
-    };
-}
-/**
- * Round 3 baseline metrics, recorded against the current `term-normalizer`
- * + `calculateRelevance` + fixture `solution-match-bootstrap.json`. Used as
- * a relative regression guard in `tests/solution-matcher-eval.test.ts` —
- * downstream PRs must not regress any field by more than `BASELINE_TOLERANCE`.
- *
- * History (chronological ascending — v1 at top, latest at bottom):
- *   - v1 (2026-04-08, fixture v1, 41+10+10 queries): 1.0 / 1.0 / 0.0 / 0.1
- *     Recorded against the original 61-query fixture, all positive queries
- *     PASS@1. Indicated a measurement plateau but masked the matcher's true
- *     ranking and false-positive weaknesses because the fixture queries were
- *     too tag-aligned.
- *
- *   - v2 (2026-04-08, fixture v2, 53+16+14 queries): 1.0 / 0.969 / 0.0 / 0.357
- *     Expanded with 12 hard positive (multi-canonical / compound-tag tug-of-
- *     war), 6 Korean subtle paraphrase, and 4 tricky negative queries. The
- *     drops are intentional and represent genuine matcher behaviour:
- *       * positive mrrAt5 1.0 → 0.959: 4 of 12 added positives rank #2-3:
- *         (1) "managing api keys and credentials safely" → secret @3 vs
- *             api-error-responses @1 — the `api` canonical in
- *             DEFAULT_MATCH_TERMS expands to {api, rest, graphql, endpoint,
- *             route}, so query `api` hits BOTH `api` AND `rest` on
- *             starter-api-error-responses (matched=['api','rest']) — a
- *             double-count numerator. starter-secret-management only scores
- *             a single weak partial match on `credential`. The compound
- *             `api-key` tag on secret-management is never reached because
- *             extractTags strips the query-side hyphen and yields
- *             ['api','keys'] (the solution-side tag remains hyphenated in
- *             the index but has no query token to intersect with). T4 IDF
- *             would down-weight both `api` and `rest`, neutralising the
- *             double-count and letting `credential` outscore the noise.
- *         (2) "avoiding hardcoded credentials in source code" → secret @2
- *             vs code-review @1 — `code` partial-matches `code-review`
- *             (len>3, code-review.includes('code')=true) at half weight.
- *             secret-management's `credential` matches by partial too but
- *             the union size differs.
- *         (3) "red green refactor cycle for new features" → tdd @2 vs
- *             refactor-safely @1 — `refactor` is a full-weight intersection
- *             with both refactor-safely's `refactor` and `리팩토링` (via
- *             the refactor canonical), giving 2 hits at 1.0 each. tdd-red-
- *             green-refactor only matches the literal compound tag
- *             `red-green-refactor` (one weighted hit) — the full-weight
- *             generic `refactor` term overpowers the compound-tag specifity.
- *         (4) "writing unit tests for a function with side effects" → tdd
- *             @2 vs separation-of-concerns @1 — both solutions have a
- *             SINGLE matching tag with weighted score 0.5: separation gets
- *             `function` (COMMON_TAG, exact intersection, weight 0.5);
- *             tdd-red-green-refactor gets `tests` partial-matching `test`
- *             (len>3, partial weight 1.0 × 0.5 = 0.5). Both numerators are
- *             identical. Separation wins because the `function` co-occurs
- *             in both promptTags and solution.tags, shrinking its Jaccard
- *             union by one element vs tdd's — a 1-element union-size
- *             advantage drives the entire ranking. starter-dependency-
- *             injection is *not* in top-5 despite having `testing`/`mock`/
- *             `dependency` tags (`tests` does not partial-match `testing`
- *             — neither is a substring of the other), so listing `di` in
- *             expectAnyOf is purely defensive recall, not a live candidate.
- *             T4 BM25 with proper length normalization would attack the
- *             union-size tie-breaker more rigorously than current Jaccard.
- *       * paraphrase mrrAt5 stays at 1.0: all 6 added Korean paraphrases
- *         rank @1 (the originally hard "테스트 먼저 작성하고 리팩토링" is
- *         documented in the fixture as legitimately matching either tdd
- *         OR refactor-safely, since starter-refactor-safely's README also
- *         covers test-first workflows — both are defensible answers).
- *       * negativeAnyResultRate 0.1 → 0.357: 4 added tricky negatives all
- *         trigger false positives via single common dev-adjacent words —
- *         "performance review meeting notes" → caching (matches
- *         `performance`), "system architecture overview document" →
- *         separation-of-concerns (matches `architecture`), "database backup
- *         recovery procedure" → n-plus-one-queries (matches `database`,
- *         `query`, `데이터베이스`), "validation of insurance claims" →
- *         error-handling (matches `validation`).
- *     The original Round 3 plan staged these for T4 (BM25 + IDF). T4 was
- *     EMPIRICALLY SKIPPED on 2026-04-08 — see
- *     `docs/plans/2026-04-08-t4-bm25-skip-adr.md` for the full decision
- *     record. Summary: BM25 prototypes (naive, hybrid Jaccard×IDF,
- *     precision filter, soft penalty) all matched or underperformed the
- *     current scorer on every metric. The starter corpus (N=15) is too
- *     small for IDF to be informative, and the false positives are
- *     semantic ("performance" is both a dev tag and an English noun) — not
- *     statistical, so no frequency-based weighting can fix them. The real
- *     follow-up candidates are tokenizer fix for compound tags, an n-gram
- *     phrase matcher, and corpus growth — all deferred to Round 4 per the
- *     ADR.
- *
- *   - v3 (2026-04-08, fixture v2 + R4-T1 compound-tag fix): 1.0 / 0.986 / 0.0 / 0.357
- *     R4-T1 added `expandCompoundTags` (solution-side) and
- *     `expandQueryBigrams` (query-side) so hyphenated solution tags like
- *     `api-key`, `code-review`, `red-green-refactor` participate in direct
- *     intersection rather than relying on the half-weight partialMatches
- *     fallback. positive `mrrAt5` improved 0.959 → 0.981 (+0.022). 2 of
- *     the 4 v2 hard positive cases were resolved (`managing api keys and
- *     credentials safely` and `red green refactor cycle for new features`
- *     now rank @1). The remaining 2 (`avoiding hardcoded credentials …`
- *     and `writing unit tests for a function with side effects`) require
- *     R4-T2 (phrase matcher) or R4-T3 (specificity classifier) — they're
- *     about query-side English semantics, not compound-tag tokenization.
- *     `negativeAnyResultRate` is unchanged at 0.357 because R4-T1 is a
- *     ranking-quality fix, not a false-positive filter.
- *
- *   - v4 (2026-04-08, fixture v2 + R4-T1 + R4-T2 phrase blocklist):
- *     1.0 / 0.986 / 0.0 / 0.143
- *     R4-T2 added `phrase-blocklist.ts` with 17 curated 2-word English
- *     non-dev compounds ("performance review", "system architecture",
- *     "database backup", etc.) and a `maskBlockedTokens` step at the
- *     top of `rankCandidates` and `searchSolutions`. When a query
- *     contains a blocked phrase, the constituent tokens are removed
- *     from the prompt tag list before bigram expansion / canonical
- *     normalization runs — so the false-positive evidence is removed
- *     at the source rather than demoted in scoring.
- *
- *     `negativeAnyResultRate` dropped 0.357 → 0.143 (3 of 5 v2 trigger
- *     negatives fully blocked):
- *       * "performance review meeting notes" — blocked via
- *         `performance review` + `meeting notes`
- *       * "system architecture overview document" — blocked via
- *         `system architecture` + `overview document`
- *       * "solar system planets astronomy" — blocked via `solar system`
- *
- *     2 false positives remain (both deferred to R4-T3 query-side
- *     specificity classifier — the residuals share a common shape:
- *     a single dev-tag homograph survives whatever masking is applied,
- *     and the term-normalizer expansion still surfaces a false match):
- *
- *       * "database backup recovery procedure" → error-handling-patterns:
- *         `database backup` is blocked, but the residual tokens
- *         {`recovery`, `procedure`} survive. `recovery` is in the
- *         `handling` canonical's matchTerms (intentional, for legitimate
- *         "error recovery handler" queries), so the masked query still
- *         hits `starter-error-handling-patterns` via the handling
- *         family. A 3-word `recovery procedure` blocklist entry was
- *         considered and rejected — it would silently mask legitimate
- *         dev SRE queries like "disaster recovery procedure" or
- *         "rollback recovery procedure" without a fixture-driven
- *         signal. The right fix is at the query-specificity layer
- *         (R4-T3): require ≥ 2 distinct dev-context signals before any
- *         match is returned, not at the phrase-blocklist layer.
- *
- *       * "validation of insurance claims" → error-handling-patterns:
- *         `insurance claim` is blocked, but the residual `validation`
- *         token IS a legitimate dev tag (input-validation,
- *         error-handling-patterns both have it). Same R4-T3 target.
- *
- *     positive/paraphrase mrrAt5 are unchanged from v3 because no
- *     legitimate dev query in the fixture contains a blocked phrase.
- *
- *   - v5 (2026-04-08, fixture v2 + R4-T1 + R4-T2 + R4-T3 specificity guards):
- *     1.0 / 0.986 / 0.0 / 0.000
- *     R4-T3 added two narrow precision rules at the ORCHESTRATION LAYER —
- *     NOT inside `calculateRelevance` (which remains a pure scoring
- *     function for test symmetry). The rules are implemented as the
- *     exported helper `shouldRejectByR4T3Rules(promptTags, matchedTags)`
- *     and called from both `rankCandidates` (hook path) and
- *     `searchSolutions` (MCP path) right after the per-solution
- *     `calculateRelevance` call:
- *       (Rule A) single-token query AND single-tag match → reject;
- *       (Rule B) single-tag match with no literal hit in the prompt
- *                (verbatim match, or substring partial length > 3, or
- *                shared prefix ≥ 4 for morphological stems) → reject.
- *     Both rules are scoped narrowly enough to fix exactly the 2 R4-T2
- *     residuals without recall regression — every fixture positive and
- *     paraphrase still ranks identically:
- *       * "validation of insurance claims" → masked to `[validation]`
- *         (length 1) with single-tag match `validation` → Rule A reject.
- *       * "database backup recovery procedure" → masked to
- *         `[recovery, procedure]` with single-tag match `handling`
- *         (zero literal hit; `handling` is reached via the `recovery`
- *         canonical-family expansion in term-normalizer) → Rule B reject.
- *     `negativeAnyResultRate` is now 0.000 — every fixture v2 negative
- *     produces zero candidates. positive/paraphrase metrics unchanged
- *     from v4 because no fixture positive matches the (single-token AND
- *     single-tag) or (all-expansion AND single-tag) shape.
- *
- *     Escape hatch: identifier-boost evidence (hook path) or name-match
- *     evidence (MCP path) BYPASSES the R4-T3 rules. A candidate with
- *     even a single weak tag match plus an identifier hit still
- *     surfaces — the precision rules only fire when the candidate's
- *     entire evidence pool is a single ambiguous tag.
- *
- *     Defensive precision note: Rule B's "shared prefix ≥ 4"
- *     morphological check is currently NOT fixture-driven (no fixture
- *     query masks down to the `caching/cache`-style morphological gap).
- *     It exists as a pre-emptive fix against silently rejecting
- *     legitimate future queries where the term-normalizer synonym
- *     expansion is the only bridge between the query token and the
- *     solution tag. If a production query surfaces a case the prefix
- *     check misses, extend it (e.g. by lowering the threshold or
- *     adding a Levenshtein-1 check) rather than removing it.
- *
- * Known matcher quirks (separate from the T4 BM25 investigation):
- *   - `term-normalizer.ts` `error` canonical contains `debug` as a matchTerm
- *     (intentional for `bug → error` recall), which causes any prompt
- *     containing `error` to expand to `debug` and over-rank
- *     `starter-debugging-systematic` on otherwise unrelated queries. This
- *     is why `async await error propagation` could not be added as a hard
- *     case — the matcher returns debugging-systematic at #1, which is
- *     defensible-but-noisy. The fix is at the normalizer level (split
- *     `debug` out of the `error` family or remove the `error → debug`
- *     edge entirely) and is queued as a Round 4 follow-up. T4 BM25 was
- *     considered as a partial mitigation but the T4 skip ADR (referenced
- *     in the Round 3 outcome paragraph above) shows it does not help.
- *
- * Long-tail caveat:
- *   - `"trying to handle authentication errors gracefully when our backend
- *     api returns inconsistent response formats from different
- *     microservices"` is a 17-word query intentionally added to exercise
- *     long-tail behaviour. Currently PASS@1. Originally flagged as BM25
- *     length-normalization sensitive, but since T4 BM25 was skipped this
- *     caveat is now informational only — no length-norm code path is
- *     planned in Round 3.
- *
- * If a PR legitimately improves a metric, update this constant in the same
- * commit so future PRs guard against the new floor.
- */
-export declare const ROUND3_BASELINE: EvalResult;
-/** Maximum allowed absolute regression per metric. 5% is tight enough to catch
- *  ~3-4 query regressions in a 69-query combined bucket (positive+paraphrase)
- *  but lenient enough that a single fixture edit won't spuriously fail the
- *  guard. */
-export declare const BASELINE_TOLERANCE = 0.05;
-/**
- * Test/diagnostic helper: evaluate one query against a fixture solution set
- * and return the top-5 ranked candidates with their relevance + matched tags.
- *
- * Exists so per-query regression tests (e.g. the R4-T1 hard-positive guards
- * in `tests/solution-matcher-eval.test.ts`) can assert specific ranking
- * outcomes without scraping aggregate metrics. Wraps `rankCandidates` so
- * the test path stays in sync with the production ranker.
- *
- * Returns the same shape as `rankCandidates` minus the generic carrier:
- * `{name, relevance, matchedTags}`. Use the names to assert "expected
- * solution at rank 1".
- */
-export declare function evaluateQuery(query: string, solutions: readonly EvalSolution[]): Array<{
-    name: string;
-    relevance: number;
-    matchedTags: string[];
-}>;
-/**
- * Evaluate the current matcher against a labeled fixture and return IR
- * metrics. This is the Round 3 baseline — each downstream PR (T2/T3/T4) must
- * not regress any of the thresholds asserted in `solution-matcher-eval.test.ts`.
- *
- * Uses `rankCandidates` (shared with `matchSolutions`) so the evaluator can't
- * silently drift from production ranking behaviour.
- *
- * Metrics are reported both aggregated (positive ∪ paraphrase) and per-bucket,
- * so paraphrase-only regressions surface in `byBucket.paraphrase` even if the
- * aggregate looks fine.
- */
-export declare function evaluateSolutionMatcher(fixture: EvalFixture): EvalResult;
 export declare function matchSolutions(prompt: string, scope: ScopeInfo, cwd: string): SolutionMatch[];