@wooojin/forgen 0.4.10 → 0.4.12

package/dist/engine/scoring-algorithms.js

@@ -0,0 +1,109 @@
+/**
+ * Stateless scoring primitives for the solution matcher.
+ *
+ * Extracted from solution-matcher.ts — these are pure functions with no
+ * filesystem or module-level state dependencies.
+ */
+/** High-frequency tags that should be weighted lower */
+export const COMMON_TAGS = new Set([
+    'typescript',
+    'ts',
+    'javascript',
+    'js',
+    'fix',
+    'update',
+    'add',
+    'change',
+    'file',
+    'code',
+    'function',
+    'import',
+    'export',
+    'error',
+    'type',
+    'string',
+    'number',
+    'object',
+    'array',
+    'return',
+    'const',
+    'class',
+    'module',
+    '코드',
+    '파일',
+    '함수',
+    '수정',
+    '추가',
+    '변경',
+    '에러',
+    '타입',
+]);
+/** Apply IDF-like weight: common tags get reduced weight */
+export function tagWeight(tag) {
+    return COMMON_TAGS.has(tag) ? 0.5 : 1.0;
+}
+/**
+ * Compute the Dice coefficient between two strings using character bigrams.
+ *
+ * Dice = 2 * |intersection| / (|A| + |B|)
+ *
+ * Both strings are lowercased and whitespace-stripped before bigram generation.
+ * Returns 0 for empty strings or single-character strings (no bigrams possible).
+ * Returns 1.0 for identical non-trivial strings.
+ *
+ * This is used as a lightweight fuzzy matching signal for borderline cases
+ * where the TF-IDF tag intersection produces a low score but the query and
+ * solution tags are character-similar (e.g., "database" vs "데이터베이스"
+ * won't match, but "database" vs "databse" will get a high score).
+ */
+export function bigramSimilarity(a, b) {
+    const na = a.toLowerCase().replace(/\s+/g, '');
+    const nb = b.toLowerCase().replace(/\s+/g, '');
+    if (na.length < 2 || nb.length < 2)
+        return 0;
+    if (na === nb)
+        return 1.0;
+    const bigramsA = new Map();
+    for (let i = 0; i < na.length - 1; i++) {
+        const bg = na.slice(i, i + 2);
+        bigramsA.set(bg, (bigramsA.get(bg) ?? 0) + 1);
+    }
+    const bigramsB = new Map();
+    for (let i = 0; i < nb.length - 1; i++) {
+        const bg = nb.slice(i, i + 2);
+        bigramsB.set(bg, (bigramsB.get(bg) ?? 0) + 1);
+    }
+    let intersectionSize = 0;
+    for (const [bg, countA] of bigramsA) {
+        const countB = bigramsB.get(bg) ?? 0;
+        intersectionSize += Math.min(countA, countB);
+    }
+    const totalA = na.length - 1;
+    const totalB = nb.length - 1;
+    return (2 * intersectionSize) / (totalA + totalB);
+}
+/**
+ * Simplified BM25 score for a single query-document pair.
+ * Uses tag overlap with term frequency normalization.
+ * k1=1.2, b=0.75 (standard BM25 parameters).
+ */
+export function bm25Score(queryTags, docTags, avgDocLength) {
+    const k1 = 1.2;
+    const b = 0.75;
+    const docLen = docTags.length;
+    if (docLen === 0 || queryTags.length === 0 || avgDocLength === 0)
+        return 0;
+    let score = 0;
+    for (const qt of queryTags) {
+        // Term frequency in document
+        const tf = docTags.filter((dt) => dt === qt || (dt.length > 3 && qt.length > 3 && (dt.includes(qt) || qt.includes(dt)))).length;
+        if (tf === 0)
+            continue;
+        // BM25 TF saturation
+        const numerator = tf * (k1 + 1);
+        const denominator = tf + k1 * (1 - b + b * (docLen / avgDocLength));
+        score += numerator / denominator;
+    }
+    // Normalize by query length
+    return score / queryTags.length;
+}

package/dist/engine/solution-matcher-eval.d.ts

@@ -0,0 +1,97 @@
+/**
+ * Bootstrap evaluator + IR metrics for the solution matcher.
+ *
+ * Extracted from solution-matcher.ts — test/CI infrastructure for measuring
+ * matcher quality against labeled fixtures. No production side-effects.
+ */
+/**
+ * In-memory solution shape for the bootstrap evaluator. Mirrors the index
+ * entry fields that `matchSolutions` consumes (tags, identifiers, confidence)
+ * but without any filesystem dependency.
+ */
+export interface EvalSolution {
+    name: string;
+    tags: string[];
+    identifiers?: string[];
+    confidence: number;
+}
+export interface EvalQuery {
+    query: string;
+    /** Names that should appear in the top-5. Empty array = expect no match (negative case). */
+    expectAnyOf: string[];
+}
+export interface EvalFixture {
+    solutions: EvalSolution[];
+    positive: EvalQuery[];
+    /** Bilingual or compound-word variants that exercise synonym expansion. */
+    paraphrase: EvalQuery[];
+    /** Unrelated queries that should not return a top-1 hit. */
+    negative: EvalQuery[];
+}
+/** Per-bucket metrics. Paraphrase and positive are reported separately so a
+ *  bilingual regression (T2 synonym change) can't hide inside the aggregate. */
+export interface BucketMetrics {
+    /** |{q : ∃i≤5, ranked[i] ∈ q.expectAnyOf}| / |q| */
+    recallAt5: number;
+    /** Σ (1 / firstMatchRank) / |q|; rank > 5 contributes 0. */
+    mrrAt5: number;
+    /** |{q : ranked is empty}| / |q| */
+    noResultRate: number;
+    /** Number of queries in this bucket. */
+    total: number;
+}
+export interface EvalResult {
+    /** Combined (positive ∪ paraphrase) metrics — backwards-compatible headline numbers. */
+    recallAt5: number;
+    mrrAt5: number;
+    noResultRate: number;
+    /**
+     * Fraction of negative queries where the matcher returned ≥ 1 candidate
+     * (regardless of rank).
+     */
+    negativeAnyResultRate: number;
+    /** Per-bucket breakdown — use these to catch paraphrase-only regressions. */
+    byBucket: {
+        positive: BucketMetrics;
+        paraphrase: BucketMetrics;
+    };
+    total: {
+        positive: number;
+        paraphrase: number;
+        negative: number;
+    };
+}
+/**
+ * Round 3 baseline metrics, recorded against the current `term-normalizer`
+ * + `calculateRelevance` + fixture `solution-match-bootstrap.json`. Used as
+ * a relative regression guard in `tests/solution-matcher-eval.test.ts` —
+ * downstream PRs must not regress any field by more than `BASELINE_TOLERANCE`.
+ *
+ * History (chronological ascending — v1 at top, latest at bottom):
+ *   - v1 (2026-04-08, fixture v1, 41+10+10 queries): 1.0 / 1.0 / 0.0 / 0.1
+ *   - v2 (2026-04-08, fixture v2, 53+16+14 queries): 1.0 / 0.969 / 0.0 / 0.357
+ *   - v3 (2026-04-08, fixture v2 + R4-T1 compound-tag fix): 1.0 / 0.986 / 0.0 / 0.357
+ *   - v4 (2026-04-08, fixture v2 + R4-T1 + R4-T2 phrase blocklist): 1.0 / 0.986 / 0.0 / 0.143
+ *   - v5 (2026-04-08, fixture v2 + R4-T1 + R4-T2 + R4-T3 specificity guards): 1.0 / 0.986 / 0.0 / 0.000
+ *
+ * If a PR legitimately improves a metric, update this constant in the same
+ * commit so future PRs guard against the new floor.
+ */
+export declare const ROUND3_BASELINE: EvalResult;
+/** Maximum allowed absolute regression per metric. */
+export declare const BASELINE_TOLERANCE = 0.05;
+/**
+ * Test/diagnostic helper: evaluate one query against a fixture solution set
+ * and return the top-5 ranked candidates with their relevance + matched tags.
+ */
+export declare function evaluateQuery(query: string, solutions: readonly EvalSolution[]): Array<{
+    name: string;
+    relevance: number;
+    matchedTags: string[];
+}>;
+/**
+ * Evaluate the current matcher against a labeled fixture and return IR
+ * metrics. Uses `rankCandidates` (shared with `matchSolutions`) so the
+ * evaluator can't silently drift from production ranking behaviour.
+ */
+export declare function evaluateSolutionMatcher(fixture: EvalFixture): EvalResult;

package/dist/engine/solution-matcher-eval.js

@@ -0,0 +1,122 @@
+/**
+ * Bootstrap evaluator + IR metrics for the solution matcher.
+ *
+ * Extracted from solution-matcher.ts — test/CI infrastructure for measuring
+ * matcher quality against labeled fixtures. No production side-effects.
+ */
+import { extractTags } from './solution-format.js';
+import { rankCandidates } from './ranking-pipeline.js';
+/**
+ * Round 3 baseline metrics, recorded against the current `term-normalizer`
+ * + `calculateRelevance` + fixture `solution-match-bootstrap.json`. Used as
+ * a relative regression guard in `tests/solution-matcher-eval.test.ts` —
+ * downstream PRs must not regress any field by more than `BASELINE_TOLERANCE`.
+ *
+ * History (chronological ascending — v1 at top, latest at bottom):
+ *   - v1 (2026-04-08, fixture v1, 41+10+10 queries): 1.0 / 1.0 / 0.0 / 0.1
+ *   - v2 (2026-04-08, fixture v2, 53+16+14 queries): 1.0 / 0.969 / 0.0 / 0.357
+ *   - v3 (2026-04-08, fixture v2 + R4-T1 compound-tag fix): 1.0 / 0.986 / 0.0 / 0.357
+ *   - v4 (2026-04-08, fixture v2 + R4-T1 + R4-T2 phrase blocklist): 1.0 / 0.986 / 0.0 / 0.143
+ *   - v5 (2026-04-08, fixture v2 + R4-T1 + R4-T2 + R4-T3 specificity guards): 1.0 / 0.986 / 0.0 / 0.000
+ *
+ * If a PR legitimately improves a metric, update this constant in the same
+ * commit so future PRs guard against the new floor.
+ */
+export const ROUND3_BASELINE = {
+    recallAt5: 1.0,
+    mrrAt5: 0.986,
+    noResultRate: 0.0,
+    negativeAnyResultRate: 0.0,
+    byBucket: {
+        positive: { recallAt5: 1.0, mrrAt5: 0.981, noResultRate: 0.0, total: 53 },
+        paraphrase: { recallAt5: 1.0, mrrAt5: 1.0, noResultRate: 0.0, total: 16 },
+    },
+    total: { positive: 53, paraphrase: 16, negative: 14 },
+};
+/** Maximum allowed absolute regression per metric. */
+export const BASELINE_TOLERANCE = 0.05;
+/** Run a single bucket through the ranking pipeline and aggregate IR metrics. */
+function computeBucketMetrics(queries, solutions) {
+    let recallHits = 0;
+    let reciprocalSum = 0;
+    let noResultCount = 0;
+    for (const q of queries) {
+        const promptTags = extractTags(q.query);
+        const ranked = rankCandidates(promptTags, q.query.toLowerCase(), solutions);
+        if (ranked.length === 0) {
+            noResultCount++;
+            continue;
+        }
+        for (let i = 0; i < ranked.length; i++) {
+            if (q.expectAnyOf.includes(ranked[i].solution.name)) {
+                recallHits++;
+                reciprocalSum += 1 / (i + 1);
+                break;
+            }
+        }
+    }
+    const total = queries.length;
+    return {
+        recallAt5: total > 0 ? recallHits / total : 0,
+        mrrAt5: total > 0 ? reciprocalSum / total : 0,
+        noResultRate: total > 0 ? noResultCount / total : 0,
+        total,
+    };
+}
+/**
+ * Test/diagnostic helper: evaluate one query against a fixture solution set
+ * and return the top-5 ranked candidates with their relevance + matched tags.
+ */
+export function evaluateQuery(query, solutions) {
+    const promptTags = extractTags(query);
+    return rankCandidates(promptTags, query.toLowerCase(), solutions).map((c) => ({
+        name: c.solution.name,
+        relevance: c.relevance,
+        matchedTags: c.matchedTags,
+    }));
+}
+/**
+ * Evaluate the current matcher against a labeled fixture and return IR
+ * metrics. Uses `rankCandidates` (shared with `matchSolutions`) so the
+ * evaluator can't silently drift from production ranking behaviour.
+ */
+export function evaluateSolutionMatcher(fixture) {
+    const positiveM = computeBucketMetrics(fixture.positive, fixture.solutions);
+    const paraphraseM = computeBucketMetrics(fixture.paraphrase, fixture.solutions);
+    const combinedTotal = positiveM.total + paraphraseM.total;
+    const recallAt5 = combinedTotal > 0
+        ? (positiveM.recallAt5 * positiveM.total + paraphraseM.recallAt5 * paraphraseM.total) /
+            combinedTotal
+        : 0;
+    const mrrAt5 = combinedTotal > 0
+        ? (positiveM.mrrAt5 * positiveM.total + paraphraseM.mrrAt5 * paraphraseM.total) /
+            combinedTotal
+        : 0;
+    const noResultRate = combinedTotal > 0
+        ? (positiveM.noResultRate * positiveM.total + paraphraseM.noResultRate * paraphraseM.total) /
+            combinedTotal
+        : 0;
+    let negAnyResult = 0;
+    for (const q of fixture.negative) {
+        const promptTags = extractTags(q.query);
+        const ranked = rankCandidates(promptTags, q.query.toLowerCase(), fixture.solutions);
+        if (ranked.length >= 1)
+            negAnyResult++;
+    }
+    const negTotal = fixture.negative.length;
+    return {
+        recallAt5,
+        mrrAt5,
+        noResultRate,
+        negativeAnyResultRate: negTotal > 0 ? negAnyResult / negTotal : 0,
+        byBucket: {
+            positive: positiveM,
+            paraphrase: paraphraseM,
+        },
+        total: {
+            positive: fixture.positive.length,
+            paraphrase: fixture.paraphrase.length,
+            negative: fixture.negative.length,
+        },
+    };
+}