@mars167/git-ai 2.3.0

package/dist/src/core/retrieval/reranker.js

@@ -0,0 +1,304 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CrossEncoderReranker = void 0;
+exports.fuseScores = fuseScores;
+exports.rerank = rerank;
+const path_1 = __importDefault(require("path"));
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const crypto_1 = require("../crypto");
+const embedding_1 = require("../embedding");
+const log_1 = require("../log");
+const cache_1 = require("./cache");
+const log = (0, log_1.createLogger)({ component: 'retrieval', kind: 'reranker' });
+function normalizeScores(values) {
+    if (values.length === 0)
+        return [];
+    const min = Math.min(...values);
+    const max = Math.max(...values);
+    const denom = max - min;
+    if (denom <= 0)
+        return values.map(() => 0);
+    return values.map((v) => (v - min) / denom);
+}
+function normalizeScore(value) {
+    if (!Number.isFinite(value))
+        return 0;
+    if (value >= 0 && value <= 1)
+        return value;
+    return sigmoid(value);
+}
+function fuseScores(originalScore, crossEncoderScore, weights) {
+    const normalized = normalizeScore(originalScore);
+    const cross = clamp(crossEncoderScore, 0, 1);
+    return weights.original * normalized + weights.crossEncoder * cross;
+}
+function clamp(value, min = 0, max = 1) {
+    if (!Number.isFinite(value))
+        return min;
+    return Math.max(min, Math.min(max, value));
+}
+function padBigInt(values, target, pad = 0n) {
+    if (values.length >= target)
+        return values.slice(0, target);
+    const out = values.slice();
+    while (out.length < target)
+        out.push(pad);
+    return out;
+}
+function findModelPath(modelName) {
+    const resolved = path_1.default.isAbsolute(modelName) ? modelName : path_1.default.join(process.cwd(), modelName);
+    const candidates = [resolved, path_1.default.join(resolved, 'model.onnx'), path_1.default.join(resolved, 'onnx', 'model.onnx')];
+    for (const c of candidates) {
+        if (fs_extra_1.default.pathExistsSync(c))
+            return c;
+    }
+    return resolved;
+}
+function sigmoid(x) {
+    if (x > 20)
+        return 1;
+    if (x < -20)
+        return 0;
+    return 1 / (1 + Math.exp(-x));
+}
+function validateRerankInput(query, candidates) {
+    const q = String(query ?? '').trim();
+    const safeCandidates = Array.isArray(candidates) ? candidates : [];
+    return { query: q, candidates: safeCandidates };
+}
+class CrossEncoderModel {
+    constructor(config, cache) {
+        this.config = config;
+        this.cache = cache;
+        this.onnxPromise = null;
+        this.sessionPromise = null;
+        this.tokenizerPromise = null;
+    }
+    async scorePairs(pairs) {
+        if (pairs.length === 0)
+            return [];
+        const cacheKey = (0, crypto_1.sha256Hex)(JSON.stringify(pairs.map((p) => [p.query, p.content])));
+        const cached = this.cache.get(cacheKey);
+        if (cached)
+            return cached.slice();
+        const modelPath = findModelPath(this.config.modelName);
+        if (!fs_extra_1.default.pathExistsSync(modelPath)) {
+            log.info('cross_encoder_model_missing', { model: modelPath });
+            const scores = pairs.map((p) => this.hashScore(p.query, p.content));
+            this.cache.set(cacheKey, scores);
+            return scores;
+        }
+        try {
+            const session = await this.getSession();
+            const tokenizer = await this.getTokenizer();
+            const batchSize = Math.max(1, this.config.batchSize);
+            const scores = new Array(pairs.length).fill(0);
+            for (let i = 0; i < pairs.length; i += batchSize) {
+                const slice = pairs.slice(i, i + batchSize);
+                const encoded = slice.map((pair) => tokenizer.encode(`${pair.query} ${pair.content}`, { maxLength: 256 }));
+                const maxLen = Math.max(2, Math.min(256, Math.max(...encoded.map((e) => e.input_ids.length))));
+                const inputIds = encoded.map((e) => padBigInt(e.input_ids, maxLen, 0n));
+                const attentionMask = encoded.map((e) => padBigInt(e.attention_mask, maxLen, 0n));
+                const feeds = await this.buildFeeds(inputIds, attentionMask, maxLen);
+                const outputs = await session.run(feeds);
+                const outputName = Object.keys(outputs)[0];
+                const output = outputs[outputName];
+                if (!output)
+                    throw new Error('ONNX output missing');
+                const data = output.data;
+                const dims = output.dims ?? [slice.length, 1];
+                const perRow = Math.max(1, dims[dims.length - 1] ?? 1);
+                for (let j = 0; j < slice.length; j++) {
+                    const raw = data[j * perRow] ?? 0;
+                    scores[i + j] = sigmoid(Number(raw));
+                }
+            }
+            this.cache.set(cacheKey, scores);
+            return scores;
+        }
+        catch (err) {
+            log.warn('cross_encoder_fallback', { err: String(err?.message ?? err) });
+            const scores = pairs.map((p) => this.hashScore(p.query, p.content));
+            this.cache.set(cacheKey, scores);
+            return scores;
+        }
+    }
+    dispose() {
+        this.onnxPromise = null;
+        this.sessionPromise = null;
+        this.tokenizerPromise = null;
+        this.cache.clear();
+    }
+    async getSession() {
+        if (!this.sessionPromise) {
+            this.sessionPromise = (async () => {
+                const onnx = await this.getOnnx();
+                const modelPath = findModelPath(this.config.modelName);
+                const providers = this.config.device === 'gpu' ? ['cuda', 'cpu'] : ['cpu'];
+                const opts = { executionProviders: providers };
+                const session = await onnx.InferenceSession.create(modelPath, opts);
+                log.info('cross_encoder_session_ready', { model: modelPath, device: this.config.device });
+                return session;
+            })();
+        }
+        return this.sessionPromise;
+    }
+    async getTokenizer() {
+        if (!this.tokenizerPromise) {
+            this.tokenizerPromise = (async () => {
+                const mod = await this.loadTokenizerModule();
+                return mod.loadTokenizer(this.config.modelName);
+            })();
+        }
+        return this.tokenizerPromise;
+    }
+    async getOnnx() {
+        if (!this.onnxPromise)
+            this.onnxPromise = this.loadOnnx();
+        return this.onnxPromise;
+    }
+    async loadOnnx() {
+        const moduleName = 'onnxruntime-node';
+        const mod = await import(moduleName);
+        return mod;
+    }
+    async loadTokenizerModule() {
+        const moduleName = '../embedding/tokenizer.js';
+        const mod = await import(moduleName);
+        return mod;
+    }
+    async buildFeeds(inputIds, attentionMask, maxLen) {
+        const onnx = await this.getOnnx();
+        const batch = inputIds.length;
+        const flattenIds = inputIds.flat();
+        const flattenMask = attentionMask.flat();
+        const idsTensor = new onnx.Tensor('int64', BigInt64Array.from(flattenIds), [batch, maxLen]);
+        const maskTensor = new onnx.Tensor('int64', BigInt64Array.from(flattenMask), [batch, maxLen]);
+        const feeds = {};
+        const inputNames = ['input_ids', 'attention_mask', 'token_type_ids'];
+        for (const name of inputNames) {
+            if (name === 'input_ids')
+                feeds[name] = idsTensor;
+            if (name === 'attention_mask')
+                feeds[name] = maskTensor;
+            if (name === 'token_type_ids') {
+                const types = new onnx.Tensor('int64', new BigInt64Array(batch * maxLen), [batch, maxLen]);
+                feeds[name] = types;
+            }
+        }
+        return feeds;
+    }
+    hashScore(query, content) {
+        const vec = (0, embedding_1.hashEmbedding)(`${query} ${content}`, { dim: 64 });
+        const sum = vec.reduce((acc, v) => acc + v, 0);
+        return sigmoid(sum);
+    }
+}
+class CrossEncoderReranker {
+    constructor(config, cache = new cache_1.LruCache(256)) {
+        this.config = config;
+        this.cache = cache;
+        this.model = new CrossEncoderModel(config, cache);
+    }
+    async rerank(query, candidates) {
+        const { query: q, candidates: items } = validateRerankInput(query, candidates);
+        if (!q || items.length === 0)
+            return [];
+        const limited = items.slice(0, Math.max(1, this.config.topK));
+        const pairs = limited.map((item) => ({ query: q, content: item.content }));
+        const scores = await this.model.scorePairs(pairs);
+        const originalScores = limited.map((c) => c.score);
+        const normalizedOriginal = normalizeScores(originalScores);
+        const results = limited.map((item, idx) => {
+            const rerankScore = clamp(scores[idx] ?? 0, 0, 1);
+            const originalScore = normalizedOriginal[idx] ?? 0;
+            const finalScore = this.config.scoreWeights.original * originalScore +
+                this.config.scoreWeights.crossEncoder * rerankScore;
+            return {
+                id: item.id,
+                content: item.content,
+                filePath: item.filePath,
+                originalScore: item.score,
+                rerankScore,
+                finalScore,
+            };
+        });
+        results.sort((a, b) => b.finalScore - a.finalScore || b.rerankScore - a.rerankScore);
+        return results;
+    }
+    async rerankBatch(queries, candidates) {
+        const batchSize = Math.min(queries.length, candidates.length);
+        const results = new Array(batchSize);
+        for (let i = 0; i < batchSize; i++) {
+            results[i] = await this.rerank(queries[i] ?? '', candidates[i] ?? []);
+        }
+        return results;
+    }
+    dispose() {
+        this.model.dispose();
+        this.cache.clear();
+    }
+}
+exports.CrossEncoderReranker = CrossEncoderReranker;
+function tokenize(text) {
+    return String(text ?? '')
+        .toLowerCase()
+        .split(/[^a-z0-9_]+/g)
+        .map((t) => t.trim())
+        .filter(Boolean);
+}
+function overlapScore(queryTokens, candidateTokens) {
+    if (queryTokens.length === 0 || candidateTokens.length === 0)
+        return 0;
+    const set = new Set(candidateTokens);
+    let hits = 0;
+    for (const t of queryTokens)
+        if (set.has(t))
+            hits += 1;
+    return hits / queryTokens.length;
+}
+function pairwiseBoost(results) {
+    const boost = new Map();
+    for (let i = 0; i < results.length; i++) {
+        for (let j = i + 1; j < results.length; j++) {
+            const a = results[i];
+            const b = results[j];
+            if (a.source === b.source)
+                continue;
+            const aKey = `${a.source}:${a.id}`;
+            const bKey = `${b.source}:${b.id}`;
+            const aText = String(a.text ?? a.metadata?.text ?? '');
+            const bText = String(b.text ?? b.metadata?.text ?? '');
+            const aTokens = new Set(tokenize(aText));
+            const overlap = overlapScore(Array.from(aTokens), tokenize(bText));
+            if (overlap > 0.2) {
+                boost.set(aKey, (boost.get(aKey) ?? 0) + 0.05);
+                boost.set(bKey, (boost.get(bKey) ?? 0) + 0.05);
+            }
+        }
+    }
+    return boost;
+}
+function rerank(query, candidates, options = {}) {
+    const qTokens = tokenize(query);
+    const limit = Math.max(1, Number(options.limit ?? 50));
+    const ranked = candidates.map((c, idx) => {
+        const normalizedScore = 'normalizedScore' in c ? c.normalizedScore : 0;
+        const fusedScore = 'fusedScore' in c ? c.fusedScore : c.score;
+        const text = String(c.text ?? c.metadata?.text ?? '');
+        const overlap = overlapScore(qTokens, tokenize(text));
+        const rerankScore = fusedScore + overlap * 0.2;
+        return { ...c, normalizedScore, fusedScore: rerankScore, rank: idx + 1 };
+    });
+    const boosts = pairwiseBoost(ranked);
+    for (const r of ranked) {
+        const key = `${r.source}:${r.id}`;
+        const boost = boosts.get(key) ?? 0;
+        r.fusedScore += boost;
+    }
+    ranked.sort((a, b) => b.fusedScore - a.fusedScore || b.score - a.score);
+    return ranked.slice(0, limit).map((r, idx) => ({ ...r, rank: idx + 1 }));
+}

package/dist/src/core/retrieval/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/src/core/retrieval/weights.js

@@ -0,0 +1,42 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.computeWeights = computeWeights;
+const BASE_WEIGHTS = {
+    semantic: { vectorWeight: 0.55, graphWeight: 0.2, dsrWeight: 0.15, symbolWeight: 0.1 },
+    structural: { vectorWeight: 0.25, graphWeight: 0.45, dsrWeight: 0.15, symbolWeight: 0.15 },
+    historical: { vectorWeight: 0.2, graphWeight: 0.15, dsrWeight: 0.5, symbolWeight: 0.15 },
+    hybrid: { vectorWeight: 0.4, graphWeight: 0.3, dsrWeight: 0.2, symbolWeight: 0.1 },
+};
+function normalize(weights) {
+    const total = weights.vectorWeight + weights.graphWeight + weights.dsrWeight + weights.symbolWeight;
+    if (total <= 0)
+        return BASE_WEIGHTS.semantic;
+    return {
+        vectorWeight: weights.vectorWeight / total,
+        graphWeight: weights.graphWeight / total,
+        dsrWeight: weights.dsrWeight / total,
+        symbolWeight: weights.symbolWeight / total,
+    };
+}
+function computeWeights(queryType, feedback) {
+    const base = { ...BASE_WEIGHTS[queryType.primary] };
+    const bias = feedback?.weightBias;
+    if (bias) {
+        base.vectorWeight += bias.vectorWeight ?? 0;
+        base.graphWeight += bias.graphWeight ?? 0;
+        base.dsrWeight += bias.dsrWeight ?? 0;
+        base.symbolWeight += bias.symbolWeight ?? 0;
+    }
+    if (feedback?.acceptedSource) {
+        const boost = 0.05;
+        if (feedback.acceptedSource === 'vector')
+            base.vectorWeight += boost;
+        if (feedback.acceptedSource === 'graph')
+            base.graphWeight += boost;
+        if (feedback.acceptedSource === 'dsr')
+            base.dsrWeight += boost;
+        if (feedback.acceptedSource === 'symbol')
+            base.symbolWeight += boost;
+    }
+    return normalize(base);
+}

package/dist/src/core/search.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildQueryVector = buildQueryVector;
+exports.scoreAgainst = scoreAgainst;
+exports.buildAdaptiveQueryPlan = buildAdaptiveQueryPlan;
+exports.runAdaptiveRetrieval = runAdaptiveRetrieval;
+const sq8_1 = require("./sq8");
+const embedding_1 = require("./embedding");
+const classifier_1 = require("./retrieval/classifier");
+const expander_1 = require("./retrieval/expander");
+const fuser_1 = require("./retrieval/fuser");
+const reranker_1 = require("./retrieval/reranker");
+const weights_1 = require("./retrieval/weights");
+function buildQueryVector(text, dim) {
+    const vec = (0, embedding_1.hashEmbedding)(text, { dim });
+    return (0, sq8_1.quantizeSQ8)(vec);
+}
+function scoreAgainst(q, item) {
+    const qf = (0, sq8_1.dequantizeSQ8)(q);
+    const vf = (0, sq8_1.dequantizeSQ8)({ dim: item.dim, scale: item.scale, q: item.qvec });
+    return (0, sq8_1.cosineSimilarity)(qf, vf);
+}
+function buildAdaptiveQueryPlan(query, feedback) {
+    const q = String(query ?? '').trim();
+    const queryType = (0, classifier_1.classifyQuery)(q);
+    const expanded = (0, expander_1.expandQuery)(q, queryType);
+    const weights = (0, weights_1.computeWeights)(queryType, feedback);
+    return { query: q, expanded, queryType, weights };
+}
+/**
+ * Runs the adaptive retrieval pipeline: classification -> expansion -> weighting -> fusion -> heuristic reranking.
+ *
+ * Note: This uses synchronous heuristic reranking. For higher quality but slower reranking using
+ * the ONNX Cross-Encoder, use the `CrossEncoderReranker` class directly (which is async).
+ */
+function runAdaptiveRetrieval(query, candidates, options = {}) {
+    const plan = buildAdaptiveQueryPlan(query, options.feedback);
+    const fused = (0, fuser_1.fuseResults)(candidates, plan.weights, options.limit);
+    const results = (0, reranker_1.rerank)(plan.query, fused, { limit: options.limit });
+    return { ...plan, results };
+}

package/dist/src/core/sq8.js

@@ -0,0 +1,65 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.quantizeSQ8 = quantizeSQ8;
+exports.quantizeToBits = quantizeToBits;
+exports.dequantizeSQ8 = dequantizeSQ8;
+exports.hnswQuantize = hnswQuantize;
+exports.cosineSimilarity = cosineSimilarity;
+function quantizeSQ8(vector, bits = 8) {
+    const dim = vector.length;
+    let maxAbs = 0;
+    for (let i = 0; i < dim; i++) {
+        const a = Math.abs(vector[i] ?? 0);
+        if (a > maxAbs)
+            maxAbs = a;
+    }
+    const { scale, q } = quantizeToBits(vector, bits, maxAbs);
+    return { dim, scale, q };
+}
+function quantizeToBits(vector, bits, maxAbs) {
+    const dim = vector.length;
+    const clampedBits = Math.max(4, Math.min(8, Math.round(bits)));
+    const range = Math.pow(2, clampedBits - 1) - 1;
+    let maxAbsLocal = maxAbs ?? 0;
+    if (maxAbsLocal === 0) {
+        for (let i = 0; i < dim; i++) {
+            const a = Math.abs(vector[i] ?? 0);
+            if (a > maxAbsLocal)
+                maxAbsLocal = a;
+        }
+    }
+    const scale = maxAbsLocal === 0 ? 1 : maxAbsLocal / range;
+    const q = new Int8Array(dim);
+    for (let i = 0; i < dim; i++) {
+        const v = (vector[i] ?? 0) / scale;
+        const r = Math.round(v);
+        const clamped = Math.max(-range, Math.min(range, r));
+        q[i] = clamped;
+    }
+    return { dim, scale, q };
+}
+function dequantizeSQ8(sq8) {
+    const out = new Float32Array(sq8.dim);
+    for (let i = 0; i < sq8.dim; i++)
+        out[i] = sq8.q[i] * sq8.scale;
+    return out;
+}
+function hnswQuantize(vector, bits) {
+    return quantizeToBits(vector, bits);
+}
+function cosineSimilarity(a, b) {
+    const dim = Math.min(a.length, b.length);
+    let dot = 0;
+    let na = 0;
+    let nb = 0;
+    for (let i = 0; i < dim; i++) {
+        const av = Number(a[i] ?? 0);
+        const bv = Number(b[i] ?? 0);
+        dot += av * bv;
+        na += av * av;
+        nb += bv * bv;
+    }
+    if (na === 0 || nb === 0)
+        return 0;
+    return dot / (Math.sqrt(na) * Math.sqrt(nb));
+}

package/dist/src/core/symbolSearch.js

@@ -0,0 +1,143 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.inferSymbolSearchMode = inferSymbolSearchMode;
+exports.pickCoarseToken = pickCoarseToken;
+exports.buildCoarseWhere = buildCoarseWhere;
+exports.filterAndRankSymbolRows = filterAndRankSymbolRows;
+function inferSymbolSearchMode(query, mode) {
+    if (mode)
+        return mode;
+    if (query.includes('*') || query.includes('?'))
+        return 'wildcard';
+    return 'substring';
+}
+function escapeQuotes(s) {
+    return s.replace(/'/g, "''");
+}
+function extractTokens(s) {
+    const tokens = s.match(/[A-Za-z0-9_$.]+/g) ?? [];
+    return tokens.map(t => t.trim()).filter(Boolean);
+}
+function pickCoarseToken(query) {
+    const tokens = extractTokens(query);
+    if (tokens.length === 0)
+        return '';
+    let best = tokens[0];
+    for (const t of tokens)
+        if (t.length > best.length)
+            best = t;
+    return best;
+}
+function buildCoarseWhere(params) {
+    const q = String(params.query ?? '');
+    const mode = inferSymbolSearchMode(q, params.mode);
+    const safe = escapeQuotes(q);
+    const likeOp = params.caseInsensitive ? 'ILIKE' : 'LIKE';
+    if (mode === 'prefix') {
+        if (!safe)
+            return null;
+        return `symbol ${likeOp} '${safe}%'`;
+    }
+    if (mode === 'substring') {
+        if (!safe)
+            return null;
+        return `symbol ${likeOp} '%${safe}%'`;
+    }
+    const token = pickCoarseToken(q);
+    if (!token)
+        return null;
+    const tokenSafe = escapeQuotes(token);
+    return `symbol ${likeOp} '%${tokenSafe}%'`;
+}
+function escapeRegex(s) {
+    return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+function globToRegex(pattern, caseInsensitive) {
+    try {
+        const body = pattern
+            .split('')
+            .map(ch => {
+            if (ch === '*')
+                return '.*';
+            if (ch === '?')
+                return '.';
+            return escapeRegex(ch);
+        })
+            .join('');
+        const flags = caseInsensitive ? 'i' : '';
+        return new RegExp(`^${body}$`, flags);
+    }
+    catch {
+        return null;
+    }
+}
+function buildRegex(pattern, caseInsensitive) {
+    try {
+        const flags = caseInsensitive ? 'i' : '';
+        return new RegExp(pattern, flags);
+    }
+    catch {
+        return null;
+    }
+}
+function normalizeForFuzzy(s) {
+    return s.toLowerCase().replace(/[^a-z0-9_$.]+/g, '');
+}
+function fuzzySubsequenceScore(needle, haystack) {
+    if (!needle)
+        return 0;
+    let i = 0;
+    let score = 0;
+    let lastMatch = -2;
+    for (let j = 0; j < haystack.length && i < needle.length; j++) {
+        if (haystack[j] === needle[i]) {
+            score += (j === lastMatch + 1) ? 2 : 1;
+            lastMatch = j;
+            i++;
+        }
+    }
+    if (i < needle.length)
+        return -1;
+    return score;
+}
+function filterAndRankSymbolRows(rows, params) {
+    const qRaw = String(params.query ?? '');
+    const mode = inferSymbolSearchMode(qRaw, params.mode);
+    const limit = Math.max(1, Number(params.limit ?? 50));
+    const caseInsensitive = Boolean(params.caseInsensitive);
+    const getSymbol = (r) => String(r?.symbol ?? r?.name ?? '');
+    if (mode === 'substring' || mode === 'prefix') {
+        const q = caseInsensitive ? qRaw.toLowerCase() : qRaw;
+        const out = rows.filter(r => {
+            const s = getSymbol(r);
+            const ss = caseInsensitive ? s.toLowerCase() : s;
+            return mode === 'prefix' ? ss.startsWith(q) : ss.includes(q);
+        });
+        return out.slice(0, limit);
+    }
+    if (mode === 'wildcard') {
+        const re = globToRegex(qRaw, caseInsensitive);
+        if (!re)
+            return [];
+        const out = rows.filter(r => re.test(getSymbol(r)));
+        return out.slice(0, limit);
+    }
+    if (mode === 'regex') {
+        const re = buildRegex(qRaw, caseInsensitive);
+        if (!re)
+            return [];
+        const out = rows.filter(r => re.test(getSymbol(r)));
+        return out.slice(0, limit);
+    }
+    const q = normalizeForFuzzy(qRaw);
+    const scored = rows
+        .map(r => {
+        const s = normalizeForFuzzy(getSymbol(r));
+        const score = fuzzySubsequenceScore(q, s);
+        return { r, score };
+    })
+        .filter(x => x.score >= 0)
+        .sort((a, b) => b.score - a.score)
+        .slice(0, limit);
+    return scored.map(x => x.r);
+}

package/dist/src/core/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });