@astro-minimax/ai 0.7.4 → 0.8.0

package/dist/search/vector-reranker.js

@@ -1,135 +0,0 @@
-/**
- * Optional TF-IDF vector reranker.
- *
- * When a pre-built vector index is available (from `vectorize.ts`),
- * uses cosine similarity to refine the ranking of search results.
- * Gracefully degrades to a no-op when no vector index is loaded.
- */
-// ── State ────────────────────────────────────────────────────
-let loadedIndex = null;
-let idfCache = null;
-export function loadVectorIndex(data) {
-    loadedIndex = data;
-    idfCache = null;
-    if (data?.vocabulary && data.chunks.length > 0) {
-        idfCache = buildIDFFromVocab(data.vocabulary, data.chunks);
-    }
-}
-export function clearVectorIndex() {
-    loadedIndex = null;
-    idfCache = null;
-}
-export function hasVectorIndex() {
-    return loadedIndex !== null && loadedIndex.chunks.length > 0;
-}
-// ── Core: Rerank ──────────────────────────────────────────────
-/**
- * Reranks article search results using vector cosine similarity.
- *
- * For each candidate article, finds the best-matching chunk in the vector
- * index and uses the cosine similarity as a reranking signal.
- *
- * Final score = original_score * (1 - alpha) + vector_score * alpha
- *
- * @param alpha - Blend factor (0 = ignore vectors, 1 = vectors only). Default 0.3
- */
-export function rerankWithVectors(query, candidates, alpha = 0.3) {
-    if (!loadedIndex || !idfCache || !loadedIndex.vocabulary) {
-        return candidates;
-    }
-    const queryVector = computeQueryVector(query, loadedIndex.vocabulary, idfCache);
-    if (!queryVector)
-        return candidates;
-    // Map postId → best chunk cosine similarity
-    const articleScores = new Map();
-    for (const chunk of loadedIndex.chunks) {
-        if (!chunk.vector)
-            continue;
-        const sim = cosineSimilarity(queryVector, chunk.vector);
-        const current = articleScores.get(chunk.postId) ?? 0;
-        if (sim > current) {
-            articleScores.set(chunk.postId, sim);
-        }
-    }
-    if (articleScores.size === 0)
-        return candidates;
-    // Normalize original scores to 0-1
-    const maxOriginal = Math.max(...candidates.map(c => c.score ?? 0), 1);
-    const reranked = candidates.map(article => {
-        const slug = extractSlugFromUrl(article.url);
-        const vectorScore = articleScores.get(slug) ?? 0;
-        const originalNorm = (article.score ?? 0) / maxOriginal;
-        const blended = originalNorm * (1 - alpha) + vectorScore * alpha;
-        return { ...article, score: blended };
-    });
-    return reranked.sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
-}
-// ── Math Utilities ────────────────────────────────────────────
-function cosineSimilarity(a, b) {
-    if (a.length !== b.length)
-        return 0;
-    let dot = 0, magA = 0, magB = 0;
-    for (let i = 0; i < a.length; i++) {
-        dot += a[i] * b[i];
-        magA += a[i] * a[i];
-        magB += b[i] * b[i];
-    }
-    const mag = Math.sqrt(magA) * Math.sqrt(magB);
-    return mag === 0 ? 0 : dot / mag;
-}
-function computeQueryVector(query, vocabulary, idf) {
-    const tokens = tokenizeForVector(query);
-    if (!tokens.length)
-        return null;
-    const tf = new Map();
-    for (const t of tokens)
-        tf.set(t, (tf.get(t) || 0) + 1);
-    const maxTf = Math.max(...tf.values(), 1);
-    const vector = vocabulary.map(term => {
-        const termTf = (tf.get(term) || 0) / maxTf;
-        const termIdf = idf.get(term) || 0;
-        return termTf * termIdf;
-    });
-    // Check if vector is all zeros
-    if (vector.every(v => v === 0))
-        return null;
-    return vector;
-}
-function tokenizeForVector(text) {
-    const CJK = /[\u4e00-\u9fff\u3400-\u4dbf]/g;
-    const cjkChars = text.match(CJK) || [];
-    const latin = text
-        .replace(CJK, ' ')
-        .toLowerCase()
-        .split(/\W+/)
-        .filter(w => w.length > 2);
-    return [...cjkChars, ...latin];
-}
-function buildIDFFromVocab(vocabulary, chunks) {
-    const N = chunks.length;
-    const df = new Map();
-    for (const chunk of chunks) {
-        const tokens = new Set(tokenizeForVector(chunk.text));
-        for (const term of vocabulary) {
-            if (tokens.has(term)) {
-                df.set(term, (df.get(term) || 0) + 1);
-            }
-        }
-    }
-    const idf = new Map();
-    for (const term of vocabulary) {
-        const docCount = df.get(term) || 0;
-        idf.set(term, Math.log(N / (docCount + 1)) + 1);
-    }
-    return idf;
-}
-/**
- * Extracts a slug/postId from a URL path like "/zh/posts/some-slug/" → "zh/some-slug"
- */
-function extractSlugFromUrl(url) {
-    const path = url.replace(/^https?:\/\/[^/]+/, '');
-    const match = path.match(/^\/([\w-]+)\/posts\/(.+?)\/?$/);
-    if (match)
-        return `${match[1]}/${match[2]}`;
-    return path.replace(/^\/|\/$/g, '');
-}