grepmind-core 0.1.0-alpha

package/dist/indexer/reranking.js

@@ -0,0 +1,256 @@
+import { pipeline } from '@xenova/transformers';
+import { createError } from '../types.js';
+/**
+ * Normalize scores to [0, 1] range.
+ */
+export function normalizeScores(scores) {
+    if (scores.length === 0)
+        return [];
+    const min = Math.min(...scores);
+    const max = Math.max(...scores);
+    if (max === min)
+        return scores.map(() => 1);
+    return scores.map((score) => (score - min) / (max - min));
+}
+/**
+ * Truncate document by character length, preferring word boundary.
+ */
+export function truncateDocument(doc, maxLength) {
+    if (doc.length <= maxLength)
+        return doc;
+    const truncated = doc.slice(0, maxLength);
+    const lastSpace = truncated.lastIndexOf(' ');
+    return lastSpace > maxLength * 0.8 ? truncated.slice(0, lastSpace) : truncated;
+}
+/**
+ * Split an array into fixed-size batches.
+ */
+export function batchArray(array, batchSize) {
+    const batches = [];
+    for (let i = 0; i < array.length; i += batchSize) {
+        batches.push(array.slice(i, i + batchSize));
+    }
+    return batches;
+}
+function sortAndSlice(results, topK) {
+    const sorted = [...results].sort((a, b) => b.score - a.score);
+    return sorted.slice(0, topK ?? sorted.length);
+}
+function extractClassificationScore(result) {
+    if (Array.isArray(result)) {
+        const first = result[0];
+        return typeof first?.score === 'number' ? first.score : 0;
+    }
+    if (typeof result === 'object' && result !== null) {
+        const maybeScore = result.score;
+        return typeof maybeScore === 'number' ? maybeScore : 0;
+    }
+    return 0;
+}
+export class WasmRerankerProvider {
+    classifier = null;
+    initPromise = null;
+    modelName;
+    maxDocumentLength;
+    constructor(modelName, maxDocumentLength) {
+        this.modelName = modelName;
+        this.maxDocumentLength = maxDocumentLength;
+    }
+    isInitialized() {
+        return this.classifier !== null;
+    }
+    getModelName() {
+        return this.modelName;
+    }
+    async getClassifier() {
+        if (this.classifier)
+            return this.classifier;
+        if (!this.initPromise) {
+            this.initPromise = pipeline('text-classification', this.modelName, {
+                quantized: true,
+            });
+        }
+        this.classifier = await this.initPromise;
+        return this.classifier;
+    }
+    async initialize() {
+        await this.getClassifier();
+    }
+    async rerank(query, documents, options) {
+        if (documents.length === 0)
+            return [];
+        try {
+            const classifier = await this.getClassifier();
+            const rawScores = [];
+            for (const doc of documents) {
+                const truncatedDoc = truncateDocument(doc, this.maxDocumentLength);
+                const result = await classifier(`${query} [SEP] ${truncatedDoc}`);
+                rawScores.push(extractClassificationScore(result));
+            }
+            const normalized = normalizeScores(rawScores);
+            const ranked = normalized.map((score, index) => ({
+                index,
+                score,
+                ...(options?.returnDocuments ? { document: documents[index] } : {}),
+            }));
+            return sortAndSlice(ranked, options?.topK);
+        }
+        catch (error) {
+            return createError('RERANK_FAILED', `WASM reranking failed: ${error.message}`, { error: error.message });
+        }
+    }
+}
+export class VoyageRerankerProvider {
+    initialized = false;
+    modelName;
+    apiKey;
+    batchSize;
+    maxDocumentLength;
+    constructor(modelName, apiKey, batchSize, maxDocumentLength) {
+        this.modelName = modelName;
+        this.apiKey = apiKey;
+        this.batchSize = batchSize;
+        this.maxDocumentLength = maxDocumentLength;
+    }
+    async initialize() {
+        this.initialized = true;
+    }
+    isInitialized() {
+        return this.initialized;
+    }
+    getModelName() {
+        return `voyage:${this.modelName}`;
+    }
+    async rerank(query, documents, options) {
+        if (documents.length === 0)
+            return [];
+        try {
+            const truncatedDocuments = documents.map((doc) => truncateDocument(doc, this.maxDocumentLength));
+            const results = [];
+            for (let offset = 0; offset < truncatedDocuments.length; offset += this.batchSize) {
+                const batch = truncatedDocuments.slice(offset, offset + this.batchSize);
+                const response = await fetch('https://api.voyageai.com/v1/rerank', {
+                    method: 'POST',
+                    headers: {
+                        Authorization: `Bearer ${this.apiKey}`,
+                        'Content-Type': 'application/json',
+                    },
+                    body: JSON.stringify({
+                        query,
+                        documents: batch,
+                        model: this.modelName,
+                        top_k: batch.length,
+                    }),
+                });
+                if (!response.ok) {
+                    const body = await response.text();
+                    return createError('RERANK_FAILED', `Voyage rerank error ${response.status}: ${body}`);
+                }
+                const data = (await response.json());
+                for (const item of data.data ?? []) {
+                    const globalIndex = offset + item.index;
+                    results.push({
+                        index: globalIndex,
+                        score: item.relevance_score,
+                        ...(options?.returnDocuments ? { document: documents[globalIndex] } : {}),
+                    });
+                }
+            }
+            return sortAndSlice(results, options?.topK);
+        }
+        catch (error) {
+            return createError('RERANK_FAILED', `Voyage reranking failed: ${error.message}`, { error: error.message });
+        }
+    }
+}
+export class LocalAIRerankerProvider {
+    initialized = false;
+    baseUrl;
+    modelName;
+    apiKey;
+    batchSize;
+    maxDocumentLength;
+    constructor(baseUrl, modelName, batchSize, maxDocumentLength, apiKey) {
+        this.baseUrl = baseUrl.replace(/\/$/, '');
+        this.modelName = modelName;
+        this.batchSize = batchSize;
+        this.maxDocumentLength = maxDocumentLength;
+        this.apiKey = apiKey;
+    }
+    async initialize() {
+        this.initialized = true;
+    }
+    isInitialized() {
+        return this.initialized;
+    }
+    getModelName() {
+        return `localai:${this.modelName}`;
+    }
+    async rerank(query, documents, options) {
+        if (documents.length === 0)
+            return [];
+        try {
+            const truncatedDocuments = documents.map((doc) => truncateDocument(doc, this.maxDocumentLength));
+            const rawScores = new Array(documents.length).fill(0);
+            for (let offset = 0; offset < truncatedDocuments.length; offset += this.batchSize) {
+                const batch = truncatedDocuments.slice(offset, offset + this.batchSize);
+                const headers = {
+                    'Content-Type': 'application/json',
+                };
+                if (this.apiKey) {
+                    headers.Authorization = `Bearer ${this.apiKey}`;
+                }
+                const response = await fetch(`${this.baseUrl}/v1/rerank`, {
+                    method: 'POST',
+                    headers,
+                    body: JSON.stringify({
+                        model: this.modelName,
+                        query,
+                        documents: batch,
+                        top_n: batch.length,
+                    }),
+                });
+                if (!response.ok) {
+                    const body = await response.text();
+                    return createError('RERANK_FAILED', `LocalAI rerank error ${response.status}: ${body}`);
+                }
+                const data = (await response.json());
+                const items = data.data ?? data.results ?? [];
+                for (const item of items) {
+                    const globalIndex = offset + item.index;
+                    const score = item.relevance_score ?? item.score ?? 0;
+                    rawScores[globalIndex] = score;
+                }
+            }
+            const normalizedScores = normalizeScores(rawScores);
+            const results = normalizedScores.map((score, index) => ({
+                index,
+                score,
+                ...(options?.returnDocuments ? { document: documents[index] } : {}),
+            }));
+            return sortAndSlice(results, options?.topK);
+        }
+        catch (error) {
+            return createError('RERANK_FAILED', `LocalAI reranking failed: ${error.message}`, { error: error.message });
+        }
+    }
+}
+export function createRerankerProvider(config, secrets) {
+    if (!config.enabled || config.provider === 'none') {
+        return null;
+    }
+    switch (config.provider) {
+        case 'wasm':
+            return new WasmRerankerProvider(config.wasmModel, config.maxDocumentLength);
+        case 'voyage':
+            if (!secrets?.voyageApiKey) {
+                throw new Error('Voyage API key required for reranking');
+            }
+            return new VoyageRerankerProvider(config.voyageModel, secrets.voyageApiKey, config.batchSize, config.maxDocumentLength);
+        case 'localai':
+            return new LocalAIRerankerProvider(config.localaiBaseUrl, config.localaiRerankModel, config.batchSize, config.maxDocumentLength, secrets?.localaiApiKey);
+        default:
+            return null;
+    }
+}
+//# sourceMappingURL=reranking.js.map

package/dist/indexer/reranking.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reranking.js","sourceRoot":"","sources":["../../src/indexer/reranking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAmC,MAAM,sBAAsB,CAAC;AAEjF,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAqC1C;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,MAAgB;IAC9C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACnC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IAChC,IAAI,GAAG,KAAK,GAAG;QAAE,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAW,EAAE,SAAiB;IAC7D,IAAI,GAAG,CAAC,MAAM,IAAI,SAAS;QAAE,OAAO,GAAG,CAAC;IACxC,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC7C,OAAO,SAAS,GAAG,SAAS,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACjF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAI,KAAU,EAAE,SAAiB;IACzD,MAAM,OAAO,GAAU,EAAE,CAAC;IAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACjD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,YAAY,CAAC,OAAuB,EAAE,IAAa;IAC1D,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9D,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,0BAA0B,CAAC,MAAe;IACjD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAoC,CAAC;QAC3D,OAAO,OAAO,KAAK,EAAE,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,CAAC;IAED,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAClD,MAAM,UAAU,GAAI,MAA8B,CAAC,KAAK,CAAC;QACzD,OAAO,OAAO,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC;AAED,MAAM,OAAO,oBAAoB;IACvB,UAAU,GAAsC,IAAI,CAAC;IACrD,WAAW,GAA+C,IAAI,CAAC;IACtD,SAAS,CAAS;IAClB,iBAAiB,CAAS;IAE3C,YAAY,SAAiB,EAAE,iBAAyB;QACtD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;IAC7C,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC;IAClC,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,IAAI,CAAC,UAAU,CAAC;QAE5C,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC,qBAAqB,EAAE,IAAI,CAAC,SAAS,EAAE;gBACjE,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC;QACzC,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,UAAU;QACd,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,SAAmB,EAAE,OAAuB;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC9C,MAAM,SAAS,GAAa,EAAE,CAAC;YAE/B,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBACnE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,KAAK,UAAU,YAAY,EAAE,CAAC,CAAC;gBAClE,SAAS,CAAC,IAAI,CAAC,0BAA0B,CAAC,MAAM,CAAC,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,UAAU,GAAG,eAAe,CAAC,SAAS,CAAC,CAAC;YAC9C,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;gBAC/C,KAAK;gBACL,KAAK;gBACL,GAAG,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACpE,CAAC,CAAC,CAAC;YAEJ,OAAO,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;QAC7C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,WAAW,CAChB,eAAe,EACf,0BAA2B,KAAe,CAAC,OAAO,EAAE,EACpD,EAAE,KAAK,EAAG,KAAe,CAAC,OAAO,EAAE,CACpC,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AAED,MAAM,OAAO,sBAAsB;IACzB,WAAW,GAAG,KAAK,CAAC;IACX,SAAS,CAAS;IAClB,MAAM,CAAS;IACf,SAAS,CAAS;IAClB,iBAAiB,CAAS;IAE3C,YAAY,SAAiB,EAAE,MAAc,EAAE,SAAiB,EAAE,iBAAyB;QACzF,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED,YAAY;QACV,OAAO,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,SAAmB,EAAE,OAAuB;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,IAAI,CAAC;YACH,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;YACjG,MAAM,OAAO,GAAmB,EAAE,CAAC;YAEnC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,kBAAkB,CAAC,MAAM,EAAE,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClF,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;gBACxE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,oCAAoC,EAAE;oBACjE,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE;wBACP,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;wBACtC,cAAc,EAAE,kBAAkB;qBACnC;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK;wBACL,SAAS,EAAE,KAAK;wBAChB,KAAK,EAAE,IAAI,CAAC,SAAS;wBACrB,KAAK,EAAE,KAAK,CAAC,MAAM;qBACpB,CAAC;iBACH,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;oBACnC,OAAO,WAAW,CAChB,eAAe,EACf,uBAAuB,QAAQ,CAAC,MAAM,KAAK,IAAI,EAAE,CAClD,CAAC;gBACJ,CAAC;gBAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAkC,CAAC;gBACtE,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC;oBACnC,MAAM,WAAW,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC;oBACxC,OAAO,CAAC,IAAI,CAAC;wBACX,KAAK,EAAE,WAAW;wBAClB,KAAK,EAAE,IAAI,CAAC,eAAe;wBAC3B,GAAG,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;qBAC1E,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO,YAAY,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,WAAW,CAChB,eAAe,EACf,4BAA6B,KAAe,CAAC,OAAO,EAAE,EACtD,EAAE,KAAK,EAAG,KAAe,CAAC,OAAO,EAAE,CACpC,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AAED,MAAM,OAAO,uBAAuB;IAC1B,WAAW,GAAG,KAAK,CAAC;IACX,OAAO,CAAS;IAChB,SAAS,CAAS;IAClB,MAAM,CAAU;IAChB,SAAS,CAAS;IAClB,iBAAiB,CAAS;IAE3C,YACE,OAAe,EACf,SAAiB,EACjB,SAAiB,EACjB,iBAAyB,EACzB,MAAe;QAEf,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAC3C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED,YAAY;QACV,OAAO,WAAW,IAAI,CAAC,SAAS,EAAE,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,SAAmB,EAAE,OAAuB;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,IAAI,CAAC;YACH,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;YACjG,MAAM,SAAS,GAAG,IAAI,KAAK,CAAS,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAE9D,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,kBAAkB,CAAC,MAAM,EAAE,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClF,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;gBACxE,MAAM,OAAO,GAA2B;oBACtC,cAAc,EAAE,kBAAkB;iBACnC,CAAC;gBACF,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;oBAChB,OAAO,CAAC,aAAa,GAAG,UAAU,IAAI,CAAC,MAAM,EAAE,CAAC;gBAClD,CAAC;gBAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,YAAY,EAAE;oBACxD,MAAM,EAAE,MAAM;oBACd,OAAO;oBACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK,EAAE,IAAI,CAAC,SAAS;wBACrB,KAAK;wBACL,SAAS,EAAE,KAAK;wBAChB,KAAK,EAAE,KAAK,CAAC,MAAM;qBACpB,CAAC;iBACH,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;oBACnC,OAAO,WAAW,CAChB,eAAe,EACf,wBAAwB,QAAQ,CAAC,MAAM,KAAK,IAAI,EAAE,CACnD,CAAC;gBACJ,CAAC;gBAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAA0B,CAAC;gBAC9D,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;gBAC9C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,MAAM,WAAW,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC;oBACxC,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;oBACtD,SAAS,CAAC,WAAW,CAAC,GAAG,KAAK,CAAC;gBACjC,CAAC;YACH,CAAC;YAED,MAAM,gBAAgB,GAAG,eAAe,CAAC,SAAS,CAAC,CAAC;YACpD,MAAM,OAAO,GAAmB,gBAAgB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;gBACtE,KAAK;gBACL,KAAK;gBACL,GAAG,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACpE,CAAC,CAAC,CAAC;YAEJ,OAAO,YAAY,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,WAAW,CAChB,eAAe,EACf,6BAA8B,KAAe,CAAC,OAAO,EAAE,EACvD,EAAE,KAAK,EAAG,KAAe,CAAC,OAAO,EAAE,CACpC,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AAED,MAAM,UAAU,sBAAsB,CACpC,MAAuB,EACvB,OAA2D;IAE3D,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;QAClD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxB,KAAK,MAAM;YACT,OAAO,IAAI,oBAAoB,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,iBAAiB,CAAC,CAAC;QAC9E,KAAK,QAAQ;YACX,IAAI,CAAC,OAAO,EAAE,YAAY,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO,IAAI,sBAAsB,CAC/B,MAAM,CAAC,WAAW,EAClB,OAAO,CAAC,YAAY,EACpB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,iBAAiB,CACzB,CAAC;QACJ,KAAK,SAAS;YACZ,OAAO,IAAI,uBAAuB,CAChC,MAAM,CAAC,cAAc,EACrB,MAAM,CAAC,kBAAkB,EACzB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,iBAAiB,EACxB,OAAO,EAAE,aAAa,CACvB,CAAC;QACJ;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC"}

package/dist/indexer/reranking.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=reranking.test.d.ts.map

package/dist/indexer/reranking.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reranking.test.d.ts","sourceRoot":"","sources":["../../src/indexer/reranking.test.ts"],"names":[],"mappings":""}

package/dist/indexer/reranking.test.js

@@ -0,0 +1,130 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { WasmRerankerProvider, VoyageRerankerProvider, LocalAIRerankerProvider, normalizeScores, truncateDocument, batchArray, createRerankerProvider, } from './reranking.js';
+import { isError } from '../types.js';
+const { pipelineMock, classifierMock } = vi.hoisted(() => ({
+    pipelineMock: vi.fn(),
+    classifierMock: vi.fn(),
+}));
+vi.mock('@xenova/transformers', () => ({
+    pipeline: pipelineMock.mockImplementation(async () => classifierMock),
+}));
+describe('RerankerProvider', () => {
+    beforeEach(() => {
+        vi.clearAllMocks();
+        classifierMock.mockImplementation(async (input) => {
+            if (input.includes('doc-high'))
+                return [{ label: 'relevant', score: 0.9 }];
+            if (input.includes('doc-mid'))
+                return [{ label: 'relevant', score: 0.5 }];
+            return [{ label: 'relevant', score: 0.1 }];
+        });
+    });
+    describe('normalizeScores', () => {
+        it('normalizes scores to [0, 1] range', () => {
+            expect(normalizeScores([1, 2, 3])).toEqual([0, 0.5, 1]);
+            expect(normalizeScores([-1, 0, 1])).toEqual([0, 0.5, 1]);
+        });
+        it('handles equal scores', () => {
+            expect(normalizeScores([5, 5, 5])).toEqual([1, 1, 1]);
+        });
+        it('handles empty array', () => {
+            expect(normalizeScores([])).toEqual([]);
+        });
+    });
+    describe('helpers', () => {
+        it('truncates long documents before scoring', () => {
+            const longDoc = `${'a '.repeat(4000)}end`;
+            const truncated = truncateDocument(longDoc, 100);
+            expect(truncated.length).toBeLessThanOrEqual(100);
+        });
+        it('batches arrays', () => {
+            const batches = batchArray([1, 2, 3, 4, 5], 2);
+            expect(batches).toEqual([[1, 2], [3, 4], [5]]);
+        });
+    });
+    describe('WasmRerankerProvider', () => {
+        it('initializes lazily', async () => {
+            const provider = new WasmRerankerProvider('Xenova/ms-marco-MiniLM-L-6-v2', 2000);
+            expect(provider.isInitialized()).toBe(false);
+            await provider.rerank('query', ['doc-low']);
+            expect(provider.isInitialized()).toBe(true);
+            expect(pipelineMock).toHaveBeenCalledTimes(1);
+        });
+        it('returns normalized scores', async () => {
+            const provider = new WasmRerankerProvider('Xenova/ms-marco-MiniLM-L-6-v2', 2000);
+            const results = await provider.rerank('query', ['doc-low', 'doc-mid', 'doc-high']);
+            expect(isError(results)).toBe(false);
+            if (!isError(results)) {
+                results.forEach((r) => {
+                    expect(r.score).toBeGreaterThanOrEqual(0);
+                    expect(r.score).toBeLessThanOrEqual(1);
+                });
+                expect(results[0].score).toBeGreaterThan(results[1].score);
+            }
+        });
+    });
+    describe('VoyageRerankerProvider', () => {
+        it('batches large document sets for API providers', async () => {
+            const fetchSpy = vi.fn(async (_url, init) => {
+                const body = JSON.parse(init?.body || '{}');
+                return new Response(JSON.stringify({
+                    data: body.documents.map((_, index) => ({
+                        index,
+                        relevance_score: 1 - index / 100,
+                    })),
+                }), { status: 200, headers: { 'Content-Type': 'application/json' } });
+            });
+            vi.stubGlobal('fetch', fetchSpy);
+            const provider = new VoyageRerankerProvider('rerank-2.5-lite', 'test-key', 10, 2000);
+            const docs = Array.from({ length: 25 }, (_, i) => `doc-${i}`);
+            const results = await provider.rerank('query', docs);
+            expect(isError(results)).toBe(false);
+            expect(fetchSpy).toHaveBeenCalledTimes(3);
+            vi.unstubAllGlobals();
+        });
+    });
+    describe('LocalAIRerankerProvider', () => {
+        it('normalizes scores from LocalAI responses', async () => {
+            const fetchSpy = vi.fn(async (_url, init) => {
+                const body = JSON.parse(init?.body || '{}');
+                return new Response(JSON.stringify({
+                    results: body.documents.map((_, index) => ({
+                        index,
+                        score: (index + 1) * 10,
+                    })),
+                }), { status: 200, headers: { 'Content-Type': 'application/json' } });
+            });
+            vi.stubGlobal('fetch', fetchSpy);
+            const provider = new LocalAIRerankerProvider('http://localhost:8080', 'jina-reranker-v1-base-en', 10, 2000);
+            const results = await provider.rerank('query', ['a', 'b']);
+            expect(isError(results)).toBe(false);
+            if (!isError(results)) {
+                expect(results[0].score).toBe(1);
+                expect(results[1].score).toBe(0);
+            }
+            vi.unstubAllGlobals();
+        });
+    });
+    describe('createRerankerProvider', () => {
+        const baseConfig = {
+            enabled: true,
+            provider: 'wasm',
+            topK: 50,
+            prerankThreshold: 0.3,
+            maxDocumentLength: 2000,
+            batchSize: 100,
+            wasmModel: 'Xenova/ms-marco-MiniLM-L-6-v2',
+            voyageModel: 'rerank-2.5-lite',
+            localaiBaseUrl: 'http://localhost:8080',
+            localaiRerankModel: 'jina-reranker-v1-base-en',
+        };
+        it('returns null when disabled', () => {
+            const provider = createRerankerProvider({ ...baseConfig, enabled: false, provider: 'none' });
+            expect(provider).toBeNull();
+        });
+        it('throws when voyage key missing', () => {
+            expect(() => createRerankerProvider({ ...baseConfig, provider: 'voyage' })).toThrow('Voyage API key required for reranking');
+        });
+    });
+});
+//# sourceMappingURL=reranking.test.js.map

package/dist/indexer/reranking.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reranking.test.js","sourceRoot":"","sources":["../../src/indexer/reranking.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9D,OAAO,EACL,oBAAoB,EACpB,sBAAsB,EACtB,uBAAuB,EACvB,eAAe,EACf,gBAAgB,EAChB,UAAU,EACV,sBAAsB,GACvB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAGtC,MAAM,EAAE,YAAY,EAAE,cAAc,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACzD,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE;IACrB,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE;CACxB,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,sBAAsB,EAAE,GAAG,EAAE,CAAC,CAAC;IACrC,QAAQ,EAAE,YAAY,CAAC,kBAAkB,CAAC,KAAK,IAAI,EAAE,CAAC,cAAc,CAAC;CACtE,CAAC,CAAC,CAAC;AAEJ,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,cAAc,CAAC,kBAAkB,CAAC,KAAK,EAAE,KAAa,EAAE,EAAE;YACxD,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC;gBAAE,OAAO,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YAC3E,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAAE,OAAO,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YAC1E,OAAO,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;YACxD,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;YAC9B,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qBAAqB,EAAE,GAAG,EAAE;YAC7B,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;YACjD,MAAM,OAAO,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;YAC1C,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YACjD,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gBAAgB,EAAE,GAAG,EAAE;YACxB,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/C,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;QACpC,EAAE,CAAC,oBAAoB,EAAE,KAAK,IAAI,EAAE;YAClC,MAAM,QAAQ,GAAG,IAAI,oBAAoB,CAAC,+BAA+B,EAAE,IAAI,CAAC,CAAC;YACjF,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7C,MAAM,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC;YAC5C,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5C,MAAM,CAAC,YAAY,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;YACzC,MAAM,QAAQ,GAAG,IAAI,oBAAoB,CAAC,+BAA+B,EAAE,IAAI,CAAC,CAAC;YACjF,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,SAAS,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;YACnF,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBACtB,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;oBACpB,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;oBAC1C,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;gBACzC,CAAC,CAAC,CAAC;gBACH,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;QACtC,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,IAAY,EAAE,IAAkB,EAAE,EAAE;gBAChE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAE,IAAI,EAAE,IAAe,IAAI,IAAI,CAA4B,CAAC;gBACnF,OAAO,IAAI,QAAQ,CACjB,IAAI,CAAC,SAAS,CAAC;oBACb,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;wBACtC,KAAK;wBACL,eAAe,EAAE,CAAC,GAAG,KAAK,GAAG,GAAG;qBACjC,CAAC,CAAC;iBACJ,CAAC,EACF,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,EAAE,CACjE,CAAC;YACJ,CAAC,CAAC,CAAC;YACH,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAEjC,MAAM,QAAQ,GAAG,IAAI,sBAAsB,CAAC,iBAAiB,EAAE,UAAU,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;YACrF,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC9D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YAErD,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,CAAC,QAAQ,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAC1C,EAAE,CAAC,gBAAgB,EAAE,CAAC;QACxB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACvC,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,IAAY,EAAE,IAAkB,EAAE,EAAE;gBAChE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAE,IAAI,EAAE,IAAe,IAAI,IAAI,CAA4B,CAAC;gBACnF,OAAO,IAAI,QAAQ,CACjB,IAAI,CAAC,SAAS,CAAC;oBACb,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;wBACzC,KAAK;wBACL,KAAK,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,EAAE;qBACxB,CAAC,CAAC;iBACJ,CAAC,EACF,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,EAAE,CACjE,CAAC;YACJ,CAAC,CAAC,CAAC;YACH,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAEjC,MAAM,QAAQ,GAAG,IAAI,uBAAuB,CAC1C,uBAAuB,EACvB,0BAA0B,EAC1B,EAAE,EACF,IAAI,CACL,CAAC;YAEF,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;YAC3D,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBACtB,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACjC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACnC,CAAC;YAED,EAAE,CAAC,gBAAgB,EAAE,CAAC;QACxB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;QACtC,MAAM,UAAU,GAAoB;YAClC,OAAO,EAAE,IAAI;YACb,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,EAAE;YACR,gBAAgB,EAAE,GAAG;YACrB,iBAAiB,EAAE,IAAI;YACvB,SAAS,EAAE,GAAG;YACd,SAAS,EAAE,+BAA+B;YAC1C,WAAW,EAAE,iBAAiB;YAC9B,cAAc,EAAE,uBAAuB;YACvC,kBAAkB,EAAE,0BAA0B;SAC/C,CAAC;QAEF,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,QAAQ,GAAG,sBAAsB,CAAC,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7F,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC9B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,CAAC,GAAG,EAAE,CACV,sBAAsB,CAAC,EAAE,GAAG,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAC9D,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/indexer/wal/file-storage.d.ts

@@ -0,0 +1,60 @@
+/**
+ * FileWalStorage - File-based implementation of WalStorage interface
+ *
+ * File structure:
+ * .lancedb/
+ * ├── sync.lock            # Global lock with PID and heartbeat
+ * ├── sync.wal             # Append-only log (bounded size via truncation)
+ * ├── sync.checkpoint      # Last checkpoint snapshot (recovery point)
+ * └── sync.manifest        # Current state + last completed task
+ */
+import type { WalStorage, LockData, WalEntry, CheckpointData, ManifestData } from './types.js';
+export declare class FileWalStorage implements WalStorage {
+    private lockPath;
+    private walPath;
+    private checkpointPath;
+    private manifestPath;
+    constructor(dir: string);
+    acquireLock(lock: LockData): Promise<boolean>;
+    releaseLock(lockId: string): Promise<void>;
+    getLock(): Promise<LockData | null>;
+    updateHeartbeat(lockId: string): Promise<void>;
+    /**
+     * Force clear the lock file.
+     * Use with caution - only for stale lock recovery.
+     */
+    forceClearLock(): Promise<void>;
+    /**
+     * Atomically clear lock only if it matches expected state.
+     * Prevents race condition where two processes detect stale lock simultaneously.
+     */
+    compareAndClearLock(expectedLock: LockData): Promise<boolean>;
+    appendEntry(entry: WalEntry): Promise<void>;
+    getEntriesFromLsn(lsn: number): Promise<WalEntry[]>;
+    getAllEntries(): Promise<WalEntry[]>;
+    truncateBeforeLsn(lsn: number): Promise<void>;
+    clearWal(): Promise<void>;
+    writeCheckpoint(data: CheckpointData): Promise<void>;
+    getCheckpoint(): Promise<CheckpointData | null>;
+    clearCheckpoint(): Promise<void>;
+    writeManifest(data: ManifestData): Promise<void>;
+    getManifest(): Promise<ManifestData | null>;
+    clearManifest(): Promise<void>;
+    clear(): Promise<void>;
+    private atomicWrite;
+    private readJsonFile;
+    /**
+     * Check if PID is still running (for stale lock detection)
+     */
+    static isPidRunning(pid: number): boolean;
+    /**
+     * Get paths for debugging/testing
+     */
+    getPaths(): {
+        lock: string;
+        wal: string;
+        checkpoint: string;
+        manifest: string;
+    };
+}
+//# sourceMappingURL=file-storage.d.ts.map

package/dist/indexer/wal/file-storage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-storage.d.ts","sourceRoot":"","sources":["../../../src/indexer/wal/file-storage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,OAAO,KAAK,EACV,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,YAAY,EACb,MAAM,YAAY,CAAC;AAEpB,qBAAa,cAAe,YAAW,UAAU;IAC/C,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,YAAY,CAAS;gBAEjB,GAAG,EAAE,MAAM;IAWjB,WAAW,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;IAyB7C,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAa1C,OAAO,IAAI,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAYnC,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQpD;;;OAGG;IACG,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAUrC;;;OAGG;IACG,mBAAmB,CAAC,YAAY,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;IAgC7D,WAAW,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAI3C,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IA2BnD,aAAa,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;IAIpC,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAS7C,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAczB,eAAe,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAIpD,aAAa,IAAI,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAI/C,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAchC,aAAa,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAIhD,WAAW,IAAI,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAI3C,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IAc9B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAad,WAAW;YAMX,YAAY;IAkB1B;;OAEG;IACH,MAAM,CAAC,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAUzC;;OAEG;IACH,QAAQ,IAAI;QACV,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;KAClB;CAQF"}