@keel_flow/kb-pipeline 0.2.0

package/dist/retrieval/index.js

@@ -0,0 +1,90 @@
+import { denseRank } from "./dense.js";
+import { bm25Rank } from "./bm25.js";
+import { reciprocalRankFusion } from "./rrf.js";
+const RRF_POOL = 50;
+function effectiveScore(chunk) {
+    return chunk.rerankScore ?? chunk.rrfScore;
+}
+export async function retrieve(opts) {
+    const start = Date.now();
+    const k = opts.k ?? 5;
+    const rerankTopN = opts.rerankTopN ?? Math.max(k * 4, 20);
+    let dense;
+    if (opts.precomputedDenseScores) {
+        // Database-side ranking (pgvector). Already sorted by the query; cap to
+        // the same pool size the in-process path uses so RRF sees equal lists.
+        dense = opts.precomputedDenseScores.slice(0, RRF_POOL);
+    }
+    else if (opts.queryEmbedding) {
+        dense = denseRank(opts.queryEmbedding, opts.candidates
+            .filter((c) => Array.isArray(c.embedding))
+            .map((c) => ({ id: c.id, embedding: c.embedding })), RRF_POOL);
+    }
+    else {
+        throw new Error("retrieve() needs queryEmbedding or precomputedDenseScores");
+    }
+    const bm25 = bm25Rank(opts.query, opts.candidates.map((c) => ({ id: c.id, content: c.contentForBm25 ?? c.content })), RRF_POOL);
+    const fused = reciprocalRankFusion([
+        { source: "dense", scores: dense },
+        { source: "bm25", scores: bm25 },
+    ]);
+    const denseById = new Map(dense.map((s) => [s.id, s.score]));
+    const bm25ById = new Map(bm25.map((s) => [s.id, s.score]));
+    const candById = new Map(opts.candidates.map((c) => [c.id, c]));
+    let rerankScores = null;
+    let finalIds;
+    if (opts.reranker) {
+        const top = fused.slice(0, rerankTopN);
+        const rerankInput = top
+            .map((r) => candById.get(r.id))
+            .filter((c) => c !== undefined)
+            .map((c) => ({ id: c.id, content: c.content }));
+        rerankScores = await opts.reranker.rerank(opts.query, rerankInput);
+        finalIds = rerankScores.slice(0, k).map((s) => s.id);
+    }
+    else {
+        finalIds = fused.slice(0, k).map((r) => r.id);
+    }
+    const rrfById = new Map(fused.map((r) => [r.id, r.rrfScore]));
+    const rerankById = new Map(rerankScores?.map((s) => [s.id, s.score]) ?? []);
+    const allChunks = finalIds
+        .map((id) => {
+        const cand = candById.get(id);
+        if (!cand)
+            return undefined;
+        return {
+            id,
+            content: cand.content,
+            rrfScore: rrfById.get(id) ?? 0,
+            denseScore: denseById.get(id),
+            bm25Score: bm25ById.get(id),
+            rerankScore: rerankById.get(id),
+        };
+    })
+        .filter((c) => c !== undefined);
+    const chunks = opts.minScore === undefined
+        ? allChunks
+        : allChunks.filter((c) => effectiveScore(c) >= opts.minScore);
+    return {
+        chunks,
+        trace: {
+            denseScores: dense,
+            bm25Scores: bm25,
+            rrfScores: fused.map((r) => ({ id: r.id, score: r.rrfScore })),
+            rerankScores,
+            finalIds: chunks.map((c) => c.id),
+            latencyMs: Date.now() - start,
+        },
+    };
+}
+export async function embedQuery(embedder, query) {
+    const [vec] = await embedder.embed([query], "query");
+    if (!vec)
+        throw new Error("embedder returned no vector for query");
+    return vec;
+}
+export { denseRank, cosineSim } from "./dense.js";
+export { bm25Rank } from "./bm25.js";
+export { reciprocalRankFusion } from "./rrf.js";
+export { createVoyageReranker, createCrossEncoderReranker, createDefaultReranker, shouldRerank, } from "./rerank.js";
+//# sourceMappingURL=index.js.map

package/dist/retrieval/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/retrieval/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AA2DhD,MAAM,QAAQ,GAAG,EAAE,CAAC;AAEpB,SAAS,cAAc,CAAC,KAA2B;IACjD,OAAO,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,QAAQ,CAAC;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAkB;IAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;IACtB,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;IAE1D,IAAI,KAA2C,CAAC;IAChD,IAAI,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAChC,wEAAwE;QACxE,uEAAuE;QACvE,KAAK,GAAG,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACzD,CAAC;SAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;QAC/B,KAAK,GAAG,SAAS,CACf,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,UAAU;aACZ,MAAM,CAAC,CAAC,CAAC,EAAqD,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;aAC5F,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,EACrD,QAAQ,CACT,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC/E,CAAC;IACD,MAAM,IAAI,GAAG,QAAQ,CACnB,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,cAAc,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,EAClF,QAAQ,CACT,CAAC;IACF,MAAM,KAAK,GAAG,oBAAoB,CAAC;QACjC,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE;QAClC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE;KACjC,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC3D,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAEhE,IAAI,YAAY,GAAgD,IAAI,CAAC;IACrE,IAAI,QAAkB,CAAC;IAEvB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QACvC,MAAM,WAAW,GAAG,GAAG;aACpB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;aAC9B,MAAM,CAAC,CAAC,CAAC,EAA2B,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC;aACvD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAClD,YAAY,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QACnE,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACvD,CAAC;SAAM,CAAC;QACN,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAChD,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC9D,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAE5E,MAAM,SAAS,GAA2B,QAAQ;SAC/C,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACV,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI;YAAE,OAAO,SAAS,CAAC;QAC5B,OAAO;YACL,EAAE;YACF,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC;YAC9B,UAAU,EAAE,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7B,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,WAAW,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;SACR,CAAC;IAC5B,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAA6B,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;IAE7D,MAAM,MAAM,GACV,IAAI,CAAC,QAAQ,KAAK,SAAS;QACzB,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAS,CAAC,CAAC;IAEnE,OAAO;QACL,MAAM;QACN,KAAK,EAAE;YACL,WAAW,EAAE,KAAK;YAClB,UAAU,EAAE,IAAI;YAChB,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC9D,YAAY;YACZ,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;SAC9B;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,QAAkB,EAAE,KAAa;IAChE,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,CAAC;IACrD,IAAI,CAAC,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IACnE,OAAO,GAAG,CAAC;AACb,CAAC;AAED,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EACL,oBAAoB,EACpB,0BAA0B,EAC1B,qBAAqB,EACrB,YAAY,GACb,MAAM,aAAa,CAAC"}

package/dist/retrieval/rerank.d.ts

@@ -0,0 +1,34 @@
+export interface RerankCandidate {
+    id: string;
+    content: string;
+}
+export interface RerankScore {
+    id: string;
+    score: number;
+}
+export interface Reranker {
+    modelId: string;
+    rerank(query: string, candidates: RerankCandidate[]): Promise<RerankScore[]>;
+}
+export interface VoyageRerankOpts {
+    apiKey?: string;
+    endpoint?: string;
+    model?: string;
+    fetchFn?: typeof fetch;
+}
+export declare function createVoyageReranker(opts?: VoyageRerankOpts): Reranker;
+export interface CrossEncoderOpts {
+    pipelineFactory?: () => Promise<(input: {
+        text: string;
+        text_pair: string;
+    }, opts: object) => Promise<{
+        data: Float32Array;
+    } | Array<{
+        score: number;
+        label: string;
+    }>>>;
+}
+export declare function createCrossEncoderReranker(opts?: CrossEncoderOpts): Reranker;
+export declare function shouldRerank(env?: NodeJS.ProcessEnv): boolean;
+export declare function createDefaultReranker(env?: NodeJS.ProcessEnv): Reranker;
+//# sourceMappingURL=rerank.d.ts.map

package/dist/retrieval/rerank.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rerank.d.ts","sourceRoot":"","sources":["../../src/retrieval/rerank.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;CAC9E;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACxB;AAID,wBAAgB,oBAAoB,CAAC,IAAI,GAAE,gBAAqB,GAAG,QAAQ,CA4C1E;AAID,MAAM,WAAW,gBAAgB;IAC/B,eAAe,CAAC,EAAE,MAAM,OAAO,CAC7B,CACE,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,EAC1C,IAAI,EAAE,MAAM,KACT,OAAO,CAAC;QAAE,IAAI,EAAE,YAAY,CAAA;KAAE,GAAG,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAC/E,CAAC;CACH;AAOD,wBAAgB,0BAA0B,CAAC,IAAI,GAAE,gBAAqB,GAAG,QAAQ,CAiDhF;AAED,wBAAgB,YAAY,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,OAAO,CAE1E;AAED,wBAAgB,qBAAqB,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,QAAQ,CAMpF"}

package/dist/retrieval/rerank.js

@@ -0,0 +1,101 @@
+const VOYAGE_RERANK_ENDPOINT = "https://api.voyageai.com/v1/rerank";
+export function createVoyageReranker(opts = {}) {
+    const apiKey = opts.apiKey ?? process.env["VOYAGE_RERANK_API_KEY"] ?? process.env["VOYAGE_API_KEY"];
+    if (!apiKey) {
+        throw new Error("VOYAGE_RERANK_API_KEY or VOYAGE_API_KEY is required for voyage-rerank-2.5");
+    }
+    const model = opts.model ?? "voyage-rerank-2.5";
+    const endpoint = opts.endpoint ?? VOYAGE_RERANK_ENDPOINT;
+    const fetchFn = opts.fetchFn ?? fetch;
+    return {
+        modelId: model,
+        async rerank(query, candidates) {
+            if (candidates.length === 0)
+                return [];
+            const res = await fetchFn(endpoint, {
+                method: "POST",
+                headers: {
+                    "Authorization": `Bearer ${apiKey}`,
+                    "Content-Type": "application/json",
+                },
+                body: JSON.stringify({
+                    model,
+                    query,
+                    documents: candidates.map((c) => c.content),
+                }),
+            });
+            if (!res.ok) {
+                const body = await res.text().catch(() => "");
+                throw new Error(`voyage rerank failed: ${res.status} ${body}`);
+            }
+            const json = (await res.json());
+            const out = [];
+            for (const row of json.data ?? []) {
+                const cand = candidates[row.index ?? -1];
+                if (!cand)
+                    continue;
+                out.push({ id: cand.id, score: row.relevance_score ?? 0 });
+            }
+            return out.sort((a, b) => b.score - a.score);
+        },
+    };
+}
+export function createCrossEncoderReranker(opts = {}) {
+    let pipelinePromise = null;
+    async function getPipeline() {
+        if (pipelinePromise)
+            return pipelinePromise;
+        pipelinePromise = (async () => {
+            if (opts.pipelineFactory)
+                return opts.pipelineFactory();
+            let rawMod;
+            try {
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any -- dynamic import of the optional @huggingface/transformers peer; the `as {...}` assertion below pins the used shape
+                rawMod = await import("@huggingface/transformers");
+            }
+            catch (cause) {
+                throw new Error("The cross-encoder reranker requires @huggingface/transformers, which is an optional peer dependency of @keel_flow/kb-pipeline. " +
+                    "Install it with: pnpm add @huggingface/transformers — " +
+                    "or set VOYAGE_RERANK_API_KEY / VOYAGE_API_KEY to use the Voyage reranker instead.", { cause });
+            }
+            const mod = rawMod;
+            return mod.pipeline("text-classification", "Xenova/ms-marco-MiniLM-L-6-v2", { revision: "main" });
+        })();
+        pipelinePromise.catch(() => {
+            pipelinePromise = null;
+        });
+        return pipelinePromise;
+    }
+    return {
+        modelId: "Xenova/ms-marco-MiniLM-L-6-v2",
+        async rerank(query, candidates) {
+            if (candidates.length === 0)
+                return [];
+            const pipeline = await getPipeline();
+            const out = [];
+            for (const cand of candidates) {
+                const r = await pipeline({ text: query, text_pair: cand.content }, { topk: 1 });
+                let score = 0;
+                if (Array.isArray(r)) {
+                    score = r[0]?.score ?? 0;
+                }
+                else if (r && "data" in r) {
+                    score = r.data[0] ?? 0;
+                }
+                out.push({ id: cand.id, score });
+            }
+            return out.sort((a, b) => b.score - a.score);
+        },
+    };
+}
+export function shouldRerank(env = process.env) {
+    return env["KEEL_KB_RERANK"] === "true";
+}
+export function createDefaultReranker(env = process.env) {
+    const voyageKey = env["VOYAGE_RERANK_API_KEY"] ?? env["VOYAGE_API_KEY"];
+    if (voyageKey) {
+        return createVoyageReranker({ apiKey: voyageKey });
+    }
+    return createCrossEncoderReranker();
+}
+//# sourceMappingURL=rerank.js.map

package/dist/retrieval/rerank.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rerank.js","sourceRoot":"","sources":["../../src/retrieval/rerank.ts"],"names":[],"mappings":"AAsBA,MAAM,sBAAsB,GAAG,oCAAoC,CAAC;AAEpE,MAAM,UAAU,oBAAoB,CAAC,OAAyB,EAAE;IAC9D,MAAM,MAAM,GACV,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IACvF,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,2EAA2E,CAC5E,CAAC;IACJ,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,mBAAmB,CAAC;IAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,sBAAsB,CAAC;IACzD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC;IAEtC,OAAO;QACL,OAAO,EAAE,KAAK;QACd,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAA6B;YACvD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YACvC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE;gBAClC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,eAAe,EAAE,UAAU,MAAM,EAAE;oBACnC,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK;oBACL,KAAK;oBACL,SAAS,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;iBAC5C,CAAC;aACH,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;gBAC9C,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC,CAAC;YACjE,CAAC;YACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAE7B,CAAC;YACF,MAAM,GAAG,GAAkB,EAAE,CAAC;YAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC;gBAClC,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC;gBACzC,IAAI,CAAC,IAAI;oBAAE,SAAS;gBACpB,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,eAAe,IAAI,CAAC,EAAE,CAAC,CAAC;YAC7D,CAAC;YACD,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAC/C,CAAC;KACF,CAAC;AACJ,CAAC;AAkBD,MAAM,UAAU,0BAA0B,CAAC,OAAyB,EAAE;IACpE,IAAI,eAAe,GAAyC,IAAI,CAAC;IAEjE,KAAK,UAAU,WAAW;QACxB,IAAI,eAAe;YAAE,OAAO,eAAe,CAAC;QAC5C,eAAe,GAAG,CAAC,KAAK,IAAmC,EAAE;YAC3D,IAAI,IAAI,CAAC,eAAe;gBAAE,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC;YACxD,IAAI,MAAe,CAAC;YACpB,IAAI,CAAC;gBACH,mLAAmL;gBACnL,MAAM,GAAG,MAAM,MAAM,CAAC,2BAAkC,CAAC,CAAC;YAC5D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CACb,iIAAiI;oBAC/H,wDAAwD;oBACxD,mFAAmF,EACrF,EAAE,KAAK,EAAE,CACV,CAAC;YACJ,CAAC;YACD,MAAM,GAAG,GAAG,MAEX,CAAC;YACF,OAAO,GAAG,CAAC,QAAQ,CAAC,qBAAqB,EAAE,+BAA+B,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;QACpG,CAAC,CAAC,EAAE,CAAC;QACL,eAAe,CAAC,KAAK,CAAC,GAAG,EAAE;YACzB,eAAe,GAAG,IAAI,CAAC;QACzB,CAAC,CAAC,CAAC;QACH,OAAO,eAAe,CAAC;IACzB,CAAC;IAED,OAAO;QACL,OAAO,EAAE,+BAA+B;QACxC,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAA6B;YACvD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YACvC,MAAM,QAAQ,GAAG,MAAM,WAAW,EAAE,CAAC;YACrC,MAAM,GAAG,GAAkB,EAAE,CAAC;YAC9B,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;gBAC9B,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;gBAChF,IAAI,KAAK,GAAG,CAAC,CAAC;gBACd,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;oBACrB,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC;gBAC3B,CAAC;qBAAM,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;oBAC5B,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACzB,CAAC;gBACD,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YACnC,CAAC;YACD,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAC/C,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAyB,OAAO,CAAC,GAAG;IAC/D,OAAO,GAAG,CAAC,gBAAgB,CAAC,KAAK,MAAM,CAAC;AAC1C,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,MAAyB,OAAO,CAAC,GAAG;IACxE,MAAM,SAAS,GAAG,GAAG,CAAC,uBAAuB,CAAC,IAAI,GAAG,CAAC,gBAAgB,CAAC,CAAC;IACxE,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,oBAAoB,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,0BAA0B,EAAE,CAAC;AACtC,CAAC"}

package/dist/retrieval/rrf.d.ts

@@ -0,0 +1,17 @@
+export interface RankedList {
+    source: string;
+    scores: Array<{
+        id: string;
+        score: number;
+    }>;
+}
+export interface RrfResult {
+    id: string;
+    rrfScore: number;
+    sources: Record<string, number>;
+}
+export interface RrfOpts {
+    k?: number;
+}
+export declare function reciprocalRankFusion(lists: RankedList[], opts?: RrfOpts): RrfResult[];
+//# sourceMappingURL=rrf.d.ts.map

package/dist/retrieval/rrf.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rrf.d.ts","sourceRoot":"","sources":["../../src/retrieval/rrf.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC9C;AAED,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,OAAO;IACtB,CAAC,CAAC,EAAE,MAAM,CAAC;CACZ;AAID,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,UAAU,EAAE,EACnB,IAAI,GAAE,OAAY,GACjB,SAAS,EAAE,CAkBb"}

package/dist/retrieval/rrf.js

@@ -0,0 +1,23 @@
+// Reciprocal Rank Fusion — combines arbitrarily many ranked lists by summing
+// 1 / (k + rank). The default k=60 matches the literature.
+export function reciprocalRankFusion(lists, opts = {}) {
+    const k = opts.k ?? 60;
+    const fused = new Map();
+    for (const list of lists) {
+        for (let rank = 0; rank < list.scores.length; rank++) {
+            const entry = list.scores[rank];
+            if (!entry)
+                continue;
+            const existing = fused.get(entry.id) ?? {
+                id: entry.id,
+                rrfScore: 0,
+                sources: {},
+            };
+            existing.rrfScore += 1 / (k + rank + 1);
+            existing.sources[list.source] = entry.score;
+            fused.set(entry.id, existing);
+        }
+    }
+    return Array.from(fused.values()).sort((a, b) => b.rrfScore - a.rrfScore);
+}
+//# sourceMappingURL=rrf.js.map

package/dist/retrieval/rrf.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rrf.js","sourceRoot":"","sources":["../../src/retrieval/rrf.ts"],"names":[],"mappings":"AAeA,6EAA6E;AAC7E,2DAA2D;AAC3D,MAAM,UAAU,oBAAoB,CAClC,KAAmB,EACnB,OAAgB,EAAE;IAElB,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;IACvB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;YACrD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAChC,IAAI,CAAC,KAAK;gBAAE,SAAS;YACrB,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI;gBACtC,EAAE,EAAE,KAAK,CAAC,EAAE;gBACZ,QAAQ,EAAE,CAAC;gBACX,OAAO,EAAE,EAAE;aACZ,CAAC;YACF,QAAQ,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;YACxC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC;YAC5C,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;AAC5E,CAAC"}

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@keel_flow/kb-pipeline",
+  "version": "0.2.0",
+  "private": false,
+  "type": "module",
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "dist"
+  ],
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "require": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "dependencies": {
+    "zod": "^3.23.0",
+    "@keel_flow/schema": "0.2.0"
+  },
+  "peerDependencies": {
+    "@huggingface/transformers": ">=4.0.0"
+  },
+  "peerDependenciesMeta": {
+    "@huggingface/transformers": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@types/node": "^25.9.1",
+    "@typescript-eslint/eslint-plugin": "^8.0.0",
+    "@typescript-eslint/parser": "^8.0.0",
+    "eslint": "^9.0.0",
+    "typescript": "^5.5.0",
+    "vitest": "^2.0.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run",
+    "lint": "eslint src"
+  }
+}