@framers/agentos 0.1.156 → 0.1.158

package/dist/rag/reranking/providers/LlmJudgeReranker.js

@@ -0,0 +1,121 @@
+/**
+ * @fileoverview LLM-as-Judge Reranker — two-phase hybrid reranking using LLM calls.
+ *
+ * Phase 1: Batch pointwise scoring with a cheap model (gpt-4o-mini, haiku).
+ *          Groups documents into batches of 10, asks LLM to score 0-10.
+ * Phase 2: Listwise final ranking with a synthesis model.
+ *          Takes top-K from phase 1, asks LLM to rank by relevance.
+ *
+ * Cognitive science: Combines absolute judgment (pointwise) with comparative
+ * judgment (listwise) — mirrors how human expert reviewers evaluate documents.
+ *
+ * References:
+ * - Sun, W., et al. (2023). "Is ChatGPT Good at Search? Investigating Large
+ *   Language Models as Re-Ranking Agents." arXiv:2304.09542
+ * - Qin, Z., et al. (2023). "Large Language Models are Effective Text Rankers
+ *   with Pairwise Ranking Prompting." arXiv:2306.17563
+ *
+ * @module agentos/rag/reranking/providers/LlmJudgeReranker
+ */
+const POINTWISE_SYSTEM = `You are a relevance scorer. Rate each document's relevance to the query on a scale of 0-10. 10 = perfectly relevant, 0 = completely irrelevant. Return ONLY a JSON array of integer scores, one per document, in the same order. Example: [8, 3, 7, 2, 9]`;
+const LISTWISE_SYSTEM = `You are a relevance ranker. Rank the documents by relevance to the query, most relevant first. Return ONLY a JSON array of document IDs in ranked order. Example: ["doc-3", "doc-1", "doc-5"]`;
+/** Two-phase LLM-based reranker: batch pointwise + listwise top-K. */
+export class LlmJudgeReranker {
+    constructor(config) {
+        this.providerId = 'llm-judge';
+        this.llmCallFn = config.llmCallFn;
+        this.scoringModel = config.scoringModel;
+        this.rankingModel = config.rankingModel;
+        this.maxPointwiseDocuments = config.maxPointwiseDocuments ?? 100;
+        this.pointwiseTopK = config.pointwiseTopK ?? 20;
+        this.batchSize = config.batchSize ?? 10;
+    }
+    async isAvailable() {
+        return typeof this.llmCallFn === 'function';
+    }
+    async rerank(input, config) {
+        const topN = config.topN ?? this.pointwiseTopK;
+        let documents = input.documents;
+        if (documents.length > this.maxPointwiseDocuments) {
+            documents = documents.slice(0, this.maxPointwiseDocuments);
+        }
+        // Phase 1: Batch pointwise scoring
+        const scored = await this.batchPointwiseScore(input.query, documents);
+        // Sort by score descending, take top-K for phase 2
+        scored.sort((a, b) => b.score - a.score);
+        const candidates = scored.slice(0, this.pointwiseTopK);
+        // Phase 2: Listwise ranking
+        let finalRanking;
+        try {
+            finalRanking = await this.listwiseRank(input.query, candidates, topN);
+        }
+        catch {
+            // Fallback: use pointwise scores
+            finalRanking = candidates.slice(0, topN).map((c, i) => ({
+                id: c.id,
+                content: c.content,
+                relevanceScore: 1 - (i / Math.max(topN, 1)),
+                originalScore: c.originalScore,
+                metadata: c.metadata,
+            }));
+        }
+        return { results: finalRanking };
+    }
+    /** Phase 1: Score documents in batches. */
+    async batchPointwiseScore(query, documents) {
+        const batches = [];
+        for (let i = 0; i < documents.length; i += this.batchSize) {
+            batches.push(documents.slice(i, i + this.batchSize));
+        }
+        const results = [];
+        for (const batch of batches) {
+            const docList = batch
+                .map((d, i) => `[${i + 1}] ${d.content.slice(0, 200)}`)
+                .join('\n');
+            const userPrompt = `Query: "${query}"\n\nDocuments:\n${docList}`;
+            try {
+                const raw = await this.llmCallFn(POINTWISE_SYSTEM, userPrompt, this.scoringModel);
+                const cleaned = raw.replace(/```json?\n?/g, '').replace(/```/g, '').trim();
+                const scores = JSON.parse(cleaned);
+                for (let i = 0; i < batch.length; i++) {
+                    results.push({
+                        ...batch[i],
+                        score: typeof scores[i] === 'number' ? scores[i] : 0,
+                    });
+                }
+            }
+            catch {
+                for (const doc of batch) {
+                    results.push({ ...doc, score: 0 });
+                }
+            }
+        }
+        return results;
+    }
+    /** Phase 2: Listwise ranking of top candidates. */
+    async listwiseRank(query, candidates, topN) {
+        const docList = candidates
+            .map((d) => `[${d.id}] ${d.content.slice(0, 200)}`)
+            .join('\n');
+        const userPrompt = `Query: "${query}"\n\nDocuments:\n${docList}`;
+        const raw = await this.llmCallFn(LISTWISE_SYSTEM, userPrompt, this.rankingModel);
+        const cleaned = raw.replace(/```json?\n?/g, '').replace(/```/g, '').trim();
+        const ranking = JSON.parse(cleaned);
+        const candidateMap = new Map(candidates.map((c) => [c.id, c]));
+        const results = [];
+        for (let i = 0; i < Math.min(ranking.length, topN); i++) {
+            const doc = candidateMap.get(ranking[i]);
+            if (!doc)
+                continue;
+            results.push({
+                id: doc.id,
+                content: doc.content,
+                relevanceScore: 1 - (i / Math.max(ranking.length, 1)),
+                originalScore: doc.originalScore,
+                metadata: doc.metadata,
+            });
+        }
+        return results;
+    }
+}
+//# sourceMappingURL=LlmJudgeReranker.js.map

package/dist/rag/reranking/providers/LlmJudgeReranker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LlmJudgeReranker.js","sourceRoot":"","sources":["../../../../src/rag/reranking/providers/LlmJudgeReranker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AA4BH,MAAM,gBAAgB,GAAG,2PAA2P,CAAC;AAErR,MAAM,eAAe,GAAG,+LAA+L,CAAC;AAExN,sEAAsE;AACtE,MAAM,OAAO,gBAAgB;IAU3B,YAAY,MAA8B;QAT1B,eAAU,GAAG,WAAoB,CAAC;QAUhD,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QAClC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;QACxC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;QACxC,IAAI,CAAC,qBAAqB,GAAG,MAAM,CAAC,qBAAqB,IAAI,GAAG,CAAC;QACjE,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,EAAE,CAAC;QAChD,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,WAAW;QACf,OAAO,OAAO,IAAI,CAAC,SAAS,KAAK,UAAU,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAoB,EAAE,MAA6B;QAC9D,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,aAAa,CAAC;QAC/C,IAAI,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;QAEhC,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAClD,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAC7D,CAAC;QAED,mCAAmC;QACnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAEtE,mDAAmD;QACnD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QAEvD,4BAA4B;QAC5B,IAAI,YAAgC,CAAC;QACrC,IAAI,CAAC;YACH,YAAY,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;QACxE,CAAC;QAAC,MAAM,CAAC;YACP,iCAAiC;YACjC,YAAY,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBACtD,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,cAAc,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;gBAC3C,aAAa,EAAE,CAAC,CAAC,aAAa;gBAC9B,QAAQ,EAAE,CAAC,CAAC,QAAQ;aACrB,CAAC,CAAC,CAAC;QACN,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;IACnC,CAAC;IAED,2CAA2C;IACnC,KAAK,CAAC,mBAAmB,CAC/B,KAAa,EACb,SAAqC;QAErC,MAAM,OAAO,GAAiC,EAAE,CAAC;QACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAC1D,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;QACvD,CAAC;QAED,MAAM,OAAO,GAAkE,EAAE,CAAC;QAElF,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,OAAO,GAAG,KAAK;iBAClB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;iBACtD,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,MAAM,UAAU,GAAG,WAAW,KAAK,oBAAoB,OAAO,EAAE,CAAC;YAEjE,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;gBAClF,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC3E,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAa,CAAC;gBAE/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACtC,OAAO,CAAC,IAAI,CAAC;wBACX,GAAG,KAAK,CAAC,CAAC,CAAC;wBACX,KAAK,EAAE,OAAO,MAAM,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;qBACrD,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;oBACxB,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,mDAAmD;IAC3C,KAAK,CAAC,YAAY,CACxB,KAAa,EACb,UAAyE,EACzE,IAAY;QAEZ,MAAM,OAAO,GAAG,UAAU;aACvB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;aAClD,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,MAAM,UAAU,GAAG,WAAW,KAAK,oBAAoB,OAAO,EAAE,CAAC;QAEjE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,EAAE,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QACjF,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3E,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAa,CAAC;QAEhD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,MAAM,OAAO,GAAuB,EAAE,CAAC;QAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACxD,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,IAAI,CAAC,GAAG;gBAAE,SAAS;YACnB,OAAO,CAAC,IAAI,CAAC;gBACX,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,cAAc,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBACrD,aAAa,EAAE,GAAG,CAAC,aAAa;gBAChC,QAAQ,EAAE,GAAG,CAAC,QAAQ;aACvB,CAAC,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@framers/agentos",
-  "version": "0.1.156",
+  "version": "0.1.158",
   "description": "Modular AgentOS orchestration library",
   "type": "module",
   "main": "dist/index.js",