@adonis-agora/agent 0.34.1 → 0.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/README.md +34 -0
- package/dist/assets/spa/assets/index-7XqfoeQH.js +9 -0
- package/dist/assets/spa/index.html +1 -1
- package/dist/providers/agent_provider.d.ts.map +1 -1
- package/dist/providers/agent_provider.js +4 -0
- package/dist/providers/agent_provider.js.map +1 -1
- package/dist/src/agent-deps-factory.d.ts +5 -0
- package/dist/src/agent-deps-factory.d.ts.map +1 -1
- package/dist/src/agent-deps-factory.js +6 -0
- package/dist/src/agent-deps-factory.js.map +1 -1
- package/dist/src/agent-deps.d.ts +5 -0
- package/dist/src/agent-deps.d.ts.map +1 -1
- package/dist/src/agent-deps.js.map +1 -1
- package/dist/src/define_config.d.ts +12 -0
- package/dist/src/define_config.d.ts.map +1 -1
- package/dist/src/define_config.js.map +1 -1
- package/dist/src/guardrails/agent.d.ts +211 -0
- package/dist/src/guardrails/agent.d.ts.map +1 -0
- package/dist/src/guardrails/agent.js +433 -0
- package/dist/src/guardrails/agent.js.map +1 -0
- package/dist/src/guardrails/detectors/custom.d.ts +9 -0
- package/dist/src/guardrails/detectors/custom.d.ts.map +1 -0
- package/dist/src/guardrails/detectors/custom.js +81 -0
- package/dist/src/guardrails/detectors/custom.js.map +1 -0
- package/dist/src/guardrails/detectors/injection.d.ts +23 -0
- package/dist/src/guardrails/detectors/injection.d.ts.map +1 -0
- package/dist/src/guardrails/detectors/injection.js +208 -0
- package/dist/src/guardrails/detectors/injection.js.map +1 -0
- package/dist/src/guardrails/detectors/pii.d.ts +23 -0
- package/dist/src/guardrails/detectors/pii.d.ts.map +1 -0
- package/dist/src/guardrails/detectors/pii.js +295 -0
- package/dist/src/guardrails/detectors/pii.js.map +1 -0
- package/dist/src/guardrails/detectors/secrets.d.ts +5 -0
- package/dist/src/guardrails/detectors/secrets.d.ts.map +1 -0
- package/dist/src/guardrails/detectors/secrets.js +110 -0
- package/dist/src/guardrails/detectors/secrets.js.map +1 -0
- package/dist/src/guardrails/detectors/tool-poisoning.d.ts +16 -0
- package/dist/src/guardrails/detectors/tool-poisoning.d.ts.map +1 -0
- package/dist/src/guardrails/detectors/tool-poisoning.js +116 -0
- package/dist/src/guardrails/detectors/tool-poisoning.js.map +1 -0
- package/dist/src/guardrails/engine.d.ts +90 -0
- package/dist/src/guardrails/engine.d.ts.map +1 -0
- package/dist/src/guardrails/engine.js +324 -0
- package/dist/src/guardrails/engine.js.map +1 -0
- package/dist/src/guardrails/glob.d.ts +4 -0
- package/dist/src/guardrails/glob.d.ts.map +1 -0
- package/dist/src/guardrails/glob.js +20 -0
- package/dist/src/guardrails/glob.js.map +1 -0
- package/dist/src/guardrails/index.d.ts +20 -0
- package/dist/src/guardrails/index.d.ts.map +1 -0
- package/dist/src/guardrails/index.js +20 -0
- package/dist/src/guardrails/index.js.map +1 -0
- package/dist/src/guardrails/segments.d.ts +14 -0
- package/dist/src/guardrails/segments.d.ts.map +1 -0
- package/dist/src/guardrails/segments.js +32 -0
- package/dist/src/guardrails/segments.js.map +1 -0
- package/dist/src/guardrails/types.d.ts +161 -0
- package/dist/src/guardrails/types.d.ts.map +1 -0
- package/dist/src/guardrails/types.js +49 -0
- package/dist/src/guardrails/types.js.map +1 -0
- package/dist/src/guardrails/vault.d.ts +44 -0
- package/dist/src/guardrails/vault.d.ts.map +1 -0
- package/dist/src/guardrails/vault.js +103 -0
- package/dist/src/guardrails/vault.js.map +1 -0
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.js +1 -1
- package/dist/src/mcp-client/index.d.ts +1 -1
- package/dist/src/mcp-client/index.d.ts.map +1 -1
- package/dist/src/mcp-client/index.js.map +1 -1
- package/dist/src/mcp-client/options.d.ts +32 -0
- package/dist/src/mcp-client/options.d.ts.map +1 -1
- package/dist/src/mcp-client/tool-source.d.ts +2 -0
- package/dist/src/mcp-client/tool-source.d.ts.map +1 -1
- package/dist/src/mcp-client/tool-source.js +27 -1
- package/dist/src/mcp-client/tool-source.js.map +1 -1
- package/dist/src/rag/http-reranker.d.ts +46 -0
- package/dist/src/rag/http-reranker.d.ts.map +1 -0
- package/dist/src/rag/http-reranker.js +83 -0
- package/dist/src/rag/http-reranker.js.map +1 -0
- package/dist/src/rag/hybrid-retriever.d.ts +6 -1
- package/dist/src/rag/hybrid-retriever.d.ts.map +1 -1
- package/dist/src/rag/hybrid-retriever.js +27 -3
- package/dist/src/rag/hybrid-retriever.js.map +1 -1
- package/dist/src/rag/index.d.ts +6 -2
- package/dist/src/rag/index.d.ts.map +1 -1
- package/dist/src/rag/index.js +6 -2
- package/dist/src/rag/index.js.map +1 -1
- package/dist/src/rag/lexical-query.d.ts +28 -0
- package/dist/src/rag/lexical-query.d.ts.map +1 -0
- package/dist/src/rag/lexical-query.js +90 -0
- package/dist/src/rag/lexical-query.js.map +1 -0
- package/dist/src/rag/lexical-retriever.d.ts +23 -0
- package/dist/src/rag/lexical-retriever.d.ts.map +1 -0
- package/dist/src/rag/lexical-retriever.js +28 -0
- package/dist/src/rag/lexical-retriever.js.map +1 -0
- package/dist/src/rag/openai-embeddings.d.ts +62 -0
- package/dist/src/rag/openai-embeddings.d.ts.map +1 -0
- package/dist/src/rag/openai-embeddings.js +106 -0
- package/dist/src/rag/openai-embeddings.js.map +1 -0
- package/dist/src/rag/pg-vector-store.d.ts +133 -6
- package/dist/src/rag/pg-vector-store.d.ts.map +1 -1
- package/dist/src/rag/pg-vector-store.js +272 -31
- package/dist/src/rag/pg-vector-store.js.map +1 -1
- package/dist/src/rag/vector-store.d.ts +15 -0
- package/dist/src/rag/vector-store.d.ts.map +1 -1
- package/dist/src/rag/vector-store.js +4 -0
- package/dist/src/rag/vector-store.js.map +1 -1
- package/dist/src/stores/factory.d.ts +25 -3
- package/dist/src/stores/factory.d.ts.map +1 -1
- package/dist/src/stores/factory.js +21 -3
- package/dist/src/stores/factory.js.map +1 -1
- package/dist/src/testing/fake-embedding-provider.d.ts +12 -0
- package/dist/src/testing/fake-embedding-provider.d.ts.map +1 -1
- package/dist/src/testing/fake-embedding-provider.js +35 -0
- package/dist/src/testing/fake-embedding-provider.js.map +1 -1
- package/dist/src/testing/index.d.ts +1 -1
- package/dist/src/testing/index.d.ts.map +1 -1
- package/dist/src/testing/index.js +1 -1
- package/dist/src/testing/index.js.map +1 -1
- package/package.json +9 -3
- package/dist/assets/spa/assets/index-DjIMQOxi.js +0 -9
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { errorMessageOf, HttpModelError } from './openai-embeddings.js';
|
|
2
|
+
/**
|
|
3
|
+
* A cross-encoder {@link Reranker} behind an HTTP `/rerank` endpoint — the Cohere-style contract
|
|
4
|
+
* most providers and self-hosted servers share. Compose it with `RerankingRetriever`:
|
|
5
|
+
*
|
|
6
|
+
* ```ts
|
|
7
|
+
* const reranker = new HttpReranker({ url: 'https://api.cohere.com/v2/rerank', model: 'rerank-v3.5', apiKey });
|
|
8
|
+
* const retriever = new RerankingRetriever(hybrid, reranker, { fetchTopK: 30 });
|
|
9
|
+
* ```
|
|
10
|
+
*
|
|
11
|
+
* Reads `results: [{ index, relevance_score }]` (Cohere, Jina), `data: [{ index, relevance_score }]`
|
|
12
|
+
* (Voyage) and a bare `[{ index, score }]` (TEI). Each passage keeps its fields and takes the new
|
|
13
|
+
* score; the list is re-sorted and cut to `topK`. Indices the server invents are ignored.
|
|
14
|
+
*/
|
|
15
|
+
export class HttpReranker {
|
|
16
|
+
options;
|
|
17
|
+
format;
|
|
18
|
+
doFetch;
|
|
19
|
+
constructor(options) {
|
|
20
|
+
this.options = options;
|
|
21
|
+
this.format = options.format ?? 'auto';
|
|
22
|
+
this.doFetch = options.fetch ?? fetch;
|
|
23
|
+
}
|
|
24
|
+
async rerank(query, passages, options = {}) {
|
|
25
|
+
if (passages.length === 0) {
|
|
26
|
+
return [];
|
|
27
|
+
}
|
|
28
|
+
const topK = options.topK ?? passages.length;
|
|
29
|
+
const documents = passages.map((passage) => passage.text);
|
|
30
|
+
const response = await this.doFetch(this.options.url, {
|
|
31
|
+
method: 'POST',
|
|
32
|
+
headers: {
|
|
33
|
+
'content-type': 'application/json',
|
|
34
|
+
...(this.options.apiKey ? { authorization: `Bearer ${this.options.apiKey}` } : {}),
|
|
35
|
+
...this.options.headers,
|
|
36
|
+
},
|
|
37
|
+
body: JSON.stringify({
|
|
38
|
+
...(this.options.model !== undefined ? { model: this.options.model } : {}),
|
|
39
|
+
query,
|
|
40
|
+
...(this.format !== 'tei' ? { documents, top_n: topK } : {}),
|
|
41
|
+
...(this.format !== 'cohere' ? { texts: documents } : {}),
|
|
42
|
+
}),
|
|
43
|
+
signal: AbortSignal.timeout(this.options.timeoutMs ?? 30_000),
|
|
44
|
+
});
|
|
45
|
+
const body = await response.text();
|
|
46
|
+
if (!response.ok) {
|
|
47
|
+
throw new HttpModelError(response.status, `Rerank request failed (${response.status}): ${errorMessageOf(body)}`);
|
|
48
|
+
}
|
|
49
|
+
const items = itemsOf(body);
|
|
50
|
+
if (items === undefined) {
|
|
51
|
+
throw new HttpModelError(502, 'Rerank response has no results');
|
|
52
|
+
}
|
|
53
|
+
const seen = new Set();
|
|
54
|
+
const reranked = [];
|
|
55
|
+
for (const item of items) {
|
|
56
|
+
const index = item.index;
|
|
57
|
+
const passage = typeof index === 'number' ? passages[index] : undefined;
|
|
58
|
+
if (passage === undefined || seen.has(index)) {
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
seen.add(index);
|
|
62
|
+
reranked.push({ ...passage, score: Number(item.relevance_score ?? item.score ?? 0) });
|
|
63
|
+
}
|
|
64
|
+
reranked.sort((a, b) => b.score - a.score);
|
|
65
|
+
return reranked.slice(0, topK);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
function itemsOf(body) {
|
|
69
|
+
let json;
|
|
70
|
+
try {
|
|
71
|
+
json = JSON.parse(body);
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
return undefined;
|
|
75
|
+
}
|
|
76
|
+
if (Array.isArray(json)) {
|
|
77
|
+
return json;
|
|
78
|
+
}
|
|
79
|
+
const wrapped = json;
|
|
80
|
+
const items = wrapped?.results ?? wrapped?.data;
|
|
81
|
+
return Array.isArray(items) ? items : undefined;
|
|
82
|
+
}
|
|
83
|
+
//# sourceMappingURL=http-reranker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"http-reranker.js","sourceRoot":"","sources":["../../../src/rag/http-reranker.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAgCxE;;;;;;;;;;;;GAYG;AACH,MAAM,OAAO,YAAY;IAIM,OAAO;IAHnB,MAAM,CAA6C;IACnD,OAAO,CAAe;IAEvC,YAA6B,OAA4B;uBAA5B,OAAO;QAClC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;QACvC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,MAAM,CACV,KAAa,EACb,QAAmB,EACnB,OAAO,GAAkB,EAAE;QAE3B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC,MAAM,CAAC;QAC7C,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC1D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE;YACpD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClF,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO;aACxB;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC1E,KAAK;gBACL,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5D,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC1D,CAAC;YACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC;SAC9D,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,cAAc,CACtB,QAAQ,CAAC,MAAM,EACf,0BAA0B,QAAQ,CAAC,MAAM,MAAM,cAAc,CAAC,IAAI,CAAC,EAAE,CACtE,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,cAAc,CAAC,GAAG,EAAE,gCAAgC,CAAC,CAAC;QAClE,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,QAAQ,GAAc,EAAE,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;YACzB,MAAM,OAAO,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACxE,IAAI,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,CAAC,KAAe,CAAC,EAAE,CAAC;gBACvD,SAAS;YACX,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,KAAe,CAAC,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QACxF,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAC3C,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACjC,CAAC;CACF;AAED,SAAS,OAAO,CAAC,IAAY;IAC3B,IAAI,IAAa,CAAC;IAClB,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO,IAAoB,CAAC;IAC9B,CAAC;IACD,MAAM,OAAO,GAAG,IAAoD,CAAC;IACrE,MAAM,KAAK,GAAG,OAAO,EAAE,OAAO,IAAI,OAAO,EAAE,IAAI,CAAC;IAChD,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAE,KAAsB,CAAC,CAAC,CAAC,SAAS,CAAC;AACpE,CAAC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Passage, RetrieveOptions, Retriever } from '../spi/retriever.js';
|
|
1
|
+
import type { Passage, RetrievalResult, RetrieveOptions, Retriever } from '../spi/retriever.js';
|
|
2
2
|
export interface HybridRetrieverOptions {
|
|
3
3
|
/** RRF constant — larger flattens the rank weighting. Default 60 (the value from the original paper). */
|
|
4
4
|
k?: number;
|
|
@@ -19,5 +19,10 @@ export declare class HybridRetriever implements Retriever {
|
|
|
19
19
|
private readonly options;
|
|
20
20
|
constructor(retrievers: Retriever[], options?: HybridRetrieverOptions);
|
|
21
21
|
retrieve(query: string, options?: RetrieveOptions): Promise<Passage[]>;
|
|
22
|
+
/**
|
|
23
|
+
* {@link HybridRetriever.retrieve} plus the embedding usage of every leg that reports one (summed), so
|
|
24
|
+
* a hybrid over an embedding leg still reaches the ledger and the quota.
|
|
25
|
+
*/
|
|
26
|
+
retrieveWithUsage(query: string, options?: RetrieveOptions): Promise<RetrievalResult>;
|
|
22
27
|
}
|
|
23
28
|
//# sourceMappingURL=hybrid-retriever.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hybrid-retriever.d.ts","sourceRoot":"","sources":["../../../src/rag/hybrid-retriever.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"hybrid-retriever.d.ts","sourceRoot":"","sources":["../../../src/rag/hybrid-retriever.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAEhG,MAAM,WAAW,sBAAsB;IACrC,yGAAyG;IACzG,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,uEAAuE;IACvE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sGAAsG;IACtG,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;;;;GAMG;AACH,qBAAa,eAAgB,YAAW,SAAS;IAE7C,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAF1B,YACmB,UAAU,EAAE,SAAS,EAAE,EACvB,OAAO,GAAE,sBAA2B,EACnD;IAEE,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAE/E;IAED;;;OAGG;IACG,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,eAAe,CAAC,CAsC9F;CACF"}
|
|
@@ -13,13 +13,25 @@ export class HybridRetriever {
|
|
|
13
13
|
this.options = options;
|
|
14
14
|
}
|
|
15
15
|
async retrieve(query, options = {}) {
|
|
16
|
+
return (await this.retrieveWithUsage(query, options)).passages;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* {@link HybridRetriever.retrieve} plus the embedding usage of every leg that reports one (summed), so
|
|
20
|
+
* a hybrid over an embedding leg still reaches the ledger and the quota.
|
|
21
|
+
*/
|
|
22
|
+
async retrieveWithUsage(query, options = {}) {
|
|
16
23
|
const k = this.options.k ?? 60;
|
|
17
24
|
const fetchTopK = this.options.fetchTopK ?? 20;
|
|
18
25
|
const weights = this.options.weights;
|
|
19
|
-
const
|
|
26
|
+
const legOptions = {
|
|
20
27
|
topK: fetchTopK,
|
|
21
28
|
...(options.filter !== undefined ? { filter: options.filter } : {}),
|
|
22
|
-
}
|
|
29
|
+
};
|
|
30
|
+
const results = await Promise.all(this.retrievers.map(async (retriever) => typeof retriever.retrieveWithUsage === 'function'
|
|
31
|
+
? retriever.retrieveWithUsage(query, legOptions)
|
|
32
|
+
: { passages: await retriever.retrieve(query, legOptions) }));
|
|
33
|
+
const lists = results.map((result) => result.passages);
|
|
34
|
+
const usage = sumUsage(results.map((result) => result.usage));
|
|
23
35
|
const fused = new Map();
|
|
24
36
|
lists.forEach((list, listIndex) => {
|
|
25
37
|
const weight = weights?.[listIndex] ?? 1;
|
|
@@ -34,10 +46,22 @@ export class HybridRetriever {
|
|
|
34
46
|
}
|
|
35
47
|
});
|
|
36
48
|
});
|
|
37
|
-
|
|
49
|
+
const passages = [...fused.values()]
|
|
38
50
|
.sort((a, b) => b.score - a.score)
|
|
39
51
|
.slice(0, options.topK ?? 5)
|
|
40
52
|
.map((entry) => ({ ...entry.passage, score: entry.score }));
|
|
53
|
+
return { passages, ...(usage !== undefined ? { usage } : {}) };
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
function sumUsage(usages) {
|
|
57
|
+
const reported = usages.filter((usage) => usage !== undefined);
|
|
58
|
+
if (reported.length === 0) {
|
|
59
|
+
return undefined;
|
|
41
60
|
}
|
|
61
|
+
const modelId = reported.find((usage) => usage.modelId !== undefined)?.modelId;
|
|
62
|
+
return {
|
|
63
|
+
inputTokens: reported.reduce((total, usage) => total + usage.inputTokens, 0),
|
|
64
|
+
...(modelId !== undefined ? { modelId } : {}),
|
|
65
|
+
};
|
|
42
66
|
}
|
|
43
67
|
//# sourceMappingURL=hybrid-retriever.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hybrid-retriever.js","sourceRoot":"","sources":["../../../src/rag/hybrid-retriever.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"hybrid-retriever.js","sourceRoot":"","sources":["../../../src/rag/hybrid-retriever.ts"],"names":[],"mappings":"AAYA;;;;;;GAMG;AACH,MAAM,OAAO,eAAe;IAEP,UAAU;IACV,OAAO;IAF1B,YACmB,UAAuB,EACvB,OAAO,GAA2B,EAAE;0BADpC,UAAU;uBACV,OAAO;IACvB,CAAC;IAEJ,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAO,GAAoB,EAAE;QACzD,OAAO,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;IACjE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,iBAAiB,CAAC,KAAa,EAAE,OAAO,GAAoB,EAAE;QAClE,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;QACrC,MAAM,UAAU,GAAoB;YAClC,IAAI,EAAE,SAAS;YACf,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpE,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,IAAI,CAAC,UAAU,CAAC,GAAG,CACjB,KAAK,EAAE,SAAS,EAA4B,EAAE,CAC5C,OAAO,SAAS,CAAC,iBAAiB,KAAK,UAAU;YAC/C,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,KAAK,EAAE,UAAU,CAAC;YAChD,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,CAChE,CACF,CAAC;QACF,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACvD,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAE9D,MAAM,KAAK,GAAG,IAAI,GAAG,EAA+C,CAAC;QACrE,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE;YAChC,MAAM,MAAM,GAAG,OAAO,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACzC,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE;gBAC7B,MAAM,YAAY,GAAG,MAAM,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;gBAC7C,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBACvC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;oBAC3B,QAAQ,CAAC,KAAK,IAAI,YAAY,CAAC;gBACjC,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;gBAC1D,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;aACjC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;aACjC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;aAC3B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC9D,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IACjE,CAAC;CACF;AAED,SAAS,QAAQ,CAAC,MAAsC;IACtD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAA2B,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IACxF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,EAAE,OAAO,CAAC;IAC/E,OAAO;QACL,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QAC5E,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9C,CAAC;AACJ,CAAC"}
|
package/dist/src/rag/index.d.ts
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
export { type ChunkOptions, chunkText } from './chunk.js';
|
|
2
2
|
export { EmbeddingRetriever } from './embedding-retriever.js';
|
|
3
3
|
export { matchesFilter } from './filter.js';
|
|
4
|
+
export { HttpReranker, type HttpRerankerOptions } from './http-reranker.js';
|
|
4
5
|
export { HybridRetriever, type HybridRetrieverOptions } from './hybrid-retriever.js';
|
|
5
6
|
export { type ChunkRecord, chunkDocuments, type IngestChunksOptions, type IngestDocument, type IngestOptions, ingestChunks, ingestDocuments, } from './ingest.js';
|
|
6
7
|
export { KeywordRetriever, type KeywordRetrieverOptions } from './keyword-retriever.js';
|
|
8
|
+
export { anyTermTsquery, DEFAULT_STOP_WORDS, hasSearchSyntax, keywordTerms, } from './lexical-query.js';
|
|
9
|
+
export { LexicalRetriever } from './lexical-retriever.js';
|
|
7
10
|
export { cosineSimilarity, MemoryVectorStore } from './memory-vector-store.js';
|
|
8
|
-
export {
|
|
11
|
+
export { HttpModelError, type OpenAiEmbeddingsOptions, openAiEmbeddings, } from './openai-embeddings.js';
|
|
12
|
+
export { type PgFullTextOptions, PgLexicalVectorStore, type PgVectorColumns, type PgVectorMetric, PgVectorRetriever, PgVectorStore, type PgVectorStoreOptions, stripNulBytes, toVectorLiteral, } from './pg-vector-store.js';
|
|
9
13
|
export { RerankingRetriever, type RerankingRetrieverOptions, } from './reranking-retriever.js';
|
|
10
|
-
export { applyMetadataPatch, assertRemovalFilter, documentIdOf, effectivePatchKeys, filterDeniesAll, type IndexedDocument, type MetadataPatch, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, } from './vector-store.js';
|
|
14
|
+
export { applyMetadataPatch, assertRemovalFilter, documentIdOf, effectivePatchKeys, filterDeniesAll, type IndexedDocument, isLexicalVectorStore, type LexicalVectorStore, type MetadataPatch, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, } from './vector-store.js';
|
|
11
15
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/rag/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,YAAY,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EACL,KAAK,WAAW,EAChB,cAAc,EACd,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,YAAY,EACZ,eAAe,GAChB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,gBAAgB,EAAE,KAAK,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACxF,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,iBAAiB,EACjB,aAAa,EACb,KAAK,oBAAoB,EACzB,aAAa,EACb,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,kBAAkB,EAClB,KAAK,yBAAyB,GAC/B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,kBAAkB,EAClB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,EAClB,eAAe,EACf,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,kBAAkB,EAClB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,WAAW,GACjB,MAAM,mBAAmB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/rag/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,YAAY,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,KAAK,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EACL,KAAK,WAAW,EAChB,cAAc,EACd,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,YAAY,EACZ,eAAe,GAChB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,gBAAgB,EAAE,KAAK,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACxF,OAAO,EACL,cAAc,EACd,kBAAkB,EAClB,eAAe,EACf,YAAY,GACb,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EACL,cAAc,EACd,KAAK,uBAAuB,EAC5B,gBAAgB,GACjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,KAAK,iBAAiB,EACtB,oBAAoB,EACpB,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,iBAAiB,EACjB,aAAa,EACb,KAAK,oBAAoB,EACzB,aAAa,EACb,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,kBAAkB,EAClB,KAAK,yBAAyB,GAC/B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,kBAAkB,EAClB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,EAClB,eAAe,EACf,KAAK,eAAe,EACpB,oBAAoB,EACpB,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,kBAAkB,EAClB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,WAAW,GACjB,MAAM,mBAAmB,CAAC"}
|
package/dist/src/rag/index.js
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
export { chunkText } from './chunk.js';
|
|
2
2
|
export { EmbeddingRetriever } from './embedding-retriever.js';
|
|
3
3
|
export { matchesFilter } from './filter.js';
|
|
4
|
+
export { HttpReranker } from './http-reranker.js';
|
|
4
5
|
export { HybridRetriever } from './hybrid-retriever.js';
|
|
5
6
|
export { chunkDocuments, ingestChunks, ingestDocuments, } from './ingest.js';
|
|
6
7
|
export { KeywordRetriever } from './keyword-retriever.js';
|
|
8
|
+
export { anyTermTsquery, DEFAULT_STOP_WORDS, hasSearchSyntax, keywordTerms, } from './lexical-query.js';
|
|
9
|
+
export { LexicalRetriever } from './lexical-retriever.js';
|
|
7
10
|
export { cosineSimilarity, MemoryVectorStore } from './memory-vector-store.js';
|
|
8
|
-
export {
|
|
11
|
+
export { HttpModelError, openAiEmbeddings, } from './openai-embeddings.js';
|
|
12
|
+
export { PgLexicalVectorStore, PgVectorRetriever, PgVectorStore, stripNulBytes, toVectorLiteral, } from './pg-vector-store.js';
|
|
9
13
|
export { RerankingRetriever, } from './reranking-retriever.js';
|
|
10
|
-
export { applyMetadataPatch, assertRemovalFilter, documentIdOf, effectivePatchKeys, filterDeniesAll, UnsafeRemovalError, } from './vector-store.js';
|
|
14
|
+
export { applyMetadataPatch, assertRemovalFilter, documentIdOf, effectivePatchKeys, filterDeniesAll, isLexicalVectorStore, UnsafeRemovalError, } from './vector-store.js';
|
|
11
15
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rag/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,SAAS,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,eAAe,EAA+B,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAEL,cAAc,EAId,YAAY,EACZ,eAAe,GAChB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,gBAAgB,EAAgC,MAAM,wBAAwB,CAAC;AACxF,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rag/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,SAAS,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,YAAY,EAA4B,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAAE,eAAe,EAA+B,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAEL,cAAc,EAId,YAAY,EACZ,eAAe,GAChB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,gBAAgB,EAAgC,MAAM,wBAAwB,CAAC;AACxF,OAAO,EACL,cAAc,EACd,kBAAkB,EAClB,eAAe,EACf,YAAY,GACb,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EACL,cAAc,EAEd,gBAAgB,GACjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,oBAAoB,EAGpB,iBAAiB,EACjB,aAAa,EAEb,aAAa,EACb,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,kBAAkB,GAEnB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,kBAAkB,EAClB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,EAClB,eAAe,EAEf,oBAAoB,EAGpB,kBAAkB,GAInB,MAAM,mBAAmB,CAAC"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Turning a natural-language question into full-text search terms, for the lexical leg of
|
|
3
|
+
* `PgLexicalVectorStore`.
|
|
4
|
+
*
|
|
5
|
+
* With the `simple` text-search configuration (no stemming, no stop words; any language) a question
|
|
6
|
+
* searched as is either matches nothing (every word required) or, as "any word", ranks chunks full
|
|
7
|
+
* of "the / of / which / de / que" first. So the query side drops the stop words of the question's
|
|
8
|
+
* language(s) and keeps the meaningful terms, which the store then ranks by how many of them a chunk
|
|
9
|
+
* holds, rare terms weighing more (an IDF over the matching rows, BM25-style).
|
|
10
|
+
*/
|
|
11
|
+
/**
|
|
12
|
+
* Postgres' Snowball stop word lists (`tsearch_data/{english,portuguese,spanish}.stop`), minus
|
|
13
|
+
* one-letter words (never terms here) and words that carry meaning in a sibling language
|
|
14
|
+
* ("estado", "sentido").
|
|
15
|
+
*/
|
|
16
|
+
export declare const DEFAULT_STOP_WORDS: Readonly<Record<string, ReadonlySet<string>>>;
|
|
17
|
+
/**
|
|
18
|
+
* The meaningful terms of a question, in order, deduplicated, at most `max`: the stop words of the
|
|
19
|
+
* language(s) the question is written in are dropped (the language(s) with the most stop words in
|
|
20
|
+
* it, so "sea"/"son" stay terms in an English question although they are Spanish stop words). A
|
|
21
|
+
* question made only of stop words keeps them all (there is nothing else to search for).
|
|
22
|
+
*/
|
|
23
|
+
export declare function keywordTerms(query: string, max?: number, stopWords?: Readonly<Record<string, ReadonlySet<string>>>): string[];
|
|
24
|
+
/** Whether the user wrote search syntax (a quoted phrase, `-exclusion`) worth honoring as such. */
|
|
25
|
+
export declare function hasSearchSyntax(query: string): boolean;
|
|
26
|
+
/** `'a' | 'b' | …` for `to_tsquery` (terms are letter/digit runs, so quoting is enough). */
|
|
27
|
+
export declare function anyTermTsquery(terms: string[]): string;
|
|
28
|
+
//# sourceMappingURL=lexical-query.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lexical-query.d.ts","sourceRoot":"","sources":["../../../src/rag/lexical-query.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CA+C5E,CAAC;AAIF;;;;;GAKG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,MAAM,EACb,GAAG,SAAK,EACR,SAAS,GAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAsB,GAC5E,MAAM,EAAE,CAUV;AAED,mGAAmG;AACnG,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAEtD;AAED,4FAA4F;AAC5F,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtD"}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Turning a natural-language question into full-text search terms, for the lexical leg of
|
|
3
|
+
* `PgLexicalVectorStore`.
|
|
4
|
+
*
|
|
5
|
+
* With the `simple` text-search configuration (no stemming, no stop words; any language) a question
|
|
6
|
+
* searched as is either matches nothing (every word required) or, as "any word", ranks chunks full
|
|
7
|
+
* of "the / of / which / de / que" first. So the query side drops the stop words of the question's
|
|
8
|
+
* language(s) and keeps the meaningful terms, which the store then ranks by how many of them a chunk
|
|
9
|
+
* holds, rare terms weighing more (an IDF over the matching rows, BM25-style).
|
|
10
|
+
*/
|
|
11
|
+
const words = (s) => new Set(s.split(/\s+/).filter(Boolean));
|
|
12
|
+
/**
|
|
13
|
+
* Postgres' Snowball stop word lists (`tsearch_data/{english,portuguese,spanish}.stop`), minus
|
|
14
|
+
* one-letter words (never terms here) and words that carry meaning in a sibling language
|
|
15
|
+
* ("estado", "sentido").
|
|
16
|
+
*/
|
|
17
|
+
export const DEFAULT_STOP_WORDS = {
|
|
18
|
+
english: words(`
|
|
19
|
+
me my myself we our ours ourselves you your yours yourself yourselves he him his himself she
|
|
20
|
+
her hers herself it its itself they them their theirs themselves what which who whom this that
|
|
21
|
+
these those am is are was were be been being have has had having do does did doing an the and
|
|
22
|
+
but if or because as until while of at by for with about against between into through during
|
|
23
|
+
before after above below to from up down in out on off over under again further then once here
|
|
24
|
+
there when where why how all any both each few more most other some such no nor not only own
|
|
25
|
+
same so than too very can will just don should now`),
|
|
26
|
+
portuguese: words(`
|
|
27
|
+
de que do da em um para com não uma os no se na por mais as dos como mas ao ele das seu sua ou
|
|
28
|
+
quando muito nos já eu também só pelo pela até isso ela entre depois sem mesmo aos seus quem
|
|
29
|
+
nas me esse eles você essa num nem suas meu às minha numa pelos elas qual nós lhe deles essas
|
|
30
|
+
esses pelas este dele tu te vocês vos lhes meus minhas teu tua teus tuas nosso nossa nossos
|
|
31
|
+
nossas dela delas esta estes estas aquele aquela aqueles aquelas isto aquilo estou está estamos
|
|
32
|
+
estão estive esteve estivemos estiveram estava estávamos estavam estivera estivéramos esteja
|
|
33
|
+
estejamos estejam estivesse estivéssemos estivessem estiver estivermos estiverem hei há havemos
|
|
34
|
+
hão houve houvemos houveram houvera houvéramos haja hajamos hajam houvesse houvéssemos
|
|
35
|
+
houvessem houver houvermos houverem houverei houverá houveremos houverão houveria houveríamos
|
|
36
|
+
houveriam sou somos são era éramos eram fui foi fomos foram fora fôramos seja sejamos sejam
|
|
37
|
+
fosse fôssemos fossem for formos forem serei será seremos serão seria seríamos seriam tenho tem
|
|
38
|
+
temos tém tinha tínhamos tinham tive teve tivemos tiveram tivera tivéramos tenha tenhamos
|
|
39
|
+
tenham tivesse tivéssemos tivessem tiver tivermos tiverem terei terá teremos terão teria
|
|
40
|
+
teríamos teriam`),
|
|
41
|
+
spanish: words(`
|
|
42
|
+
de la que el en los del se las por un para con no una su al lo como más pero sus le ya este sí
|
|
43
|
+
porque esta entre cuando muy sin sobre también me hasta hay donde quien desde todo nos durante
|
|
44
|
+
todos uno les ni contra otros ese eso ante ellos esto mí antes algunos qué unos yo otro otras
|
|
45
|
+
otra él tanto esa estos mucho quienes nada muchos cual poco ella estar estas algunas algo
|
|
46
|
+
nosotros mi mis tú te ti tu tus ellas nosotras vosotros vosotras os mío mía míos mías tuyo
|
|
47
|
+
tuya tuyos tuyas suyo suya suyos suyas nuestro nuestra nuestros nuestras vuestro vuestra
|
|
48
|
+
vuestros vuestras esos esas estoy estás está estamos estáis están esté estés estemos estéis
|
|
49
|
+
estén estaré estarás estará estaremos estaréis estarán estaría estarías estaríamos estaríais
|
|
50
|
+
estarían estaba estabas estábamos estabais estaban estuve estuviste estuvo estuvimos
|
|
51
|
+
estuvisteis estuvieron estuviera estuvieras estuviéramos estuvierais estuvieran estuviese
|
|
52
|
+
estuvieses estuviésemos estuvieseis estuviesen estando estad he has ha hemos habéis han haya
|
|
53
|
+
hayas hayamos hayáis hayan habré habrás habrá habremos habréis habrán habría habrías habríamos
|
|
54
|
+
habríais habrían había habías habíamos habíais habían hube hubiste hubo hubimos hubisteis
|
|
55
|
+
hubieron hubiera hubieras hubiéramos hubierais hubieran hubiese hubieses hubiésemos hubieseis
|
|
56
|
+
hubiesen habiendo habido habida habidos habidas soy eres es somos sois son sea seas seamos
|
|
57
|
+
seáis sean seré serás será seremos seréis serán sería serías seríamos seríais serían era eras
|
|
58
|
+
éramos erais eran fui fuiste fue fuimos fuisteis fueron fuera fueras fuéramos fuerais fueran
|
|
59
|
+
fuese fueses fuésemos fueseis fuesen tengo tienes tiene tenemos tenéis tienen tenga tengas
|
|
60
|
+
tengamos tengáis tengan tendré tendrás tendrá tendremos tendréis tendrán tendría tendrías
|
|
61
|
+
tendríamos tendríais tendrían tenía tenías teníamos teníais tenían tuve tuviste tuvo tuvimos
|
|
62
|
+
tuvisteis tuvieron tuviera tuvieras tuviéramos tuvierais tuvieran tuviese tuvieses tuviésemos
|
|
63
|
+
tuvieseis tuviesen teniendo tenido tenida tenidos tenidas tened`),
|
|
64
|
+
};
|
|
65
|
+
const TOKEN = /[\p{L}\p{N}]{2,}/gu;
|
|
66
|
+
/**
|
|
67
|
+
* The meaningful terms of a question, in order, deduplicated, at most `max`: the stop words of the
|
|
68
|
+
* language(s) the question is written in are dropped (the language(s) with the most stop words in
|
|
69
|
+
* it, so "sea"/"son" stay terms in an English question although they are Spanish stop words). A
|
|
70
|
+
* question made only of stop words keeps them all (there is nothing else to search for).
|
|
71
|
+
*/
|
|
72
|
+
export function keywordTerms(query, max = 24, stopWords = DEFAULT_STOP_WORDS) {
|
|
73
|
+
const tokens = [...new Set(query.toLowerCase().normalize('NFC').match(TOKEN) ?? [])];
|
|
74
|
+
const counts = Object.entries(stopWords).map(([, stop]) => [stop, tokens.filter((t) => stop.has(t)).length]);
|
|
75
|
+
const best = Math.max(0, ...counts.map(([, n]) => n));
|
|
76
|
+
if (best === 0)
|
|
77
|
+
return tokens.slice(0, max);
|
|
78
|
+
const stop = counts.filter(([, n]) => n === best).map(([s]) => s);
|
|
79
|
+
const terms = tokens.filter((t) => !stop.some((s) => s.has(t)));
|
|
80
|
+
return (terms.length > 0 ? terms : tokens).slice(0, max);
|
|
81
|
+
}
|
|
82
|
+
/** Whether the user wrote search syntax (a quoted phrase, `-exclusion`) worth honoring as such. */
|
|
83
|
+
export function hasSearchSyntax(query) {
|
|
84
|
+
return /"[^"]+"/.test(query) || /(^|\s)-[\p{L}\p{N}]/u.test(query);
|
|
85
|
+
}
|
|
86
|
+
/** `'a' | 'b' | …` for `to_tsquery` (terms are letter/digit runs, so quoting is enough). */
|
|
87
|
+
export function anyTermTsquery(terms) {
|
|
88
|
+
return terms.map((t) => `'${t.replace(/'/g, '')}'`).join(' | ');
|
|
89
|
+
}
|
|
90
|
+
//# sourceMappingURL=lexical-query.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lexical-query.js","sourceRoot":"","sources":["../../../src/rag/lexical-query.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;AAErE;;;;GAIG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAkD;IAC/E,OAAO,EAAE,KAAK,CAAC;;;;;;;uDAOsC,CAAC;IACtD,UAAU,EAAE,KAAK,CAAC;;;;;;;;;;;;;;oBAcA,CAAC;IACnB,OAAO,EAAE,KAAK,CAAC;;;;;;;;;;;;;;;;;;;;;;oEAsBmD,CAAC;CACpE,CAAC;AAEF,MAAM,KAAK,GAAG,oBAAoB,CAAC;AAEnC;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAC1B,KAAa,EACb,GAAG,GAAG,EAAE,EACR,SAAS,GAAkD,kBAAkB;IAE7E,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACrF,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,GAAG,CAC1C,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAU,CACxE,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACtD,IAAI,IAAI,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAClE,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC3D,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,eAAe,CAAC,KAAa;IAC3C,OAAO,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,sBAAsB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,4FAA4F;AAC5F,MAAM,UAAU,cAAc,CAAC,KAAe;IAC5C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAClE,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { Passage, RetrieveOptions, Retriever } from '../spi/retriever.js';
|
|
2
|
+
import type { LexicalVectorStore } from './vector-store.js';
|
|
3
|
+
/**
|
|
4
|
+
* A {@link Retriever} over a store's own full-text index — the lexical half of a hybrid search, next to
|
|
5
|
+
* an {@link import('./embedding-retriever.js').EmbeddingRetriever} over the same store:
|
|
6
|
+
*
|
|
7
|
+
* ```ts
|
|
8
|
+
* const store = new PgLexicalVectorStore(db, { dimension: 1536 })
|
|
9
|
+
* const retriever = new HybridRetriever([
|
|
10
|
+
* new EmbeddingRetriever(embedder, store),
|
|
11
|
+
* new LexicalRetriever(store),
|
|
12
|
+
* ])
|
|
13
|
+
* ```
|
|
14
|
+
*
|
|
15
|
+
* Embeds nothing, so it reports no usage. `minScore` is not forwarded: a text rank is not a similarity,
|
|
16
|
+
* and a floor tuned for cosine would mean nothing on it.
|
|
17
|
+
*/
|
|
18
|
+
export declare class LexicalRetriever implements Retriever {
|
|
19
|
+
private readonly store;
|
|
20
|
+
constructor(store: LexicalVectorStore);
|
|
21
|
+
retrieve(query: string, options?: RetrieveOptions): Promise<Passage[]>;
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=lexical-retriever.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lexical-retriever.d.ts","sourceRoot":"","sources":["../../../src/rag/lexical-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAE5D;;;;;;;;;;;;;;GAcG;AACH,qBAAa,gBAAiB,YAAW,SAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,KAAK;IAAlC,YAA6B,KAAK,EAAE,kBAAkB,EAAI;IAEpD,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAK/E;CACF"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A {@link Retriever} over a store's own full-text index — the lexical half of a hybrid search, next to
|
|
3
|
+
* an {@link import('./embedding-retriever.js').EmbeddingRetriever} over the same store:
|
|
4
|
+
*
|
|
5
|
+
* ```ts
|
|
6
|
+
* const store = new PgLexicalVectorStore(db, { dimension: 1536 })
|
|
7
|
+
* const retriever = new HybridRetriever([
|
|
8
|
+
* new EmbeddingRetriever(embedder, store),
|
|
9
|
+
* new LexicalRetriever(store),
|
|
10
|
+
* ])
|
|
11
|
+
* ```
|
|
12
|
+
*
|
|
13
|
+
* Embeds nothing, so it reports no usage. `minScore` is not forwarded: a text rank is not a similarity,
|
|
14
|
+
* and a floor tuned for cosine would mean nothing on it.
|
|
15
|
+
*/
|
|
16
|
+
export class LexicalRetriever {
|
|
17
|
+
store;
|
|
18
|
+
constructor(store) {
|
|
19
|
+
this.store = store;
|
|
20
|
+
}
|
|
21
|
+
async retrieve(query, options = {}) {
|
|
22
|
+
return this.store.searchText(query, {
|
|
23
|
+
topK: options.topK ?? 5,
|
|
24
|
+
...(options.filter !== undefined ? { filter: options.filter } : {}),
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=lexical-retriever.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lexical-retriever.js","sourceRoot":"","sources":["../../../src/rag/lexical-retriever.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,gBAAgB;IACE,KAAK;IAAlC,YAA6B,KAAyB;qBAAzB,KAAK;IAAuB,CAAC;IAE1D,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAO,GAAoB,EAAE;QACzD,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE;YAClC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC;YACvB,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpE,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import type { EmbeddingProvider } from '../spi/embedding-provider.js';
|
|
2
|
+
/** A non-2xx answer (or a malformed one) from an embeddings or rerank endpoint. */
|
|
3
|
+
export declare class HttpModelError extends Error {
|
|
4
|
+
/** HTTP status; `502` when the endpoint answered 2xx with a body that doesn't fit the contract. */
|
|
5
|
+
readonly status: number;
|
|
6
|
+
constructor(
|
|
7
|
+
/** HTTP status; `502` when the endpoint answered 2xx with a body that doesn't fit the contract. */
|
|
8
|
+
status: number, message: string);
|
|
9
|
+
}
|
|
10
|
+
export interface OpenAiEmbeddingsOptions {
|
|
11
|
+
/** The embedding model id, sent as `model`. */
|
|
12
|
+
model: string;
|
|
13
|
+
/**
|
|
14
|
+
* Base URL up to and including `/v1` — the client posts to `<baseUrl>/embeddings`. Default
|
|
15
|
+
* `https://api.openai.com/v1`. Point it at any OpenAI-compatible server: Azure/OpenRouter/an LLM
|
|
16
|
+
* gateway, Ollama (`http://localhost:11434/v1`), Hugging Face TEI, vLLM, LocalAI, LM Studio…
|
|
17
|
+
*/
|
|
18
|
+
baseUrl?: string;
|
|
19
|
+
/** Sent as `Authorization: Bearer <apiKey>` when set. Local servers usually need none. */
|
|
20
|
+
apiKey?: string;
|
|
21
|
+
/** Output width for models that support shortening (`text-embedding-3-*`), sent as `dimensions`. */
|
|
22
|
+
dimensions?: number;
|
|
23
|
+
/** Inputs per request. Default 64. */
|
|
24
|
+
batchSize?: number;
|
|
25
|
+
/**
|
|
26
|
+
* Inputs longer than this many characters are truncated before sending, so one oversized chunk
|
|
27
|
+
* doesn't fail a whole batch on the model's context limit. Default 24 000 (≈ 6–8k tokens); set
|
|
28
|
+
* `Infinity` to send inputs as they are.
|
|
29
|
+
*/
|
|
30
|
+
maxInputChars?: number;
|
|
31
|
+
/** Extra request headers (an org/project id, a gateway's attribution header). */
|
|
32
|
+
headers?: Record<string, string>;
|
|
33
|
+
/** Per-request timeout. Default 120 000 ms. */
|
|
34
|
+
timeoutMs?: number;
|
|
35
|
+
/**
|
|
36
|
+
* Called with the provider-reported token count of every request. The same counts also reach the
|
|
37
|
+
* ledger and quota through `embedWithUsage`; this is for your own metering on top.
|
|
38
|
+
*/
|
|
39
|
+
onUsage?: (tokens: number) => void;
|
|
40
|
+
/** `fetch` to use. Default the global one. */
|
|
41
|
+
fetch?: typeof fetch;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* An {@link EmbeddingProvider} over any OpenAI-compatible `POST /v1/embeddings` endpoint, with no SDK
|
|
45
|
+
* dependency: OpenAI itself, or a gateway/local server speaking the same protocol (TEI, Ollama,
|
|
46
|
+
* vLLM, LocalAI…). For a Vercel AI SDK model use `aiSdkEmbedding` from `-ai-sdk` instead.
|
|
47
|
+
*
|
|
48
|
+
* Batches by {@link OpenAiEmbeddingsOptions.batchSize}, restores input order from each item's
|
|
49
|
+
* `index`, and fails loudly (an {@link HttpModelError}) on a non-2xx answer or a vector count that
|
|
50
|
+
* doesn't match the inputs. Empty strings are sent as a single space, since OpenAI rejects them.
|
|
51
|
+
* Implements `embedWithUsage` with the provider-reported tokens (summed over the batches) and model, so
|
|
52
|
+
* embedding spend reaches the ledger and the quota.
|
|
53
|
+
*
|
|
54
|
+
* ```ts
|
|
55
|
+
* const embedder = openAiEmbeddings({ model: 'text-embedding-3-small', apiKey: process.env.OPENAI_API_KEY });
|
|
56
|
+
* const local = openAiEmbeddings({ baseUrl: 'http://tei:8080/v1', model: 'BAAI/bge-m3' });
|
|
57
|
+
* ```
|
|
58
|
+
*/
|
|
59
|
+
export declare function openAiEmbeddings(options: OpenAiEmbeddingsOptions): EmbeddingProvider;
|
|
60
|
+
/** The human part of an error body: OpenAI's `error.message`, a bare `message`/`error`, or the text. */
|
|
61
|
+
export declare function errorMessageOf(body: string): string;
|
|
62
|
+
//# sourceMappingURL=openai-embeddings.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai-embeddings.d.ts","sourceRoot":"","sources":["../../../src/rag/openai-embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAmB,MAAM,8BAA8B,CAAC;AAEvF,mFAAmF;AACnF,qBAAa,cAAe,SAAQ,KAAK;IAErC,mGAAmG;IACnG,QAAQ,CAAC,MAAM,EAAE,MAAM;IAFzB;IACE,mGAAmG;IAC1F,MAAM,EAAE,MAAM,EACvB,OAAO,EAAE,MAAM,EAIhB;CACF;AAED,MAAM,WAAW,uBAAuB;IACtC,+CAA+C;IAC/C,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,0FAA0F;IAC1F,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,oGAAoG;IACpG,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,sCAAsC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iFAAiF;IACjF,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,8CAA8C;IAC9C,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACtB;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,uBAAuB,GAAG,iBAAiB,CA6EpF;AAUD,wGAAwG;AACxG,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAUnD"}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/** A non-2xx answer (or a malformed one) from an embeddings or rerank endpoint. */
|
|
2
|
+
export class HttpModelError extends Error {
|
|
3
|
+
status;
|
|
4
|
+
constructor(
|
|
5
|
+
/** HTTP status; `502` when the endpoint answered 2xx with a body that doesn't fit the contract. */
|
|
6
|
+
status, message) {
|
|
7
|
+
super(message);
|
|
8
|
+
this.status = status;
|
|
9
|
+
this.name = 'HttpModelError';
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* An {@link EmbeddingProvider} over any OpenAI-compatible `POST /v1/embeddings` endpoint, with no SDK
|
|
14
|
+
* dependency: OpenAI itself, or a gateway/local server speaking the same protocol (TEI, Ollama,
|
|
15
|
+
* vLLM, LocalAI…). For a Vercel AI SDK model use `aiSdkEmbedding` from `-ai-sdk` instead.
|
|
16
|
+
*
|
|
17
|
+
* Batches by {@link OpenAiEmbeddingsOptions.batchSize}, restores input order from each item's
|
|
18
|
+
* `index`, and fails loudly (an {@link HttpModelError}) on a non-2xx answer or a vector count that
|
|
19
|
+
* doesn't match the inputs. Empty strings are sent as a single space, since OpenAI rejects them.
|
|
20
|
+
* Implements `embedWithUsage` with the provider-reported tokens (summed over the batches) and model, so
|
|
21
|
+
* embedding spend reaches the ledger and the quota.
|
|
22
|
+
*
|
|
23
|
+
* ```ts
|
|
24
|
+
* const embedder = openAiEmbeddings({ model: 'text-embedding-3-small', apiKey: process.env.OPENAI_API_KEY });
|
|
25
|
+
* const local = openAiEmbeddings({ baseUrl: 'http://tei:8080/v1', model: 'BAAI/bge-m3' });
|
|
26
|
+
* ```
|
|
27
|
+
*/
|
|
28
|
+
export function openAiEmbeddings(options) {
|
|
29
|
+
const url = `${(options.baseUrl ?? 'https://api.openai.com/v1').replace(/\/+$/, '')}/embeddings`;
|
|
30
|
+
const batchSize = Math.max(1, Math.floor(options.batchSize ?? 64));
|
|
31
|
+
const maxInputChars = options.maxInputChars ?? 24_000;
|
|
32
|
+
const timeoutMs = options.timeoutMs ?? 120_000;
|
|
33
|
+
const doFetch = options.fetch ?? fetch;
|
|
34
|
+
const provider = {
|
|
35
|
+
async embed(texts) {
|
|
36
|
+
return (await provider.embedWithUsage(texts)).vectors;
|
|
37
|
+
},
|
|
38
|
+
async embedWithUsage(texts) {
|
|
39
|
+
const vectors = [];
|
|
40
|
+
let inputTokens;
|
|
41
|
+
let modelId;
|
|
42
|
+
for (let start = 0; start < texts.length; start += batchSize) {
|
|
43
|
+
const input = texts
|
|
44
|
+
.slice(start, start + batchSize)
|
|
45
|
+
.map((text) => (text.length > maxInputChars ? text.slice(0, maxInputChars) : text) || ' ');
|
|
46
|
+
const response = await doFetch(url, {
|
|
47
|
+
method: 'POST',
|
|
48
|
+
headers: {
|
|
49
|
+
'content-type': 'application/json',
|
|
50
|
+
...(options.apiKey ? { authorization: `Bearer ${options.apiKey}` } : {}),
|
|
51
|
+
...options.headers,
|
|
52
|
+
},
|
|
53
|
+
body: JSON.stringify({
|
|
54
|
+
model: options.model,
|
|
55
|
+
input,
|
|
56
|
+
...(options.dimensions !== undefined ? { dimensions: options.dimensions } : {}),
|
|
57
|
+
}),
|
|
58
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
59
|
+
});
|
|
60
|
+
const body = await response.text();
|
|
61
|
+
if (!response.ok) {
|
|
62
|
+
throw new HttpModelError(response.status, `Embeddings request failed (${response.status}): ${errorMessageOf(body)}`);
|
|
63
|
+
}
|
|
64
|
+
const json = parseJson(body);
|
|
65
|
+
const items = [...(json?.data ?? [])].sort((a, b) => (a.index ?? 0) - (b.index ?? 0));
|
|
66
|
+
if (items.length !== input.length ||
|
|
67
|
+
!items.every((item) => Array.isArray(item.embedding))) {
|
|
68
|
+
throw new HttpModelError(502, `Embeddings returned ${items.length} vectors for ${input.length} inputs`);
|
|
69
|
+
}
|
|
70
|
+
for (const item of items) {
|
|
71
|
+
vectors.push(item.embedding);
|
|
72
|
+
}
|
|
73
|
+
const tokens = json?.usage?.total_tokens ?? json?.usage?.prompt_tokens;
|
|
74
|
+
if (typeof tokens === 'number' && Number.isFinite(tokens)) {
|
|
75
|
+
inputTokens = (inputTokens ?? 0) + tokens;
|
|
76
|
+
options.onUsage?.(tokens);
|
|
77
|
+
}
|
|
78
|
+
modelId ??= typeof json?.model === 'string' ? json.model : undefined;
|
|
79
|
+
}
|
|
80
|
+
return {
|
|
81
|
+
vectors,
|
|
82
|
+
...(inputTokens !== undefined
|
|
83
|
+
? { usage: { inputTokens, modelId: modelId ?? options.model } }
|
|
84
|
+
: {}),
|
|
85
|
+
};
|
|
86
|
+
},
|
|
87
|
+
};
|
|
88
|
+
return provider;
|
|
89
|
+
}
|
|
90
|
+
function parseJson(body) {
|
|
91
|
+
try {
|
|
92
|
+
return JSON.parse(body);
|
|
93
|
+
}
|
|
94
|
+
catch {
|
|
95
|
+
return null;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
/** The human part of an error body: OpenAI's `error.message`, a bare `message`/`error`, or the text. */
|
|
99
|
+
export function errorMessageOf(body) {
|
|
100
|
+
const json = parseJson(body);
|
|
101
|
+
const message = typeof json?.error === 'string'
|
|
102
|
+
? json.error
|
|
103
|
+
: (json?.error?.message ?? json?.message ?? undefined);
|
|
104
|
+
return typeof message === 'string' ? message : body.slice(0, 300);
|
|
105
|
+
}
|
|
106
|
+
//# sourceMappingURL=openai-embeddings.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai-embeddings.js","sourceRoot":"","sources":["../../../src/rag/openai-embeddings.ts"],"names":[],"mappings":"AAEA,mFAAmF;AACnF,MAAM,OAAO,cAAe,SAAQ,KAAK;IAG5B,MAAM;IAFjB;IACE,mGAAmG;IAC1F,MAAc,EACvB,OAAe;QAEf,KAAK,CAAC,OAAO,CAAC,CAAC;sBAHN,MAAM;QAIf,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF;AAoCD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAgC;IAC/D,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,2BAA2B,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,aAAa,CAAC;IACjG,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC;IACnE,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,MAAM,CAAC;IACtD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC;IAC/C,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC;IAEvC,MAAM,QAAQ,GAAG;QACf,KAAK,CAAC,KAAK,CAAC,KAAe;YACzB,OAAO,CAAC,MAAM,QAAQ,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC;QACxD,CAAC;QAED,KAAK,CAAC,cAAc,CAAC,KAAe;YAClC,MAAM,OAAO,GAAe,EAAE,CAAC;YAC/B,IAAI,WAA+B,CAAC;YACpC,IAAI,OAA2B,CAAC;YAChC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,IAAI,SAAS,EAAE,CAAC;gBAC7D,MAAM,KAAK,GAAG,KAAK;qBAChB,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,SAAS,CAAC;qBAC/B,GAAG,CACF,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,CACrF,CAAC;gBACJ,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE;oBAClC,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;wBACxE,GAAG,OAAO,CAAC,OAAO;qBACnB;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK,EAAE,OAAO,CAAC,KAAK;wBACpB,KAAK;wBACL,GAAG,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;qBAChF,CAAC;oBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC;iBACvC,CAAC,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,MAAM,IAAI,cAAc,CACtB,QAAQ,CAAC,MAAM,EACf,8BAA8B,QAAQ,CAAC,MAAM,MAAM,cAAc,CAAC,IAAI,CAAC,EAAE,CAC1E,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAInB,CAAC;gBACT,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC;gBACtF,IACE,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;oBAC7B,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EACrD,CAAC;oBACD,MAAM,IAAI,cAAc,CACtB,GAAG,EACH,uBAAuB,KAAK,CAAC,MAAM,gBAAgB,KAAK,CAAC,MAAM,SAAS,CACzE,CAAC;gBACJ,CAAC;gBACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAqB,CAAC,CAAC;gBAC3C,CAAC;gBACD,MAAM,MAAM,GAAG,IAAI,EAAE,KAAK,EAAE,YAAY,IAAI,IAAI,EAAE,KAAK,EAAE,aAAa,CAAC;gBACvE,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC1D,WAAW,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC;oBAC1C,OAAO,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC;gBAC5B,CAAC;gBACD,OAAO,KAAK,OAAO,IAAI,EAAE,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YACvE,CAAC;YACD,OAAO;gBACL,OAAO;gBACP,GAAG,CAAC,WAAW,KAAK,SAAS;oBAC3B,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,EAAE;oBAC/D,CAAC,CAAC,EAAE,CAAC;aACR,CAAC;QACJ,CAAC;KACF,CAAC;IACF,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,wGAAwG;AACxG,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAGnB,CAAC;IACT,MAAM,OAAO,GACX,OAAO,IAAI,EAAE,KAAK,KAAK,QAAQ;QAC7B,CAAC,CAAC,IAAI,CAAC,KAAK;QACZ,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,IAAI,IAAI,EAAE,OAAO,IAAI,SAAS,CAAC,CAAC;IAC3D,OAAO,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACpE,CAAC"}
|