@adonis-agora/agent 0.34.1 → 0.36.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/CHANGELOG.md +24 -0
  2. package/README.md +34 -0
  3. package/dist/assets/spa/assets/index-7XqfoeQH.js +9 -0
  4. package/dist/assets/spa/index.html +1 -1
  5. package/dist/providers/agent_provider.d.ts.map +1 -1
  6. package/dist/providers/agent_provider.js +4 -0
  7. package/dist/providers/agent_provider.js.map +1 -1
  8. package/dist/src/agent-deps-factory.d.ts +5 -0
  9. package/dist/src/agent-deps-factory.d.ts.map +1 -1
  10. package/dist/src/agent-deps-factory.js +6 -0
  11. package/dist/src/agent-deps-factory.js.map +1 -1
  12. package/dist/src/agent-deps.d.ts +5 -0
  13. package/dist/src/agent-deps.d.ts.map +1 -1
  14. package/dist/src/agent-deps.js.map +1 -1
  15. package/dist/src/define_config.d.ts +12 -0
  16. package/dist/src/define_config.d.ts.map +1 -1
  17. package/dist/src/define_config.js.map +1 -1
  18. package/dist/src/guardrails/agent.d.ts +211 -0
  19. package/dist/src/guardrails/agent.d.ts.map +1 -0
  20. package/dist/src/guardrails/agent.js +433 -0
  21. package/dist/src/guardrails/agent.js.map +1 -0
  22. package/dist/src/guardrails/detectors/custom.d.ts +9 -0
  23. package/dist/src/guardrails/detectors/custom.d.ts.map +1 -0
  24. package/dist/src/guardrails/detectors/custom.js +81 -0
  25. package/dist/src/guardrails/detectors/custom.js.map +1 -0
  26. package/dist/src/guardrails/detectors/injection.d.ts +23 -0
  27. package/dist/src/guardrails/detectors/injection.d.ts.map +1 -0
  28. package/dist/src/guardrails/detectors/injection.js +208 -0
  29. package/dist/src/guardrails/detectors/injection.js.map +1 -0
  30. package/dist/src/guardrails/detectors/pii.d.ts +23 -0
  31. package/dist/src/guardrails/detectors/pii.d.ts.map +1 -0
  32. package/dist/src/guardrails/detectors/pii.js +295 -0
  33. package/dist/src/guardrails/detectors/pii.js.map +1 -0
  34. package/dist/src/guardrails/detectors/secrets.d.ts +5 -0
  35. package/dist/src/guardrails/detectors/secrets.d.ts.map +1 -0
  36. package/dist/src/guardrails/detectors/secrets.js +110 -0
  37. package/dist/src/guardrails/detectors/secrets.js.map +1 -0
  38. package/dist/src/guardrails/detectors/tool-poisoning.d.ts +16 -0
  39. package/dist/src/guardrails/detectors/tool-poisoning.d.ts.map +1 -0
  40. package/dist/src/guardrails/detectors/tool-poisoning.js +116 -0
  41. package/dist/src/guardrails/detectors/tool-poisoning.js.map +1 -0
  42. package/dist/src/guardrails/engine.d.ts +90 -0
  43. package/dist/src/guardrails/engine.d.ts.map +1 -0
  44. package/dist/src/guardrails/engine.js +324 -0
  45. package/dist/src/guardrails/engine.js.map +1 -0
  46. package/dist/src/guardrails/glob.d.ts +4 -0
  47. package/dist/src/guardrails/glob.d.ts.map +1 -0
  48. package/dist/src/guardrails/glob.js +20 -0
  49. package/dist/src/guardrails/glob.js.map +1 -0
  50. package/dist/src/guardrails/index.d.ts +20 -0
  51. package/dist/src/guardrails/index.d.ts.map +1 -0
  52. package/dist/src/guardrails/index.js +20 -0
  53. package/dist/src/guardrails/index.js.map +1 -0
  54. package/dist/src/guardrails/segments.d.ts +14 -0
  55. package/dist/src/guardrails/segments.d.ts.map +1 -0
  56. package/dist/src/guardrails/segments.js +32 -0
  57. package/dist/src/guardrails/segments.js.map +1 -0
  58. package/dist/src/guardrails/types.d.ts +161 -0
  59. package/dist/src/guardrails/types.d.ts.map +1 -0
  60. package/dist/src/guardrails/types.js +49 -0
  61. package/dist/src/guardrails/types.js.map +1 -0
  62. package/dist/src/guardrails/vault.d.ts +44 -0
  63. package/dist/src/guardrails/vault.d.ts.map +1 -0
  64. package/dist/src/guardrails/vault.js +103 -0
  65. package/dist/src/guardrails/vault.js.map +1 -0
  66. package/dist/src/index.d.ts +1 -1
  67. package/dist/src/index.js +1 -1
  68. package/dist/src/mcp-client/index.d.ts +1 -1
  69. package/dist/src/mcp-client/index.d.ts.map +1 -1
  70. package/dist/src/mcp-client/index.js.map +1 -1
  71. package/dist/src/mcp-client/options.d.ts +32 -0
  72. package/dist/src/mcp-client/options.d.ts.map +1 -1
  73. package/dist/src/mcp-client/tool-source.d.ts +2 -0
  74. package/dist/src/mcp-client/tool-source.d.ts.map +1 -1
  75. package/dist/src/mcp-client/tool-source.js +27 -1
  76. package/dist/src/mcp-client/tool-source.js.map +1 -1
  77. package/dist/src/rag/http-reranker.d.ts +46 -0
  78. package/dist/src/rag/http-reranker.d.ts.map +1 -0
  79. package/dist/src/rag/http-reranker.js +83 -0
  80. package/dist/src/rag/http-reranker.js.map +1 -0
  81. package/dist/src/rag/hybrid-retriever.d.ts +6 -1
  82. package/dist/src/rag/hybrid-retriever.d.ts.map +1 -1
  83. package/dist/src/rag/hybrid-retriever.js +27 -3
  84. package/dist/src/rag/hybrid-retriever.js.map +1 -1
  85. package/dist/src/rag/index.d.ts +6 -2
  86. package/dist/src/rag/index.d.ts.map +1 -1
  87. package/dist/src/rag/index.js +6 -2
  88. package/dist/src/rag/index.js.map +1 -1
  89. package/dist/src/rag/lexical-query.d.ts +28 -0
  90. package/dist/src/rag/lexical-query.d.ts.map +1 -0
  91. package/dist/src/rag/lexical-query.js +90 -0
  92. package/dist/src/rag/lexical-query.js.map +1 -0
  93. package/dist/src/rag/lexical-retriever.d.ts +23 -0
  94. package/dist/src/rag/lexical-retriever.d.ts.map +1 -0
  95. package/dist/src/rag/lexical-retriever.js +28 -0
  96. package/dist/src/rag/lexical-retriever.js.map +1 -0
  97. package/dist/src/rag/openai-embeddings.d.ts +62 -0
  98. package/dist/src/rag/openai-embeddings.d.ts.map +1 -0
  99. package/dist/src/rag/openai-embeddings.js +106 -0
  100. package/dist/src/rag/openai-embeddings.js.map +1 -0
  101. package/dist/src/rag/pg-vector-store.d.ts +133 -6
  102. package/dist/src/rag/pg-vector-store.d.ts.map +1 -1
  103. package/dist/src/rag/pg-vector-store.js +272 -31
  104. package/dist/src/rag/pg-vector-store.js.map +1 -1
  105. package/dist/src/rag/vector-store.d.ts +15 -0
  106. package/dist/src/rag/vector-store.d.ts.map +1 -1
  107. package/dist/src/rag/vector-store.js +4 -0
  108. package/dist/src/rag/vector-store.js.map +1 -1
  109. package/dist/src/stores/factory.d.ts +25 -3
  110. package/dist/src/stores/factory.d.ts.map +1 -1
  111. package/dist/src/stores/factory.js +21 -3
  112. package/dist/src/stores/factory.js.map +1 -1
  113. package/dist/src/testing/fake-embedding-provider.d.ts +12 -0
  114. package/dist/src/testing/fake-embedding-provider.d.ts.map +1 -1
  115. package/dist/src/testing/fake-embedding-provider.js +35 -0
  116. package/dist/src/testing/fake-embedding-provider.js.map +1 -1
  117. package/dist/src/testing/index.d.ts +1 -1
  118. package/dist/src/testing/index.d.ts.map +1 -1
  119. package/dist/src/testing/index.js +1 -1
  120. package/dist/src/testing/index.js.map +1 -1
  121. package/package.json +9 -3
  122. package/dist/assets/spa/assets/index-DjIMQOxi.js +0 -9
@@ -0,0 +1,83 @@
1
+ import { errorMessageOf, HttpModelError } from './openai-embeddings.js';
2
+ /**
3
+ * A cross-encoder {@link Reranker} behind an HTTP `/rerank` endpoint — the Cohere-style contract
4
+ * most providers and self-hosted servers share. Compose it with `RerankingRetriever`:
5
+ *
6
+ * ```ts
7
+ * const reranker = new HttpReranker({ url: 'https://api.cohere.com/v2/rerank', model: 'rerank-v3.5', apiKey });
8
+ * const retriever = new RerankingRetriever(hybrid, reranker, { fetchTopK: 30 });
9
+ * ```
10
+ *
11
+ * Reads `results: [{ index, relevance_score }]` (Cohere, Jina), `data: [{ index, relevance_score }]`
12
+ * (Voyage) and a bare `[{ index, score }]` (TEI). Each passage keeps its fields and takes the new
13
+ * score; the list is re-sorted and cut to `topK`. Indices the server invents are ignored.
14
+ */
15
+ export class HttpReranker {
16
+ options;
17
+ format;
18
+ doFetch;
19
+ constructor(options) {
20
+ this.options = options;
21
+ this.format = options.format ?? 'auto';
22
+ this.doFetch = options.fetch ?? fetch;
23
+ }
24
+ async rerank(query, passages, options = {}) {
25
+ if (passages.length === 0) {
26
+ return [];
27
+ }
28
+ const topK = options.topK ?? passages.length;
29
+ const documents = passages.map((passage) => passage.text);
30
+ const response = await this.doFetch(this.options.url, {
31
+ method: 'POST',
32
+ headers: {
33
+ 'content-type': 'application/json',
34
+ ...(this.options.apiKey ? { authorization: `Bearer ${this.options.apiKey}` } : {}),
35
+ ...this.options.headers,
36
+ },
37
+ body: JSON.stringify({
38
+ ...(this.options.model !== undefined ? { model: this.options.model } : {}),
39
+ query,
40
+ ...(this.format !== 'tei' ? { documents, top_n: topK } : {}),
41
+ ...(this.format !== 'cohere' ? { texts: documents } : {}),
42
+ }),
43
+ signal: AbortSignal.timeout(this.options.timeoutMs ?? 30_000),
44
+ });
45
+ const body = await response.text();
46
+ if (!response.ok) {
47
+ throw new HttpModelError(response.status, `Rerank request failed (${response.status}): ${errorMessageOf(body)}`);
48
+ }
49
+ const items = itemsOf(body);
50
+ if (items === undefined) {
51
+ throw new HttpModelError(502, 'Rerank response has no results');
52
+ }
53
+ const seen = new Set();
54
+ const reranked = [];
55
+ for (const item of items) {
56
+ const index = item.index;
57
+ const passage = typeof index === 'number' ? passages[index] : undefined;
58
+ if (passage === undefined || seen.has(index)) {
59
+ continue;
60
+ }
61
+ seen.add(index);
62
+ reranked.push({ ...passage, score: Number(item.relevance_score ?? item.score ?? 0) });
63
+ }
64
+ reranked.sort((a, b) => b.score - a.score);
65
+ return reranked.slice(0, topK);
66
+ }
67
+ }
68
+ function itemsOf(body) {
69
+ let json;
70
+ try {
71
+ json = JSON.parse(body);
72
+ }
73
+ catch {
74
+ return undefined;
75
+ }
76
+ if (Array.isArray(json)) {
77
+ return json;
78
+ }
79
+ const wrapped = json;
80
+ const items = wrapped?.results ?? wrapped?.data;
81
+ return Array.isArray(items) ? items : undefined;
82
+ }
83
+ //# sourceMappingURL=http-reranker.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"http-reranker.js","sourceRoot":"","sources":["../../../src/rag/http-reranker.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAgCxE;;;;;;;;;;;;GAYG;AACH,MAAM,OAAO,YAAY;IAIM,OAAO;IAHnB,MAAM,CAA6C;IACnD,OAAO,CAAe;IAEvC,YAA6B,OAA4B;uBAA5B,OAAO;QAClC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;QACvC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,MAAM,CACV,KAAa,EACb,QAAmB,EACnB,OAAO,GAAkB,EAAE;QAE3B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC,MAAM,CAAC;QAC7C,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC1D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE;YACpD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClF,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO;aACxB;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC1E,KAAK;gBACL,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5D,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC1D,CAAC;YACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC;SAC9D,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,cAAc,CACtB,QAAQ,CAAC,MAAM,EACf,0BAA0B,QAAQ,CAAC,MAAM,MAAM,cAAc,CAAC,IAAI,CAAC,EAAE,CACtE,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,cAAc,CAAC,GAAG,EAAE,gCAAgC,CAAC,CAAC;QAClE,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,QAAQ,GAAc,EAAE,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;YACzB,MAAM,OAAO,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACxE,IAAI,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,CAAC,KAAe,CAAC,EAAE,CAAC;gBACvD,SAAS;YACX,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,KAAe,CAAC,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QACxF,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAC3C,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACjC,CAAC;CACF;AAED,SAAS,OAAO,CAAC,IAAY;IAC3B,IAAI,IAAa,CAAC;IAClB,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO,IAAoB,CAAC;IAC9B,CAAC;IACD,MAAM,OAAO,GAAG,IAAoD,CAAC;IACrE,MAAM,KAAK,GAAG,OAAO,EAAE,OAAO,IAAI,OAAO,EAAE,IAAI,CAAC;IAChD,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAE,KAAsB,CAAC,CAAC,CAAC,SAAS,CAAC;AACpE,CAAC"}
@@ -1,4 +1,4 @@
1
- import type { Passage, RetrieveOptions, Retriever } from '../spi/retriever.js';
1
+ import type { Passage, RetrievalResult, RetrieveOptions, Retriever } from '../spi/retriever.js';
2
2
  export interface HybridRetrieverOptions {
3
3
  /** RRF constant — larger flattens the rank weighting. Default 60 (the value from the original paper). */
4
4
  k?: number;
@@ -19,5 +19,10 @@ export declare class HybridRetriever implements Retriever {
19
19
  private readonly options;
20
20
  constructor(retrievers: Retriever[], options?: HybridRetrieverOptions);
21
21
  retrieve(query: string, options?: RetrieveOptions): Promise<Passage[]>;
22
+ /**
23
+ * {@link HybridRetriever.retrieve} plus the embedding usage of every leg that reports one (summed), so
24
+ * a hybrid over an embedding leg still reaches the ledger and the quota.
25
+ */
26
+ retrieveWithUsage(query: string, options?: RetrieveOptions): Promise<RetrievalResult>;
22
27
  }
23
28
  //# sourceMappingURL=hybrid-retriever.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"hybrid-retriever.d.ts","sourceRoot":"","sources":["../../../src/rag/hybrid-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAE/E,MAAM,WAAW,sBAAsB;IACrC,yGAAyG;IACzG,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,uEAAuE;IACvE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sGAAsG;IACtG,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;;;;GAMG;AACH,qBAAa,eAAgB,YAAW,SAAS;IAE7C,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAF1B,YACmB,UAAU,EAAE,SAAS,EAAE,EACvB,OAAO,GAAE,sBAA2B,EACnD;IAEE,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CA+B/E;CACF"}
1
+ {"version":3,"file":"hybrid-retriever.d.ts","sourceRoot":"","sources":["../../../src/rag/hybrid-retriever.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAEhG,MAAM,WAAW,sBAAsB;IACrC,yGAAyG;IACzG,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,uEAAuE;IACvE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sGAAsG;IACtG,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;;;;GAMG;AACH,qBAAa,eAAgB,YAAW,SAAS;IAE7C,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAF1B,YACmB,UAAU,EAAE,SAAS,EAAE,EACvB,OAAO,GAAE,sBAA2B,EACnD;IAEE,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAE/E;IAED;;;OAGG;IACG,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,eAAe,CAAC,CAsC9F;CACF"}
@@ -13,13 +13,25 @@ export class HybridRetriever {
13
13
  this.options = options;
14
14
  }
15
15
  async retrieve(query, options = {}) {
16
+ return (await this.retrieveWithUsage(query, options)).passages;
17
+ }
18
+ /**
19
+ * {@link HybridRetriever.retrieve} plus the embedding usage of every leg that reports one (summed), so
20
+ * a hybrid over an embedding leg still reaches the ledger and the quota.
21
+ */
22
+ async retrieveWithUsage(query, options = {}) {
16
23
  const k = this.options.k ?? 60;
17
24
  const fetchTopK = this.options.fetchTopK ?? 20;
18
25
  const weights = this.options.weights;
19
- const lists = await Promise.all(this.retrievers.map((retriever) => retriever.retrieve(query, {
26
+ const legOptions = {
20
27
  topK: fetchTopK,
21
28
  ...(options.filter !== undefined ? { filter: options.filter } : {}),
22
- })));
29
+ };
30
+ const results = await Promise.all(this.retrievers.map(async (retriever) => typeof retriever.retrieveWithUsage === 'function'
31
+ ? retriever.retrieveWithUsage(query, legOptions)
32
+ : { passages: await retriever.retrieve(query, legOptions) }));
33
+ const lists = results.map((result) => result.passages);
34
+ const usage = sumUsage(results.map((result) => result.usage));
23
35
  const fused = new Map();
24
36
  lists.forEach((list, listIndex) => {
25
37
  const weight = weights?.[listIndex] ?? 1;
@@ -34,10 +46,22 @@ export class HybridRetriever {
34
46
  }
35
47
  });
36
48
  });
37
- return [...fused.values()]
49
+ const passages = [...fused.values()]
38
50
  .sort((a, b) => b.score - a.score)
39
51
  .slice(0, options.topK ?? 5)
40
52
  .map((entry) => ({ ...entry.passage, score: entry.score }));
53
+ return { passages, ...(usage !== undefined ? { usage } : {}) };
54
+ }
55
+ }
56
+ function sumUsage(usages) {
57
+ const reported = usages.filter((usage) => usage !== undefined);
58
+ if (reported.length === 0) {
59
+ return undefined;
41
60
  }
61
+ const modelId = reported.find((usage) => usage.modelId !== undefined)?.modelId;
62
+ return {
63
+ inputTokens: reported.reduce((total, usage) => total + usage.inputTokens, 0),
64
+ ...(modelId !== undefined ? { modelId } : {}),
65
+ };
42
66
  }
43
67
  //# sourceMappingURL=hybrid-retriever.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"hybrid-retriever.js","sourceRoot":"","sources":["../../../src/rag/hybrid-retriever.ts"],"names":[],"mappings":"AAWA;;;;;;GAMG;AACH,MAAM,OAAO,eAAe;IAEP,UAAU;IACV,OAAO;IAF1B,YACmB,UAAuB,EACvB,OAAO,GAA2B,EAAE;0BADpC,UAAU;uBACV,OAAO;IACvB,CAAC;IAEJ,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAO,GAAoB,EAAE;QACzD,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;QACrC,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAChC,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE;YACxB,IAAI,EAAE,SAAS;YACf,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpE,CAAC,CACH,CACF,CAAC;QAEF,MAAM,KAAK,GAAG,IAAI,GAAG,EAA+C,CAAC;QACrE,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE;YAChC,MAAM,MAAM,GAAG,OAAO,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACzC,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE;gBAC7B,MAAM,YAAY,GAAG,MAAM,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;gBAC7C,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBACvC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;oBAC3B,QAAQ,CAAC,KAAK,IAAI,YAAY,CAAC;gBACjC,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;gBAC1D,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;aACvB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;aACjC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;aAC3B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAChE,CAAC;CACF"}
1
+ {"version":3,"file":"hybrid-retriever.js","sourceRoot":"","sources":["../../../src/rag/hybrid-retriever.ts"],"names":[],"mappings":"AAYA;;;;;;GAMG;AACH,MAAM,OAAO,eAAe;IAEP,UAAU;IACV,OAAO;IAF1B,YACmB,UAAuB,EACvB,OAAO,GAA2B,EAAE;0BADpC,UAAU;uBACV,OAAO;IACvB,CAAC;IAEJ,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAO,GAAoB,EAAE;QACzD,OAAO,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;IACjE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,iBAAiB,CAAC,KAAa,EAAE,OAAO,GAAoB,EAAE;QAClE,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;QACrC,MAAM,UAAU,GAAoB;YAClC,IAAI,EAAE,SAAS;YACf,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpE,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,IAAI,CAAC,UAAU,CAAC,GAAG,CACjB,KAAK,EAAE,SAAS,EAA4B,EAAE,CAC5C,OAAO,SAAS,CAAC,iBAAiB,KAAK,UAAU;YAC/C,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,KAAK,EAAE,UAAU,CAAC;YAChD,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,CAChE,CACF,CAAC;QACF,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACvD,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAE9D,MAAM,KAAK,GAAG,IAAI,GAAG,EAA+C,CAAC;QACrE,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE;YAChC,MAAM,MAAM,GAAG,OAAO,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACzC,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE;gBAC7B,MAAM,YAAY,GAAG,MAAM,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;gBAC7C,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBACvC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;oBAC3B,QAAQ,CAAC,KAAK,IAAI,YAAY,CAAC;gBACjC,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;gBAC1D,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;aACjC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;aACjC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;aAC3B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC9D,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IACjE,CAAC;CACF;AAED,SAAS,QAAQ,CAAC,MAAsC;IACtD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAA2B,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IACxF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,EAAE,OAAO,CAAC;IAC/E,OAAO;QACL,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QAC5E,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9C,CAAC;AACJ,CAAC"}
@@ -1,11 +1,15 @@
1
1
  export { type ChunkOptions, chunkText } from './chunk.js';
2
2
  export { EmbeddingRetriever } from './embedding-retriever.js';
3
3
  export { matchesFilter } from './filter.js';
4
+ export { HttpReranker, type HttpRerankerOptions } from './http-reranker.js';
4
5
  export { HybridRetriever, type HybridRetrieverOptions } from './hybrid-retriever.js';
5
6
  export { type ChunkRecord, chunkDocuments, type IngestChunksOptions, type IngestDocument, type IngestOptions, ingestChunks, ingestDocuments, } from './ingest.js';
6
7
  export { KeywordRetriever, type KeywordRetrieverOptions } from './keyword-retriever.js';
8
+ export { anyTermTsquery, DEFAULT_STOP_WORDS, hasSearchSyntax, keywordTerms, } from './lexical-query.js';
9
+ export { LexicalRetriever } from './lexical-retriever.js';
7
10
  export { cosineSimilarity, MemoryVectorStore } from './memory-vector-store.js';
8
- export { type PgVectorColumns, type PgVectorMetric, PgVectorRetriever, PgVectorStore, type PgVectorStoreOptions, stripNulBytes, toVectorLiteral, } from './pg-vector-store.js';
11
+ export { HttpModelError, type OpenAiEmbeddingsOptions, openAiEmbeddings, } from './openai-embeddings.js';
12
+ export { type PgFullTextOptions, PgLexicalVectorStore, type PgVectorColumns, type PgVectorMetric, PgVectorRetriever, PgVectorStore, type PgVectorStoreOptions, stripNulBytes, toVectorLiteral, } from './pg-vector-store.js';
9
13
  export { RerankingRetriever, type RerankingRetrieverOptions, } from './reranking-retriever.js';
10
- export { applyMetadataPatch, assertRemovalFilter, documentIdOf, effectivePatchKeys, filterDeniesAll, type IndexedDocument, type MetadataPatch, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, } from './vector-store.js';
14
+ export { applyMetadataPatch, assertRemovalFilter, documentIdOf, effectivePatchKeys, filterDeniesAll, type IndexedDocument, isLexicalVectorStore, type LexicalVectorStore, type MetadataPatch, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, } from './vector-store.js';
11
15
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/rag/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,YAAY,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EACL,KAAK,WAAW,EAChB,cAAc,EACd,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,YAAY,EACZ,eAAe,GAChB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,gBAAgB,EAAE,KAAK,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACxF,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,iBAAiB,EACjB,aAAa,EACb,KAAK,oBAAoB,EACzB,aAAa,EACb,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,kBAAkB,EAClB,KAAK,yBAAyB,GAC/B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,kBAAkB,EAClB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,EAClB,eAAe,EACf,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,kBAAkB,EAClB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,WAAW,GACjB,MAAM,mBAAmB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/rag/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,YAAY,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,KAAK,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EACL,KAAK,WAAW,EAChB,cAAc,EACd,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,YAAY,EACZ,eAAe,GAChB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,gBAAgB,EAAE,KAAK,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACxF,OAAO,EACL,cAAc,EACd,kBAAkB,EAClB,eAAe,EACf,YAAY,GACb,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EACL,cAAc,EACd,KAAK,uBAAuB,EAC5B,gBAAgB,GACjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,KAAK,iBAAiB,EACtB,oBAAoB,EACpB,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,iBAAiB,EACjB,aAAa,EACb,KAAK,oBAAoB,EACzB,aAAa,EACb,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,kBAAkB,EAClB,KAAK,yBAAyB,GAC/B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,kBAAkB,EAClB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,EAClB,eAAe,EACf,KAAK,eAAe,EACpB,oBAAoB,EACpB,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,kBAAkB,EAClB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,WAAW,GACjB,MAAM,mBAAmB,CAAC"}
@@ -1,11 +1,15 @@
1
1
  export { chunkText } from './chunk.js';
2
2
  export { EmbeddingRetriever } from './embedding-retriever.js';
3
3
  export { matchesFilter } from './filter.js';
4
+ export { HttpReranker } from './http-reranker.js';
4
5
  export { HybridRetriever } from './hybrid-retriever.js';
5
6
  export { chunkDocuments, ingestChunks, ingestDocuments, } from './ingest.js';
6
7
  export { KeywordRetriever } from './keyword-retriever.js';
8
+ export { anyTermTsquery, DEFAULT_STOP_WORDS, hasSearchSyntax, keywordTerms, } from './lexical-query.js';
9
+ export { LexicalRetriever } from './lexical-retriever.js';
7
10
  export { cosineSimilarity, MemoryVectorStore } from './memory-vector-store.js';
8
- export { PgVectorRetriever, PgVectorStore, stripNulBytes, toVectorLiteral, } from './pg-vector-store.js';
11
+ export { HttpModelError, openAiEmbeddings, } from './openai-embeddings.js';
12
+ export { PgLexicalVectorStore, PgVectorRetriever, PgVectorStore, stripNulBytes, toVectorLiteral, } from './pg-vector-store.js';
9
13
  export { RerankingRetriever, } from './reranking-retriever.js';
10
- export { applyMetadataPatch, assertRemovalFilter, documentIdOf, effectivePatchKeys, filterDeniesAll, UnsafeRemovalError, } from './vector-store.js';
14
+ export { applyMetadataPatch, assertRemovalFilter, documentIdOf, effectivePatchKeys, filterDeniesAll, isLexicalVectorStore, UnsafeRemovalError, } from './vector-store.js';
11
15
  //# sourceMappingURL=index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rag/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,SAAS,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,eAAe,EAA+B,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAEL,cAAc,EAId,YAAY,EACZ,eAAe,GAChB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,gBAAgB,EAAgC,MAAM,wBAAwB,CAAC;AACxF,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EAGL,iBAAiB,EACjB,aAAa,EAEb,aAAa,EACb,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,kBAAkB,GAEnB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,kBAAkB,EAClB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,EAClB,eAAe,EAGf,kBAAkB,GAInB,MAAM,mBAAmB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rag/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,SAAS,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,YAAY,EAA4B,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAAE,eAAe,EAA+B,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAEL,cAAc,EAId,YAAY,EACZ,eAAe,GAChB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,gBAAgB,EAAgC,MAAM,wBAAwB,CAAC;AACxF,OAAO,EACL,cAAc,EACd,kBAAkB,EAClB,eAAe,EACf,YAAY,GACb,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EACL,cAAc,EAEd,gBAAgB,GACjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,oBAAoB,EAGpB,iBAAiB,EACjB,aAAa,EAEb,aAAa,EACb,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,kBAAkB,GAEnB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,kBAAkB,EAClB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,EAClB,eAAe,EAEf,oBAAoB,EAGpB,kBAAkB,GAInB,MAAM,mBAAmB,CAAC"}
@@ -0,0 +1,28 @@
1
+ /**
2
+ * Turning a natural-language question into full-text search terms, for the lexical leg of
3
+ * `PgLexicalVectorStore`.
4
+ *
5
+ * With the `simple` text-search configuration (no stemming, no stop words; any language) a question
6
+ * searched as is either matches nothing (every word required) or, as "any word", ranks chunks full
7
+ * of "the / of / which / de / que" first. So the query side drops the stop words of the question's
8
+ * language(s) and keeps the meaningful terms, which the store then ranks by how many of them a chunk
9
+ * holds, rare terms weighing more (an IDF over the matching rows, BM25-style).
10
+ */
11
+ /**
12
+ * Postgres' Snowball stop word lists (`tsearch_data/{english,portuguese,spanish}.stop`), minus
13
+ * one-letter words (never terms here) and words that carry meaning in a sibling language
14
+ * ("estado", "sentido").
15
+ */
16
+ export declare const DEFAULT_STOP_WORDS: Readonly<Record<string, ReadonlySet<string>>>;
17
+ /**
18
+ * The meaningful terms of a question, in order, deduplicated, at most `max`: the stop words of the
19
+ * language(s) the question is written in are dropped (the language(s) with the most stop words in
20
+ * it, so "sea"/"son" stay terms in an English question although they are Spanish stop words). A
21
+ * question made only of stop words keeps them all (there is nothing else to search for).
22
+ */
23
+ export declare function keywordTerms(query: string, max?: number, stopWords?: Readonly<Record<string, ReadonlySet<string>>>): string[];
24
+ /** Whether the user wrote search syntax (a quoted phrase, `-exclusion`) worth honoring as such. */
25
+ export declare function hasSearchSyntax(query: string): boolean;
26
+ /** `'a' | 'b' | …` for `to_tsquery` (terms are letter/digit runs, so quoting is enough). */
27
+ export declare function anyTermTsquery(terms: string[]): string;
28
+ //# sourceMappingURL=lexical-query.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"lexical-query.d.ts","sourceRoot":"","sources":["../../../src/rag/lexical-query.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CA+C5E,CAAC;AAIF;;;;;GAKG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,MAAM,EACb,GAAG,SAAK,EACR,SAAS,GAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAsB,GAC5E,MAAM,EAAE,CAUV;AAED,mGAAmG;AACnG,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAEtD;AAED,4FAA4F;AAC5F,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtD"}
@@ -0,0 +1,90 @@
1
+ /**
2
+ * Turning a natural-language question into full-text search terms, for the lexical leg of
3
+ * `PgLexicalVectorStore`.
4
+ *
5
+ * With the `simple` text-search configuration (no stemming, no stop words; any language) a question
6
+ * searched as is either matches nothing (every word required) or, as "any word", ranks chunks full
7
+ * of "the / of / which / de / que" first. So the query side drops the stop words of the question's
8
+ * language(s) and keeps the meaningful terms, which the store then ranks by how many of them a chunk
9
+ * holds, rare terms weighing more (an IDF over the matching rows, BM25-style).
10
+ */
11
+ const words = (s) => new Set(s.split(/\s+/).filter(Boolean));
12
+ /**
13
+ * Postgres' Snowball stop word lists (`tsearch_data/{english,portuguese,spanish}.stop`), minus
14
+ * one-letter words (never terms here) and words that carry meaning in a sibling language
15
+ * ("estado", "sentido").
16
+ */
17
+ export const DEFAULT_STOP_WORDS = {
18
+ english: words(`
19
+ me my myself we our ours ourselves you your yours yourself yourselves he him his himself she
20
+ her hers herself it its itself they them their theirs themselves what which who whom this that
21
+ these those am is are was were be been being have has had having do does did doing an the and
22
+ but if or because as until while of at by for with about against between into through during
23
+ before after above below to from up down in out on off over under again further then once here
24
+ there when where why how all any both each few more most other some such no nor not only own
25
+ same so than too very can will just don should now`),
26
+ portuguese: words(`
27
+ de que do da em um para com não uma os no se na por mais as dos como mas ao ele das seu sua ou
28
+ quando muito nos já eu também só pelo pela até isso ela entre depois sem mesmo aos seus quem
29
+ nas me esse eles você essa num nem suas meu às minha numa pelos elas qual nós lhe deles essas
30
+ esses pelas este dele tu te vocês vos lhes meus minhas teu tua teus tuas nosso nossa nossos
31
+ nossas dela delas esta estes estas aquele aquela aqueles aquelas isto aquilo estou está estamos
32
+ estão estive esteve estivemos estiveram estava estávamos estavam estivera estivéramos esteja
33
+ estejamos estejam estivesse estivéssemos estivessem estiver estivermos estiverem hei há havemos
34
+ hão houve houvemos houveram houvera houvéramos haja hajamos hajam houvesse houvéssemos
35
+ houvessem houver houvermos houverem houverei houverá houveremos houverão houveria houveríamos
36
+ houveriam sou somos são era éramos eram fui foi fomos foram fora fôramos seja sejamos sejam
37
+ fosse fôssemos fossem for formos forem serei será seremos serão seria seríamos seriam tenho tem
38
+ temos tém tinha tínhamos tinham tive teve tivemos tiveram tivera tivéramos tenha tenhamos
39
+ tenham tivesse tivéssemos tivessem tiver tivermos tiverem terei terá teremos terão teria
40
+ teríamos teriam`),
41
+ spanish: words(`
42
+ de la que el en los del se las por un para con no una su al lo como más pero sus le ya este sí
43
+ porque esta entre cuando muy sin sobre también me hasta hay donde quien desde todo nos durante
44
+ todos uno les ni contra otros ese eso ante ellos esto mí antes algunos qué unos yo otro otras
45
+ otra él tanto esa estos mucho quienes nada muchos cual poco ella estar estas algunas algo
46
+ nosotros mi mis tú te ti tu tus ellas nosotras vosotros vosotras os mío mía míos mías tuyo
47
+ tuya tuyos tuyas suyo suya suyos suyas nuestro nuestra nuestros nuestras vuestro vuestra
48
+ vuestros vuestras esos esas estoy estás está estamos estáis están esté estés estemos estéis
49
+ estén estaré estarás estará estaremos estaréis estarán estaría estarías estaríamos estaríais
50
+ estarían estaba estabas estábamos estabais estaban estuve estuviste estuvo estuvimos
51
+ estuvisteis estuvieron estuviera estuvieras estuviéramos estuvierais estuvieran estuviese
52
+ estuvieses estuviésemos estuvieseis estuviesen estando estad he has ha hemos habéis han haya
53
+ hayas hayamos hayáis hayan habré habrás habrá habremos habréis habrán habría habrías habríamos
54
+ habríais habrían había habías habíamos habíais habían hube hubiste hubo hubimos hubisteis
55
+ hubieron hubiera hubieras hubiéramos hubierais hubieran hubiese hubieses hubiésemos hubieseis
56
+ hubiesen habiendo habido habida habidos habidas soy eres es somos sois son sea seas seamos
57
+ seáis sean seré serás será seremos seréis serán sería serías seríamos seríais serían era eras
58
+ éramos erais eran fui fuiste fue fuimos fuisteis fueron fuera fueras fuéramos fuerais fueran
59
+ fuese fueses fuésemos fueseis fuesen tengo tienes tiene tenemos tenéis tienen tenga tengas
60
+ tengamos tengáis tengan tendré tendrás tendrá tendremos tendréis tendrán tendría tendrías
61
+ tendríamos tendríais tendrían tenía tenías teníamos teníais tenían tuve tuviste tuvo tuvimos
62
+ tuvisteis tuvieron tuviera tuvieras tuviéramos tuvierais tuvieran tuviese tuvieses tuviésemos
63
+ tuvieseis tuviesen teniendo tenido tenida tenidos tenidas tened`),
64
+ };
65
+ const TOKEN = /[\p{L}\p{N}]{2,}/gu;
66
+ /**
67
+ * The meaningful terms of a question, in order, deduplicated, at most `max`: the stop words of the
68
+ * language(s) the question is written in are dropped (the language(s) with the most stop words in
69
+ * it, so "sea"/"son" stay terms in an English question although they are Spanish stop words). A
70
+ * question made only of stop words keeps them all (there is nothing else to search for).
71
+ */
72
+ export function keywordTerms(query, max = 24, stopWords = DEFAULT_STOP_WORDS) {
73
+ const tokens = [...new Set(query.toLowerCase().normalize('NFC').match(TOKEN) ?? [])];
74
+ const counts = Object.entries(stopWords).map(([, stop]) => [stop, tokens.filter((t) => stop.has(t)).length]);
75
+ const best = Math.max(0, ...counts.map(([, n]) => n));
76
+ if (best === 0)
77
+ return tokens.slice(0, max);
78
+ const stop = counts.filter(([, n]) => n === best).map(([s]) => s);
79
+ const terms = tokens.filter((t) => !stop.some((s) => s.has(t)));
80
+ return (terms.length > 0 ? terms : tokens).slice(0, max);
81
+ }
82
+ /** Whether the user wrote search syntax (a quoted phrase, `-exclusion`) worth honoring as such. */
83
+ export function hasSearchSyntax(query) {
84
+ return /"[^"]+"/.test(query) || /(^|\s)-[\p{L}\p{N}]/u.test(query);
85
+ }
86
+ /** `'a' | 'b' | …` for `to_tsquery` (terms are letter/digit runs, so quoting is enough). */
87
+ export function anyTermTsquery(terms) {
88
+ return terms.map((t) => `'${t.replace(/'/g, '')}'`).join(' | ');
89
+ }
90
+ //# sourceMappingURL=lexical-query.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"lexical-query.js","sourceRoot":"","sources":["../../../src/rag/lexical-query.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;AAErE;;;;GAIG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAkD;IAC/E,OAAO,EAAE,KAAK,CAAC;;;;;;;uDAOsC,CAAC;IACtD,UAAU,EAAE,KAAK,CAAC;;;;;;;;;;;;;;oBAcA,CAAC;IACnB,OAAO,EAAE,KAAK,CAAC;;;;;;;;;;;;;;;;;;;;;;oEAsBmD,CAAC;CACpE,CAAC;AAEF,MAAM,KAAK,GAAG,oBAAoB,CAAC;AAEnC;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAC1B,KAAa,EACb,GAAG,GAAG,EAAE,EACR,SAAS,GAAkD,kBAAkB;IAE7E,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACrF,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,GAAG,CAC1C,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAU,CACxE,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACtD,IAAI,IAAI,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAClE,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC3D,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,eAAe,CAAC,KAAa;IAC3C,OAAO,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,sBAAsB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,4FAA4F;AAC5F,MAAM,UAAU,cAAc,CAAC,KAAe;IAC5C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAClE,CAAC"}
@@ -0,0 +1,23 @@
1
+ import type { Passage, RetrieveOptions, Retriever } from '../spi/retriever.js';
2
+ import type { LexicalVectorStore } from './vector-store.js';
3
+ /**
4
+ * A {@link Retriever} over a store's own full-text index — the lexical half of a hybrid search, next to
5
+ * an {@link import('./embedding-retriever.js').EmbeddingRetriever} over the same store:
6
+ *
7
+ * ```ts
8
+ * const store = new PgLexicalVectorStore(db, { dimension: 1536 })
9
+ * const retriever = new HybridRetriever([
10
+ * new EmbeddingRetriever(embedder, store),
11
+ * new LexicalRetriever(store),
12
+ * ])
13
+ * ```
14
+ *
15
+ * Embeds nothing, so it reports no usage. `minScore` is not forwarded: a text rank is not a similarity,
16
+ * and a floor tuned for cosine would mean nothing on it.
17
+ */
18
+ export declare class LexicalRetriever implements Retriever {
19
+ private readonly store;
20
+ constructor(store: LexicalVectorStore);
21
+ retrieve(query: string, options?: RetrieveOptions): Promise<Passage[]>;
22
+ }
23
+ //# sourceMappingURL=lexical-retriever.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"lexical-retriever.d.ts","sourceRoot":"","sources":["../../../src/rag/lexical-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAC/E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAE5D;;;;;;;;;;;;;;GAcG;AACH,qBAAa,gBAAiB,YAAW,SAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,KAAK;IAAlC,YAA6B,KAAK,EAAE,kBAAkB,EAAI;IAEpD,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAK/E;CACF"}
@@ -0,0 +1,28 @@
1
+ /**
2
+ * A {@link Retriever} over a store's own full-text index — the lexical half of a hybrid search, next to
3
+ * an {@link import('./embedding-retriever.js').EmbeddingRetriever} over the same store:
4
+ *
5
+ * ```ts
6
+ * const store = new PgLexicalVectorStore(db, { dimension: 1536 })
7
+ * const retriever = new HybridRetriever([
8
+ * new EmbeddingRetriever(embedder, store),
9
+ * new LexicalRetriever(store),
10
+ * ])
11
+ * ```
12
+ *
13
+ * Embeds nothing, so it reports no usage. `minScore` is not forwarded: a text rank is not a similarity,
14
+ * and a floor tuned for cosine would mean nothing on it.
15
+ */
16
+ export class LexicalRetriever {
17
+ store;
18
+ constructor(store) {
19
+ this.store = store;
20
+ }
21
+ async retrieve(query, options = {}) {
22
+ return this.store.searchText(query, {
23
+ topK: options.topK ?? 5,
24
+ ...(options.filter !== undefined ? { filter: options.filter } : {}),
25
+ });
26
+ }
27
+ }
28
+ //# sourceMappingURL=lexical-retriever.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"lexical-retriever.js","sourceRoot":"","sources":["../../../src/rag/lexical-retriever.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,gBAAgB;IACE,KAAK;IAAlC,YAA6B,KAAyB;qBAAzB,KAAK;IAAuB,CAAC;IAE1D,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAO,GAAoB,EAAE;QACzD,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE;YAClC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC;YACvB,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpE,CAAC,CAAC;IACL,CAAC;CACF"}
@@ -0,0 +1,62 @@
1
+ import type { EmbeddingProvider } from '../spi/embedding-provider.js';
2
+ /** A non-2xx answer (or a malformed one) from an embeddings or rerank endpoint. */
3
+ export declare class HttpModelError extends Error {
4
+ /** HTTP status; `502` when the endpoint answered 2xx with a body that doesn't fit the contract. */
5
+ readonly status: number;
6
+ constructor(
7
+ /** HTTP status; `502` when the endpoint answered 2xx with a body that doesn't fit the contract. */
8
+ status: number, message: string);
9
+ }
10
+ export interface OpenAiEmbeddingsOptions {
11
+ /** The embedding model id, sent as `model`. */
12
+ model: string;
13
+ /**
14
+ * Base URL up to and including `/v1` — the client posts to `<baseUrl>/embeddings`. Default
15
+ * `https://api.openai.com/v1`. Point it at any OpenAI-compatible server: Azure/OpenRouter/an LLM
16
+ * gateway, Ollama (`http://localhost:11434/v1`), Hugging Face TEI, vLLM, LocalAI, LM Studio…
17
+ */
18
+ baseUrl?: string;
19
+ /** Sent as `Authorization: Bearer <apiKey>` when set. Local servers usually need none. */
20
+ apiKey?: string;
21
+ /** Output width for models that support shortening (`text-embedding-3-*`), sent as `dimensions`. */
22
+ dimensions?: number;
23
+ /** Inputs per request. Default 64. */
24
+ batchSize?: number;
25
+ /**
26
+ * Inputs longer than this many characters are truncated before sending, so one oversized chunk
27
+ * doesn't fail a whole batch on the model's context limit. Default 24 000 (≈ 6–8k tokens); set
28
+ * `Infinity` to send inputs as they are.
29
+ */
30
+ maxInputChars?: number;
31
+ /** Extra request headers (an org/project id, a gateway's attribution header). */
32
+ headers?: Record<string, string>;
33
+ /** Per-request timeout. Default 120 000 ms. */
34
+ timeoutMs?: number;
35
+ /**
36
+ * Called with the provider-reported token count of every request. The same counts also reach the
37
+ * ledger and quota through `embedWithUsage`; this is for your own metering on top.
38
+ */
39
+ onUsage?: (tokens: number) => void;
40
+ /** `fetch` to use. Default the global one. */
41
+ fetch?: typeof fetch;
42
+ }
43
+ /**
44
+ * An {@link EmbeddingProvider} over any OpenAI-compatible `POST /v1/embeddings` endpoint, with no SDK
45
+ * dependency: OpenAI itself, or a gateway/local server speaking the same protocol (TEI, Ollama,
46
+ * vLLM, LocalAI…). For a Vercel AI SDK model use `aiSdkEmbedding` from `-ai-sdk` instead.
47
+ *
48
+ * Batches by {@link OpenAiEmbeddingsOptions.batchSize}, restores input order from each item's
49
+ * `index`, and fails loudly (an {@link HttpModelError}) on a non-2xx answer or a vector count that
50
+ * doesn't match the inputs. Empty strings are sent as a single space, since OpenAI rejects them.
51
+ * Implements `embedWithUsage` with the provider-reported tokens (summed over the batches) and model, so
52
+ * embedding spend reaches the ledger and the quota.
53
+ *
54
+ * ```ts
55
+ * const embedder = openAiEmbeddings({ model: 'text-embedding-3-small', apiKey: process.env.OPENAI_API_KEY });
56
+ * const local = openAiEmbeddings({ baseUrl: 'http://tei:8080/v1', model: 'BAAI/bge-m3' });
57
+ * ```
58
+ */
59
+ export declare function openAiEmbeddings(options: OpenAiEmbeddingsOptions): EmbeddingProvider;
60
+ /** The human part of an error body: OpenAI's `error.message`, a bare `message`/`error`, or the text. */
61
+ export declare function errorMessageOf(body: string): string;
62
+ //# sourceMappingURL=openai-embeddings.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"openai-embeddings.d.ts","sourceRoot":"","sources":["../../../src/rag/openai-embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAmB,MAAM,8BAA8B,CAAC;AAEvF,mFAAmF;AACnF,qBAAa,cAAe,SAAQ,KAAK;IAErC,mGAAmG;IACnG,QAAQ,CAAC,MAAM,EAAE,MAAM;IAFzB;IACE,mGAAmG;IAC1F,MAAM,EAAE,MAAM,EACvB,OAAO,EAAE,MAAM,EAIhB;CACF;AAED,MAAM,WAAW,uBAAuB;IACtC,+CAA+C;IAC/C,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,0FAA0F;IAC1F,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,oGAAoG;IACpG,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,sCAAsC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iFAAiF;IACjF,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,8CAA8C;IAC9C,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACtB;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,uBAAuB,GAAG,iBAAiB,CA6EpF;AAUD,wGAAwG;AACxG,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAUnD"}
@@ -0,0 +1,106 @@
1
+ /** A non-2xx answer (or a malformed one) from an embeddings or rerank endpoint. */
2
+ export class HttpModelError extends Error {
3
+ status;
4
+ constructor(
5
+ /** HTTP status; `502` when the endpoint answered 2xx with a body that doesn't fit the contract. */
6
+ status, message) {
7
+ super(message);
8
+ this.status = status;
9
+ this.name = 'HttpModelError';
10
+ }
11
+ }
12
+ /**
13
+ * An {@link EmbeddingProvider} over any OpenAI-compatible `POST /v1/embeddings` endpoint, with no SDK
14
+ * dependency: OpenAI itself, or a gateway/local server speaking the same protocol (TEI, Ollama,
15
+ * vLLM, LocalAI…). For a Vercel AI SDK model use `aiSdkEmbedding` from `-ai-sdk` instead.
16
+ *
17
+ * Batches by {@link OpenAiEmbeddingsOptions.batchSize}, restores input order from each item's
18
+ * `index`, and fails loudly (an {@link HttpModelError}) on a non-2xx answer or a vector count that
19
+ * doesn't match the inputs. Empty strings are sent as a single space, since OpenAI rejects them.
20
+ * Implements `embedWithUsage` with the provider-reported tokens (summed over the batches) and model, so
21
+ * embedding spend reaches the ledger and the quota.
22
+ *
23
+ * ```ts
24
+ * const embedder = openAiEmbeddings({ model: 'text-embedding-3-small', apiKey: process.env.OPENAI_API_KEY });
25
+ * const local = openAiEmbeddings({ baseUrl: 'http://tei:8080/v1', model: 'BAAI/bge-m3' });
26
+ * ```
27
+ */
28
+ export function openAiEmbeddings(options) {
29
+ const url = `${(options.baseUrl ?? 'https://api.openai.com/v1').replace(/\/+$/, '')}/embeddings`;
30
+ const batchSize = Math.max(1, Math.floor(options.batchSize ?? 64));
31
+ const maxInputChars = options.maxInputChars ?? 24_000;
32
+ const timeoutMs = options.timeoutMs ?? 120_000;
33
+ const doFetch = options.fetch ?? fetch;
34
+ const provider = {
35
+ async embed(texts) {
36
+ return (await provider.embedWithUsage(texts)).vectors;
37
+ },
38
+ async embedWithUsage(texts) {
39
+ const vectors = [];
40
+ let inputTokens;
41
+ let modelId;
42
+ for (let start = 0; start < texts.length; start += batchSize) {
43
+ const input = texts
44
+ .slice(start, start + batchSize)
45
+ .map((text) => (text.length > maxInputChars ? text.slice(0, maxInputChars) : text) || ' ');
46
+ const response = await doFetch(url, {
47
+ method: 'POST',
48
+ headers: {
49
+ 'content-type': 'application/json',
50
+ ...(options.apiKey ? { authorization: `Bearer ${options.apiKey}` } : {}),
51
+ ...options.headers,
52
+ },
53
+ body: JSON.stringify({
54
+ model: options.model,
55
+ input,
56
+ ...(options.dimensions !== undefined ? { dimensions: options.dimensions } : {}),
57
+ }),
58
+ signal: AbortSignal.timeout(timeoutMs),
59
+ });
60
+ const body = await response.text();
61
+ if (!response.ok) {
62
+ throw new HttpModelError(response.status, `Embeddings request failed (${response.status}): ${errorMessageOf(body)}`);
63
+ }
64
+ const json = parseJson(body);
65
+ const items = [...(json?.data ?? [])].sort((a, b) => (a.index ?? 0) - (b.index ?? 0));
66
+ if (items.length !== input.length ||
67
+ !items.every((item) => Array.isArray(item.embedding))) {
68
+ throw new HttpModelError(502, `Embeddings returned ${items.length} vectors for ${input.length} inputs`);
69
+ }
70
+ for (const item of items) {
71
+ vectors.push(item.embedding);
72
+ }
73
+ const tokens = json?.usage?.total_tokens ?? json?.usage?.prompt_tokens;
74
+ if (typeof tokens === 'number' && Number.isFinite(tokens)) {
75
+ inputTokens = (inputTokens ?? 0) + tokens;
76
+ options.onUsage?.(tokens);
77
+ }
78
+ modelId ??= typeof json?.model === 'string' ? json.model : undefined;
79
+ }
80
+ return {
81
+ vectors,
82
+ ...(inputTokens !== undefined
83
+ ? { usage: { inputTokens, modelId: modelId ?? options.model } }
84
+ : {}),
85
+ };
86
+ },
87
+ };
88
+ return provider;
89
+ }
90
+ function parseJson(body) {
91
+ try {
92
+ return JSON.parse(body);
93
+ }
94
+ catch {
95
+ return null;
96
+ }
97
+ }
98
+ /** The human part of an error body: OpenAI's `error.message`, a bare `message`/`error`, or the text. */
99
+ export function errorMessageOf(body) {
100
+ const json = parseJson(body);
101
+ const message = typeof json?.error === 'string'
102
+ ? json.error
103
+ : (json?.error?.message ?? json?.message ?? undefined);
104
+ return typeof message === 'string' ? message : body.slice(0, 300);
105
+ }
106
+ //# sourceMappingURL=openai-embeddings.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"openai-embeddings.js","sourceRoot":"","sources":["../../../src/rag/openai-embeddings.ts"],"names":[],"mappings":"AAEA,mFAAmF;AACnF,MAAM,OAAO,cAAe,SAAQ,KAAK;IAG5B,MAAM;IAFjB;IACE,mGAAmG;IAC1F,MAAc,EACvB,OAAe;QAEf,KAAK,CAAC,OAAO,CAAC,CAAC;sBAHN,MAAM;QAIf,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF;AAoCD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAgC;IAC/D,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,2BAA2B,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,aAAa,CAAC;IACjG,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC;IACnE,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,MAAM,CAAC;IACtD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC;IAC/C,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC;IAEvC,MAAM,QAAQ,GAAG;QACf,KAAK,CAAC,KAAK,CAAC,KAAe;YACzB,OAAO,CAAC,MAAM,QAAQ,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC;QACxD,CAAC;QAED,KAAK,CAAC,cAAc,CAAC,KAAe;YAClC,MAAM,OAAO,GAAe,EAAE,CAAC;YAC/B,IAAI,WAA+B,CAAC;YACpC,IAAI,OAA2B,CAAC;YAChC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,IAAI,SAAS,EAAE,CAAC;gBAC7D,MAAM,KAAK,GAAG,KAAK;qBAChB,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,SAAS,CAAC;qBAC/B,GAAG,CACF,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,CACrF,CAAC;gBACJ,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE;oBAClC,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;wBACxE,GAAG,OAAO,CAAC,OAAO;qBACnB;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK,EAAE,OAAO,CAAC,KAAK;wBACpB,KAAK;wBACL,GAAG,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;qBAChF,CAAC;oBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC;iBACvC,CAAC,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,MAAM,IAAI,cAAc,CACtB,QAAQ,CAAC,MAAM,EACf,8BAA8B,QAAQ,CAAC,MAAM,MAAM,cAAc,CAAC,IAAI,CAAC,EAAE,CAC1E,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAInB,CAAC;gBACT,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC;gBACtF,IACE,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;oBAC7B,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EACrD,CAAC;oBACD,MAAM,IAAI,cAAc,CACtB,GAAG,EACH,uBAAuB,KAAK,CAAC,MAAM,gBAAgB,KAAK,CAAC,MAAM,SAAS,CACzE,CAAC;gBACJ,CAAC;gBACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAqB,CAAC,CAAC;gBAC3C,CAAC;gBACD,MAAM,MAAM,GAAG,IAAI,EAAE,KAAK,EAAE,YAAY,IAAI,IAAI,EAAE,KAAK,EAAE,aAAa,CAAC;gBACvE,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC1D,WAAW,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC;oBAC1C,OAAO,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC;gBAC5B,CAAC;gBACD,OAAO,KAAK,OAAO,IAAI,EAAE,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YACvE,CAAC;YACD,OAAO;gBACL,OAAO;gBACP,GAAG,CAAC,WAAW,KAAK,SAAS;oBAC3B,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,IAAI,OAAO,CAAC,KAAK,EAAE,EAAE;oBAC/D,CAAC,CAAC,EAAE,CAAC;aACR,CAAC;QACJ,CAAC;KACF,CAAC;IACF,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,wGAAwG;AACxG,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAGnB,CAAC;IACT,MAAM,OAAO,GACX,OAAO,IAAI,EAAE,KAAK,KAAK,QAAQ;QAC7B,CAAC,CAAC,IAAI,CAAC,KAAK;QACZ,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,IAAI,IAAI,EAAE,OAAO,IAAI,SAAS,CAAC,CAAC;IAC3D,OAAO,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACpE,CAAC"}