@dudousxd/nestjs-agent-rag 0.11.0 → 0.12.0

package/README.md

@@ -230,9 +230,13 @@ const store = new PgLexicalVectorStore(db, {
 });
 ```
 
-Ranking is `ts_rank_cd`; the query goes through `websearch_to_tsquery` (every word must appear), and
-when that finds nothing it retries with any word (`anyTermFallback`, default on) so a long question
-still matches. Without `column`, `ensureSchema`/`schemaStatements` add a GIN index over
+A question is searched by its meaningful terms: its stop words are dropped (English, Portuguese and
+Spanish lists by default, only the language(s) the question is written in; `stopWords` to change or
+`false` to keep every word), and rows holding any remaining term are ranked by the sum of the
+matched terms' IDF (BM25 without term frequencies), `ts_rank_cd` breaking ties, so three rare terms
+beat a row full of "the/of/which". Quoted phrases and `-word` go through `websearch_to_tsquery` as
+written first; `anyTermFallback: false` keeps only that (every word must appear). `keywordTerms()`
+is exported for other stores. Without `column`, `ensureSchema`/`schemaStatements` add a GIN index over
 `to_tsvector(config, text)` — the exact expression the query uses. On a populated table create it
 `CONCURRENTLY` in a migration rather than at boot (see below).
 

package/dist/index.cjs

@@ -21,6 +21,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
+  DEFAULT_STOP_WORDS: () => DEFAULT_STOP_WORDS,
   EmbeddingRetriever: () => EmbeddingRetriever,
   FilteredRetriever: () => FilteredRetriever,
   HttpModelError: () => HttpModelError,
@@ -38,6 +39,7 @@ __export(index_exports, {
   RedisVectorStore: () => RedisVectorStore,
   RerankingRetriever: () => RerankingRetriever,
   UnsafeRemovalError: () => UnsafeRemovalError,
+  anyTermTsquery: () => anyTermTsquery,
   applyMetadataPatch: () => applyMetadataPatch,
   chunkDocuments: () => chunkDocuments,
   chunkIndexOf: () => chunkIndexOf,
@@ -49,10 +51,12 @@ __export(index_exports, {
   describesRetrieval: () => describesRetrieval,
   documentIdOf: () => documentIdOf,
   emitRagRetrieval: () => emitRagRetrieval,
+  hasSearchSyntax: () => hasSearchSyntax,
   ingestChunks: () => ingestChunks,
   ingestDocuments: () => ingestDocuments,
   instrumentRetriever: () => instrumentRetriever,
   isLexicalVectorStore: () => isLexicalVectorStore,
+  keywordTerms: () => keywordTerms,
   openAiEmbeddings: () => openAiEmbeddings,
   toPgClient: () => toPgClient
 });
@@ -413,6 +417,81 @@ function cosineSimilarity(a, b) {
 }
 __name(cosineSimilarity, "cosineSimilarity");
 
+// src/lexical-query.ts
+var words = /* @__PURE__ */ __name((s) => new Set(s.split(/\s+/).filter(Boolean)), "words");
+var DEFAULT_STOP_WORDS = {
+  english: words(`
+    me my myself we our ours ourselves you your yours yourself yourselves he him his himself she
+    her hers herself it its itself they them their theirs themselves what which who whom this that
+    these those am is are was were be been being have has had having do does did doing an the and
+    but if or because as until while of at by for with about against between into through during
+    before after above below to from up down in out on off over under again further then once here
+    there when where why how all any both each few more most other some such no nor not only own
+    same so than too very can will just don should now`),
+  portuguese: words(`
+    de que do da em um para com n\xE3o uma os no se na por mais as dos como mas ao ele das seu sua ou
+    quando muito nos j\xE1 eu tamb\xE9m s\xF3 pelo pela at\xE9 isso ela entre depois sem mesmo aos seus quem
+    nas me esse eles voc\xEA essa num nem suas meu \xE0s minha numa pelos elas qual n\xF3s lhe deles essas
+    esses pelas este dele tu te voc\xEAs vos lhes meus minhas teu tua teus tuas nosso nossa nossos
+    nossas dela delas esta estes estas aquele aquela aqueles aquelas isto aquilo estou est\xE1 estamos
+    est\xE3o estive esteve estivemos estiveram estava est\xE1vamos estavam estivera estiv\xE9ramos esteja
+    estejamos estejam estivesse estiv\xE9ssemos estivessem estiver estivermos estiverem hei h\xE1 havemos
+    h\xE3o houve houvemos houveram houvera houv\xE9ramos haja hajamos hajam houvesse houv\xE9ssemos
+    houvessem houver houvermos houverem houverei houver\xE1 houveremos houver\xE3o houveria houver\xEDamos
+    houveriam sou somos s\xE3o era \xE9ramos eram fui foi fomos foram fora f\xF4ramos seja sejamos sejam
+    fosse f\xF4ssemos fossem for formos forem serei ser\xE1 seremos ser\xE3o seria ser\xEDamos seriam tenho tem
+    temos t\xE9m tinha t\xEDnhamos tinham tive teve tivemos tiveram tivera tiv\xE9ramos tenha tenhamos
+    tenham tivesse tiv\xE9ssemos tivessem tiver tivermos tiverem terei ter\xE1 teremos ter\xE3o teria
+    ter\xEDamos teriam`),
+  spanish: words(`
+    de la que el en los del se las por un para con no una su al lo como m\xE1s pero sus le ya este s\xED
+    porque esta entre cuando muy sin sobre tambi\xE9n me hasta hay donde quien desde todo nos durante
+    todos uno les ni contra otros ese eso ante ellos esto m\xED antes algunos qu\xE9 unos yo otro otras
+    otra \xE9l tanto esa estos mucho quienes nada muchos cual poco ella estar estas algunas algo
+    nosotros mi mis t\xFA te ti tu tus ellas nosotras vosotros vosotras os m\xEDo m\xEDa m\xEDos m\xEDas tuyo
+    tuya tuyos tuyas suyo suya suyos suyas nuestro nuestra nuestros nuestras vuestro vuestra
+    vuestros vuestras esos esas estoy est\xE1s est\xE1 estamos est\xE1is est\xE1n est\xE9 est\xE9s estemos est\xE9is
+    est\xE9n estar\xE9 estar\xE1s estar\xE1 estaremos estar\xE9is estar\xE1n estar\xEDa estar\xEDas estar\xEDamos estar\xEDais
+    estar\xEDan estaba estabas est\xE1bamos estabais estaban estuve estuviste estuvo estuvimos
+    estuvisteis estuvieron estuviera estuvieras estuvi\xE9ramos estuvierais estuvieran estuviese
+    estuvieses estuvi\xE9semos estuvieseis estuviesen estando estad he has ha hemos hab\xE9is han haya
+    hayas hayamos hay\xE1is hayan habr\xE9 habr\xE1s habr\xE1 habremos habr\xE9is habr\xE1n habr\xEDa habr\xEDas habr\xEDamos
+    habr\xEDais habr\xEDan hab\xEDa hab\xEDas hab\xEDamos hab\xEDais hab\xEDan hube hubiste hubo hubimos hubisteis
+    hubieron hubiera hubieras hubi\xE9ramos hubierais hubieran hubiese hubieses hubi\xE9semos hubieseis
+    hubiesen habiendo habido habida habidos habidas soy eres es somos sois son sea seas seamos
+    se\xE1is sean ser\xE9 ser\xE1s ser\xE1 seremos ser\xE9is ser\xE1n ser\xEDa ser\xEDas ser\xEDamos ser\xEDais ser\xEDan era eras
+    \xE9ramos erais eran fui fuiste fue fuimos fuisteis fueron fuera fueras fu\xE9ramos fuerais fueran
+    fuese fueses fu\xE9semos fueseis fuesen tengo tienes tiene tenemos ten\xE9is tienen tenga tengas
+    tengamos teng\xE1is tengan tendr\xE9 tendr\xE1s tendr\xE1 tendremos tendr\xE9is tendr\xE1n tendr\xEDa tendr\xEDas
+    tendr\xEDamos tendr\xEDais tendr\xEDan ten\xEDa ten\xEDas ten\xEDamos ten\xEDais ten\xEDan tuve tuviste tuvo tuvimos
+    tuvisteis tuvieron tuviera tuvieras tuvi\xE9ramos tuvierais tuvieran tuviese tuvieses tuvi\xE9semos
+    tuvieseis tuviesen teniendo tenido tenida tenidos tenidas tened`)
+};
+var TOKEN = /[\p{L}\p{N}]{2,}/gu;
+function keywordTerms(query, max = 24, stopWords = DEFAULT_STOP_WORDS) {
+  const tokens = [
+    ...new Set(query.toLowerCase().normalize("NFC").match(TOKEN) ?? [])
+  ];
+  const counts = Object.entries(stopWords).map(([, stop2]) => [
+    stop2,
+    tokens.filter((t) => stop2.has(t)).length
+  ]);
+  const best = Math.max(0, ...counts.map(([, n]) => n));
+  if (best === 0) return tokens.slice(0, max);
+  const stop = counts.filter(([, n]) => n === best).map(([s]) => s);
+  const terms = tokens.filter((t) => !stop.some((s) => s.has(t)));
+  return (terms.length > 0 ? terms : tokens).slice(0, max);
+}
+__name(keywordTerms, "keywordTerms");
+function hasSearchSyntax(query) {
+  return /"[^"]+"/.test(query) || /(^|\s)-[\p{L}\p{N}]/u.test(query);
+}
+__name(hasSearchSyntax, "hasSearchSyntax");
+function anyTermTsquery(terms) {
+  return terms.map((t) => `'${t.replace(/'/g, "")}'`).join(" | ");
+}
+__name(anyTermTsquery, "anyTermTsquery");
+
 // src/pg-client.ts
 var ADAPTED = /* @__PURE__ */ Symbol.for("@dudousxd/nestjs-agent-rag/pg-client");
 function toPgClient(source) {
@@ -871,6 +950,7 @@ var PgLexicalVectorStore = class extends PgVectorStore {
   tsConfig;
   tsColumn;
   anyTermFallback;
+  stopWords;
   constructor(client, options = {}) {
     super(client, options);
     const config = options.fullText?.config ?? "simple";
@@ -884,6 +964,8 @@ var PgLexicalVectorStore = class extends PgVectorStore {
     this.tsConfig = config;
     this.tsColumn = column;
     this.anyTermFallback = options.fullText?.anyTermFallback ?? true;
+    const stopWords = options.fullText?.stopWords;
+    this.stopWords = stopWords === false ? {} : stopWords ?? DEFAULT_STOP_WORDS;
   }
   schemaStatements() {
     const statements = super.schemaStatements();
@@ -897,19 +979,18 @@ var PgLexicalVectorStore = class extends PgVectorStore {
     if (filterMatchesNothing(options.filter) || query.trim() === "") {
       return [];
     }
-    const all = await this.runTextSearch(`websearch_to_tsquery('${this.tsConfig}', $1)`, query, options);
-    if (all.length > 0 || !this.anyTermFallback) {
-      return all;
-    }
-    const terms = [
-      ...new Set(query.toLowerCase().match(/[\p{L}\p{N}]{2,}/gu) ?? [])
-    ].slice(0, 24);
-    if (terms.length < 2) {
-      return all;
+    if (!this.anyTermFallback || hasSearchSyntax(query)) {
+      const strict = await this.runTextSearch(query, options);
+      if (strict.length > 0 || !this.anyTermFallback) {
+        return strict;
+      }
     }
-    return this.runTextSearch(`to_tsquery('${this.tsConfig}', $1)`, terms.map((term) => `'${term}'`).join(" | "), options);
+    const terms = keywordTerms(query, 24, this.stopWords);
+    return terms.length === 0 ? [] : this.runRankedTermSearch(terms, options);
   }
-  async runTextSearch(tsquery, query, options) {
+  /** `websearch_to_tsquery` on the query as written: every word, quoted phrases, `-exclusions`. */
+  async runTextSearch(query, options) {
+    const tsquery = `websearch_to_tsquery('${this.tsConfig}', $1)`;
     const params = [
       query,
       options.topK
@@ -925,6 +1006,47 @@ var PgLexicalVectorStore = class extends PgVectorStore {
        LIMIT $2`, params);
     return rows.map(toPassage);
   }
+  /**
+  * Rows holding any of `terms`, ranked by the sum of the matched terms' IDF among those rows,
+  * `ln(1 + (N - df + 0.5) / (df + 0.5))`, then `ts_rank_cd`. Each (row, term) match is evaluated
+  * once, and `ts_rank_cd` only for the rows ranked within `topK` (ties included): a question full
+  * of common words can match most of the table.
+  */
+  async runRankedTermSearch(terms, options) {
+    const any = `to_tsquery('${this.tsConfig}', $1)`;
+    const params = [
+      anyTermTsquery(terms),
+      options.topK,
+      terms
+    ];
+    const conditions = [
+      `${this.tsvector()} @@ ${any}`,
+      ...this.whereConditions(options.filter, params)
+    ];
+    const rows = await this.client.query(`WITH __terms AS (
+         SELECT DISTINCT to_tsquery('${this.tsConfig}', quote_literal(t)) AS q
+         FROM unnest($3::text[]) AS t
+       ), __cand AS MATERIALIZED (
+         SELECT id, text, source, metadata, ${this.tsvector()} AS __tsv
+         FROM ${this.table}
+         WHERE ${conditions.join(" AND ")}
+       ), __hits AS MATERIALIZED (
+         SELECT __cand.id AS __id, __terms.q FROM __cand JOIN __terms ON __cand.__tsv @@ __terms.q
+       ), __idf AS (
+         SELECT q, ln(1 + ((SELECT count(*) FROM __cand) - count(*) + 0.5) / (count(*) + 0.5)) AS w
+         FROM __hits GROUP BY q
+       ), __scored AS (
+         SELECT __hits.__id, sum(__idf.w) AS s, rank() OVER (ORDER BY sum(__idf.w) DESC) AS r
+         FROM __hits JOIN __idf USING (q) GROUP BY __hits.__id
+       )
+       SELECT __cand.id, __cand.text, __cand.source, __cand.metadata,
+              __scored.s + ts_rank_cd(__cand.__tsv, ${any}, 32) AS score
+       FROM __scored JOIN __cand ON __cand.id = __scored.__id
+       WHERE __scored.r <= $2
+       ORDER BY __scored.s DESC, ts_rank_cd(__cand.__tsv, ${any}, 32) DESC, __cand.id
+       LIMIT $2`, params);
+    return rows.map(toPassage);
+  }
   /** Must be character-for-character the index expression, or the planner won't use the index. */
   tsvector() {
     return this.tsColumn ?? `to_tsvector('${this.tsConfig}'::regconfig, text)`;
@@ -2546,6 +2668,7 @@ function itemsOf(body) {
 __name(itemsOf, "itemsOf");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  DEFAULT_STOP_WORDS,
   EmbeddingRetriever,
   FilteredRetriever,
   HttpModelError,
@@ -2563,6 +2686,7 @@ __name(itemsOf, "itemsOf");
   RedisVectorStore,
   RerankingRetriever,
   UnsafeRemovalError,
+  anyTermTsquery,
   applyMetadataPatch,
   chunkDocuments,
   chunkIndexOf,
@@ -2574,10 +2698,12 @@ __name(itemsOf, "itemsOf");
   describesRetrieval,
   documentIdOf,
   emitRagRetrieval,
+  hasSearchSyntax,
   ingestChunks,
   ingestDocuments,
   instrumentRetriever,
   isLexicalVectorStore,
+  keywordTerms,
   openAiEmbeddings,
   toPgClient
 });