@dudousxd/nestjs-agent-rag 0.11.0 → 0.12.0

package/dist/index.d.cts

@@ -432,6 +432,34 @@ declare class MemoryVectorStore implements VectorStore {
     search(embedding: number[], options: VectorSearchOptions): Promise<Passage[]>;
 }
 
+/**
+ * Turning a natural-language question into full-text search terms, for the lexical leg of
+ * `PgLexicalVectorStore`.
+ *
+ * With the `simple` text-search configuration (no stemming, no stop words; any language) a question
+ * searched as is either matches nothing (every word required) or, as "any word", ranks chunks full
+ * of "the / of / which / de / que" first. So the query side drops the stop words of the question's
+ * language(s) and keeps the meaningful terms, which the store then ranks by how many of them a chunk
+ * holds, rare terms weighing more (an IDF over the matching rows, BM25-style).
+ */
+/**
+ * Postgres' Snowball stop word lists (`tsearch_data/{english,portuguese,spanish}.stop`), minus
+ * one-letter words (never terms here) and words that carry meaning in a sibling language
+ * ("estado", "sentido").
+ */
+declare const DEFAULT_STOP_WORDS: Readonly<Record<string, ReadonlySet<string>>>;
+/**
+ * The meaningful terms of a question, in order, deduplicated, at most `max`: the stop words of the
+ * language(s) the question is written in are dropped (the language(s) with the most stop words in
+ * it, so "sea"/"son" stay terms in an English question although they are Spanish stop words). A
+ * question made only of stop words keeps them all (there is nothing else to search for).
+ */
+declare function keywordTerms(query: string, max?: number, stopWords?: Readonly<Record<string, ReadonlySet<string>>>): string[];
+/** Whether the user wrote search syntax (a quoted phrase, `-exclusion`) worth honoring as such. */
+declare function hasSearchSyntax(query: string): boolean;
+/** `'a' | 'b' | …` for `to_tsquery` (terms are letter/digit runs, so quoting is enough). */
+declare function anyTermTsquery(terms: string[]): string;
+
 /**
  * The minimal Postgres surface {@link import('./pg-vector-store.js').PgVectorStore} needs — adapt your
  * own `pg` / `postgres.js` client to it, so this package pulls in NO driver (bring your own, like the
@@ -502,11 +530,22 @@ interface PgFullTextOptions {
      */
     column?: string;
     /**
-     * When the question matches nothing as a whole (`websearch_to_tsquery`: every word must appear),
-     * retry with any word (`w1 | w2 | …`, at most 24 distinct words) so a long natural-language
-     * question still finds the chunks sharing its rare terms. Default `true`.
+     * Search natural-language questions by their meaningful terms (default `true`): the question's
+     * stop words are dropped (see {@link PgFullTextOptions.stopWords}) and rows holding any remaining
+     * term (at most 24) are ranked by the sum of the matched terms' IDF among those rows (BM25
+     * without term frequencies), `ts_rank_cd` breaking ties; a question with explicit search syntax
+     * (a quoted phrase, `-word`) is first tried as written. `false`: only `websearch_to_tsquery` on
+     * the query as written (every word must appear).
      */
     anyTermFallback?: boolean;
+    /**
+     * Stop word lists dropped from questions, by language; only the list(s) with the most words in
+     * the question apply, so a word that is a stop word in another language stays a term. Default
+     * {@link DEFAULT_STOP_WORDS} (English, Portuguese, Spanish: Postgres' Snowball lists). `false`
+     * keeps every word. With a language `config` (`english`) Postgres drops that language's stop
+     * words as well.
+     */
+    stopWords?: Readonly<Record<string, ReadonlySet<string>>> | false;
 }
 interface PgVectorStoreOptions {
     /** Table name. Default `agent_rag_chunks`. */
@@ -669,9 +708,10 @@ declare class PgVectorStore implements VectorStore {
  * `LexicalRetriever` and `HybridRetriever` work on Postgres exactly as on RediSearch — the dense leg
  * over pgvector, the lexical leg over a `tsvector`, both on the same rows and ids.
  *
- * Ranking is `ts_rank_cd` (cover density). The query is parsed with `websearch_to_tsquery` (quoted
- * phrases, `-exclusions`, `or`), which requires every word; when that matches nothing it retries
- * with any word (see {@link PgFullTextOptions.anyTermFallback}).
+ * A question is searched by its meaningful terms: its stop words dropped, rows holding any remaining
+ * term ranked by IDF-weighted coverage, `ts_rank_cd` (cover density) breaking ties (see
+ * {@link PgFullTextOptions.anyTermFallback}); explicit syntax (quoted phrases, `-exclusions`) goes
+ * through `websearch_to_tsquery` first.
  *
  * The `tsvector` is either a column you own ({@link PgFullTextOptions.column}) or the expression
  * `to_tsvector(config, text)`, for which {@link PgVectorStore.schemaStatements} adds a GIN
@@ -682,12 +722,21 @@ declare class PgLexicalVectorStore extends PgVectorStore implements LexicalVecto
     private readonly tsConfig;
     private readonly tsColumn;
     private readonly anyTermFallback;
+    private readonly stopWords;
     constructor(client: PgClientSource, options?: PgVectorStoreOptions & {
         fullText?: PgFullTextOptions;
     });
     schemaStatements(): string[];
     searchText(query: string, options: VectorSearchOptions): Promise<Passage[]>;
+    /** `websearch_to_tsquery` on the query as written: every word, quoted phrases, `-exclusions`. */
     private runTextSearch;
+    /**
+     * Rows holding any of `terms`, ranked by the sum of the matched terms' IDF among those rows,
+     * `ln(1 + (N - df + 0.5) / (df + 0.5))`, then `ts_rank_cd`. Each (row, term) match is evaluated
+     * once, and `ts_rank_cd` only for the rows ranked within `topK` (ties included): a question full
+     * of common words can match most of the table.
+     */
+    private runRankedTermSearch;
     /** Must be character-for-character the index expression, or the planner won't use the index. */
     private tsvector;
 }
@@ -1476,4 +1525,4 @@ declare class HttpReranker implements Reranker {
     rerank(query: string, passages: Passage[], options?: RerankOptions): Promise<Passage[]>;
 }
 
-export { type ChunkOptions, type ChunkRecord, type DescribesRetrieval, type DrizzleLike, EmbeddingRetriever, type EmbeddingRetrieverOptions, FilteredRetriever, HttpModelError, HttpReranker, type HttpRerankerOptions, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, type InstrumentRetrieverOptions, KeywordRetriever, type KeywordRetrieverOptions, LexicalRetriever, type LexicalVectorStore, type ListChunksOptions, MemoryVectorStore, type MetadataPatch, type NodePgQueryable, type OpenAiEmbeddingsOptions, type PgClient, type PgClientSource, type PgFullTextOptions, PgLexicalVectorStore, PgVectorStore, type PgVectorStoreOptions, type PostgresJsSql, RAG_DIAGNOSTIC_LIB, RAG_RETRIEVAL_CHANNEL, RAG_RETRIEVAL_EVENT, type RagRetrievalEvent, type RedisSearchClient, RedisVectorSchemaMismatchError, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalDescriptor, type RetrievalTool, type RetrievalToolOptions, type RetrieverKind, type StoredChunk, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, type VectorStoreKind, applyMetadataPatch, chunkDocuments, chunkIndexOf, chunkText, createRetrievalTool, describeRetrieval, describeSharedSource, describeSource, describesRetrieval, documentIdOf, emitRagRetrieval, ingestChunks, ingestDocuments, instrumentRetriever, isLexicalVectorStore, openAiEmbeddings, toPgClient };
+export { type ChunkOptions, type ChunkRecord, DEFAULT_STOP_WORDS, type DescribesRetrieval, type DrizzleLike, EmbeddingRetriever, type EmbeddingRetrieverOptions, FilteredRetriever, HttpModelError, HttpReranker, type HttpRerankerOptions, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, type InstrumentRetrieverOptions, KeywordRetriever, type KeywordRetrieverOptions, LexicalRetriever, type LexicalVectorStore, type ListChunksOptions, MemoryVectorStore, type MetadataPatch, type NodePgQueryable, type OpenAiEmbeddingsOptions, type PgClient, type PgClientSource, type PgFullTextOptions, PgLexicalVectorStore, PgVectorStore, type PgVectorStoreOptions, type PostgresJsSql, RAG_DIAGNOSTIC_LIB, RAG_RETRIEVAL_CHANNEL, RAG_RETRIEVAL_EVENT, type RagRetrievalEvent, type RedisSearchClient, RedisVectorSchemaMismatchError, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalDescriptor, type RetrievalTool, type RetrievalToolOptions, type RetrieverKind, type StoredChunk, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, type VectorStoreKind, anyTermTsquery, applyMetadataPatch, chunkDocuments, chunkIndexOf, chunkText, createRetrievalTool, describeRetrieval, describeSharedSource, describeSource, describesRetrieval, documentIdOf, emitRagRetrieval, hasSearchSyntax, ingestChunks, ingestDocuments, instrumentRetriever, isLexicalVectorStore, keywordTerms, openAiEmbeddings, toPgClient };

package/dist/index.d.ts

@@ -432,6 +432,34 @@ declare class MemoryVectorStore implements VectorStore {
     search(embedding: number[], options: VectorSearchOptions): Promise<Passage[]>;
 }
 
+/**
+ * Turning a natural-language question into full-text search terms, for the lexical leg of
+ * `PgLexicalVectorStore`.
+ *
+ * With the `simple` text-search configuration (no stemming, no stop words; any language) a question
+ * searched as is either matches nothing (every word required) or, as "any word", ranks chunks full
+ * of "the / of / which / de / que" first. So the query side drops the stop words of the question's
+ * language(s) and keeps the meaningful terms, which the store then ranks by how many of them a chunk
+ * holds, rare terms weighing more (an IDF over the matching rows, BM25-style).
+ */
+/**
+ * Postgres' Snowball stop word lists (`tsearch_data/{english,portuguese,spanish}.stop`), minus
+ * one-letter words (never terms here) and words that carry meaning in a sibling language
+ * ("estado", "sentido").
+ */
+declare const DEFAULT_STOP_WORDS: Readonly<Record<string, ReadonlySet<string>>>;
+/**
+ * The meaningful terms of a question, in order, deduplicated, at most `max`: the stop words of the
+ * language(s) the question is written in are dropped (the language(s) with the most stop words in
+ * it, so "sea"/"son" stay terms in an English question although they are Spanish stop words). A
+ * question made only of stop words keeps them all (there is nothing else to search for).
+ */
+declare function keywordTerms(query: string, max?: number, stopWords?: Readonly<Record<string, ReadonlySet<string>>>): string[];
+/** Whether the user wrote search syntax (a quoted phrase, `-exclusion`) worth honoring as such. */
+declare function hasSearchSyntax(query: string): boolean;
+/** `'a' | 'b' | …` for `to_tsquery` (terms are letter/digit runs, so quoting is enough). */
+declare function anyTermTsquery(terms: string[]): string;
+
 /**
  * The minimal Postgres surface {@link import('./pg-vector-store.js').PgVectorStore} needs — adapt your
  * own `pg` / `postgres.js` client to it, so this package pulls in NO driver (bring your own, like the
@@ -502,11 +530,22 @@ interface PgFullTextOptions {
      */
     column?: string;
     /**
-     * When the question matches nothing as a whole (`websearch_to_tsquery`: every word must appear),
-     * retry with any word (`w1 | w2 | …`, at most 24 distinct words) so a long natural-language
-     * question still finds the chunks sharing its rare terms. Default `true`.
+     * Search natural-language questions by their meaningful terms (default `true`): the question's
+     * stop words are dropped (see {@link PgFullTextOptions.stopWords}) and rows holding any remaining
+     * term (at most 24) are ranked by the sum of the matched terms' IDF among those rows (BM25
+     * without term frequencies), `ts_rank_cd` breaking ties; a question with explicit search syntax
+     * (a quoted phrase, `-word`) is first tried as written. `false`: only `websearch_to_tsquery` on
+     * the query as written (every word must appear).
      */
     anyTermFallback?: boolean;
+    /**
+     * Stop word lists dropped from questions, by language; only the list(s) with the most words in
+     * the question apply, so a word that is a stop word in another language stays a term. Default
+     * {@link DEFAULT_STOP_WORDS} (English, Portuguese, Spanish: Postgres' Snowball lists). `false`
+     * keeps every word. With a language `config` (`english`) Postgres drops that language's stop
+     * words as well.
+     */
+    stopWords?: Readonly<Record<string, ReadonlySet<string>>> | false;
 }
 interface PgVectorStoreOptions {
     /** Table name. Default `agent_rag_chunks`. */
@@ -669,9 +708,10 @@ declare class PgVectorStore implements VectorStore {
  * `LexicalRetriever` and `HybridRetriever` work on Postgres exactly as on RediSearch — the dense leg
  * over pgvector, the lexical leg over a `tsvector`, both on the same rows and ids.
  *
- * Ranking is `ts_rank_cd` (cover density). The query is parsed with `websearch_to_tsquery` (quoted
- * phrases, `-exclusions`, `or`), which requires every word; when that matches nothing it retries
- * with any word (see {@link PgFullTextOptions.anyTermFallback}).
+ * A question is searched by its meaningful terms: its stop words dropped, rows holding any remaining
+ * term ranked by IDF-weighted coverage, `ts_rank_cd` (cover density) breaking ties (see
+ * {@link PgFullTextOptions.anyTermFallback}); explicit syntax (quoted phrases, `-exclusions`) goes
+ * through `websearch_to_tsquery` first.
  *
  * The `tsvector` is either a column you own ({@link PgFullTextOptions.column}) or the expression
  * `to_tsvector(config, text)`, for which {@link PgVectorStore.schemaStatements} adds a GIN
@@ -682,12 +722,21 @@ declare class PgLexicalVectorStore extends PgVectorStore implements LexicalVecto
     private readonly tsConfig;
     private readonly tsColumn;
     private readonly anyTermFallback;
+    private readonly stopWords;
     constructor(client: PgClientSource, options?: PgVectorStoreOptions & {
         fullText?: PgFullTextOptions;
     });
     schemaStatements(): string[];
     searchText(query: string, options: VectorSearchOptions): Promise<Passage[]>;
+    /** `websearch_to_tsquery` on the query as written: every word, quoted phrases, `-exclusions`. */
     private runTextSearch;
+    /**
+     * Rows holding any of `terms`, ranked by the sum of the matched terms' IDF among those rows,
+     * `ln(1 + (N - df + 0.5) / (df + 0.5))`, then `ts_rank_cd`. Each (row, term) match is evaluated
+     * once, and `ts_rank_cd` only for the rows ranked within `topK` (ties included): a question full
+     * of common words can match most of the table.
+     */
+    private runRankedTermSearch;
     /** Must be character-for-character the index expression, or the planner won't use the index. */
     private tsvector;
 }
@@ -1476,4 +1525,4 @@ declare class HttpReranker implements Reranker {
     rerank(query: string, passages: Passage[], options?: RerankOptions): Promise<Passage[]>;
 }
 
-export { type ChunkOptions, type ChunkRecord, type DescribesRetrieval, type DrizzleLike, EmbeddingRetriever, type EmbeddingRetrieverOptions, FilteredRetriever, HttpModelError, HttpReranker, type HttpRerankerOptions, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, type InstrumentRetrieverOptions, KeywordRetriever, type KeywordRetrieverOptions, LexicalRetriever, type LexicalVectorStore, type ListChunksOptions, MemoryVectorStore, type MetadataPatch, type NodePgQueryable, type OpenAiEmbeddingsOptions, type PgClient, type PgClientSource, type PgFullTextOptions, PgLexicalVectorStore, PgVectorStore, type PgVectorStoreOptions, type PostgresJsSql, RAG_DIAGNOSTIC_LIB, RAG_RETRIEVAL_CHANNEL, RAG_RETRIEVAL_EVENT, type RagRetrievalEvent, type RedisSearchClient, RedisVectorSchemaMismatchError, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalDescriptor, type RetrievalTool, type RetrievalToolOptions, type RetrieverKind, type StoredChunk, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, type VectorStoreKind, applyMetadataPatch, chunkDocuments, chunkIndexOf, chunkText, createRetrievalTool, describeRetrieval, describeSharedSource, describeSource, describesRetrieval, documentIdOf, emitRagRetrieval, ingestChunks, ingestDocuments, instrumentRetriever, isLexicalVectorStore, openAiEmbeddings, toPgClient };
+export { type ChunkOptions, type ChunkRecord, DEFAULT_STOP_WORDS, type DescribesRetrieval, type DrizzleLike, EmbeddingRetriever, type EmbeddingRetrieverOptions, FilteredRetriever, HttpModelError, HttpReranker, type HttpRerankerOptions, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, type InstrumentRetrieverOptions, KeywordRetriever, type KeywordRetrieverOptions, LexicalRetriever, type LexicalVectorStore, type ListChunksOptions, MemoryVectorStore, type MetadataPatch, type NodePgQueryable, type OpenAiEmbeddingsOptions, type PgClient, type PgClientSource, type PgFullTextOptions, PgLexicalVectorStore, PgVectorStore, type PgVectorStoreOptions, type PostgresJsSql, RAG_DIAGNOSTIC_LIB, RAG_RETRIEVAL_CHANNEL, RAG_RETRIEVAL_EVENT, type RagRetrievalEvent, type RedisSearchClient, RedisVectorSchemaMismatchError, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalDescriptor, type RetrievalTool, type RetrievalToolOptions, type RetrieverKind, type StoredChunk, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, type VectorStoreKind, anyTermTsquery, applyMetadataPatch, chunkDocuments, chunkIndexOf, chunkText, createRetrievalTool, describeRetrieval, describeSharedSource, describeSource, describesRetrieval, documentIdOf, emitRagRetrieval, hasSearchSyntax, ingestChunks, ingestDocuments, instrumentRetriever, isLexicalVectorStore, keywordTerms, openAiEmbeddings, toPgClient };

package/dist/index.js

@@ -356,6 +356,81 @@ function cosineSimilarity(a, b) {
 }
 __name(cosineSimilarity, "cosineSimilarity");
 
+// src/lexical-query.ts
+var words = /* @__PURE__ */ __name((s) => new Set(s.split(/\s+/).filter(Boolean)), "words");
+var DEFAULT_STOP_WORDS = {
+  english: words(`
+    me my myself we our ours ourselves you your yours yourself yourselves he him his himself she
+    her hers herself it its itself they them their theirs themselves what which who whom this that
+    these those am is are was were be been being have has had having do does did doing an the and
+    but if or because as until while of at by for with about against between into through during
+    before after above below to from up down in out on off over under again further then once here
+    there when where why how all any both each few more most other some such no nor not only own
+    same so than too very can will just don should now`),
+  portuguese: words(`
+    de que do da em um para com n\xE3o uma os no se na por mais as dos como mas ao ele das seu sua ou
+    quando muito nos j\xE1 eu tamb\xE9m s\xF3 pelo pela at\xE9 isso ela entre depois sem mesmo aos seus quem
+    nas me esse eles voc\xEA essa num nem suas meu \xE0s minha numa pelos elas qual n\xF3s lhe deles essas
+    esses pelas este dele tu te voc\xEAs vos lhes meus minhas teu tua teus tuas nosso nossa nossos
+    nossas dela delas esta estes estas aquele aquela aqueles aquelas isto aquilo estou est\xE1 estamos
+    est\xE3o estive esteve estivemos estiveram estava est\xE1vamos estavam estivera estiv\xE9ramos esteja
+    estejamos estejam estivesse estiv\xE9ssemos estivessem estiver estivermos estiverem hei h\xE1 havemos
+    h\xE3o houve houvemos houveram houvera houv\xE9ramos haja hajamos hajam houvesse houv\xE9ssemos
+    houvessem houver houvermos houverem houverei houver\xE1 houveremos houver\xE3o houveria houver\xEDamos
+    houveriam sou somos s\xE3o era \xE9ramos eram fui foi fomos foram fora f\xF4ramos seja sejamos sejam
+    fosse f\xF4ssemos fossem for formos forem serei ser\xE1 seremos ser\xE3o seria ser\xEDamos seriam tenho tem
+    temos t\xE9m tinha t\xEDnhamos tinham tive teve tivemos tiveram tivera tiv\xE9ramos tenha tenhamos
+    tenham tivesse tiv\xE9ssemos tivessem tiver tivermos tiverem terei ter\xE1 teremos ter\xE3o teria
+    ter\xEDamos teriam`),
+  spanish: words(`
+    de la que el en los del se las por un para con no una su al lo como m\xE1s pero sus le ya este s\xED
+    porque esta entre cuando muy sin sobre tambi\xE9n me hasta hay donde quien desde todo nos durante
+    todos uno les ni contra otros ese eso ante ellos esto m\xED antes algunos qu\xE9 unos yo otro otras
+    otra \xE9l tanto esa estos mucho quienes nada muchos cual poco ella estar estas algunas algo
+    nosotros mi mis t\xFA te ti tu tus ellas nosotras vosotros vosotras os m\xEDo m\xEDa m\xEDos m\xEDas tuyo
+    tuya tuyos tuyas suyo suya suyos suyas nuestro nuestra nuestros nuestras vuestro vuestra
+    vuestros vuestras esos esas estoy est\xE1s est\xE1 estamos est\xE1is est\xE1n est\xE9 est\xE9s estemos est\xE9is
+    est\xE9n estar\xE9 estar\xE1s estar\xE1 estaremos estar\xE9is estar\xE1n estar\xEDa estar\xEDas estar\xEDamos estar\xEDais
+    estar\xEDan estaba estabas est\xE1bamos estabais estaban estuve estuviste estuvo estuvimos
+    estuvisteis estuvieron estuviera estuvieras estuvi\xE9ramos estuvierais estuvieran estuviese
+    estuvieses estuvi\xE9semos estuvieseis estuviesen estando estad he has ha hemos hab\xE9is han haya
+    hayas hayamos hay\xE1is hayan habr\xE9 habr\xE1s habr\xE1 habremos habr\xE9is habr\xE1n habr\xEDa habr\xEDas habr\xEDamos
+    habr\xEDais habr\xEDan hab\xEDa hab\xEDas hab\xEDamos hab\xEDais hab\xEDan hube hubiste hubo hubimos hubisteis
+    hubieron hubiera hubieras hubi\xE9ramos hubierais hubieran hubiese hubieses hubi\xE9semos hubieseis
+    hubiesen habiendo habido habida habidos habidas soy eres es somos sois son sea seas seamos
+    se\xE1is sean ser\xE9 ser\xE1s ser\xE1 seremos ser\xE9is ser\xE1n ser\xEDa ser\xEDas ser\xEDamos ser\xEDais ser\xEDan era eras
+    \xE9ramos erais eran fui fuiste fue fuimos fuisteis fueron fuera fueras fu\xE9ramos fuerais fueran
+    fuese fueses fu\xE9semos fueseis fuesen tengo tienes tiene tenemos ten\xE9is tienen tenga tengas
+    tengamos teng\xE1is tengan tendr\xE9 tendr\xE1s tendr\xE1 tendremos tendr\xE9is tendr\xE1n tendr\xEDa tendr\xEDas
+    tendr\xEDamos tendr\xEDais tendr\xEDan ten\xEDa ten\xEDas ten\xEDamos ten\xEDais ten\xEDan tuve tuviste tuvo tuvimos
+    tuvisteis tuvieron tuviera tuvieras tuvi\xE9ramos tuvierais tuvieran tuviese tuvieses tuvi\xE9semos
+    tuvieseis tuviesen teniendo tenido tenida tenidos tenidas tened`)
+};
+var TOKEN = /[\p{L}\p{N}]{2,}/gu;
+function keywordTerms(query, max = 24, stopWords = DEFAULT_STOP_WORDS) {
+  const tokens = [
+    ...new Set(query.toLowerCase().normalize("NFC").match(TOKEN) ?? [])
+  ];
+  const counts = Object.entries(stopWords).map(([, stop2]) => [
+    stop2,
+    tokens.filter((t) => stop2.has(t)).length
+  ]);
+  const best = Math.max(0, ...counts.map(([, n]) => n));
+  if (best === 0) return tokens.slice(0, max);
+  const stop = counts.filter(([, n]) => n === best).map(([s]) => s);
+  const terms = tokens.filter((t) => !stop.some((s) => s.has(t)));
+  return (terms.length > 0 ? terms : tokens).slice(0, max);
+}
+__name(keywordTerms, "keywordTerms");
+function hasSearchSyntax(query) {
+  return /"[^"]+"/.test(query) || /(^|\s)-[\p{L}\p{N}]/u.test(query);
+}
+__name(hasSearchSyntax, "hasSearchSyntax");
+function anyTermTsquery(terms) {
+  return terms.map((t) => `'${t.replace(/'/g, "")}'`).join(" | ");
+}
+__name(anyTermTsquery, "anyTermTsquery");
+
 // src/pg-client.ts
 var ADAPTED = /* @__PURE__ */ Symbol.for("@dudousxd/nestjs-agent-rag/pg-client");
 function toPgClient(source) {
@@ -814,6 +889,7 @@ var PgLexicalVectorStore = class extends PgVectorStore {
   tsConfig;
   tsColumn;
   anyTermFallback;
+  stopWords;
   constructor(client, options = {}) {
     super(client, options);
     const config = options.fullText?.config ?? "simple";
@@ -827,6 +903,8 @@ var PgLexicalVectorStore = class extends PgVectorStore {
     this.tsConfig = config;
     this.tsColumn = column;
     this.anyTermFallback = options.fullText?.anyTermFallback ?? true;
+    const stopWords = options.fullText?.stopWords;
+    this.stopWords = stopWords === false ? {} : stopWords ?? DEFAULT_STOP_WORDS;
   }
   schemaStatements() {
     const statements = super.schemaStatements();
@@ -840,19 +918,18 @@ var PgLexicalVectorStore = class extends PgVectorStore {
     if (filterMatchesNothing(options.filter) || query.trim() === "") {
       return [];
     }
-    const all = await this.runTextSearch(`websearch_to_tsquery('${this.tsConfig}', $1)`, query, options);
-    if (all.length > 0 || !this.anyTermFallback) {
-      return all;
-    }
-    const terms = [
-      ...new Set(query.toLowerCase().match(/[\p{L}\p{N}]{2,}/gu) ?? [])
-    ].slice(0, 24);
-    if (terms.length < 2) {
-      return all;
+    if (!this.anyTermFallback || hasSearchSyntax(query)) {
+      const strict = await this.runTextSearch(query, options);
+      if (strict.length > 0 || !this.anyTermFallback) {
+        return strict;
+      }
     }
-    return this.runTextSearch(`to_tsquery('${this.tsConfig}', $1)`, terms.map((term) => `'${term}'`).join(" | "), options);
+    const terms = keywordTerms(query, 24, this.stopWords);
+    return terms.length === 0 ? [] : this.runRankedTermSearch(terms, options);
   }
-  async runTextSearch(tsquery, query, options) {
+  /** `websearch_to_tsquery` on the query as written: every word, quoted phrases, `-exclusions`. */
+  async runTextSearch(query, options) {
+    const tsquery = `websearch_to_tsquery('${this.tsConfig}', $1)`;
     const params = [
       query,
       options.topK
@@ -868,6 +945,47 @@ var PgLexicalVectorStore = class extends PgVectorStore {
        LIMIT $2`, params);
     return rows.map(toPassage);
   }
+  /**
+  * Rows holding any of `terms`, ranked by the sum of the matched terms' IDF among those rows,
+  * `ln(1 + (N - df + 0.5) / (df + 0.5))`, then `ts_rank_cd`. Each (row, term) match is evaluated
+  * once, and `ts_rank_cd` only for the rows ranked within `topK` (ties included): a question full
+  * of common words can match most of the table.
+  */
+  async runRankedTermSearch(terms, options) {
+    const any = `to_tsquery('${this.tsConfig}', $1)`;
+    const params = [
+      anyTermTsquery(terms),
+      options.topK,
+      terms
+    ];
+    const conditions = [
+      `${this.tsvector()} @@ ${any}`,
+      ...this.whereConditions(options.filter, params)
+    ];
+    const rows = await this.client.query(`WITH __terms AS (
+         SELECT DISTINCT to_tsquery('${this.tsConfig}', quote_literal(t)) AS q
+         FROM unnest($3::text[]) AS t
+       ), __cand AS MATERIALIZED (
+         SELECT id, text, source, metadata, ${this.tsvector()} AS __tsv
+         FROM ${this.table}
+         WHERE ${conditions.join(" AND ")}
+       ), __hits AS MATERIALIZED (
+         SELECT __cand.id AS __id, __terms.q FROM __cand JOIN __terms ON __cand.__tsv @@ __terms.q
+       ), __idf AS (
+         SELECT q, ln(1 + ((SELECT count(*) FROM __cand) - count(*) + 0.5) / (count(*) + 0.5)) AS w
+         FROM __hits GROUP BY q
+       ), __scored AS (
+         SELECT __hits.__id, sum(__idf.w) AS s, rank() OVER (ORDER BY sum(__idf.w) DESC) AS r
+         FROM __hits JOIN __idf USING (q) GROUP BY __hits.__id
+       )
+       SELECT __cand.id, __cand.text, __cand.source, __cand.metadata,
+              __scored.s + ts_rank_cd(__cand.__tsv, ${any}, 32) AS score
+       FROM __scored JOIN __cand ON __cand.id = __scored.__id
+       WHERE __scored.r <= $2
+       ORDER BY __scored.s DESC, ts_rank_cd(__cand.__tsv, ${any}, 32) DESC, __cand.id
+       LIMIT $2`, params);
+    return rows.map(toPassage);
+  }
   /** Must be character-for-character the index expression, or the planner won't use the index. */
   tsvector() {
     return this.tsColumn ?? `to_tsvector('${this.tsConfig}'::regconfig, text)`;
@@ -2488,6 +2606,7 @@ function itemsOf(body) {
 }
 __name(itemsOf, "itemsOf");
 export {
+  DEFAULT_STOP_WORDS,
   EmbeddingRetriever,
   FilteredRetriever,
   HttpModelError,
@@ -2505,6 +2624,7 @@ export {
   RedisVectorStore,
   RerankingRetriever,
   UnsafeRemovalError,
+  anyTermTsquery,
   applyMetadataPatch,
   chunkDocuments,
   chunkIndexOf,
@@ -2516,10 +2636,12 @@ export {
   describesRetrieval,
   documentIdOf,
   emitRagRetrieval,
+  hasSearchSyntax,
   ingestChunks,
   ingestDocuments,
   instrumentRetriever,
   isLexicalVectorStore,
+  keywordTerms,
   openAiEmbeddings,
   toPgClient
 };