@dudousxd/nestjs-agent-rag 0.4.1 → 0.5.0

package/dist/index.d.ts

@@ -54,6 +54,37 @@ interface VectorStore {
      */
     listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
 }
+/**
+ * An **optional** capability a {@link VectorStore} may also implement: full-text (lexical) search
+ * over the chunk text it already stores, using the backing engine's own keyword index — RediSearch's
+ * BM25 over the `TEXT` field, say. Its point is the multi-process deployment: when ingestion runs on
+ * a worker and search runs on an API pod, the API process never sees the chunks, so the in-process
+ * {@link import('./keyword-retriever.js').KeywordRetriever} has nothing to index. A store that
+ * implements this needs no second index, no corpus in JS heap, and no refresh window — a chunk is
+ * lexically findable the moment it is upserted.
+ *
+ * Wrap it as a core `Retriever` with {@link import('./lexical-retriever.js').LexicalRetriever}.
+ *
+ * Implementations MUST apply `options.filter` with exactly the same semantics as
+ * {@link VectorStore.search} — including the empty-array deny primitive — because the filter is
+ * routinely an access-control boundary, not a hint.
+ */
+interface LexicalVectorStore extends VectorStore {
+    /**
+     * Rank stored chunks against a natural-language `query` by keyword relevance. `options.filter`
+     * narrows exactly as in {@link VectorStore.search}. The returned `score` is the engine's own
+     * relevance score (BM25), which shares no scale with `search`'s cosine similarity — fuse the two
+     * with {@link import('./hybrid-retriever.js').HybridRetriever}, whose RRF is rank-based and so
+     * needs no common scale.
+     */
+    searchText(query: string, options: VectorSearchOptions): Promise<Passage[]>;
+}
+/**
+ * Does this store carry the optional {@link LexicalVectorStore} capability? Lets a consumer pick the
+ * store-backed lexical leg when available and fall back to `KeywordRetriever` (or to dense-only)
+ * when it is not, without knowing which adapter it was handed.
+ */
+declare function isLexicalVectorStore(store: VectorStore): store is LexicalVectorStore;
 /** A distinct source document as seen by the index — its id plus a representative chunk's metadata. */
 interface IndexedDocument {
     id: string;
@@ -93,6 +124,24 @@ interface PgVectorStoreOptions {
  * A pgvector-backed {@link VectorStore} — the production reference adapter. Cosine distance via the
  * `<=>` operator over an HNSW index; metadata in a `jsonb` column filtered with `@>`. Call
  * {@link PgVectorStore.ensureSchema} once at boot to create the extension, table, and index.
+ *
+ * This store does **not** implement the optional
+ * {@link import('./vector-store.js').LexicalVectorStore} capability, unlike
+ * {@link import('./redis-vector-store.js').RedisVectorStore}. That is a deliberate omission, not an
+ * oversight: RediSearch already indexes the chunk text (the schema declares it `TEXT`), so BM25 is
+ * free there, whereas the table below indexes nothing lexically — `text` is a plain column with no
+ * `tsvector` and no GIN index. Adding one is not a transparent upgrade:
+ *
+ * - it is new DDL in `ensureSchema`, and `CREATE INDEX` (non-concurrently, as this method must, since
+ *   `CONCURRENTLY` cannot run in a transaction) takes a write lock on an already-populated chunks
+ *   table — a boot-time stall proportional to corpus size for every existing deployment;
+ * - it forces a text-search-configuration choice (`english` vs `simple` vs …) that must match between
+ *   the index expression and every query, or Postgres silently ignores the index and sequentially
+ *   scans the corpus — a failure mode that stays *correct* while quietly getting slower with scale.
+ *
+ * Both want a migration a consumer runs and observes, not a side effect of upgrading the library. The
+ * capability interface is open, so a `PgLexicalVectorStore` (or an option here, gated on an
+ * explicitly-created index) can be added later without changing anything else.
  */
 declare class PgVectorStore implements VectorStore {
     private readonly client;
@@ -157,19 +206,30 @@ interface RedisVectorStoreOptions {
      * front, so only keys listed here can be passed in `search`'s `filter`. Default none.
      */
     filterableFields?: string[];
+    /**
+     * RediSearch scorer used by {@link RedisVectorStore.searchText}. Default `BM25` — the standard
+     * lexical ranking, and the same family the in-process `KeywordRetriever` implements. Override for
+     * a server that names it differently (`BM25STD` on newer builds) or to pick `TFIDF`/`DISMAX`.
+     */
+    lexicalScorer?: string;
 }
 /**
  * A RediSearch-backed {@link VectorStore} — HNSW + cosine over Redis, the ecosystem-native option
  * for anyone already running Redis (see `-transport-redis`). Chunks are hashes under `prefix`, the
  * embedding a FLOAT32 vector field; `search` is a KNN query. Call {@link RedisVectorStore.ensureSchema}
  * once at boot to create the index.
+ *
+ * The chunk text is declared as a RediSearch `TEXT` field, so this store also satisfies
+ * {@link LexicalVectorStore}: {@link RedisVectorStore.searchText} is BM25 over that same index —
+ * the lexical half of hybrid search with no second index to build, feed, or invalidate.
  */
-declare class RedisVectorStore implements VectorStore {
+declare class RedisVectorStore implements LexicalVectorStore {
     private readonly client;
     private readonly index;
     private readonly prefix;
     private readonly dimensions;
     private readonly filterableFields;
+    private readonly lexicalScorer;
     constructor(client: RedisSearchClient, options?: RedisVectorStoreOptions);
     /**
      * Idempotent, and drift-aware: create the HNSW/cosine index if `FT.INFO` says it doesn't exist —
@@ -202,6 +262,18 @@ declare class RedisVectorStore implements VectorStore {
     remove(documentId: string): Promise<void>;
     listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
     search(embedding: number[], options: VectorSearchOptions): Promise<Passage[]>;
+    /**
+     * Lexical (BM25) search over the same index `search` uses — the {@link LexicalVectorStore} half of
+     * hybrid retrieval, served by RediSearch's own keyword index rather than an in-process copy of the
+     * corpus. `options.filter` is ANDed onto the text clause with the identical TAG semantics as
+     * `search` (array = OR, empty array = deny), so a lexical hit can never escape a filter a vector
+     * hit would have respected.
+     *
+     * A query with no searchable terms — empty, whitespace, or pure punctuation — returns `[]`. That is
+     * deliberate: RediSearch's `*` would match the entire corpus, which is the opposite of what a
+     * relevance query with nothing to be relevant to should do.
+     */
+    searchText(query: string, options: VectorSearchOptions): Promise<Passage[]>;
 }
 
 /**
@@ -291,6 +363,35 @@ declare class KeywordRetriever implements Retriever {
     retrieve(query: string, options?: RetrieveOptions): Promise<Passage[]>;
 }
 
+/**
+ * The keyword half of hybrid search, served by the store's own full-text index — the mirror of
+ * {@link import('./embedding-retriever.js').EmbeddingRetriever}, which serves the dense half. It
+ * holds no state: every query goes to the store, so any process that can reach the store can run
+ * lexical search, whether or not it did the ingestion.
+ *
+ * Compose it exactly like the dense leg:
+ *
+ * ```ts
+ * const retriever = new HybridRetriever([
+ *   new EmbeddingRetriever(embedder, store),
+ *   new LexicalRetriever(store),
+ * ]);
+ * ```
+ *
+ * Both legs read the same chunks from the same index, so their ids line up by construction and
+ * `HybridRetriever`'s RRF fuses them without either side needing a score on a common scale.
+ *
+ * Prefer this over {@link import('./keyword-retriever.js').KeywordRetriever} whenever the store
+ * supports it (see {@link import('./vector-store.js').isLexicalVectorStore}) and ingestion and
+ * search can run in different processes; `KeywordRetriever` remains the answer for `MemoryVectorStore`
+ * and single-process deployments.
+ */
+declare class LexicalRetriever implements Retriever {
+    private readonly store;
+    constructor(store: LexicalVectorStore);
+    retrieve(query: string, options?: RetrieveOptions): Promise<Passage[]>;
+}
+
 interface HybridRetrieverOptions {
     /** RRF constant — larger flattens the rank weighting. Default 60 (the value from the original paper). */
     k?: number;
@@ -369,4 +470,4 @@ interface RetrievalTool {
  */
 declare function createRetrievalTool(retriever: Retriever, options?: RetrievalToolOptions): RetrievalTool;
 
-export { type ChunkOptions, type ChunkRecord, EmbeddingRetriever, FilteredRetriever, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, KeywordRetriever, type KeywordRetrieverOptions, MemoryVectorStore, type PgClient, PgVectorStore, type PgVectorStoreOptions, type RedisSearchClient, RedisVectorSchemaMismatchError, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalTool, type RetrievalToolOptions, type VectorRecord, type VectorSearchOptions, type VectorStore, chunkDocuments, chunkText, createRetrievalTool, documentIdOf, ingestChunks, ingestDocuments };
+export { type ChunkOptions, type ChunkRecord, EmbeddingRetriever, FilteredRetriever, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, KeywordRetriever, type KeywordRetrieverOptions, LexicalRetriever, type LexicalVectorStore, MemoryVectorStore, type PgClient, PgVectorStore, type PgVectorStoreOptions, type RedisSearchClient, RedisVectorSchemaMismatchError, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalTool, type RetrievalToolOptions, type VectorRecord, type VectorSearchOptions, type VectorStore, chunkDocuments, chunkText, createRetrievalTool, documentIdOf, ingestChunks, ingestDocuments, isLexicalVectorStore };

package/dist/index.js

@@ -40,6 +40,10 @@ function chunkText(text, options = {}) {
 __name(chunkText, "chunkText");
 
 // src/vector-store.ts
+function isLexicalVectorStore(store) {
+  return typeof store.searchText === "function";
+}
+__name(isLexicalVectorStore, "isLexicalVectorStore");
 function documentIdOf(chunkId) {
   return chunkId.replace(/#\d+$/, "");
 }
@@ -291,12 +295,14 @@ var RedisVectorStore = class {
   prefix;
   dimensions;
   filterableFields;
+  lexicalScorer;
   constructor(client, options = {}) {
     this.client = client;
     this.index = options.index ?? "agent_rag_idx";
     this.prefix = options.prefix ?? "agent_rag:";
     this.dimensions = options.dimensions ?? 1536;
     this.filterableFields = options.filterableFields ?? [];
+    this.lexicalScorer = options.lexicalScorer ?? "BM25";
   }
   /**
   * Idempotent, and drift-aware: create the HNSW/cosine index if `FT.INFO` says it doesn't exist —
@@ -518,6 +524,47 @@ var RedisVectorStore = class {
     ]);
     return parseSearchReply(reply, this.prefix);
   }
+  /**
+  * Lexical (BM25) search over the same index `search` uses — the {@link LexicalVectorStore} half of
+  * hybrid retrieval, served by RediSearch's own keyword index rather than an in-process copy of the
+  * corpus. `options.filter` is ANDed onto the text clause with the identical TAG semantics as
+  * `search` (array = OR, empty array = deny), so a lexical hit can never escape a filter a vector
+  * hit would have respected.
+  *
+  * A query with no searchable terms — empty, whitespace, or pure punctuation — returns `[]`. That is
+  * deliberate: RediSearch's `*` would match the entire corpus, which is the opposite of what a
+  * relevance query with nothing to be relevant to should do.
+  */
+  async searchText(query, options) {
+    if (filterMatchesNothing(options.filter)) {
+      return [];
+    }
+    const terms = tokenizeQuery(query);
+    if (terms.length === 0) {
+      return [];
+    }
+    const filterQuery = buildFilter(options.filter);
+    const textQuery = `@text:(${terms.join("|")})`;
+    const reply = await this.client.sendCommand([
+      "FT.SEARCH",
+      this.index,
+      filterQuery === "*" ? textQuery : `(${filterQuery} ${textQuery})`,
+      "SCORER",
+      this.lexicalScorer,
+      "WITHSCORES",
+      "RETURN",
+      "3",
+      "text",
+      "source",
+      "metadata_json",
+      "DIALECT",
+      "2",
+      "LIMIT",
+      "0",
+      String(options.topK)
+    ]);
+    return parseSearchReply(reply, this.prefix, "lexical");
+  }
 };
 function parseIndexAttributes(reply) {
   const attributes = /* @__PURE__ */ new Map();
@@ -596,6 +643,10 @@ function filterMatchesNothing(filter) {
   return Object.values(filter).some((value) => Array.isArray(value) && value.length === 0);
 }
 __name(filterMatchesNothing, "filterMatchesNothing");
+function tokenizeQuery(query) {
+  return query.match(/[\p{L}\p{N}_]+/gu) ?? [];
+}
+__name(tokenizeQuery, "tokenizeQuery");
 function escapeTag(value) {
   return value.replace(/[^a-zA-Z0-9_]/g, (char) => `\\${char}`);
 }
@@ -632,17 +683,19 @@ function toStr(value) {
   return Buffer.isBuffer(value) ? value.toString("utf8") : String(value);
 }
 __name(toStr, "toStr");
-function parseSearchReply(reply, prefix) {
+function parseSearchReply(reply, prefix, scoreSource = "vector") {
+  const stride = scoreSource === "lexical" ? 3 : 2;
   if (Array.isArray(reply)) {
     const passages = [];
-    for (let index = 1; index + 1 < reply.length; index += 2) {
-      const rawFields = reply[index + 1];
+    for (let index = 1; index + stride - 1 < reply.length; index += stride) {
+      const rawFields = reply[index + stride - 1];
       if (Array.isArray(rawFields)) {
         const attrs = {};
         for (let field = 0; field + 1 < rawFields.length; field += 2) {
           attrs[toStr(rawFields[field])] = toStr(rawFields[field + 1]);
         }
-        passages.push(toPassage(toStr(reply[index]), attrs, prefix));
+        const inlineScore = scoreSource === "lexical" ? toStr(reply[index + 1]) : void 0;
+        passages.push(toPassage(toStr(reply[index]), attrs, prefix, scoreSource, inlineScore));
       }
     }
     return passages;
@@ -650,12 +703,19 @@ function parseSearchReply(reply, prefix) {
   if (typeof reply === "object" && reply !== null && "results" in reply) {
     const results = reply.results;
     if (Array.isArray(results)) {
-      return results.map((result) => toPassage(readId(result), readAttributes(result), prefix));
+      return results.map((result) => toPassage(readId(result), readAttributes(result), prefix, scoreSource, scoreSource === "lexical" ? readScore(result) : void 0));
     }
   }
   return [];
 }
 __name(parseSearchReply, "parseSearchReply");
+function readScore(result) {
+  if (typeof result === "object" && result !== null && "score" in result) {
+    return toStr(result.score);
+  }
+  return void 0;
+}
+__name(readScore, "readScore");
 function readId(result) {
   if (typeof result === "object" && result !== null && "id" in result) {
     return toStr(result.id);
@@ -676,13 +736,13 @@ function readAttributes(result) {
   return attrs;
 }
 __name(readAttributes, "readAttributes");
-function toPassage(rawId, attrs, prefix) {
-  const distance = Number(attrs.vector_score ?? "1");
+function toPassage(rawId, attrs, prefix, scoreSource = "vector", inlineScore) {
+  const score = scoreSource === "lexical" ? Number(inlineScore ?? "0") : 1 - Number(attrs.vector_score ?? "1");
   const source = attrs.source;
   return {
     id: rawId.startsWith(prefix) ? rawId.slice(prefix.length) : rawId,
     text: attrs.text ?? "",
-    score: 1 - distance,
+    score: Number.isFinite(score) ? score : 0,
     ...source !== void 0 ? {
       source
     } : {},
@@ -856,6 +916,25 @@ function tokenize(text) {
 }
 __name(tokenize, "tokenize");
 
+// src/lexical-retriever.ts
+var LexicalRetriever = class {
+  static {
+    __name(this, "LexicalRetriever");
+  }
+  store;
+  constructor(store) {
+    this.store = store;
+  }
+  async retrieve(query, options = {}) {
+    return this.store.searchText(query, {
+      topK: options.topK ?? 5,
+      ...options.filter !== void 0 ? {
+        filter: options.filter
+      } : {}
+    });
+  }
+};
+
 // src/hybrid-retriever.ts
 var HybridRetriever = class {
   static {
@@ -1042,6 +1121,7 @@ export {
   FilteredRetriever,
   HybridRetriever,
   KeywordRetriever,
+  LexicalRetriever,
   MemoryVectorStore,
   PgVectorStore,
   RedisVectorSchemaMismatchError,
@@ -1052,6 +1132,7 @@ export {
   createRetrievalTool,
   documentIdOf,
   ingestChunks,
-  ingestDocuments
+  ingestDocuments,
+  isLexicalVectorStore
 };
 //# sourceMappingURL=index.js.map