@dudousxd/nestjs-agent-rag 0.4.0 → 0.5.0

package/dist/index.js

@@ -40,6 +40,10 @@ function chunkText(text, options = {}) {
 __name(chunkText, "chunkText");
 
 // src/vector-store.ts
+function isLexicalVectorStore(store) {
+  return typeof store.searchText === "function";
+}
+__name(isLexicalVectorStore, "isLexicalVectorStore");
 function documentIdOf(chunkId) {
   return chunkId.replace(/#\d+$/, "");
 }
@@ -269,6 +273,19 @@ function toVectorLiteral(embedding) {
 __name(toVectorLiteral, "toVectorLiteral");
 
 // src/redis-vector-store.ts
+var RedisVectorSchemaMismatchError = class extends Error {
+  static {
+    __name(this, "RedisVectorSchemaMismatchError");
+  }
+  index;
+  field;
+  expected;
+  actual;
+  constructor(index, field, expected, actual, remedy) {
+    super(`RediSearch index "${index}" disagrees with this RedisVectorStore's configuration: field "${field}" is ${actual}, expected ${expected}. ${remedy}`), this.index = index, this.field = field, this.expected = expected, this.actual = actual;
+    this.name = "RedisVectorSchemaMismatchError";
+  }
+};
 var RedisVectorStore = class {
   static {
     __name(this, "RedisVectorStore");
@@ -278,23 +295,83 @@ var RedisVectorStore = class {
   prefix;
   dimensions;
   filterableFields;
+  lexicalScorer;
   constructor(client, options = {}) {
     this.client = client;
     this.index = options.index ?? "agent_rag_idx";
     this.prefix = options.prefix ?? "agent_rag:";
     this.dimensions = options.dimensions ?? 1536;
     this.filterableFields = options.filterableFields ?? [];
+    this.lexicalScorer = options.lexicalScorer ?? "BM25";
   }
-  /** Idempotent: create the HNSW/cosine index if `FT.INFO` says it doesn't exist yet. */
+  /**
+  * Idempotent, and drift-aware: create the HNSW/cosine index if `FT.INFO` says it doesn't exist —
+  * and if it DOES exist, compare what's there against this store's configuration instead of
+  * assuming they agree. An index that outlives a config change is the normal case (schemas live in
+  * Redis, config lives in a redeployed process), and the two failure modes it produces are both
+  * invisible at runtime: a `filterableFields` entry added later has no `meta_*` TAG, so every search
+  * filtering on it matches *nothing*; a changed `dimensions` leaves the index on the old width, so
+  * writes are rejected or KNN ranks garbage. Neither raises anything on its own.
+  *
+  * The two are repaired differently, because only one of them CAN be repaired:
+  *
+  * - **A missing filterable TAG is additive** → `FT.ALTER … SCHEMA ADD`. Nothing that already works
+  *   changes, so doing it automatically is safe. Note the chunks written *before* the alter carry no
+  *   `meta_*` hash field at all (`upsert` only writes the fields configured at the time), so they
+  *   only become filterable on that key once they're re-ingested — the schema is repaired here, the
+  *   backfill is the host's call.
+  * - **A dimension (or field-type) change is not repairable** → {@link RedisVectorSchemaMismatchError}.
+  *   It needs a drop + full reindex, which would destroy the host's corpus; failing loudly at boot is
+  *   the only honest option.
+  *
+  * If `FT.INFO` comes back in a shape this parser doesn't recognise, no drift is inferred — an
+  * unreadable reply must not be turned into a false alarm that blocks a boot.
+  */
   async ensureSchema() {
+    let info;
     try {
-      await this.client.sendCommand([
+      info = await this.client.sendCommand([
         "FT.INFO",
         this.index
       ]);
-      return;
     } catch {
+      await this.createIndex();
+      return;
+    }
+    await this.reconcileSchema(info);
+  }
+  /** Repair what can be repaired in place; throw on what can't. */
+  async reconcileSchema(info) {
+    const attributes = parseIndexAttributes(info);
+    if (attributes.size === 0) {
+      return;
+    }
+    const vector = attributes.get("embedding");
+    if (vector?.dimensions !== void 0 && vector.dimensions !== this.dimensions) {
+      throw new RedisVectorSchemaMismatchError(this.index, "embedding", `DIM ${this.dimensions}`, `DIM ${vector.dimensions}`, `A vector field's width is fixed at FT.CREATE and every stored embedding has the old one, so this cannot be altered in place: drop the index (FT.DROPINDEX ${this.index} DD) and re-ingest with the new embedding model.`);
+    }
+    const missing = [];
+    for (const field of this.filterableFields) {
+      const name = `meta_${field}`;
+      const existing = attributes.get(name);
+      if (existing === void 0) {
+        missing.push(name);
+      } else if (existing.type !== "TAG") {
+        throw new RedisVectorSchemaMismatchError(this.index, name, "TAG", existing.type, `A field's type cannot be changed in place: drop the index (FT.DROPINDEX ${this.index} DD) and re-ingest, or rename the metadata key.`);
+      }
+    }
+    for (const name of missing) {
+      await this.client.sendCommand([
+        "FT.ALTER",
+        this.index,
+        "SCHEMA",
+        "ADD",
+        name,
+        "TAG"
+      ]);
     }
+  }
+  async createIndex() {
     const schema = [
       "text",
       "TEXT",
@@ -447,7 +524,100 @@ var RedisVectorStore = class {
     ]);
     return parseSearchReply(reply, this.prefix);
   }
+  /**
+  * Lexical (BM25) search over the same index `search` uses — the {@link LexicalVectorStore} half of
+  * hybrid retrieval, served by RediSearch's own keyword index rather than an in-process copy of the
+  * corpus. `options.filter` is ANDed onto the text clause with the identical TAG semantics as
+  * `search` (array = OR, empty array = deny), so a lexical hit can never escape a filter a vector
+  * hit would have respected.
+  *
+  * A query with no searchable terms — empty, whitespace, or pure punctuation — returns `[]`. That is
+  * deliberate: RediSearch's `*` would match the entire corpus, which is the opposite of what a
+  * relevance query with nothing to be relevant to should do.
+  */
+  async searchText(query, options) {
+    if (filterMatchesNothing(options.filter)) {
+      return [];
+    }
+    const terms = tokenizeQuery(query);
+    if (terms.length === 0) {
+      return [];
+    }
+    const filterQuery = buildFilter(options.filter);
+    const textQuery = `@text:(${terms.join("|")})`;
+    const reply = await this.client.sendCommand([
+      "FT.SEARCH",
+      this.index,
+      filterQuery === "*" ? textQuery : `(${filterQuery} ${textQuery})`,
+      "SCORER",
+      this.lexicalScorer,
+      "WITHSCORES",
+      "RETURN",
+      "3",
+      "text",
+      "source",
+      "metadata_json",
+      "DIALECT",
+      "2",
+      "LIMIT",
+      "0",
+      String(options.topK)
+    ]);
+    return parseSearchReply(reply, this.prefix, "lexical");
+  }
 };
+function parseIndexAttributes(reply) {
+  const attributes = /* @__PURE__ */ new Map();
+  for (const entry of readInfoField(reply, "attributes")) {
+    const fields = readKeyValues(entry);
+    const name = fields.attribute ?? fields.identifier;
+    if (name === void 0) {
+      continue;
+    }
+    const dim = Number(fields.dim);
+    attributes.set(name, {
+      type: (fields.type ?? "").toUpperCase(),
+      ...fields.dim !== void 0 && Number.isFinite(dim) ? {
+        dimensions: dim
+      } : {}
+    });
+  }
+  return attributes;
+}
+__name(parseIndexAttributes, "parseIndexAttributes");
+function readInfoField(reply, key) {
+  if (Array.isArray(reply)) {
+    for (let index = 0; index + 1 < reply.length; index += 2) {
+      if (toStr(reply[index]) === key) {
+        const value = reply[index + 1];
+        return Array.isArray(value) ? value : [];
+      }
+    }
+    return [];
+  }
+  if (typeof reply === "object" && reply !== null && key in reply) {
+    const value = reply[key];
+    return Array.isArray(value) ? value : [];
+  }
+  return [];
+}
+__name(readInfoField, "readInfoField");
+function readKeyValues(entry) {
+  const fields = {};
+  if (Array.isArray(entry)) {
+    for (let index = 0; index + 1 < entry.length; index += 2) {
+      fields[toStr(entry[index]).toLowerCase()] = toStr(entry[index + 1]);
+    }
+    return fields;
+  }
+  if (typeof entry === "object" && entry !== null) {
+    for (const [key, value] of Object.entries(entry)) {
+      fields[key.toLowerCase()] = toStr(value);
+    }
+  }
+  return fields;
+}
+__name(readKeyValues, "readKeyValues");
 function encodeVector(embedding) {
   return Buffer.from(new Float32Array(embedding).buffer);
 }
@@ -473,6 +643,10 @@ function filterMatchesNothing(filter) {
   return Object.values(filter).some((value) => Array.isArray(value) && value.length === 0);
 }
 __name(filterMatchesNothing, "filterMatchesNothing");
+function tokenizeQuery(query) {
+  return query.match(/[\p{L}\p{N}_]+/gu) ?? [];
+}
+__name(tokenizeQuery, "tokenizeQuery");
 function escapeTag(value) {
   return value.replace(/[^a-zA-Z0-9_]/g, (char) => `\\${char}`);
 }
@@ -509,17 +683,19 @@ function toStr(value) {
   return Buffer.isBuffer(value) ? value.toString("utf8") : String(value);
 }
 __name(toStr, "toStr");
-function parseSearchReply(reply, prefix) {
+function parseSearchReply(reply, prefix, scoreSource = "vector") {
+  const stride = scoreSource === "lexical" ? 3 : 2;
   if (Array.isArray(reply)) {
     const passages = [];
-    for (let index = 1; index + 1 < reply.length; index += 2) {
-      const rawFields = reply[index + 1];
+    for (let index = 1; index + stride - 1 < reply.length; index += stride) {
+      const rawFields = reply[index + stride - 1];
       if (Array.isArray(rawFields)) {
         const attrs = {};
         for (let field = 0; field + 1 < rawFields.length; field += 2) {
           attrs[toStr(rawFields[field])] = toStr(rawFields[field + 1]);
         }
-        passages.push(toPassage(toStr(reply[index]), attrs, prefix));
+        const inlineScore = scoreSource === "lexical" ? toStr(reply[index + 1]) : void 0;
+        passages.push(toPassage(toStr(reply[index]), attrs, prefix, scoreSource, inlineScore));
       }
     }
     return passages;
@@ -527,12 +703,19 @@ function parseSearchReply(reply, prefix) {
   if (typeof reply === "object" && reply !== null && "results" in reply) {
     const results = reply.results;
     if (Array.isArray(results)) {
-      return results.map((result) => toPassage(readId(result), readAttributes(result), prefix));
+      return results.map((result) => toPassage(readId(result), readAttributes(result), prefix, scoreSource, scoreSource === "lexical" ? readScore(result) : void 0));
     }
   }
   return [];
 }
 __name(parseSearchReply, "parseSearchReply");
+function readScore(result) {
+  if (typeof result === "object" && result !== null && "score" in result) {
+    return toStr(result.score);
+  }
+  return void 0;
+}
+__name(readScore, "readScore");
 function readId(result) {
   if (typeof result === "object" && result !== null && "id" in result) {
     return toStr(result.id);
@@ -553,13 +736,13 @@ function readAttributes(result) {
   return attrs;
 }
 __name(readAttributes, "readAttributes");
-function toPassage(rawId, attrs, prefix) {
-  const distance = Number(attrs.vector_score ?? "1");
+function toPassage(rawId, attrs, prefix, scoreSource = "vector", inlineScore) {
+  const score = scoreSource === "lexical" ? Number(inlineScore ?? "0") : 1 - Number(attrs.vector_score ?? "1");
   const source = attrs.source;
   return {
     id: rawId.startsWith(prefix) ? rawId.slice(prefix.length) : rawId,
     text: attrs.text ?? "",
-    score: 1 - distance,
+    score: Number.isFinite(score) ? score : 0,
     ...source !== void 0 ? {
       source
     } : {},
@@ -625,10 +808,14 @@ var KeywordRetriever = class {
     this.k1 = options.k1 ?? 1.5;
     this.b = options.b ?? 0.75;
   }
+  /** How many chunks are currently indexed. */
+  get size() {
+    return this.docs.size;
+  }
   /** Index (or re-index) records. Re-adding an existing id replaces its posting and updates stats. */
   add(records) {
     for (const record of records) {
-      this.remove(record.id);
+      this.removeChunk(record.id);
       const tokens = tokenize(record.text);
       const frequencies = /* @__PURE__ */ new Map();
       for (const token of tokens) {
@@ -645,7 +832,30 @@ var KeywordRetriever = class {
       this.totalLength += tokens.length;
     }
   }
-  remove(id) {
+  /**
+  * Forget a source document — every chunk whose id is `${documentId}` or `${documentId}#<n>`, the
+  * same id scheme (and the same {@link documentIdOf} collapse) the vector stores use for their
+  * `remove`. The keyword half of a hybrid index keeps its own copy of the chunk text, so a document
+  * dropped from the vector store stays fully retrievable here until this is called. Unknown ids are
+  * a no-op.
+  */
+  remove(documentId) {
+    for (const id of [
+      ...this.docs.keys()
+    ]) {
+      if (id === documentId || documentIdOf(id) === documentId) {
+        this.removeChunk(id);
+      }
+    }
+  }
+  /** Drop every indexed chunk, resetting the corpus statistics with it. */
+  clear() {
+    this.docs.clear();
+    this.documentFrequency.clear();
+    this.totalLength = 0;
+  }
+  /** Drop ONE chunk by its exact id, decrementing the corpus statistics it contributed. */
+  removeChunk(id) {
     const existing = this.docs.get(id);
     if (existing === void 0) {
       return;
@@ -706,6 +916,25 @@ function tokenize(text) {
 }
 __name(tokenize, "tokenize");
 
+// src/lexical-retriever.ts
+var LexicalRetriever = class {
+  static {
+    __name(this, "LexicalRetriever");
+  }
+  store;
+  constructor(store) {
+    this.store = store;
+  }
+  async retrieve(query, options = {}) {
+    return this.store.searchText(query, {
+      topK: options.topK ?? 5,
+      ...options.filter !== void 0 ? {
+        filter: options.filter
+      } : {}
+    });
+  }
+};
+
 // src/hybrid-retriever.ts
 var HybridRetriever = class {
   static {
@@ -892,8 +1121,10 @@ export {
   FilteredRetriever,
   HybridRetriever,
   KeywordRetriever,
+  LexicalRetriever,
   MemoryVectorStore,
   PgVectorStore,
+  RedisVectorSchemaMismatchError,
   RedisVectorStore,
   RerankingRetriever,
   chunkDocuments,
@@ -901,6 +1132,7 @@ export {
   createRetrievalTool,
   documentIdOf,
   ingestChunks,
-  ingestDocuments
+  ingestDocuments,
+  isLexicalVectorStore
 };
 //# sourceMappingURL=index.js.map