@dudousxd/nestjs-agent-rag 0.3.10 → 0.4.1

package/README.md

@@ -69,10 +69,45 @@ const store = new PgVectorStore(
 await store.ensureSchema(); // CREATE EXTENSION vector + table + HNSW cosine index
 ```
 
+## Hybrid search — and deleting from it
+
+Hybrid search means chunking once and feeding the SAME chunks to both halves, so their ids line up
+and `HybridRetriever` can fuse the rankings:
+
+```ts
+const chunks = chunkDocuments(docs);
+await ingestChunks(chunks, { embedder, store });
+keyword.add(chunks);
+
+const retriever = new HybridRetriever([new EmbeddingRetriever(embedder, store), keyword]);
+```
+
+Because there are two indexes, **deletes have to hit both**. `KeywordRetriever` keeps its own copy of
+each chunk's text, so a document dropped from the vector store alone stays fully retrievable —
+`retrieve` will keep returning the removed passage:
+
+```ts
+await store.remove(documentId);
+keyword.remove(documentId); // ← the other half. Same document id; chunk ids collapse for you.
+```
+
+`KeywordRetriever` also exposes `clear()` and `size` for rebuilding the index wholesale.
+
+## RediSearch store
+
+`RedisVectorStore` needs `ensureSchema()` at boot. It creates the index if missing — and if the index
+already exists, it compares it against your configuration rather than assuming they agree. A
+`filterableFields` entry added after the index was created is repaired in place with
+`FT.ALTER … SCHEMA ADD` (chunks written before the repair only become filterable on that key once
+re-ingested). A changed `dimensions` — a model swap — can't be repaired without a full reindex, so it
+throws `RedisVectorSchemaMismatchError` instead of silently leaving the index on the old width.
+
 ## API
 
 - `chunkText(text, { chunkSize?, overlap? })` — overlapping, boundary-aware chunks.
 - `ingestDocuments(docs, { embedder, store, chunkSize?, overlap? })` — chunk → embed → upsert.
 - `EmbeddingRetriever(embedder, store)` — the core `Retriever` from an embedder + vector store.
-- `MemoryVectorStore` / `PgVectorStore` — `VectorStore` adapters.
+- `KeywordRetriever` — BM25 lexical half of hybrid search: `add(chunks)`, `remove(documentId)`,
+  `clear()`, `size`.
+- `MemoryVectorStore` / `PgVectorStore` / `RedisVectorStore` — `VectorStore` adapters.
 - `createRetrievalTool(retriever, { name?, description?, topK? })` — the agentic-retrieval tool.

package/dist/index.cjs

@@ -27,6 +27,7 @@ __export(index_exports, {
   KeywordRetriever: () => KeywordRetriever,
   MemoryVectorStore: () => MemoryVectorStore,
   PgVectorStore: () => PgVectorStore,
+  RedisVectorSchemaMismatchError: () => RedisVectorSchemaMismatchError,
   RedisVectorStore: () => RedisVectorStore,
   RerankingRetriever: () => RerankingRetriever,
   chunkDocuments: () => chunkDocuments,
@@ -84,10 +85,17 @@ __name(documentIdOf, "documentIdOf");
 
 // src/filter.ts
 function matchesFilter(metadata, filter) {
+  const entries = Object.entries(filter);
   if (metadata === void 0) {
-    return Object.keys(filter).length === 0;
+    return entries.length === 0;
   }
-  return Object.entries(filter).every(([key, value]) => metadata[key] === value);
+  return entries.every(([key, expected]) => {
+    if (Array.isArray(expected)) {
+      const actual = metadata[key];
+      return Array.isArray(actual) ? actual.some((value) => expected.includes(value)) : expected.includes(actual);
+    }
+    return metadata[key] === expected;
+  });
 }
 __name(matchesFilter, "matchesFilter");
 
@@ -216,13 +224,11 @@ var PgVectorStore = class {
     ]);
   }
   async listDocuments(filter) {
-    const hasFilter = filter !== void 0 && Object.keys(filter).length > 0;
+    const where = buildWhere(filter, 1);
     const rows = await this.client.query(`SELECT DISTINCT ON (${DOCUMENT_ID_FROM_CHUNK}) ${DOCUMENT_ID_FROM_CHUNK} AS doc_id, metadata
        FROM ${this.table}
-       ${hasFilter ? "WHERE metadata @> $1::jsonb" : ""}
-       ORDER BY ${DOCUMENT_ID_FROM_CHUNK}`, hasFilter ? [
-      JSON.stringify(filter)
-    ] : []);
+       ${where.sql}
+       ORDER BY ${DOCUMENT_ID_FROM_CHUNK}`, where.params);
     return rows.map((row) => ({
       id: row.doc_id,
       ...row.metadata !== null ? {
@@ -232,18 +238,15 @@ var PgVectorStore = class {
   }
   async search(embedding, options) {
     const vector = toVectorLiteral(embedding);
-    const hasFilter = options.filter !== void 0 && Object.keys(options.filter).length > 0;
+    const where = buildWhere(options.filter, 3);
     const rows = await this.client.query(`SELECT id, text, source, metadata, 1 - (embedding <=> $1::vector) AS score
        FROM ${this.table}
-       ${hasFilter ? "WHERE metadata @> $3::jsonb" : ""}
+       ${where.sql}
        ORDER BY embedding <=> $1::vector
-       LIMIT $2`, hasFilter ? [
+       LIMIT $2`, [
       vector,
       options.topK,
-      JSON.stringify(options.filter)
-    ] : [
-      vector,
-      options.topK
+      ...where.params
     ]);
     return rows.map((row) => ({
       id: row.id,
@@ -258,6 +261,45 @@ var PgVectorStore = class {
     }));
   }
 };
+function buildWhere(filter, startIndex) {
+  if (filter === void 0 || Object.keys(filter).length === 0) {
+    return {
+      sql: "",
+      params: []
+    };
+  }
+  const clauses = [];
+  const params = [];
+  const scalar = {};
+  let index = startIndex;
+  for (const [key, value] of Object.entries(filter)) {
+    if (Array.isArray(value)) {
+      if (value.length === 0) {
+        clauses.push("false");
+        continue;
+      }
+      const keyParam = `$${index}`;
+      params.push(key);
+      index += 1;
+      const arrParam = `$${index}`;
+      params.push(value.map(String));
+      index += 1;
+      clauses.push(`(CASE WHEN jsonb_typeof(metadata->${keyParam}) = 'array' THEN metadata->${keyParam} ELSE jsonb_build_array(metadata->${keyParam}) END) ?| ${arrParam}::text[]`);
+    } else {
+      scalar[key] = value;
+    }
+  }
+  if (Object.keys(scalar).length > 0) {
+    clauses.push(`metadata @> $${index}::jsonb`);
+    params.push(JSON.stringify(scalar));
+    index += 1;
+  }
+  return {
+    sql: `WHERE ${clauses.join(" AND ")}`,
+    params
+  };
+}
+__name(buildWhere, "buildWhere");
 var DOCUMENT_ID_FROM_CHUNK = "regexp_replace(id, '#[0-9]+$', '')";
 function toVectorLiteral(embedding) {
   return `[${embedding.join(",")}]`;
@@ -265,6 +307,19 @@ function toVectorLiteral(embedding) {
 __name(toVectorLiteral, "toVectorLiteral");
 
 // src/redis-vector-store.ts
+var RedisVectorSchemaMismatchError = class extends Error {
+  static {
+    __name(this, "RedisVectorSchemaMismatchError");
+  }
+  index;
+  field;
+  expected;
+  actual;
+  constructor(index, field, expected, actual, remedy) {
+    super(`RediSearch index "${index}" disagrees with this RedisVectorStore's configuration: field "${field}" is ${actual}, expected ${expected}. ${remedy}`), this.index = index, this.field = field, this.expected = expected, this.actual = actual;
+    this.name = "RedisVectorSchemaMismatchError";
+  }
+};
 var RedisVectorStore = class {
   static {
     __name(this, "RedisVectorStore");
@@ -281,16 +336,74 @@ var RedisVectorStore = class {
     this.dimensions = options.dimensions ?? 1536;
     this.filterableFields = options.filterableFields ?? [];
   }
-  /** Idempotent: create the HNSW/cosine index if `FT.INFO` says it doesn't exist yet. */
+  /**
+  * Idempotent, and drift-aware: create the HNSW/cosine index if `FT.INFO` says it doesn't exist —
+  * and if it DOES exist, compare what's there against this store's configuration instead of
+  * assuming they agree. An index that outlives a config change is the normal case (schemas live in
+  * Redis, config lives in a redeployed process), and the two failure modes it produces are both
+  * invisible at runtime: a `filterableFields` entry added later has no `meta_*` TAG, so every search
+  * filtering on it matches *nothing*; a changed `dimensions` leaves the index on the old width, so
+  * writes are rejected or KNN ranks garbage. Neither raises anything on its own.
+  *
+  * The two are repaired differently, because only one of them CAN be repaired:
+  *
+  * - **A missing filterable TAG is additive** → `FT.ALTER … SCHEMA ADD`. Nothing that already works
+  *   changes, so doing it automatically is safe. Note the chunks written *before* the alter carry no
+  *   `meta_*` hash field at all (`upsert` only writes the fields configured at the time), so they
+  *   only become filterable on that key once they're re-ingested — the schema is repaired here, the
+  *   backfill is the host's call.
+  * - **A dimension (or field-type) change is not repairable** → {@link RedisVectorSchemaMismatchError}.
+  *   It needs a drop + full reindex, which would destroy the host's corpus; failing loudly at boot is
+  *   the only honest option.
+  *
+  * If `FT.INFO` comes back in a shape this parser doesn't recognise, no drift is inferred — an
+  * unreadable reply must not be turned into a false alarm that blocks a boot.
+  */
   async ensureSchema() {
+    let info;
     try {
-      await this.client.sendCommand([
+      info = await this.client.sendCommand([
         "FT.INFO",
         this.index
       ]);
-      return;
     } catch {
+      await this.createIndex();
+      return;
+    }
+    await this.reconcileSchema(info);
+  }
+  /** Repair what can be repaired in place; throw on what can't. */
+  async reconcileSchema(info) {
+    const attributes = parseIndexAttributes(info);
+    if (attributes.size === 0) {
+      return;
+    }
+    const vector = attributes.get("embedding");
+    if (vector?.dimensions !== void 0 && vector.dimensions !== this.dimensions) {
+      throw new RedisVectorSchemaMismatchError(this.index, "embedding", `DIM ${this.dimensions}`, `DIM ${vector.dimensions}`, `A vector field's width is fixed at FT.CREATE and every stored embedding has the old one, so this cannot be altered in place: drop the index (FT.DROPINDEX ${this.index} DD) and re-ingest with the new embedding model.`);
+    }
+    const missing = [];
+    for (const field of this.filterableFields) {
+      const name = `meta_${field}`;
+      const existing = attributes.get(name);
+      if (existing === void 0) {
+        missing.push(name);
+      } else if (existing.type !== "TAG") {
+        throw new RedisVectorSchemaMismatchError(this.index, name, "TAG", existing.type, `A field's type cannot be changed in place: drop the index (FT.DROPINDEX ${this.index} DD) and re-ingest, or rename the metadata key.`);
+      }
+    }
+    for (const name of missing) {
+      await this.client.sendCommand([
+        "FT.ALTER",
+        this.index,
+        "SCHEMA",
+        "ADD",
+        name,
+        "TAG"
+      ]);
     }
+  }
+  async createIndex() {
     const schema = [
       "text",
       "TEXT",
@@ -334,7 +447,8 @@ var RedisVectorStore = class {
         for (const field of this.filterableFields) {
           const value = record.metadata[field];
           if (value !== void 0) {
-            args.push(`meta_${field}`, String(value));
+            const tag = Array.isArray(value) ? value.map(String).join(",") : String(value);
+            args.push(`meta_${field}`, tag);
           }
         }
       }
@@ -370,6 +484,9 @@ var RedisVectorStore = class {
     }
   }
   async listDocuments(filter) {
+    if (filterMatchesNothing(filter)) {
+      return [];
+    }
     const query = buildFilter(filter);
     const documents = /* @__PURE__ */ new Map();
     const batchSize = 1e3;
@@ -410,6 +527,9 @@ var RedisVectorStore = class {
     ];
   }
   async search(embedding, options) {
+    if (filterMatchesNothing(options.filter)) {
+      return [];
+    }
     const query = `${buildFilter(options.filter)}=>[KNN ${options.topK} @embedding $BLOB AS vector_score]`;
     const reply = await this.client.sendCommand([
       "FT.SEARCH",
@@ -437,6 +557,58 @@ var RedisVectorStore = class {
     return parseSearchReply(reply, this.prefix);
   }
 };
+function parseIndexAttributes(reply) {
+  const attributes = /* @__PURE__ */ new Map();
+  for (const entry of readInfoField(reply, "attributes")) {
+    const fields = readKeyValues(entry);
+    const name = fields.attribute ?? fields.identifier;
+    if (name === void 0) {
+      continue;
+    }
+    const dim = Number(fields.dim);
+    attributes.set(name, {
+      type: (fields.type ?? "").toUpperCase(),
+      ...fields.dim !== void 0 && Number.isFinite(dim) ? {
+        dimensions: dim
+      } : {}
+    });
+  }
+  return attributes;
+}
+__name(parseIndexAttributes, "parseIndexAttributes");
+function readInfoField(reply, key) {
+  if (Array.isArray(reply)) {
+    for (let index = 0; index + 1 < reply.length; index += 2) {
+      if (toStr(reply[index]) === key) {
+        const value = reply[index + 1];
+        return Array.isArray(value) ? value : [];
+      }
+    }
+    return [];
+  }
+  if (typeof reply === "object" && reply !== null && key in reply) {
+    const value = reply[key];
+    return Array.isArray(value) ? value : [];
+  }
+  return [];
+}
+__name(readInfoField, "readInfoField");
+function readKeyValues(entry) {
+  const fields = {};
+  if (Array.isArray(entry)) {
+    for (let index = 0; index + 1 < entry.length; index += 2) {
+      fields[toStr(entry[index]).toLowerCase()] = toStr(entry[index + 1]);
+    }
+    return fields;
+  }
+  if (typeof entry === "object" && entry !== null) {
+    for (const [key, value] of Object.entries(entry)) {
+      fields[key.toLowerCase()] = toStr(value);
+    }
+  }
+  return fields;
+}
+__name(readKeyValues, "readKeyValues");
 function encodeVector(embedding) {
   return Buffer.from(new Float32Array(embedding).buffer);
 }
@@ -445,10 +617,23 @@ function buildFilter(filter) {
   if (filter === void 0 || Object.keys(filter).length === 0) {
     return "*";
   }
-  const clauses = Object.entries(filter).map(([key, value]) => `@meta_${key}:{${escapeTag(String(value))}}`);
+  const clauses = Object.entries(filter).map(([key, value]) => {
+    const values = Array.isArray(value) ? value : [
+      value
+    ];
+    const alternation = values.map((entry) => escapeTag(String(entry))).join("|");
+    return `@meta_${key}:{${alternation}}`;
+  });
   return `(${clauses.join(" ")})`;
 }
 __name(buildFilter, "buildFilter");
+function filterMatchesNothing(filter) {
+  if (filter === void 0) {
+    return false;
+  }
+  return Object.values(filter).some((value) => Array.isArray(value) && value.length === 0);
+}
+__name(filterMatchesNothing, "filterMatchesNothing");
 function escapeTag(value) {
   return value.replace(/[^a-zA-Z0-9_]/g, (char) => `\\${char}`);
 }
@@ -601,10 +786,14 @@ var KeywordRetriever = class {
     this.k1 = options.k1 ?? 1.5;
     this.b = options.b ?? 0.75;
   }
+  /** How many chunks are currently indexed. */
+  get size() {
+    return this.docs.size;
+  }
   /** Index (or re-index) records. Re-adding an existing id replaces its posting and updates stats. */
   add(records) {
     for (const record of records) {
-      this.remove(record.id);
+      this.removeChunk(record.id);
       const tokens = tokenize(record.text);
       const frequencies = /* @__PURE__ */ new Map();
       for (const token of tokens) {
@@ -621,7 +810,30 @@ var KeywordRetriever = class {
       this.totalLength += tokens.length;
     }
   }
-  remove(id) {
+  /**
+  * Forget a source document — every chunk whose id is `${documentId}` or `${documentId}#<n>`, the
+  * same id scheme (and the same {@link documentIdOf} collapse) the vector stores use for their
+  * `remove`. The keyword half of a hybrid index keeps its own copy of the chunk text, so a document
+  * dropped from the vector store stays fully retrievable here until this is called. Unknown ids are
+  * a no-op.
+  */
+  remove(documentId) {
+    for (const id of [
+      ...this.docs.keys()
+    ]) {
+      if (id === documentId || documentIdOf(id) === documentId) {
+        this.removeChunk(id);
+      }
+    }
+  }
+  /** Drop every indexed chunk, resetting the corpus statistics with it. */
+  clear() {
+    this.docs.clear();
+    this.documentFrequency.clear();
+    this.totalLength = 0;
+  }
+  /** Drop ONE chunk by its exact id, decrementing the corpus statistics it contributed. */
+  removeChunk(id) {
     const existing = this.docs.get(id);
     if (existing === void 0) {
       return;
@@ -871,6 +1083,7 @@ __name(createRetrievalTool, "createRetrievalTool");
   KeywordRetriever,
   MemoryVectorStore,
   PgVectorStore,
+  RedisVectorSchemaMismatchError,
   RedisVectorStore,
   RerankingRetriever,
   chunkDocuments,