@dudousxd/nestjs-agent-rag 0.4.0 → 0.4.1

package/README.md

@@ -69,10 +69,45 @@ const store = new PgVectorStore(
 await store.ensureSchema(); // CREATE EXTENSION vector + table + HNSW cosine index
 ```
 
+## Hybrid search — and deleting from it
+
+Hybrid search means chunking once and feeding the SAME chunks to both halves, so their ids line up
+and `HybridRetriever` can fuse the rankings:
+
+```ts
+const chunks = chunkDocuments(docs);
+await ingestChunks(chunks, { embedder, store });
+keyword.add(chunks);
+
+const retriever = new HybridRetriever([new EmbeddingRetriever(embedder, store), keyword]);
+```
+
+Because there are two indexes, **deletes have to hit both**. `KeywordRetriever` keeps its own copy of
+each chunk's text, so a document dropped from the vector store alone stays fully retrievable —
+`retrieve` will keep returning the removed passage:
+
+```ts
+await store.remove(documentId);
+keyword.remove(documentId); // ← the other half. Same document id; chunk ids collapse for you.
+```
+
+`KeywordRetriever` also exposes `clear()` and `size` for rebuilding the index wholesale.
+
+## RediSearch store
+
+`RedisVectorStore` needs `ensureSchema()` at boot. It creates the index if missing — and if the index
+already exists, it compares it against your configuration rather than assuming they agree. A
+`filterableFields` entry added after the index was created is repaired in place with
+`FT.ALTER … SCHEMA ADD` (chunks written before the repair only become filterable on that key once
+re-ingested). A changed `dimensions` — a model swap — can't be repaired without a full reindex, so it
+throws `RedisVectorSchemaMismatchError` instead of silently leaving the index on the old width.
+
 ## API
 
 - `chunkText(text, { chunkSize?, overlap? })` — overlapping, boundary-aware chunks.
 - `ingestDocuments(docs, { embedder, store, chunkSize?, overlap? })` — chunk → embed → upsert.
 - `EmbeddingRetriever(embedder, store)` — the core `Retriever` from an embedder + vector store.
-- `MemoryVectorStore` / `PgVectorStore` — `VectorStore` adapters.
+- `KeywordRetriever` — BM25 lexical half of hybrid search: `add(chunks)`, `remove(documentId)`,
+  `clear()`, `size`.
+- `MemoryVectorStore` / `PgVectorStore` / `RedisVectorStore` — `VectorStore` adapters.
 - `createRetrievalTool(retriever, { name?, description?, topK? })` — the agentic-retrieval tool.

package/dist/index.cjs

@@ -27,6 +27,7 @@ __export(index_exports, {
   KeywordRetriever: () => KeywordRetriever,
   MemoryVectorStore: () => MemoryVectorStore,
   PgVectorStore: () => PgVectorStore,
+  RedisVectorSchemaMismatchError: () => RedisVectorSchemaMismatchError,
   RedisVectorStore: () => RedisVectorStore,
   RerankingRetriever: () => RerankingRetriever,
   chunkDocuments: () => chunkDocuments,
@@ -306,6 +307,19 @@ function toVectorLiteral(embedding) {
 __name(toVectorLiteral, "toVectorLiteral");
 
 // src/redis-vector-store.ts
+var RedisVectorSchemaMismatchError = class extends Error {
+  static {
+    __name(this, "RedisVectorSchemaMismatchError");
+  }
+  index;
+  field;
+  expected;
+  actual;
+  constructor(index, field, expected, actual, remedy) {
+    super(`RediSearch index "${index}" disagrees with this RedisVectorStore's configuration: field "${field}" is ${actual}, expected ${expected}. ${remedy}`), this.index = index, this.field = field, this.expected = expected, this.actual = actual;
+    this.name = "RedisVectorSchemaMismatchError";
+  }
+};
 var RedisVectorStore = class {
   static {
     __name(this, "RedisVectorStore");
@@ -322,16 +336,74 @@ var RedisVectorStore = class {
     this.dimensions = options.dimensions ?? 1536;
     this.filterableFields = options.filterableFields ?? [];
   }
-  /** Idempotent: create the HNSW/cosine index if `FT.INFO` says it doesn't exist yet. */
+  /**
+  * Idempotent, and drift-aware: create the HNSW/cosine index if `FT.INFO` says it doesn't exist —
+  * and if it DOES exist, compare what's there against this store's configuration instead of
+  * assuming they agree. An index that outlives a config change is the normal case (schemas live in
+  * Redis, config lives in a redeployed process), and the two failure modes it produces are both
+  * invisible at runtime: a `filterableFields` entry added later has no `meta_*` TAG, so every search
+  * filtering on it matches *nothing*; a changed `dimensions` leaves the index on the old width, so
+  * writes are rejected or KNN ranks garbage. Neither raises anything on its own.
+  *
+  * The two are repaired differently, because only one of them CAN be repaired:
+  *
+  * - **A missing filterable TAG is additive** → `FT.ALTER … SCHEMA ADD`. Nothing that already works
+  *   changes, so doing it automatically is safe. Note the chunks written *before* the alter carry no
+  *   `meta_*` hash field at all (`upsert` only writes the fields configured at the time), so they
+  *   only become filterable on that key once they're re-ingested — the schema is repaired here, the
+  *   backfill is the host's call.
+  * - **A dimension (or field-type) change is not repairable** → {@link RedisVectorSchemaMismatchError}.
+  *   It needs a drop + full reindex, which would destroy the host's corpus; failing loudly at boot is
+  *   the only honest option.
+  *
+  * If `FT.INFO` comes back in a shape this parser doesn't recognise, no drift is inferred — an
+  * unreadable reply must not be turned into a false alarm that blocks a boot.
+  */
   async ensureSchema() {
+    let info;
     try {
-      await this.client.sendCommand([
+      info = await this.client.sendCommand([
         "FT.INFO",
         this.index
       ]);
-      return;
     } catch {
+      await this.createIndex();
+      return;
+    }
+    await this.reconcileSchema(info);
+  }
+  /** Repair what can be repaired in place; throw on what can't. */
+  async reconcileSchema(info) {
+    const attributes = parseIndexAttributes(info);
+    if (attributes.size === 0) {
+      return;
     }
+    const vector = attributes.get("embedding");
+    if (vector?.dimensions !== void 0 && vector.dimensions !== this.dimensions) {
+      throw new RedisVectorSchemaMismatchError(this.index, "embedding", `DIM ${this.dimensions}`, `DIM ${vector.dimensions}`, `A vector field's width is fixed at FT.CREATE and every stored embedding has the old one, so this cannot be altered in place: drop the index (FT.DROPINDEX ${this.index} DD) and re-ingest with the new embedding model.`);
+    }
+    const missing = [];
+    for (const field of this.filterableFields) {
+      const name = `meta_${field}`;
+      const existing = attributes.get(name);
+      if (existing === void 0) {
+        missing.push(name);
+      } else if (existing.type !== "TAG") {
+        throw new RedisVectorSchemaMismatchError(this.index, name, "TAG", existing.type, `A field's type cannot be changed in place: drop the index (FT.DROPINDEX ${this.index} DD) and re-ingest, or rename the metadata key.`);
+      }
+    }
+    for (const name of missing) {
+      await this.client.sendCommand([
+        "FT.ALTER",
+        this.index,
+        "SCHEMA",
+        "ADD",
+        name,
+        "TAG"
+      ]);
+    }
+  }
+  async createIndex() {
     const schema = [
       "text",
       "TEXT",
@@ -485,6 +557,58 @@ var RedisVectorStore = class {
     return parseSearchReply(reply, this.prefix);
   }
 };
+function parseIndexAttributes(reply) {
+  const attributes = /* @__PURE__ */ new Map();
+  for (const entry of readInfoField(reply, "attributes")) {
+    const fields = readKeyValues(entry);
+    const name = fields.attribute ?? fields.identifier;
+    if (name === void 0) {
+      continue;
+    }
+    const dim = Number(fields.dim);
+    attributes.set(name, {
+      type: (fields.type ?? "").toUpperCase(),
+      ...fields.dim !== void 0 && Number.isFinite(dim) ? {
+        dimensions: dim
+      } : {}
+    });
+  }
+  return attributes;
+}
+__name(parseIndexAttributes, "parseIndexAttributes");
+function readInfoField(reply, key) {
+  if (Array.isArray(reply)) {
+    for (let index = 0; index + 1 < reply.length; index += 2) {
+      if (toStr(reply[index]) === key) {
+        const value = reply[index + 1];
+        return Array.isArray(value) ? value : [];
+      }
+    }
+    return [];
+  }
+  if (typeof reply === "object" && reply !== null && key in reply) {
+    const value = reply[key];
+    return Array.isArray(value) ? value : [];
+  }
+  return [];
+}
+__name(readInfoField, "readInfoField");
+function readKeyValues(entry) {
+  const fields = {};
+  if (Array.isArray(entry)) {
+    for (let index = 0; index + 1 < entry.length; index += 2) {
+      fields[toStr(entry[index]).toLowerCase()] = toStr(entry[index + 1]);
+    }
+    return fields;
+  }
+  if (typeof entry === "object" && entry !== null) {
+    for (const [key, value] of Object.entries(entry)) {
+      fields[key.toLowerCase()] = toStr(value);
+    }
+  }
+  return fields;
+}
+__name(readKeyValues, "readKeyValues");
 function encodeVector(embedding) {
   return Buffer.from(new Float32Array(embedding).buffer);
 }
@@ -662,10 +786,14 @@ var KeywordRetriever = class {
     this.k1 = options.k1 ?? 1.5;
     this.b = options.b ?? 0.75;
   }
+  /** How many chunks are currently indexed. */
+  get size() {
+    return this.docs.size;
+  }
   /** Index (or re-index) records. Re-adding an existing id replaces its posting and updates stats. */
   add(records) {
     for (const record of records) {
-      this.remove(record.id);
+      this.removeChunk(record.id);
       const tokens = tokenize(record.text);
       const frequencies = /* @__PURE__ */ new Map();
       for (const token of tokens) {
@@ -682,7 +810,30 @@ var KeywordRetriever = class {
       this.totalLength += tokens.length;
     }
   }
-  remove(id) {
+  /**
+  * Forget a source document — every chunk whose id is `${documentId}` or `${documentId}#<n>`, the
+  * same id scheme (and the same {@link documentIdOf} collapse) the vector stores use for their
+  * `remove`. The keyword half of a hybrid index keeps its own copy of the chunk text, so a document
+  * dropped from the vector store stays fully retrievable here until this is called. Unknown ids are
+  * a no-op.
+  */
+  remove(documentId) {
+    for (const id of [
+      ...this.docs.keys()
+    ]) {
+      if (id === documentId || documentIdOf(id) === documentId) {
+        this.removeChunk(id);
+      }
+    }
+  }
+  /** Drop every indexed chunk, resetting the corpus statistics with it. */
+  clear() {
+    this.docs.clear();
+    this.documentFrequency.clear();
+    this.totalLength = 0;
+  }
+  /** Drop ONE chunk by its exact id, decrementing the corpus statistics it contributed. */
+  removeChunk(id) {
     const existing = this.docs.get(id);
     if (existing === void 0) {
       return;
@@ -932,6 +1083,7 @@ __name(createRetrievalTool, "createRetrievalTool");
   KeywordRetriever,
   MemoryVectorStore,
   PgVectorStore,
+  RedisVectorSchemaMismatchError,
   RedisVectorStore,
   RerankingRetriever,
   chunkDocuments,