@dudousxd/nestjs-agent-rag 0.4.1 → 0.6.0

package/dist/index.js

@@ -40,11 +40,60 @@ function chunkText(text, options = {}) {
 __name(chunkText, "chunkText");
 
 // src/vector-store.ts
+function isLexicalVectorStore(store) {
+  return typeof store.searchText === "function";
+}
+__name(isLexicalVectorStore, "isLexicalVectorStore");
+var UnsafeRemovalError = class extends Error {
+  static {
+    __name(this, "UnsafeRemovalError");
+  }
+  reason;
+  constructor(reason, message) {
+    super(message), this.reason = reason;
+    this.name = "UnsafeRemovalError";
+  }
+};
 function documentIdOf(chunkId) {
   return chunkId.replace(/#\d+$/, "");
 }
 __name(documentIdOf, "documentIdOf");
 
+// src/metadata-patch.ts
+function splitMetadataPatch(patch) {
+  const set = {};
+  const remove = [];
+  for (const [key, value] of Object.entries(patch)) {
+    if (value === null) {
+      remove.push(key);
+    } else if (value !== void 0) {
+      set[key] = value;
+    }
+  }
+  return {
+    set,
+    remove
+  };
+}
+__name(splitMetadataPatch, "splitMetadataPatch");
+function isEmptyMetadataPatch(patch) {
+  const { set, remove } = splitMetadataPatch(patch);
+  return Object.keys(set).length === 0 && remove.length === 0;
+}
+__name(isEmptyMetadataPatch, "isEmptyMetadataPatch");
+function applyMetadataPatch(metadata, patch) {
+  const { set, remove } = splitMetadataPatch(patch);
+  const next = {
+    ...metadata,
+    ...set
+  };
+  for (const key of remove) {
+    delete next[key];
+  }
+  return next;
+}
+__name(applyMetadataPatch, "applyMetadataPatch");
+
 // src/filter.ts
 function matchesFilter(metadata, filter) {
   const entries = Object.entries(filter);
@@ -60,6 +109,13 @@ function matchesFilter(metadata, filter) {
   });
 }
 __name(matchesFilter, "matchesFilter");
+function filterMatchesNothing(filter) {
+  if (filter === void 0) {
+    return false;
+  }
+  return Object.values(filter).some((value) => Array.isArray(value) && value.length === 0);
+}
+__name(filterMatchesNothing, "filterMatchesNothing");
 
 // src/memory-vector-store.ts
 var MemoryVectorStore = class {
@@ -79,6 +135,29 @@ var MemoryVectorStore = class {
       }
     }
   }
+  /**
+  * Merge `patch` into every chunk of `documentId`, leaving `text` and `embedding` untouched. The
+  * record is *replaced* rather than mutated: `upsert` stores the caller's own object, and patching
+  * it in place would edit an object the caller still holds — a surprise no out-of-process store
+  * could reproduce, and therefore one this reference adapter must not teach.
+  */
+  async updateMetadata(documentId, patch) {
+    if (isEmptyMetadataPatch(patch)) {
+      return 0;
+    }
+    let updated = 0;
+    for (const [id, record] of this.records) {
+      if (documentIdOf(id) !== documentId) {
+        continue;
+      }
+      this.records.set(id, {
+        ...record,
+        metadata: applyMetadataPatch(record.metadata, patch)
+      });
+      updated += 1;
+    }
+    return updated;
+  }
   async listDocuments(filter) {
     const documents = /* @__PURE__ */ new Map();
     for (const record of this.records.values()) {
@@ -99,6 +178,66 @@ var MemoryVectorStore = class {
       ...documents.values()
     ];
   }
+  async listDocumentIds(filter) {
+    if (filterMatchesNothing(filter)) {
+      return [];
+    }
+    const documents = /* @__PURE__ */ new Set();
+    for (const record of this.records.values()) {
+      if (filter !== void 0 && !matchesFilter(record.metadata, filter)) {
+        continue;
+      }
+      documents.add(documentIdOf(record.id));
+    }
+    return [
+      ...documents
+    ];
+  }
+  async removeMany(documentIds) {
+    if (documentIds.length === 0) {
+      return;
+    }
+    const wanted = new Set(documentIds);
+    for (const id of this.records.keys()) {
+      if (wanted.has(documentIdOf(id))) {
+        this.records.delete(id);
+      }
+    }
+  }
+  /**
+  * See {@link VectorStore.removeWhere}. The empty-array short-circuit below is redundant
+  * against `matchesFilter` (which already matches nothing for an empty array) and is kept anyway:
+  * this is the method where "the filter accidentally means everything" destroys data, so the deny is
+  * stated where a reader can see it rather than inferred from another module's truth table.
+  */
+  async removeWhere(filter) {
+    if (Object.keys(filter).length === 0) {
+      throw new UnsafeRemovalError("empty-filter", "removeWhere() refuses an empty filter: it would delete every chunk in the store. Pass a filter that scopes the removal, or delete deliberately with removeMany(await store.listDocumentIds()).");
+    }
+    if (filterMatchesNothing(filter)) {
+      return 0;
+    }
+    let removed = 0;
+    for (const [id, record] of this.records) {
+      if (matchesFilter(record.metadata, filter)) {
+        this.records.delete(id);
+        removed += 1;
+      }
+    }
+    return removed;
+  }
+  async countChunks(filter) {
+    if (filterMatchesNothing(filter)) {
+      return 0;
+    }
+    let count = 0;
+    for (const record of this.records.values()) {
+      if (filter === void 0 || matchesFilter(record.metadata, filter)) {
+        count += 1;
+      }
+    }
+    return count;
+  }
   async search(embedding, options) {
     const scored = [];
     for (const record of this.records.values()) {
@@ -185,6 +324,33 @@ var PgVectorStore = class {
       documentId
     ]);
   }
+  /**
+  * Rewrite a document's metadata without re-embedding it. See {@link VectorStore.updateMetadata}
+  * for the semantics; here the whole thing is one statement, because jsonb already *is* the merge:
+  * `||` is a shallow key-wise merge (right side wins, arrays replaced wholesale — precisely
+  * {@link MetadataPatch}) and `- text[]` drops the keys the patch removed. `COALESCE` covers a chunk
+  * ingested with no metadata at all, so a patch can create the object rather than no-op on `NULL`.
+  *
+  * Unlike RediSearch there is no second representation to keep in step — the `metadata` column is
+  * both what `search` filters on and what it returns — so the correctness trap the Redis adapter has
+  * to work for simply does not exist here. `RETURNING id` is what supplies the chunk count, since
+  * {@link PgClient} is a rows-only surface with no `rowCount`.
+  */
+  async updateMetadata(documentId, patch) {
+    if (isEmptyMetadataPatch(patch)) {
+      return 0;
+    }
+    const { set, remove } = splitMetadataPatch(patch);
+    const rows = await this.client.query(`UPDATE ${this.table}
+          SET metadata = (COALESCE(metadata, '{}'::jsonb) || $2::jsonb) - $3::text[]
+        WHERE ${DOCUMENT_ID_FROM_CHUNK} = $1
+        RETURNING id`, [
+      documentId,
+      JSON.stringify(set),
+      remove
+    ]);
+    return rows.length;
+  }
   async listDocuments(filter) {
     const where = buildWhere(filter, 1);
     const rows = await this.client.query(`SELECT DISTINCT ON (${DOCUMENT_ID_FROM_CHUNK}) ${DOCUMENT_ID_FROM_CHUNK} AS doc_id, metadata
@@ -198,6 +364,51 @@ var PgVectorStore = class {
       } : {}
     }));
   }
+  /** {@link listDocuments} without the metadata: one `DISTINCT` over the derived document id. */
+  async listDocumentIds(filter) {
+    if (filterMatchesNothing(filter)) {
+      return [];
+    }
+    const where = buildWhere(filter, 1);
+    const rows = await this.client.query(`SELECT DISTINCT ${DOCUMENT_ID_FROM_CHUNK} AS doc_id
+       FROM ${this.table}
+       ${where.sql}
+       ORDER BY doc_id`, where.params);
+    return rows.map((row) => row.doc_id);
+  }
+  /** N documents in one statement — the set-based form of {@link PgVectorStore.remove}. */
+  async removeMany(documentIds) {
+    if (documentIds.length === 0) {
+      return;
+    }
+    await this.client.query(`DELETE FROM ${this.table} WHERE ${DOCUMENT_ID_FROM_CHUNK} = ANY($1::text[])`, [
+      documentIds
+    ]);
+  }
+  /**
+  * See {@link VectorStore.removeWhere}: empty filter object throws, empty-array value
+  * deletes nothing. The count comes from `RETURNING id` rather than a driver-specific `rowCount`,
+  * because {@link PgClient} is deliberately just "run SQL, get rows".
+  */
+  async removeWhere(filter) {
+    if (Object.keys(filter).length === 0) {
+      throw new UnsafeRemovalError("empty-filter", "removeWhere() refuses an empty filter: it would delete every chunk in the table. Pass a filter that scopes the removal, or delete deliberately with removeMany(await store.listDocumentIds()).");
+    }
+    if (filterMatchesNothing(filter)) {
+      return 0;
+    }
+    const where = buildWhere(filter, 1);
+    const rows = await this.client.query(`DELETE FROM ${this.table} ${where.sql} RETURNING id`, where.params);
+    return rows.length;
+  }
+  async countChunks(filter) {
+    if (filterMatchesNothing(filter)) {
+      return 0;
+    }
+    const where = buildWhere(filter, 1);
+    const rows = await this.client.query(`SELECT count(*)::int AS count FROM ${this.table} ${where.sql}`, where.params);
+    return Number(rows[0]?.count ?? 0);
+  }
   async search(embedding, options) {
     const vector = toVectorLiteral(embedding);
     const where = buildWhere(options.filter, 3);
@@ -291,12 +502,14 @@ var RedisVectorStore = class {
   prefix;
   dimensions;
   filterableFields;
+  lexicalScorer;
   constructor(client, options = {}) {
     this.client = client;
     this.index = options.index ?? "agent_rag_idx";
     this.prefix = options.prefix ?? "agent_rag:";
     this.dimensions = options.dimensions ?? 1536;
     this.filterableFields = options.filterableFields ?? [];
+    this.lexicalScorer = options.lexicalScorer ?? "BM25";
   }
   /**
   * Idempotent, and drift-aware: create the HNSW/cosine index if `FT.INFO` says it doesn't exist —
@@ -409,18 +622,129 @@ var RedisVectorStore = class {
         for (const field of this.filterableFields) {
           const value = record.metadata[field];
           if (value !== void 0) {
-            const tag = Array.isArray(value) ? value.map(String).join(",") : String(value);
-            args.push(`meta_${field}`, tag);
+            args.push(`meta_${field}`, encodeTag(value));
           }
         }
       }
       await this.client.sendCommand(args);
     }
   }
+  /**
+  * Single-document deletion, by `SCAN … MATCH`.
+  *
+  * This deliberately stays a keyspace scan even though the rest of this class no longer needs one.
+  * The obvious optimisation is to stamp each chunk with its document id as an implicit filterable
+  * TAG at {@link RedisVectorStore.upsert} and turn `remove` into an `FT.SEARCH` — and that is exactly
+  * the change that must not ship, because chunks written *before* it carry no such field. `FT.SEARCH`
+  * would find zero of them, `remove` would return happily, and a document that has been "deleted"
+  * would go on being retrieved. A delete that silently stops deleting is worse than a slow one, and
+  * the library cannot tell a fully-migrated corpus from an unmigrated one: both look like "this
+  * document has no indexed chunks". Nor is a search-then-fall-back-to-scan hybrid safe — a document
+  * re-ingested after the upgrade has *some* stamped chunks, so the search finds hits, the fallback
+  * never runs, and the unstamped tail survives. That is the same silent failure with a smaller blast
+  * radius, which makes it harder to notice rather than less wrong.
+  *
+  * The consumer pain this was measured against — deleting a whole collection, one `remove` per
+  * document — is answered instead by {@link RedisVectorStore.removeWhere} (one filtered query, no
+  * keyspace scan at all) and {@link RedisVectorStore.removeMany} (one scan for N documents rather
+  * than N scans). Neither depends on a field that older chunks lack, so neither has a migration.
+  */
   async remove(documentId) {
-    const keys = /* @__PURE__ */ new Set([
-      `${this.prefix}${documentId}`
+    const keys = await this.chunkKeys(documentId);
+    if (keys.length > 0) {
+      await this.client.sendCommand([
+        "DEL",
+        ...keys
+      ]);
+    }
+  }
+  /**
+  * Rewrite the metadata of every chunk of `documentId` in place, leaving `text` and `embedding`
+  * alone. See {@link VectorStore.updateMetadata} for the merge semantics and the return value.
+  *
+  * The care this needs is entirely about the fact that a chunk stores its metadata **twice**: as the
+  * `metadata_json` blob (what {@link RedisVectorStore.search} reads back onto a `Passage`) and as
+  * `meta_<field>` TAG fields (what RediSearch actually filters on, and only for the declared
+  * `filterableFields`). Move one without the other and the store lies to itself — a chunk that
+  * filters as the old value but reports the new one, or the reverse — so this method rewrites both
+  * from the same merged object:
+  *
+  * - a patched key that is **not** filterable lands in `metadata_json` only, and no stray `meta_*`
+  *   field is invented for it (it would be dead weight the index does not know about);
+  * - a patched key that **is** filterable also rewrites its TAG, using the identical encoding
+  *   `upsert` uses, so an updated chunk is indistinguishable from a re-ingested one;
+  * - a filterable key the patch **removes** has its TAG `HDEL`-ed, not left behind as a tag matching
+  *   a value the document no longer carries.
+  *
+  * A plain `HSET` is all the index needs: RediSearch re-indexes a hash the moment it is written, so
+  * the very next `FT.SEARCH` sees the new TAG and no longer the old one — no reindex command, and
+  * no need to rewrite the (untouched, and expensive) vector to nudge it. `HDEL` on an indexed field
+  * likewise drops it from the index. Verified against a real Redis Stack in
+  * `redis-vector-store.db.spec.ts`, not assumed.
+  *
+  * Removals are issued **before** the write, so the only window a concurrent reader can observe is
+  * "TAG already gone, `metadata_json` not yet updated" — which fails *closed* on a filter carrying
+  * an ACL. The reverse order would briefly leave a document tagged with a capability its metadata no
+  * longer claims. Read-modify-write across chunks is not transactional; concurrent writers to the
+  * same chunk are last-writer-wins, exactly as two concurrent `upsert`s already are.
+  */
+  async updateMetadata(documentId, patch) {
+    if (isEmptyMetadataPatch(patch)) {
+      return 0;
+    }
+    const keys = await this.chunkKeys(documentId);
+    let updated = 0;
+    for (const key of keys) {
+      const raw = await this.client.sendCommand([
+        "HGET",
+        key,
+        "metadata_json"
+      ]);
+      const current = parseMetadata(raw === null || raw === void 0 ? void 0 : toStr(raw));
+      const next = applyMetadataPatch(current.metadata, patch);
+      const args = [
+        "HSET",
+        key,
+        "metadata_json",
+        JSON.stringify(next)
+      ];
+      const stale = [];
+      for (const field of this.filterableFields) {
+        const value = next[field];
+        if (value !== void 0) {
+          args.push(`meta_${field}`, encodeTag(value));
+        } else if (current.metadata?.[field] !== void 0) {
+          stale.push(`meta_${field}`);
+        }
+      }
+      if (stale.length > 0) {
+        await this.client.sendCommand([
+          "HDEL",
+          key,
+          ...stale
+        ]);
+      }
+      await this.client.sendCommand(args);
+      updated += 1;
+    }
+    return updated;
+  }
+  /**
+  * Every chunk key that currently EXISTS for `documentId` — the bare `${prefix}${documentId}` plus
+  * the `${documentId}#<n>` chunks found by `SCAN`. One definition of "chunk belongs to document",
+  * shared by `remove` and `updateMetadata`, so the two can never disagree about which keys a
+  * document owns. The glob is escaped so `gone` does not reach into `goner#0`.
+  */
+  async chunkKeys(documentId) {
+    const keys = /* @__PURE__ */ new Set();
+    const bare = `${this.prefix}${documentId}`;
+    const exists = await this.client.sendCommand([
+      "EXISTS",
+      bare
     ]);
+    if (Number(toStr(exists)) > 0) {
+      keys.add(bare);
+    }
     const pattern = `${this.prefix}${escapeGlob(documentId)}#*`;
     let cursor = "0";
     do {
@@ -438,12 +762,9 @@ var RedisVectorStore = class {
         keys.add(key);
       }
     } while (cursor !== "0");
-    if (keys.size > 0) {
-      await this.client.sendCommand([
-        "DEL",
-        ...keys
-      ]);
-    }
+    return [
+      ...keys
+    ];
   }
   async listDocuments(filter) {
     if (filterMatchesNothing(filter)) {
@@ -488,6 +809,186 @@ var RedisVectorStore = class {
       ...documents.values()
     ];
   }
+  /**
+  * The id-only half of {@link RedisVectorStore.listDocuments}, served by `FT.SEARCH … NOCONTENT`:
+  * RediSearch returns bare keys, so nothing carries `metadata_json` over the wire and nothing is
+  * JSON-parsed. `listDocuments` has to parse one metadata blob per *chunk* — thousands of parses to
+  * produce a few hundred ids it then throws the metadata away from — and this does none of it.
+  *
+  * The collapse to document ids still happens client-side. RediSearch can group server-side
+  * (`FT.AGGREGATE … GROUPBY`), but only over an indexed *field*, and a chunk's document id lives in
+  * its key, not in a field — deriving it would mean stamping one at write time, which is the
+  * migration {@link RedisVectorStore.remove} explains this class refuses. Deduplicating a page of
+  * keys in JS is free by comparison.
+  */
+  async listDocumentIds(filter) {
+    if (filterMatchesNothing(filter)) {
+      return [];
+    }
+    const query = buildFilter(filter);
+    const documents = /* @__PURE__ */ new Set();
+    let offset = 0;
+    for (; ; ) {
+      const reply = await this.client.sendCommand([
+        "FT.SEARCH",
+        this.index,
+        query,
+        "NOCONTENT",
+        "LIMIT",
+        String(offset),
+        String(SCAN_PAGE),
+        "DIALECT",
+        "2"
+      ]);
+      const keys = parseSearchKeys(reply);
+      for (const key of keys) {
+        documents.add(documentIdOf(stripPrefix(key, this.prefix)));
+      }
+      if (keys.length < SCAN_PAGE) {
+        break;
+      }
+      offset += SCAN_PAGE;
+    }
+    return [
+      ...documents
+    ];
+  }
+  /**
+  * Bulk deletion by document id — **one** keyspace scan for N documents, where N calls to
+  * {@link RedisVectorStore.remove} would be N of them. The scan is `MATCH ${prefix}*`, so Redis still
+  * filters to this store's own keys server-side; membership is then decided in JS against a `Set`,
+  * which is what lets a single pass serve any number of ids.
+  *
+  * It deletes by key, exactly as `remove` does, so it is correct on chunks written by any version of
+  * this library — there is no field it needs the corpus to already carry.
+  */
+  async removeMany(documentIds) {
+    if (documentIds.length === 0) {
+      return;
+    }
+    const wanted = new Set(documentIds);
+    let pending = documentIds.map((id) => `${this.prefix}${id}`);
+    let cursor = "0";
+    do {
+      const reply = await this.client.sendCommand([
+        "SCAN",
+        cursor,
+        "MATCH",
+        `${escapeGlob(this.prefix)}*`,
+        "COUNT",
+        String(SCAN_PAGE)
+      ]);
+      const page = parseScanReply(reply);
+      cursor = page.cursor;
+      for (const key of page.keys) {
+        if (wanted.has(documentIdOf(stripPrefix(key, this.prefix)))) {
+          pending.push(key);
+        }
+      }
+      if (pending.length >= DELETE_BATCH) {
+        await this.client.sendCommand([
+          "DEL",
+          ...pending
+        ]);
+        pending = [];
+      }
+    } while (cursor !== "0");
+    if (pending.length > 0) {
+      await this.client.sendCommand([
+        "DEL",
+        ...pending
+      ]);
+    }
+  }
+  /**
+  * Delete every chunk matching `filter`, via `FT.SEARCH … NOCONTENT` + `DEL` — no keyspace scan at
+  * all, so dropping a collection costs one indexed query per page instead of one full pass per
+  * document. Resolves the number of chunk keys actually deleted.
+  *
+  * The two refusals, in this order, before any query is issued:
+  *
+  * 1. **An empty filter object throws** {@link UnsafeRemovalError}. `buildFilter({})` is `*`, and `*`
+  *    here would mean "delete the corpus" — far too destructive an outcome for the shape a filter
+  *    degenerates to when it is built wrong.
+  * 2. **An empty-array value deletes nothing** and resolves `0`. This is the package's deny primitive
+  *    (an actor with no capability tokens), and it is the one that has to be gated *explicitly*: a
+  *    plausible-looking "normalise the filter first" refactor that drops empty arrays turns a deny
+  *    into `*` and wipes everything the deny existed to protect.
+  *
+  * Then one more, which is specific to RediSearch: every filter key must be a declared
+  * `filterableFields` entry. `search` lets the engine reject an unknown field, which is fine for a
+  * read; for a delete the check is worth making up front and by name.
+  *
+  * Deletion happens page by page, always re-querying from offset 0 — the rows just deleted are gone
+  * from the index, so paging forward would skip their replacements. If a page comes back non-empty
+  * but `DEL` reports nothing removed (keys expired between query and delete, or the index is stale),
+  * the loop stops rather than spinning on results it can never consume.
+  */
+  async removeWhere(filter) {
+    const keys = Object.keys(filter);
+    if (keys.length === 0) {
+      throw new UnsafeRemovalError("empty-filter", "removeWhere() refuses an empty filter: it would delete every chunk in the index. Pass a filter that scopes the removal, or delete deliberately with removeMany(await store.listDocumentIds()).");
+    }
+    const unindexed = keys.filter((key) => !this.filterableFields.includes(key));
+    if (unindexed.length > 0) {
+      const named = unindexed.map((key) => `"${key}"`).join(", ");
+      const declared = this.filterableFields.join(", ") || "none";
+      throw new UnsafeRemovalError("unindexed-field", `removeWhere() cannot scope on ${named}: not declared in this store's filterableFields (${declared}), so RediSearch has no TAG to match on.`);
+    }
+    if (filterMatchesNothing(filter)) {
+      return 0;
+    }
+    const query = buildFilter(filter);
+    let removed = 0;
+    for (; ; ) {
+      const reply = await this.client.sendCommand([
+        "FT.SEARCH",
+        this.index,
+        query,
+        "NOCONTENT",
+        "LIMIT",
+        "0",
+        String(DELETE_BATCH),
+        "DIALECT",
+        "2"
+      ]);
+      const page = parseSearchKeys(reply);
+      if (page.length === 0) {
+        break;
+      }
+      const deleted = Number(await this.client.sendCommand([
+        "DEL",
+        ...page
+      ]));
+      if (!Number.isFinite(deleted) || deleted === 0) {
+        break;
+      }
+      removed += deleted;
+    }
+    return removed;
+  }
+  /**
+  * Count matching chunks with `FT.SEARCH … LIMIT 0 0` — RediSearch returns the total and no
+  * documents, so the cost is the query, not the corpus. An empty-array filter value denies and
+  * yields `0`, same as every other filtered path here.
+  */
+  async countChunks(filter) {
+    if (filterMatchesNothing(filter)) {
+      return 0;
+    }
+    const reply = await this.client.sendCommand([
+      "FT.SEARCH",
+      this.index,
+      buildFilter(filter),
+      "NOCONTENT",
+      "LIMIT",
+      "0",
+      "0",
+      "DIALECT",
+      "2"
+    ]);
+    return parseSearchTotal(reply);
+  }
   async search(embedding, options) {
     if (filterMatchesNothing(options.filter)) {
       return [];
@@ -518,7 +1019,77 @@ var RedisVectorStore = class {
     ]);
     return parseSearchReply(reply, this.prefix);
   }
+  /**
+  * Lexical (BM25) search over the same index `search` uses — the {@link LexicalVectorStore} half of
+  * hybrid retrieval, served by RediSearch's own keyword index rather than an in-process copy of the
+  * corpus. `options.filter` is ANDed onto the text clause with the identical TAG semantics as
+  * `search` (array = OR, empty array = deny), so a lexical hit can never escape a filter a vector
+  * hit would have respected.
+  *
+  * A query with no searchable terms — empty, whitespace, or pure punctuation — returns `[]`. That is
+  * deliberate: RediSearch's `*` would match the entire corpus, which is the opposite of what a
+  * relevance query with nothing to be relevant to should do.
+  */
+  async searchText(query, options) {
+    if (filterMatchesNothing(options.filter)) {
+      return [];
+    }
+    const terms = tokenizeQuery(query);
+    if (terms.length === 0) {
+      return [];
+    }
+    const filterQuery = buildFilter(options.filter);
+    const textQuery = `@text:(${terms.join("|")})`;
+    const reply = await this.client.sendCommand([
+      "FT.SEARCH",
+      this.index,
+      filterQuery === "*" ? textQuery : `(${filterQuery} ${textQuery})`,
+      "SCORER",
+      this.lexicalScorer,
+      "WITHSCORES",
+      "RETURN",
+      "3",
+      "text",
+      "source",
+      "metadata_json",
+      "DIALECT",
+      "2",
+      "LIMIT",
+      "0",
+      String(options.topK)
+    ]);
+    return parseSearchReply(reply, this.prefix, "lexical");
+  }
 };
+var SCAN_PAGE = 1e3;
+var DELETE_BATCH = 512;
+function stripPrefix(key, prefix) {
+  return key.startsWith(prefix) ? key.slice(prefix.length) : key;
+}
+__name(stripPrefix, "stripPrefix");
+function parseSearchKeys(reply) {
+  if (Array.isArray(reply)) {
+    return reply.slice(1).map(toStr);
+  }
+  if (typeof reply === "object" && reply !== null && "results" in reply) {
+    const results = reply.results;
+    return Array.isArray(results) ? results.map(readId) : [];
+  }
+  return [];
+}
+__name(parseSearchKeys, "parseSearchKeys");
+function parseSearchTotal(reply) {
+  if (Array.isArray(reply)) {
+    const total = Number(toStr(reply[0]));
+    return Number.isFinite(total) ? total : 0;
+  }
+  if (typeof reply === "object" && reply !== null && "total_results" in reply) {
+    const total = Number(reply.total_results);
+    return Number.isFinite(total) ? total : 0;
+  }
+  return 0;
+}
+__name(parseSearchTotal, "parseSearchTotal");
 function parseIndexAttributes(reply) {
   const attributes = /* @__PURE__ */ new Map();
   for (const entry of readInfoField(reply, "attributes")) {
@@ -589,13 +1160,14 @@ function buildFilter(filter) {
   return `(${clauses.join(" ")})`;
 }
 __name(buildFilter, "buildFilter");
-function filterMatchesNothing(filter) {
-  if (filter === void 0) {
-    return false;
-  }
-  return Object.values(filter).some((value) => Array.isArray(value) && value.length === 0);
+function tokenizeQuery(query) {
+  return query.match(/[\p{L}\p{N}_]+/gu) ?? [];
 }
-__name(filterMatchesNothing, "filterMatchesNothing");
+__name(tokenizeQuery, "tokenizeQuery");
+function encodeTag(value) {
+  return Array.isArray(value) ? value.map(String).join(",") : String(value);
+}
+__name(encodeTag, "encodeTag");
 function escapeTag(value) {
   return value.replace(/[^a-zA-Z0-9_]/g, (char) => `\\${char}`);
 }
@@ -632,17 +1204,19 @@ function toStr(value) {
   return Buffer.isBuffer(value) ? value.toString("utf8") : String(value);
 }
 __name(toStr, "toStr");
-function parseSearchReply(reply, prefix) {
+function parseSearchReply(reply, prefix, scoreSource = "vector") {
+  const stride = scoreSource === "lexical" ? 3 : 2;
   if (Array.isArray(reply)) {
     const passages = [];
-    for (let index = 1; index + 1 < reply.length; index += 2) {
-      const rawFields = reply[index + 1];
+    for (let index = 1; index + stride - 1 < reply.length; index += stride) {
+      const rawFields = reply[index + stride - 1];
       if (Array.isArray(rawFields)) {
         const attrs = {};
         for (let field = 0; field + 1 < rawFields.length; field += 2) {
           attrs[toStr(rawFields[field])] = toStr(rawFields[field + 1]);
         }
-        passages.push(toPassage(toStr(reply[index]), attrs, prefix));
+        const inlineScore = scoreSource === "lexical" ? toStr(reply[index + 1]) : void 0;
+        passages.push(toPassage(toStr(reply[index]), attrs, prefix, scoreSource, inlineScore));
       }
     }
     return passages;
@@ -650,12 +1224,19 @@ function parseSearchReply(reply, prefix) {
   if (typeof reply === "object" && reply !== null && "results" in reply) {
     const results = reply.results;
     if (Array.isArray(results)) {
-      return results.map((result) => toPassage(readId(result), readAttributes(result), prefix));
+      return results.map((result) => toPassage(readId(result), readAttributes(result), prefix, scoreSource, scoreSource === "lexical" ? readScore(result) : void 0));
     }
   }
   return [];
 }
 __name(parseSearchReply, "parseSearchReply");
+function readScore(result) {
+  if (typeof result === "object" && result !== null && "score" in result) {
+    return toStr(result.score);
+  }
+  return void 0;
+}
+__name(readScore, "readScore");
 function readId(result) {
   if (typeof result === "object" && result !== null && "id" in result) {
     return toStr(result.id);
@@ -676,13 +1257,13 @@ function readAttributes(result) {
   return attrs;
 }
 __name(readAttributes, "readAttributes");
-function toPassage(rawId, attrs, prefix) {
-  const distance = Number(attrs.vector_score ?? "1");
+function toPassage(rawId, attrs, prefix, scoreSource = "vector", inlineScore) {
+  const score = scoreSource === "lexical" ? Number(inlineScore ?? "0") : 1 - Number(attrs.vector_score ?? "1");
   const source = attrs.source;
   return {
     id: rawId.startsWith(prefix) ? rawId.slice(prefix.length) : rawId,
     text: attrs.text ?? "",
-    score: 1 - distance,
+    score: Number.isFinite(score) ? score : 0,
     ...source !== void 0 ? {
       source
     } : {},
@@ -856,6 +1437,25 @@ function tokenize(text) {
 }
 __name(tokenize, "tokenize");
 
+// src/lexical-retriever.ts
+var LexicalRetriever = class {
+  static {
+    __name(this, "LexicalRetriever");
+  }
+  store;
+  constructor(store) {
+    this.store = store;
+  }
+  async retrieve(query, options = {}) {
+    return this.store.searchText(query, {
+      topK: options.topK ?? 5,
+      ...options.filter !== void 0 ? {
+        filter: options.filter
+      } : {}
+    });
+  }
+};
+
 // src/hybrid-retriever.ts
 var HybridRetriever = class {
   static {
@@ -1042,16 +1642,20 @@ export {
   FilteredRetriever,
   HybridRetriever,
   KeywordRetriever,
+  LexicalRetriever,
   MemoryVectorStore,
   PgVectorStore,
   RedisVectorSchemaMismatchError,
   RedisVectorStore,
   RerankingRetriever,
+  UnsafeRemovalError,
+  applyMetadataPatch,
   chunkDocuments,
   chunkText,
   createRetrievalTool,
   documentIdOf,
   ingestChunks,
-  ingestDocuments
+  ingestDocuments,
+  isLexicalVectorStore
 };
 //# sourceMappingURL=index.js.map