@dudousxd/nestjs-agent-rag 0.4.1 → 0.6.0

package/dist/index.d.ts

@@ -14,6 +14,45 @@ interface ChunkOptions {
  */
 declare function chunkText(text: string, options?: ChunkOptions): string[];
 
+/**
+ * A partial update to a document's metadata, as taken by
+ * {@link import('./vector-store.js').VectorStore.updateMetadata}. Deliberately *partial*: the useful
+ * shape is "this one dimension was reclassified", not "here is the whole metadata object again" —
+ * a caller that had the whole object would have to reconstruct fields it does not own (the ingestion
+ * fingerprint, the source, the mime type) just to change one string, and would silently drop them
+ * the day the library adds another.
+ *
+ * The semantics are RFC 7386 (JSON Merge Patch), with one deliberate narrowing:
+ *
+ * - **Merge, not replace.** Keys present in the patch are written; keys absent from it keep their
+ *   stored value.
+ * - **`null` removes the key.** That is the RFC's sentinel and it is the *only* way to remove one.
+ *   The consequence — a literal `null` cannot be stored as a value — is intended: metadata is a
+ *   filter surface, and "the key is absent" and "the key is null" would filter identically anyway.
+ * - **`undefined` is ignored** — the key is left exactly as stored, as if the patch had not mentioned
+ *   it. `undefined` does not survive `JSON.stringify`, and a patch routinely crosses a JSON boundary
+ *   (an HTTP body, a durable-workflow step payload) on its way here; honouring it would make
+ *   `{ audience: undefined }` mean *delete* on one side of that boundary and *no-op* on the other.
+ *   It is also what `{ ...doc, audience: doc.audience }` produces by accident, and an accidental
+ *   deletion of an access-control dimension is the worst outcome this API can have. Removing a key
+ *   is therefore always spelled `null`.
+ * - **Shallow.** A value replaces wholesale; it is not merged into the stored value. This is the
+ *   narrowing versus RFC 7386, which recurses into nested objects. It matters most for **arrays**,
+ *   where wholesale replacement is certainly what a caller means: `{ bases: ['A', 'C'] }` sets the
+ *   document's bases to exactly A and C — it does not append to, or union with, what was there. An
+ *   array-valued key is a *set-valued dimension* (the multi-valued TAG a store filters on), and the
+ *   only sane way to shrink such a set is to state the new one.
+ */
+type MetadataPatch = Record<string, unknown>;
+/**
+ * Apply a {@link MetadataPatch} to a document's stored metadata, returning a NEW object — the
+ * reference implementation of the semantics documented on `MetadataPatch`, so every adapter that
+ * merges in JS agrees with every other one. Never mutates its input: `MemoryVectorStore` hands out
+ * the very object a caller upserted, and quietly rewriting it under them would be a surprise no
+ * other store reproduces.
+ */
+declare function applyMetadataPatch(metadata: Record<string, unknown> | undefined, patch: MetadataPatch): Record<string, unknown>;
+
 /** A stored, embedded chunk. `embedding` length must match the store's configured dimensions. */
 interface VectorRecord {
     id: string;
@@ -33,6 +72,31 @@ interface VectorSearchOptions {
  * (in-JS cosine, tests + small scale) and `PgVectorStore` (pgvector); any impl works. Pair one with
  * an `EmbeddingProvider` via {@link import('./embedding-retriever.js').EmbeddingRetriever} to get a
  * core `Retriever`.
+ *
+ * Every method here is **required**, the enumeration and bulk-deletion half included. The rule for
+ * where a new method goes, so the next person adding one has it rather than inferring it from
+ * precedent:
+ *
+ * - **Required, on this interface**, when every backend *can* implement it from what it already
+ *   stores. {@link VectorStore.updateMetadata}, {@link VectorStore.listDocumentIds},
+ *   {@link VectorStore.removeMany}, {@link VectorStore.removeWhere} and
+ *   {@link VectorStore.countChunks} are CRUD over the records the store already holds — the same
+ *   footing as {@link VectorStore.remove} and
+ *   {@link VectorStore.listDocuments}. Making them optional would buy nothing but a permanent
+ *   `if (supported)` at every call site plus an unreachable "and if it isn't?" branch.
+ * - **Optional, in a capability interface**, when some backend genuinely *cannot* without
+ *   infrastructure the consumer has to adopt. The only such capability today is
+ *   {@link LexicalVectorStore.searchText}: RediSearch has BM25 natively, Postgres would need a
+ *   `tsvector` column plus a GIN index plus a migration the consumer must run, and
+ *   `MemoryVectorStore` would have to implement BM25 itself.
+ *
+ * The test is whether a backend can do it at all — not whether adding it would inconvenience an
+ * implementer who has already shipped.
+ *
+ * Implementations MUST apply every filter with exactly the same semantics as
+ * {@link VectorStore.search} — including the empty-array deny primitive — because the filter is
+ * routinely an access-control boundary, not a hint. On {@link VectorStore.removeWhere} that is not a
+ * correctness nicety but the single most destructive thing an implementation could get wrong.
  */
 interface VectorStore {
     upsert(records: VectorRecord[]): Promise<void>;
@@ -45,6 +109,38 @@ interface VectorStore {
      * without a preceding `remove` leaves the old tail orphaned.
      */
     remove(documentId: string): Promise<void>;
+    /**
+     * Rewrite a document's metadata **without touching its text or its embeddings** — the update path
+     * for a dimension that is genuinely mutable (which bases a document is relevant to, who may see it,
+     * what it was reclassified as) rather than derived from its content.
+     *
+     * Without this, a consumer whose documents get re-classified has exactly two options, and both are
+     * bad: re-embed the whole document — paying the model bill, and burning the ingestion budget, to
+     * change a string — or refuse to stamp the dimension onto chunks at all and resolve it at query
+     * time instead, which turns a filter the index could have applied into a join the caller has to.
+     *
+     * `patch` is applied to **every chunk** of the document (metadata is a document-level property that
+     * happens to be stored per chunk), with the merge semantics documented on {@link MetadataPatch}:
+     * partial merge, `null` to remove a key, `undefined` ignored, values — including arrays — replaced
+     * wholesale.
+     *
+     * Returns **the number of chunks written**. An unknown `documentId` updates nothing and returns
+     * `0`; it does not throw. That is deliberate and matches {@link VectorStore.remove}, which is
+     * likewise silent on a document that isn't there: this is a reconciliation-shaped API, called by a
+     * loop diffing a source of truth against the index, and such a loop races with ingestion and
+     * deletion by construction. Turning "the document was removed between the diff and the write" into
+     * an exception would make the ordinary case an error, and the caller would have to catch and
+     * swallow it to get back the behaviour it wanted. The count gives the same information without
+     * that: `0` means nothing was there. (An empty patch also writes nothing and returns `0` — the
+     * return counts chunks *written*, not chunks *matched*.)
+     *
+     * Implementations MUST keep every representation of the metadata in step. A store that keeps both
+     * a filterable projection and a returned blob (RediSearch's `meta_*` TAGs beside its
+     * `metadata_json`) and updates only one makes the store lie to itself: a chunk that *filters* as
+     * one value but *reports* another, where which half you notice depends on whether you searched or
+     * read.
+     */
+    updateMetadata(documentId: string, patch: MetadataPatch): Promise<number>;
     /**
      * List the distinct source documents currently indexed (chunk ids collapsed back to their document
      * by stripping the trailing `#<n>`), each with a representative chunk's `metadata`, optionally
@@ -53,6 +149,97 @@ interface VectorStore {
      * `size` changed), or {@link VectorStore.remove remove} (orphaned).
      */
     listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
+    /**
+     * The distinct source document ids currently indexed, optionally narrowed by a metadata `filter` —
+     * {@link VectorStore.listDocuments} without the metadata. Use it when you only need the id set (a
+     * reconciliation diff against your source of truth, a count, a delete list): implementations are
+     * expected to skip fetching and parsing per-chunk metadata entirely, which is most of the cost of
+     * `listDocuments` on a large corpus.
+     *
+     * An empty-array filter value denies, yielding `[]`.
+     */
+    listDocumentIds(filter?: Record<string, unknown>): Promise<string[]>;
+    /**
+     * Delete every chunk of every one of `documentIds`, with the same per-document semantics as
+     * {@link VectorStore.remove} — but as one bulk operation rather than N independent ones. An empty
+     * array is a no-op.
+     *
+     * The gap this closes is asymptotic, not cosmetic: on the Redis adapter each `remove` is a `SCAN`
+     * that walks the whole keyspace, which in a real deployment also holds queues, workflow state and
+     * the app cache. Two thousand documents is two thousand full passes; this is one filtered query.
+     */
+    removeMany(documentIds: string[]): Promise<void>;
+    /**
+     * Delete every chunk matching a metadata `filter`, and resolve the number of **chunks** removed
+     * (not documents). The collection-maintenance primitive: dropping a collection, evicting a tenant,
+     * purging a retired audience.
+     *
+     * Two guard rails, because this is the one method here that destroys data:
+     *
+     * - **The empty-array deny is honoured, and it means "delete nothing".** `removeWhere({ audience: [] })`
+     *   removes zero chunks. Treating a deny as "no filter" would delete the entire corpus, which is
+     *   exactly inverted from what the caller asked for.
+     * - **The filter must be non-empty.** `removeWhere({})` throws {@link UnsafeRemovalError} rather
+     *   than wiping the store, because an empty object is overwhelmingly more likely to be a filter that
+     *   got built wrong (a dropped key, an `undefined` scope) than a deliberate request to delete
+     *   everything. Deliberate mass deletion stays available and stays explicit:
+     *   `removeMany(await store.listDocumentIds())`, or the backend's own drop.
+     *
+     * What it removes is exactly what a {@link VectorStore.search} carrying the same filter could
+     * reach — no more (it cannot escape the filter) and no less.
+     */
+    removeWhere(filter: Record<string, unknown>): Promise<number>;
+    /**
+     * How many **chunks** match `filter` (all of them when it is omitted) — a counted query, without
+     * transferring the chunks. Sizing a reindex, reporting a collection's footprint, and confirming a
+     * {@link VectorStore.removeWhere} did what you expected. An empty-array filter value denies,
+     * yielding `0`.
+     */
+    countChunks(filter?: Record<string, unknown>): Promise<number>;
+}
+/**
+ * An **optional** capability a {@link VectorStore} may also implement: full-text (lexical) search
+ * over the chunk text it already stores, using the backing engine's own keyword index — RediSearch's
+ * BM25 over the `TEXT` field, say. Its point is the multi-process deployment: when ingestion runs on
+ * a worker and search runs on an API pod, the API process never sees the chunks, so the in-process
+ * {@link import('./keyword-retriever.js').KeywordRetriever} has nothing to index. A store that
+ * implements this needs no second index, no corpus in JS heap, and no refresh window — a chunk is
+ * lexically findable the moment it is upserted.
+ *
+ * Wrap it as a core `Retriever` with {@link import('./lexical-retriever.js').LexicalRetriever}.
+ *
+ * Implementations MUST apply `options.filter` with exactly the same semantics as
+ * {@link VectorStore.search} — including the empty-array deny primitive — because the filter is
+ * routinely an access-control boundary, not a hint.
+ */
+interface LexicalVectorStore extends VectorStore {
+    /**
+     * Rank stored chunks against a natural-language `query` by keyword relevance. `options.filter`
+     * narrows exactly as in {@link VectorStore.search}. The returned `score` is the engine's own
+     * relevance score (BM25), which shares no scale with `search`'s cosine similarity — fuse the two
+     * with {@link import('./hybrid-retriever.js').HybridRetriever}, whose RRF is rank-based and so
+     * needs no common scale.
+     */
+    searchText(query: string, options: VectorSearchOptions): Promise<Passage[]>;
+}
+/**
+ * Does this store carry the optional {@link LexicalVectorStore} capability? Lets a consumer pick the
+ * store-backed lexical leg when available and fall back to `KeywordRetriever` (or to dense-only)
+ * when it is not, without knowing which adapter it was handed.
+ */
+declare function isLexicalVectorStore(store: VectorStore): store is LexicalVectorStore;
+/**
+ * A destructive call was refused because its scope could not be trusted. Thrown by
+ * {@link VectorStore.removeWhere} — never for something that merely matched nothing (that
+ * is a legitimate result of `0`), only for a filter that would have deleted *more* than the caller
+ * plausibly meant.
+ */
+declare class UnsafeRemovalError extends Error {
+    /** Why the removal was refused. */
+    readonly reason: 'empty-filter' | 'unindexed-field';
+    constructor(
+    /** Why the removal was refused. */
+    reason: 'empty-filter' | 'unindexed-field', message: string);
 }
 /** A distinct source document as seen by the index — its id plus a representative chunk's metadata. */
 interface IndexedDocument {
@@ -65,12 +252,35 @@ declare function documentIdOf(chunkId: string): string;
 /**
  * An in-process {@link VectorStore} — cosine similarity over a Map, no infra. The reference adapter
  * for tests and small/embedded corpora; for production scale use `PgVectorStore` (or your own).
+ *
+ * It implements the enumeration and bulk-deletion half of {@link VectorStore} too. There is no
+ * performance argument for it here — every method below is a loop over the same Map — but a store that tests are
+ * written against has to behave identically to the one production runs, and the semantics that matter
+ * (`removeWhere` refusing an empty filter, an empty-array filter deleting nothing) are behavioural,
+ * not incidental to Redis.
  */
 declare class MemoryVectorStore implements VectorStore {
     private readonly records;
     upsert(records: VectorRecord[]): Promise<void>;
     remove(documentId: string): Promise<void>;
+    /**
+     * Merge `patch` into every chunk of `documentId`, leaving `text` and `embedding` untouched. The
+     * record is *replaced* rather than mutated: `upsert` stores the caller's own object, and patching
+     * it in place would edit an object the caller still holds — a surprise no out-of-process store
+     * could reproduce, and therefore one this reference adapter must not teach.
+     */
+    updateMetadata(documentId: string, patch: MetadataPatch): Promise<number>;
     listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
+    listDocumentIds(filter?: Record<string, unknown>): Promise<string[]>;
+    removeMany(documentIds: string[]): Promise<void>;
+    /**
+     * See {@link VectorStore.removeWhere}. The empty-array short-circuit below is redundant
+     * against `matchesFilter` (which already matches nothing for an empty array) and is kept anyway:
+     * this is the method where "the filter accidentally means everything" destroys data, so the deny is
+     * stated where a reader can see it rather than inferred from another module's truth table.
+     */
+    removeWhere(filter: Record<string, unknown>): Promise<number>;
+    countChunks(filter?: Record<string, unknown>): Promise<number>;
     search(embedding: number[], options: VectorSearchOptions): Promise<Passage[]>;
 }
 
@@ -93,6 +303,28 @@ interface PgVectorStoreOptions {
  * A pgvector-backed {@link VectorStore} — the production reference adapter. Cosine distance via the
  * `<=>` operator over an HNSW index; metadata in a `jsonb` column filtered with `@>`. Call
  * {@link PgVectorStore.ensureSchema} once at boot to create the extension, table, and index.
+ *
+ * This store does **not** implement the optional
+ * {@link import('./vector-store.js').LexicalVectorStore} capability, unlike
+ * {@link import('./redis-vector-store.js').RedisVectorStore}. That is a deliberate omission, not an
+ * oversight: RediSearch already indexes the chunk text (the schema declares it `TEXT`), so BM25 is
+ * free there, whereas the table below indexes nothing lexically — `text` is a plain column with no
+ * `tsvector` and no GIN index. Adding one is not a transparent upgrade:
+ *
+ * - it is new DDL in `ensureSchema`, and `CREATE INDEX` (non-concurrently, as this method must, since
+ *   `CONCURRENTLY` cannot run in a transaction) takes a write lock on an already-populated chunks
+ *   table — a boot-time stall proportional to corpus size for every existing deployment;
+ * - it forces a text-search-configuration choice (`english` vs `simple` vs …) that must match between
+ *   the index expression and every query, or Postgres silently ignores the index and sequentially
+ *   scans the corpus — a failure mode that stays *correct* while quietly getting slower with scale.
+ *
+ * Both want a migration a consumer runs and observes, not a side effect of upgrading the library. The
+ * capability interface is open, so a `PgLexicalVectorStore` (or an option here, gated on an
+ * explicitly-created index) can be added later without changing anything else.
+ *
+ * The enumeration and bulk-deletion methods {@link VectorStore} requires need nothing new here,
+ * unlike the lexical capability above: enumeration and bulk deletion are `DISTINCT`, `= ANY($1)`, `DELETE … WHERE`
+ * and `count(*)` over the table and index that already exist. No DDL, no boot-time lock, no migration.
  */
 declare class PgVectorStore implements VectorStore {
     private readonly client;
@@ -103,7 +335,31 @@ declare class PgVectorStore implements VectorStore {
     ensureSchema(): Promise<void>;
     upsert(records: VectorRecord[]): Promise<void>;
     remove(documentId: string): Promise<void>;
+    /**
+     * Rewrite a document's metadata without re-embedding it. See {@link VectorStore.updateMetadata}
+     * for the semantics; here the whole thing is one statement, because jsonb already *is* the merge:
+     * `||` is a shallow key-wise merge (right side wins, arrays replaced wholesale — precisely
+     * {@link MetadataPatch}) and `- text[]` drops the keys the patch removed. `COALESCE` covers a chunk
+     * ingested with no metadata at all, so a patch can create the object rather than no-op on `NULL`.
+     *
+     * Unlike RediSearch there is no second representation to keep in step — the `metadata` column is
+     * both what `search` filters on and what it returns — so the correctness trap the Redis adapter has
+     * to work for simply does not exist here. `RETURNING id` is what supplies the chunk count, since
+     * {@link PgClient} is a rows-only surface with no `rowCount`.
+     */
+    updateMetadata(documentId: string, patch: MetadataPatch): Promise<number>;
     listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
+    /** {@link listDocuments} without the metadata: one `DISTINCT` over the derived document id. */
+    listDocumentIds(filter?: Record<string, unknown>): Promise<string[]>;
+    /** N documents in one statement — the set-based form of {@link PgVectorStore.remove}. */
+    removeMany(documentIds: string[]): Promise<void>;
+    /**
+     * See {@link VectorStore.removeWhere}: empty filter object throws, empty-array value
+     * deletes nothing. The count comes from `RETURNING id` rather than a driver-specific `rowCount`,
+     * because {@link PgClient} is deliberately just "run SQL, get rows".
+     */
+    removeWhere(filter: Record<string, unknown>): Promise<number>;
+    countChunks(filter?: Record<string, unknown>): Promise<number>;
     search(embedding: number[], options: VectorSearchOptions): Promise<Passage[]>;
 }
 
@@ -157,19 +413,30 @@ interface RedisVectorStoreOptions {
      * front, so only keys listed here can be passed in `search`'s `filter`. Default none.
      */
     filterableFields?: string[];
+    /**
+     * RediSearch scorer used by {@link RedisVectorStore.searchText}. Default `BM25` — the standard
+     * lexical ranking, and the same family the in-process `KeywordRetriever` implements. Override for
+     * a server that names it differently (`BM25STD` on newer builds) or to pick `TFIDF`/`DISMAX`.
+     */
+    lexicalScorer?: string;
 }
 /**
  * A RediSearch-backed {@link VectorStore} — HNSW + cosine over Redis, the ecosystem-native option
  * for anyone already running Redis (see `-transport-redis`). Chunks are hashes under `prefix`, the
  * embedding a FLOAT32 vector field; `search` is a KNN query. Call {@link RedisVectorStore.ensureSchema}
  * once at boot to create the index.
+ *
+ * The chunk text is declared as a RediSearch `TEXT` field, so this store also satisfies
+ * {@link LexicalVectorStore}: {@link RedisVectorStore.searchText} is BM25 over that same index —
+ * the lexical half of hybrid search with no second index to build, feed, or invalidate.
  */
-declare class RedisVectorStore implements VectorStore {
+declare class RedisVectorStore implements LexicalVectorStore {
     private readonly client;
     private readonly index;
     private readonly prefix;
     private readonly dimensions;
     private readonly filterableFields;
+    private readonly lexicalScorer;
     constructor(client: RedisSearchClient, options?: RedisVectorStoreOptions);
     /**
      * Idempotent, and drift-aware: create the HNSW/cosine index if `FT.INFO` says it doesn't exist —
@@ -199,9 +466,133 @@ declare class RedisVectorStore implements VectorStore {
     private reconcileSchema;
     private createIndex;
     upsert(records: VectorRecord[]): Promise<void>;
+    /**
+     * Single-document deletion, by `SCAN … MATCH`.
+     *
+     * This deliberately stays a keyspace scan even though the rest of this class no longer needs one.
+     * The obvious optimisation is to stamp each chunk with its document id as an implicit filterable
+     * TAG at {@link RedisVectorStore.upsert} and turn `remove` into an `FT.SEARCH` — and that is exactly
+     * the change that must not ship, because chunks written *before* it carry no such field. `FT.SEARCH`
+     * would find zero of them, `remove` would return happily, and a document that has been "deleted"
+     * would go on being retrieved. A delete that silently stops deleting is worse than a slow one, and
+     * the library cannot tell a fully-migrated corpus from an unmigrated one: both look like "this
+     * document has no indexed chunks". Nor is a search-then-fall-back-to-scan hybrid safe — a document
+     * re-ingested after the upgrade has *some* stamped chunks, so the search finds hits, the fallback
+     * never runs, and the unstamped tail survives. That is the same silent failure with a smaller blast
+     * radius, which makes it harder to notice rather than less wrong.
+     *
+     * The consumer pain this was measured against — deleting a whole collection, one `remove` per
+     * document — is answered instead by {@link RedisVectorStore.removeWhere} (one filtered query, no
+     * keyspace scan at all) and {@link RedisVectorStore.removeMany} (one scan for N documents rather
+     * than N scans). Neither depends on a field that older chunks lack, so neither has a migration.
+     */
     remove(documentId: string): Promise<void>;
+    /**
+     * Rewrite the metadata of every chunk of `documentId` in place, leaving `text` and `embedding`
+     * alone. See {@link VectorStore.updateMetadata} for the merge semantics and the return value.
+     *
+     * The care this needs is entirely about the fact that a chunk stores its metadata **twice**: as the
+     * `metadata_json` blob (what {@link RedisVectorStore.search} reads back onto a `Passage`) and as
+     * `meta_<field>` TAG fields (what RediSearch actually filters on, and only for the declared
+     * `filterableFields`). Move one without the other and the store lies to itself — a chunk that
+     * filters as the old value but reports the new one, or the reverse — so this method rewrites both
+     * from the same merged object:
+     *
+     * - a patched key that is **not** filterable lands in `metadata_json` only, and no stray `meta_*`
+     *   field is invented for it (it would be dead weight the index does not know about);
+     * - a patched key that **is** filterable also rewrites its TAG, using the identical encoding
+     *   `upsert` uses, so an updated chunk is indistinguishable from a re-ingested one;
+     * - a filterable key the patch **removes** has its TAG `HDEL`-ed, not left behind as a tag matching
+     *   a value the document no longer carries.
+     *
+     * A plain `HSET` is all the index needs: RediSearch re-indexes a hash the moment it is written, so
+     * the very next `FT.SEARCH` sees the new TAG and no longer the old one — no reindex command, and
+     * no need to rewrite the (untouched, and expensive) vector to nudge it. `HDEL` on an indexed field
+     * likewise drops it from the index. Verified against a real Redis Stack in
+     * `redis-vector-store.db.spec.ts`, not assumed.
+     *
+     * Removals are issued **before** the write, so the only window a concurrent reader can observe is
+     * "TAG already gone, `metadata_json` not yet updated" — which fails *closed* on a filter carrying
+     * an ACL. The reverse order would briefly leave a document tagged with a capability its metadata no
+     * longer claims. Read-modify-write across chunks is not transactional; concurrent writers to the
+     * same chunk are last-writer-wins, exactly as two concurrent `upsert`s already are.
+     */
+    updateMetadata(documentId: string, patch: MetadataPatch): Promise<number>;
+    /**
+     * Every chunk key that currently EXISTS for `documentId` — the bare `${prefix}${documentId}` plus
+     * the `${documentId}#<n>` chunks found by `SCAN`. One definition of "chunk belongs to document",
+     * shared by `remove` and `updateMetadata`, so the two can never disagree about which keys a
+     * document owns. The glob is escaped so `gone` does not reach into `goner#0`.
+     */
+    private chunkKeys;
     listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
+    /**
+     * The id-only half of {@link RedisVectorStore.listDocuments}, served by `FT.SEARCH … NOCONTENT`:
+     * RediSearch returns bare keys, so nothing carries `metadata_json` over the wire and nothing is
+     * JSON-parsed. `listDocuments` has to parse one metadata blob per *chunk* — thousands of parses to
+     * produce a few hundred ids it then throws the metadata away from — and this does none of it.
+     *
+     * The collapse to document ids still happens client-side. RediSearch can group server-side
+     * (`FT.AGGREGATE … GROUPBY`), but only over an indexed *field*, and a chunk's document id lives in
+     * its key, not in a field — deriving it would mean stamping one at write time, which is the
+     * migration {@link RedisVectorStore.remove} explains this class refuses. Deduplicating a page of
+     * keys in JS is free by comparison.
+     */
+    listDocumentIds(filter?: Record<string, unknown>): Promise<string[]>;
+    /**
+     * Bulk deletion by document id — **one** keyspace scan for N documents, where N calls to
+     * {@link RedisVectorStore.remove} would be N of them. The scan is `MATCH ${prefix}*`, so Redis still
+     * filters to this store's own keys server-side; membership is then decided in JS against a `Set`,
+     * which is what lets a single pass serve any number of ids.
+     *
+     * It deletes by key, exactly as `remove` does, so it is correct on chunks written by any version of
+     * this library — there is no field it needs the corpus to already carry.
+     */
+    removeMany(documentIds: string[]): Promise<void>;
+    /**
+     * Delete every chunk matching `filter`, via `FT.SEARCH … NOCONTENT` + `DEL` — no keyspace scan at
+     * all, so dropping a collection costs one indexed query per page instead of one full pass per
+     * document. Resolves the number of chunk keys actually deleted.
+     *
+     * The two refusals, in this order, before any query is issued:
+     *
+     * 1. **An empty filter object throws** {@link UnsafeRemovalError}. `buildFilter({})` is `*`, and `*`
+     *    here would mean "delete the corpus" — far too destructive an outcome for the shape a filter
+     *    degenerates to when it is built wrong.
+     * 2. **An empty-array value deletes nothing** and resolves `0`. This is the package's deny primitive
+     *    (an actor with no capability tokens), and it is the one that has to be gated *explicitly*: a
+     *    plausible-looking "normalise the filter first" refactor that drops empty arrays turns a deny
+     *    into `*` and wipes everything the deny existed to protect.
+     *
+     * Then one more, which is specific to RediSearch: every filter key must be a declared
+     * `filterableFields` entry. `search` lets the engine reject an unknown field, which is fine for a
+     * read; for a delete the check is worth making up front and by name.
+     *
+     * Deletion happens page by page, always re-querying from offset 0 — the rows just deleted are gone
+     * from the index, so paging forward would skip their replacements. If a page comes back non-empty
+     * but `DEL` reports nothing removed (keys expired between query and delete, or the index is stale),
+     * the loop stops rather than spinning on results it can never consume.
+     */
+    removeWhere(filter: Record<string, unknown>): Promise<number>;
+    /**
+     * Count matching chunks with `FT.SEARCH … LIMIT 0 0` — RediSearch returns the total and no
+     * documents, so the cost is the query, not the corpus. An empty-array filter value denies and
+     * yields `0`, same as every other filtered path here.
+     */
+    countChunks(filter?: Record<string, unknown>): Promise<number>;
     search(embedding: number[], options: VectorSearchOptions): Promise<Passage[]>;
+    /**
+     * Lexical (BM25) search over the same index `search` uses — the {@link LexicalVectorStore} half of
+     * hybrid retrieval, served by RediSearch's own keyword index rather than an in-process copy of the
+     * corpus. `options.filter` is ANDed onto the text clause with the identical TAG semantics as
+     * `search` (array = OR, empty array = deny), so a lexical hit can never escape a filter a vector
+     * hit would have respected.
+     *
+     * A query with no searchable terms — empty, whitespace, or pure punctuation — returns `[]`. That is
+     * deliberate: RediSearch's `*` would match the entire corpus, which is the opposite of what a
+     * relevance query with nothing to be relevant to should do.
+     */
+    searchText(query: string, options: VectorSearchOptions): Promise<Passage[]>;
 }
 
 /**
@@ -291,6 +682,35 @@ declare class KeywordRetriever implements Retriever {
     retrieve(query: string, options?: RetrieveOptions): Promise<Passage[]>;
 }
 
+/**
+ * The keyword half of hybrid search, served by the store's own full-text index — the mirror of
+ * {@link import('./embedding-retriever.js').EmbeddingRetriever}, which serves the dense half. It
+ * holds no state: every query goes to the store, so any process that can reach the store can run
+ * lexical search, whether or not it did the ingestion.
+ *
+ * Compose it exactly like the dense leg:
+ *
+ * ```ts
+ * const retriever = new HybridRetriever([
+ *   new EmbeddingRetriever(embedder, store),
+ *   new LexicalRetriever(store),
+ * ]);
+ * ```
+ *
+ * Both legs read the same chunks from the same index, so their ids line up by construction and
+ * `HybridRetriever`'s RRF fuses them without either side needing a score on a common scale.
+ *
+ * Prefer this over {@link import('./keyword-retriever.js').KeywordRetriever} whenever the store
+ * supports it (see {@link import('./vector-store.js').isLexicalVectorStore}) and ingestion and
+ * search can run in different processes; `KeywordRetriever` remains the answer for `MemoryVectorStore`
+ * and single-process deployments.
+ */
+declare class LexicalRetriever implements Retriever {
+    private readonly store;
+    constructor(store: LexicalVectorStore);
+    retrieve(query: string, options?: RetrieveOptions): Promise<Passage[]>;
+}
+
 interface HybridRetrieverOptions {
     /** RRF constant — larger flattens the rank weighting. Default 60 (the value from the original paper). */
     k?: number;
@@ -369,4 +789,4 @@ interface RetrievalTool {
  */
 declare function createRetrievalTool(retriever: Retriever, options?: RetrievalToolOptions): RetrievalTool;
 
-export { type ChunkOptions, type ChunkRecord, EmbeddingRetriever, FilteredRetriever, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, KeywordRetriever, type KeywordRetrieverOptions, MemoryVectorStore, type PgClient, PgVectorStore, type PgVectorStoreOptions, type RedisSearchClient, RedisVectorSchemaMismatchError, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalTool, type RetrievalToolOptions, type VectorRecord, type VectorSearchOptions, type VectorStore, chunkDocuments, chunkText, createRetrievalTool, documentIdOf, ingestChunks, ingestDocuments };
+export { type ChunkOptions, type ChunkRecord, EmbeddingRetriever, FilteredRetriever, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, KeywordRetriever, type KeywordRetrieverOptions, LexicalRetriever, type LexicalVectorStore, MemoryVectorStore, type MetadataPatch, type PgClient, PgVectorStore, type PgVectorStoreOptions, type RedisSearchClient, RedisVectorSchemaMismatchError, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalTool, type RetrievalToolOptions, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, applyMetadataPatch, chunkDocuments, chunkText, createRetrievalTool, documentIdOf, ingestChunks, ingestDocuments, isLexicalVectorStore };