@dudousxd/nestjs-agent-rag 0.14.1 → 0.14.3

package/dist/index.d.cts

@@ -281,8 +281,8 @@ interface LexicalVectorStore extends VectorStore {
 /**
  * An **optional** capability: swap all of a document's chunks safely under concurrent calls, so two
  * writers can't leave a mix of versions behind. Postgres also makes the swap atomic for searches and
- * crashes; Redis can't, and only guarantees the end state. {@link replaceDocumentChunks} falls back to
- * `remove` plus `upsert` for the others.
+ * crashes, and so does Redis up to a size limit (see `RedisVectorStore.replaceDocument`).
+ * {@link replaceDocumentChunks} falls back to `remove` plus `upsert` for the others.
  */
 interface AtomicReplaceVectorStore extends VectorStore {
     replaceDocument(documentId: string, records: VectorRecord[]): Promise<void>;
@@ -407,8 +407,9 @@ declare function describeSharedSource(legs: readonly unknown[]): Omit<RetrievalD
  * (`removeWhere` refusing an empty filter, an empty-array filter deleting nothing) are behavioural,
  * not incidental to Redis.
  */
-declare class MemoryVectorStore implements VectorStore {
+declare class MemoryVectorStore implements AtomicReplaceVectorStore {
     private readonly records;
+    private readonly replacing;
     /**
      * Telemetry self-description. No `collection`: there is no namespace to name — the corpus IS this
      * instance, and two `MemoryVectorStore`s in one process are two corpora with nothing to tell them
@@ -416,7 +417,11 @@ declare class MemoryVectorStore implements VectorStore {
      */
     describeRetrieval(): RetrievalDescriptor;
     upsert(records: VectorRecord[]): Promise<void>;
+    /** A replace and a removal of one document take turns, in the order they were called. */
+    replaceDocument(documentId: string, records: VectorRecord[]): Promise<void>;
     remove(documentId: string): Promise<void>;
+    private inTurn;
+    private deleteChunks;
     /**
      * Merge `patch` into every chunk of `documentId`, leaving `text` and `embedding` untouched. The
      * record is *replaced* rather than mutated: `upsert` stores the caller's own object, and patching
@@ -878,18 +883,28 @@ declare class RedisVectorStore implements LexicalVectorStore, AtomicReplaceVecto
     private reconcileSchema;
     private createIndex;
     upsert(records: VectorRecord[]): Promise<void>;
+    private chunkFields;
     /**
-     * Replaces a document's chunks, one writer at a time per document. Redis can't swap them in one step
-     * without scanning the keyspace inside a script, so a search may briefly see old and new chunks
-     * together, but concurrent calls can't leave them mixed. The new chunks go in before the old ones
-     * come out, so a crash leaves the document searchable and the next call repairs it.
+     * Replaces a document's chunks, one writer at a time per document. The swap is one script, so a search
+     * sees the old version or the new one, and a crash leaves the old one. The keys to drop are found
+     * first, under the lock, because a script can't scan the keyspace.
+     *
+     * A document past {@link SWAP_MAX_CHUNKS} would block Redis for too long that way. Its new chunks go in
+     * first and the old ones come out after, so a search may briefly see both, and a crash leaves the
+     * document searchable for the next call to repair.
      */
     replaceDocument(documentId: string, records: VectorRecord[]): Promise<void>;
+    private swapChunks;
+    private lockKey;
     /**
-     * A lease, so a crashed writer can't block the document forever. It is renewed while the work runs,
-     * which keeps a large document from outliving it.
+     * A lease on each document, so a crashed writer can't block one forever. They are renewed while the
+     * work runs, which keeps a large document from outliving them.
+     *
+     * Several documents are locked in sorted order, without duplicates. Every other caller either holds
+     * one lock or takes them in that same order, so no two calls can wait on each other.
      */
-    private withDocumentLock;
+    private withDocumentLocks;
+    private acquireLock;
     /**
      * Single-document deletion, by `SCAN … MATCH`.
      *
@@ -990,6 +1005,9 @@ declare class RedisVectorStore implements LexicalVectorStore, AtomicReplaceVecto
      *
      * It deletes by key, exactly as `remove` does, so it is correct on chunks written by any version of
      * this library — there is no field it needs the corpus to already carry.
+     *
+     * It holds every document's lock for the whole scan, so it waits for a `replaceDocument` in progress
+     * and one can't start partway through. That adds two commands per document to the one scan.
      */
     removeMany(documentIds: string[]): Promise<void>;
     /**
@@ -1015,6 +1033,10 @@ declare class RedisVectorStore implements LexicalVectorStore, AtomicReplaceVecto
      * from the index, so paging forward would skip their replacements. If a page comes back non-empty
      * but `DEL` reports nothing removed (keys expired between query and delete, or the index is stale),
      * the loop stops rather than spinning on results it can never consume.
+     *
+     * The documents aren't known until a page comes back, so each page locks the documents it touches, one
+     * page at a time, and asks again with `INKEYS` once it holds them. A `replaceDocument` that finished in
+     * between may have dropped those chunks, or rewritten them to no longer match.
      */
     removeWhere(filter: Record<string, unknown>): Promise<number>;
     /**
@@ -1711,9 +1733,10 @@ declare class RagIndex<Row, Scope extends RagScopeDefinition<Row>> {
     search(query: string, options: RagSearchOptions<Scope>): Promise<Passage[]>;
     /**
      * Replaces the row's chunks and returns how many were written. Embeddings come first: if the
-     * provider fails, the previous chunks are still there. The swap is one transaction on Postgres. Redis
-     * adds the new chunks before dropping the old, so a search can briefly see both. On a store without
-     * `replaceDocument`, a crash mid-swap loses the document until the next `index`.
+     * provider fails, the previous chunks are still there. The swap is one transaction on Postgres and one
+     * script on Redis, where a document too large for that adds the new chunks before dropping the old, so
+     * a search can briefly see both. On a store without `replaceDocument`, a crash mid-swap loses the
+     * document until the next `index`.
      */
     index(row: Row): Promise<number>;
     /** Takes the id itself once the row is gone. */

package/dist/index.d.ts

@@ -281,8 +281,8 @@ interface LexicalVectorStore extends VectorStore {
 /**
  * An **optional** capability: swap all of a document's chunks safely under concurrent calls, so two
  * writers can't leave a mix of versions behind. Postgres also makes the swap atomic for searches and
- * crashes; Redis can't, and only guarantees the end state. {@link replaceDocumentChunks} falls back to
- * `remove` plus `upsert` for the others.
+ * crashes, and so does Redis up to a size limit (see `RedisVectorStore.replaceDocument`).
+ * {@link replaceDocumentChunks} falls back to `remove` plus `upsert` for the others.
  */
 interface AtomicReplaceVectorStore extends VectorStore {
     replaceDocument(documentId: string, records: VectorRecord[]): Promise<void>;
@@ -407,8 +407,9 @@ declare function describeSharedSource(legs: readonly unknown[]): Omit<RetrievalD
  * (`removeWhere` refusing an empty filter, an empty-array filter deleting nothing) are behavioural,
  * not incidental to Redis.
  */
-declare class MemoryVectorStore implements VectorStore {
+declare class MemoryVectorStore implements AtomicReplaceVectorStore {
     private readonly records;
+    private readonly replacing;
     /**
      * Telemetry self-description. No `collection`: there is no namespace to name — the corpus IS this
      * instance, and two `MemoryVectorStore`s in one process are two corpora with nothing to tell them
@@ -416,7 +417,11 @@ declare class MemoryVectorStore implements VectorStore {
      */
     describeRetrieval(): RetrievalDescriptor;
     upsert(records: VectorRecord[]): Promise<void>;
+    /** A replace and a removal of one document take turns, in the order they were called. */
+    replaceDocument(documentId: string, records: VectorRecord[]): Promise<void>;
     remove(documentId: string): Promise<void>;
+    private inTurn;
+    private deleteChunks;
     /**
      * Merge `patch` into every chunk of `documentId`, leaving `text` and `embedding` untouched. The
      * record is *replaced* rather than mutated: `upsert` stores the caller's own object, and patching
@@ -878,18 +883,28 @@ declare class RedisVectorStore implements LexicalVectorStore, AtomicReplaceVecto
     private reconcileSchema;
     private createIndex;
     upsert(records: VectorRecord[]): Promise<void>;
+    private chunkFields;
     /**
-     * Replaces a document's chunks, one writer at a time per document. Redis can't swap them in one step
-     * without scanning the keyspace inside a script, so a search may briefly see old and new chunks
-     * together, but concurrent calls can't leave them mixed. The new chunks go in before the old ones
-     * come out, so a crash leaves the document searchable and the next call repairs it.
+     * Replaces a document's chunks, one writer at a time per document. The swap is one script, so a search
+     * sees the old version or the new one, and a crash leaves the old one. The keys to drop are found
+     * first, under the lock, because a script can't scan the keyspace.
+     *
+     * A document past {@link SWAP_MAX_CHUNKS} would block Redis for too long that way. Its new chunks go in
+     * first and the old ones come out after, so a search may briefly see both, and a crash leaves the
+     * document searchable for the next call to repair.
      */
     replaceDocument(documentId: string, records: VectorRecord[]): Promise<void>;
+    private swapChunks;
+    private lockKey;
     /**
-     * A lease, so a crashed writer can't block the document forever. It is renewed while the work runs,
-     * which keeps a large document from outliving it.
+     * A lease on each document, so a crashed writer can't block one forever. They are renewed while the
+     * work runs, which keeps a large document from outliving them.
+     *
+     * Several documents are locked in sorted order, without duplicates. Every other caller either holds
+     * one lock or takes them in that same order, so no two calls can wait on each other.
      */
-    private withDocumentLock;
+    private withDocumentLocks;
+    private acquireLock;
     /**
      * Single-document deletion, by `SCAN … MATCH`.
      *
@@ -990,6 +1005,9 @@ declare class RedisVectorStore implements LexicalVectorStore, AtomicReplaceVecto
      *
      * It deletes by key, exactly as `remove` does, so it is correct on chunks written by any version of
      * this library — there is no field it needs the corpus to already carry.
+     *
+     * It holds every document's lock for the whole scan, so it waits for a `replaceDocument` in progress
+     * and one can't start partway through. That adds two commands per document to the one scan.
      */
     removeMany(documentIds: string[]): Promise<void>;
     /**
@@ -1015,6 +1033,10 @@ declare class RedisVectorStore implements LexicalVectorStore, AtomicReplaceVecto
      * from the index, so paging forward would skip their replacements. If a page comes back non-empty
      * but `DEL` reports nothing removed (keys expired between query and delete, or the index is stale),
      * the loop stops rather than spinning on results it can never consume.
+     *
+     * The documents aren't known until a page comes back, so each page locks the documents it touches, one
+     * page at a time, and asks again with `INKEYS` once it holds them. A `replaceDocument` that finished in
+     * between may have dropped those chunks, or rewritten them to no longer match.
      */
     removeWhere(filter: Record<string, unknown>): Promise<number>;
     /**
@@ -1711,9 +1733,10 @@ declare class RagIndex<Row, Scope extends RagScopeDefinition<Row>> {
     search(query: string, options: RagSearchOptions<Scope>): Promise<Passage[]>;
     /**
      * Replaces the row's chunks and returns how many were written. Embeddings come first: if the
-     * provider fails, the previous chunks are still there. The swap is one transaction on Postgres. Redis
-     * adds the new chunks before dropping the old, so a search can briefly see both. On a store without
-     * `replaceDocument`, a crash mid-swap loses the document until the next `index`.
+     * provider fails, the previous chunks are still there. The swap is one transaction on Postgres and one
+     * script on Redis, where a document too large for that adds the new chunks before dropping the old, so
+     * a search can briefly see both. On a store without `replaceDocument`, a crash mid-swap loses the
+     * document until the next `index`.
      */
     index(row: Row): Promise<number>;
     /** Takes the id itself once the row is gone. */