@dudousxd/nestjs-agent-rag 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -3
- package/dist/index.cjs +163 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +113 -1
- package/dist/index.d.ts +113 -1
- package/dist/index.js +162 -0
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.d.cts
CHANGED
|
@@ -5,6 +5,31 @@ interface ChunkOptions {
|
|
|
5
5
|
chunkSize?: number;
|
|
6
6
|
/** Characters of overlap carried from the end of one chunk into the next. Default 100. */
|
|
7
7
|
overlap?: number;
|
|
8
|
+
/**
|
|
9
|
+
* Treat the text as records joined by this string, and never split one across chunks.
|
|
10
|
+
*
|
|
11
|
+
* Default chunking is structure-blind: it breaks at the *latest* paragraph/sentence/word boundary
|
|
12
|
+
* in the window, which for record-shaped text means a cut lands mid-record routinely. That is fine
|
|
13
|
+
* for prose, where the two halves are still sentences. It is not fine for a table flattened to one
|
|
14
|
+
* record per row, because the half that keeps the row identifier and the half that keeps the value
|
|
15
|
+
* end up in different chunks, and neither can answer a question about that row on its own.
|
|
16
|
+
*
|
|
17
|
+
* With a separator set, whole records are packed greedily up to `chunkSize` and a record is never
|
|
18
|
+
* cut — except one longer than `chunkSize`, which has nowhere to go and is character-split as
|
|
19
|
+
* usual, since no chunker can avoid that.
|
|
20
|
+
*
|
|
21
|
+
* `overlap` is IGNORED in this mode, deliberately. Overlap exists to rescue a sentence that a
|
|
22
|
+
* boundary cut in half; a boundary that never falls inside a record has nothing to rescue, and
|
|
23
|
+
* carrying the tail of the previous chunk in would duplicate whole records into neighbouring chunks
|
|
24
|
+
* — paying for the same rows twice at embedding time and letting one row match from two places.
|
|
25
|
+
*
|
|
26
|
+
* Measured on a 200-row spreadsheet flattened to one field-labelled record per row (15 columns,
|
|
27
|
+
* ~800-char budget): blind chunking left 27% of records split, and on questions targeting a field
|
|
28
|
+
* on the far side of such a cut, BM25 answered 54/66 (MRR 0.818) against 66/66 (MRR 1.000) with
|
|
29
|
+
* records kept whole. The dense leg scored 7/66 either way — 200 near-identical rows produce
|
|
30
|
+
* near-identical vectors, so that leg cannot do row lookup at all and chunking does not change it.
|
|
31
|
+
*/
|
|
32
|
+
separator?: string;
|
|
8
33
|
}
|
|
9
34
|
/**
|
|
10
35
|
* Splits text into overlapping chunks for embedding. Greedy up to `chunkSize`, but prefers to break
|
|
@@ -149,6 +174,32 @@ interface VectorStore {
|
|
|
149
174
|
* `size` changed), or {@link VectorStore.remove remove} (orphaned).
|
|
150
175
|
*/
|
|
151
176
|
listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
|
|
177
|
+
/**
|
|
178
|
+
* Read one document's chunks back — id, text, metadata — ordered by their position in the
|
|
179
|
+
* document, optionally paged with `limit`/`offset`.
|
|
180
|
+
*
|
|
181
|
+
* The gap this closes is that **the text a store holds is otherwise write-only**. Every other way
|
|
182
|
+
* back to it is a ranked query: {@link VectorStore.search} needs an embedding,
|
|
183
|
+
* {@link LexicalVectorStore.searchText} needs a query string, and both return whatever *matched*,
|
|
184
|
+
* in relevance order, capped at `topK`. Neither can answer "show me what was indexed for this
|
|
185
|
+
* document, in order" — the question you ask when retrieval returns the wrong passage and you need
|
|
186
|
+
* to know whether the chunker cut badly or the retriever chose badly. Without this, the only way to
|
|
187
|
+
* see stored text is to reach around the adapter into the backing engine, which is exactly the
|
|
188
|
+
* abstraction break a consumer adopts a store to avoid.
|
|
189
|
+
*
|
|
190
|
+
* Document-scoped rather than filter-scoped, matching {@link VectorStore.remove} and
|
|
191
|
+
* {@link VectorStore.updateMetadata}: ordering is the whole point, and "position within the
|
|
192
|
+
* document" is only meaningful once a document has been picked. Enumerate documents with
|
|
193
|
+
* {@link VectorStore.listDocuments} first when you need to sweep.
|
|
194
|
+
*
|
|
195
|
+
* An unknown `documentId` yields `[]`; it does not throw, matching `remove` and `updateMetadata`.
|
|
196
|
+
*
|
|
197
|
+
* NOT an access-control seam. It takes no metadata `filter` and applies none, so it cannot enforce
|
|
198
|
+
* one — a caller that gates retrieval on a metadata field (tenant, audience, collection) MUST
|
|
199
|
+
* resolve that gate itself before calling, exactly as it would before `remove`. It is deliberately
|
|
200
|
+
* shaped for an operator/debug surface, not for serving end users.
|
|
201
|
+
*/
|
|
202
|
+
listChunks(documentId: string, options?: ListChunksOptions): Promise<StoredChunk[]>;
|
|
152
203
|
/**
|
|
153
204
|
* The distinct source document ids currently indexed, optionally narrowed by a metadata `filter` —
|
|
154
205
|
* {@link VectorStore.listDocuments} without the metadata. Use it when you only need the id set (a
|
|
@@ -246,6 +297,25 @@ interface IndexedDocument {
|
|
|
246
297
|
id: string;
|
|
247
298
|
metadata?: Record<string, unknown>;
|
|
248
299
|
}
|
|
300
|
+
/** One stored chunk, read back verbatim — what {@link VectorStore.listChunks} yields. */
|
|
301
|
+
interface StoredChunk {
|
|
302
|
+
/** The chunk id: `${documentId}#<n>`, or the bare document id for a single-chunk document. */
|
|
303
|
+
id: string;
|
|
304
|
+
/** Zero-based position within the document, parsed from the id. A bare id is chunk `0`. */
|
|
305
|
+
index: number;
|
|
306
|
+
/** The chunk text as it was indexed — the exact string the embedding was computed from. */
|
|
307
|
+
text: string;
|
|
308
|
+
metadata?: Record<string, unknown>;
|
|
309
|
+
}
|
|
310
|
+
/** Paging for {@link VectorStore.listChunks}. Omit both to read the whole document. */
|
|
311
|
+
interface ListChunksOptions {
|
|
312
|
+
/** Max chunks to return, applied after ordering by {@link StoredChunk.index}. */
|
|
313
|
+
limit?: number;
|
|
314
|
+
/** Chunks to skip, applied after ordering. */
|
|
315
|
+
offset?: number;
|
|
316
|
+
}
|
|
317
|
+
/** Chunk `n` of `${documentId}#<n>`; a bare document id (single-chunk document) is `0`. */
|
|
318
|
+
declare function chunkIndexOf(chunkId: string): number;
|
|
249
319
|
/** Collapse a chunk id (`${documentId}#<n>`) back to its source document id. */
|
|
250
320
|
declare function documentIdOf(chunkId: string): string;
|
|
251
321
|
|
|
@@ -271,6 +341,16 @@ declare class MemoryVectorStore implements VectorStore {
|
|
|
271
341
|
*/
|
|
272
342
|
updateMetadata(documentId: string, patch: MetadataPatch): Promise<number>;
|
|
273
343
|
listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
|
|
344
|
+
/**
|
|
345
|
+
* The chunks of one document, in document order. See {@link VectorStore.listChunks}.
|
|
346
|
+
*
|
|
347
|
+
* Insertion order is not document order — `upsert` stores whatever the caller passed, and a
|
|
348
|
+
* re-ingest that rewrites chunk 3 alone moves it to the end of the Map. So this sorts by the index
|
|
349
|
+
* parsed from the id, the same thing the SQL and Redis adapters order by, rather than trusting the
|
|
350
|
+
* Map. A reference adapter that returned "usually ordered" would let a consumer ship a bug that
|
|
351
|
+
* only shows up on the store tests are NOT written against.
|
|
352
|
+
*/
|
|
353
|
+
listChunks(documentId: string, options?: ListChunksOptions): Promise<StoredChunk[]>;
|
|
274
354
|
listDocumentIds(filter?: Record<string, unknown>): Promise<string[]>;
|
|
275
355
|
removeMany(documentIds: string[]): Promise<void>;
|
|
276
356
|
/**
|
|
@@ -349,6 +429,19 @@ declare class PgVectorStore implements VectorStore {
|
|
|
349
429
|
*/
|
|
350
430
|
updateMetadata(documentId: string, patch: MetadataPatch): Promise<number>;
|
|
351
431
|
listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
|
|
432
|
+
/**
|
|
433
|
+
* The chunks of one document, in document order. See {@link VectorStore.listChunks}.
|
|
434
|
+
*
|
|
435
|
+
* The order is computed from the id rather than stored: `#12` must sort after `#2`, which a plain
|
|
436
|
+
* `ORDER BY id` gets wrong on every document past ten chunks because it compares text. So the
|
|
437
|
+
* trailing number is extracted and cast to `int`, and a bare id (a single-chunk document, which
|
|
438
|
+
* carries no `#n` suffix) collapses to `0` — the same rule `chunkIndexOf` applies in the other two
|
|
439
|
+
* adapters.
|
|
440
|
+
*
|
|
441
|
+
* `embedding` is left out of the projection: it is the widest column in the table and useless to a
|
|
442
|
+
* caller reading text back.
|
|
443
|
+
*/
|
|
444
|
+
listChunks(documentId: string, options?: ListChunksOptions): Promise<StoredChunk[]>;
|
|
352
445
|
/** {@link listDocuments} without the metadata: one `DISTINCT` over the derived document id. */
|
|
353
446
|
listDocumentIds(filter?: Record<string, unknown>): Promise<string[]>;
|
|
354
447
|
/** N documents in one statement — the set-based form of {@link PgVectorStore.remove}. */
|
|
@@ -525,6 +618,25 @@ declare class RedisVectorStore implements LexicalVectorStore {
|
|
|
525
618
|
* document owns. The glob is escaped so `gone` does not reach into `goner#0`.
|
|
526
619
|
*/
|
|
527
620
|
private chunkKeys;
|
|
621
|
+
/**
|
|
622
|
+
* The chunks of one document, in document order. See {@link VectorStore.listChunks}.
|
|
623
|
+
*
|
|
624
|
+
* Reads the hashes directly (`HMGET text metadata_json`) rather than going through `FT.SEARCH`.
|
|
625
|
+
* Two reasons, and the second is the one that matters: the query language has no way to ask for
|
|
626
|
+
* "every chunk of this document" — `id` is the key name, not an indexed field — and `FT.SEARCH`
|
|
627
|
+
* caps at `LIMIT`, so paging a document through it would mean ordering by a field the index does
|
|
628
|
+
* not hold. `chunkKeys` is the same enumeration {@link RedisVectorStore.remove} and
|
|
629
|
+
* {@link RedisVectorStore.updateMetadata} already use, so "the chunks this returns" and "the chunks
|
|
630
|
+
* a delete would take" cannot drift apart.
|
|
631
|
+
*
|
|
632
|
+
* It inherits that method's cost too: one keyspace `SCAN` per call. Acceptable here because this
|
|
633
|
+
* is an inspection path (one document, on demand) and not something retrieval runs — but it is why
|
|
634
|
+
* this must not be put on a hot path or called in a loop over a corpus.
|
|
635
|
+
*
|
|
636
|
+
* `embedding` is deliberately not fetched: it is by far the largest field on the hash and nothing
|
|
637
|
+
* a caller reading text back can do with it.
|
|
638
|
+
*/
|
|
639
|
+
listChunks(documentId: string, options?: ListChunksOptions): Promise<StoredChunk[]>;
|
|
528
640
|
listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
|
|
529
641
|
/**
|
|
530
642
|
* The id-only half of {@link RedisVectorStore.listDocuments}, served by `FT.SEARCH … NOCONTENT`:
|
|
@@ -843,4 +955,4 @@ interface RetrievalTool {
|
|
|
843
955
|
*/
|
|
844
956
|
declare function createRetrievalTool(retriever: Retriever, options?: RetrievalToolOptions): RetrievalTool;
|
|
845
957
|
|
|
846
|
-
export { type ChunkOptions, type ChunkRecord, EmbeddingRetriever, type EmbeddingRetrieverOptions, FilteredRetriever, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, KeywordRetriever, type KeywordRetrieverOptions, LexicalRetriever, type LexicalVectorStore, MemoryVectorStore, type MetadataPatch, type PgClient, PgVectorStore, type PgVectorStoreOptions, type RedisSearchClient, RedisVectorSchemaMismatchError, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalTool, type RetrievalToolOptions, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, applyMetadataPatch, chunkDocuments, chunkText, createRetrievalTool, documentIdOf, ingestChunks, ingestDocuments, isLexicalVectorStore };
|
|
958
|
+
export { type ChunkOptions, type ChunkRecord, EmbeddingRetriever, type EmbeddingRetrieverOptions, FilteredRetriever, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, KeywordRetriever, type KeywordRetrieverOptions, LexicalRetriever, type LexicalVectorStore, type ListChunksOptions, MemoryVectorStore, type MetadataPatch, type PgClient, PgVectorStore, type PgVectorStoreOptions, type RedisSearchClient, RedisVectorSchemaMismatchError, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalTool, type RetrievalToolOptions, type StoredChunk, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, applyMetadataPatch, chunkDocuments, chunkIndexOf, chunkText, createRetrievalTool, documentIdOf, ingestChunks, ingestDocuments, isLexicalVectorStore };
|
package/dist/index.d.ts
CHANGED
|
@@ -5,6 +5,31 @@ interface ChunkOptions {
|
|
|
5
5
|
chunkSize?: number;
|
|
6
6
|
/** Characters of overlap carried from the end of one chunk into the next. Default 100. */
|
|
7
7
|
overlap?: number;
|
|
8
|
+
/**
|
|
9
|
+
* Treat the text as records joined by this string, and never split one across chunks.
|
|
10
|
+
*
|
|
11
|
+
* Default chunking is structure-blind: it breaks at the *latest* paragraph/sentence/word boundary
|
|
12
|
+
* in the window, which for record-shaped text means a cut lands mid-record routinely. That is fine
|
|
13
|
+
* for prose, where the two halves are still sentences. It is not fine for a table flattened to one
|
|
14
|
+
* record per row, because the half that keeps the row identifier and the half that keeps the value
|
|
15
|
+
* end up in different chunks, and neither can answer a question about that row on its own.
|
|
16
|
+
*
|
|
17
|
+
* With a separator set, whole records are packed greedily up to `chunkSize` and a record is never
|
|
18
|
+
* cut — except one longer than `chunkSize`, which has nowhere to go and is character-split as
|
|
19
|
+
* usual, since no chunker can avoid that.
|
|
20
|
+
*
|
|
21
|
+
* `overlap` is IGNORED in this mode, deliberately. Overlap exists to rescue a sentence that a
|
|
22
|
+
* boundary cut in half; a boundary that never falls inside a record has nothing to rescue, and
|
|
23
|
+
* carrying the tail of the previous chunk in would duplicate whole records into neighbouring chunks
|
|
24
|
+
* — paying for the same rows twice at embedding time and letting one row match from two places.
|
|
25
|
+
*
|
|
26
|
+
* Measured on a 200-row spreadsheet flattened to one field-labelled record per row (15 columns,
|
|
27
|
+
* ~800-char budget): blind chunking left 27% of records split, and on questions targeting a field
|
|
28
|
+
* on the far side of such a cut, BM25 answered 54/66 (MRR 0.818) against 66/66 (MRR 1.000) with
|
|
29
|
+
* records kept whole. The dense leg scored 7/66 either way — 200 near-identical rows produce
|
|
30
|
+
* near-identical vectors, so that leg cannot do row lookup at all and chunking does not change it.
|
|
31
|
+
*/
|
|
32
|
+
separator?: string;
|
|
8
33
|
}
|
|
9
34
|
/**
|
|
10
35
|
* Splits text into overlapping chunks for embedding. Greedy up to `chunkSize`, but prefers to break
|
|
@@ -149,6 +174,32 @@ interface VectorStore {
|
|
|
149
174
|
* `size` changed), or {@link VectorStore.remove remove} (orphaned).
|
|
150
175
|
*/
|
|
151
176
|
listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
|
|
177
|
+
/**
|
|
178
|
+
* Read one document's chunks back — id, text, metadata — ordered by their position in the
|
|
179
|
+
* document, optionally paged with `limit`/`offset`.
|
|
180
|
+
*
|
|
181
|
+
* The gap this closes is that **the text a store holds is otherwise write-only**. Every other way
|
|
182
|
+
* back to it is a ranked query: {@link VectorStore.search} needs an embedding,
|
|
183
|
+
* {@link LexicalVectorStore.searchText} needs a query string, and both return whatever *matched*,
|
|
184
|
+
* in relevance order, capped at `topK`. Neither can answer "show me what was indexed for this
|
|
185
|
+
* document, in order" — the question you ask when retrieval returns the wrong passage and you need
|
|
186
|
+
* to know whether the chunker cut badly or the retriever chose badly. Without this, the only way to
|
|
187
|
+
* see stored text is to reach around the adapter into the backing engine, which is exactly the
|
|
188
|
+
* abstraction break a consumer adopts a store to avoid.
|
|
189
|
+
*
|
|
190
|
+
* Document-scoped rather than filter-scoped, matching {@link VectorStore.remove} and
|
|
191
|
+
* {@link VectorStore.updateMetadata}: ordering is the whole point, and "position within the
|
|
192
|
+
* document" is only meaningful once a document has been picked. Enumerate documents with
|
|
193
|
+
* {@link VectorStore.listDocuments} first when you need to sweep.
|
|
194
|
+
*
|
|
195
|
+
* An unknown `documentId` yields `[]`; it does not throw, matching `remove` and `updateMetadata`.
|
|
196
|
+
*
|
|
197
|
+
* NOT an access-control seam. It takes no metadata `filter` and applies none, so it cannot enforce
|
|
198
|
+
* one — a caller that gates retrieval on a metadata field (tenant, audience, collection) MUST
|
|
199
|
+
* resolve that gate itself before calling, exactly as it would before `remove`. It is deliberately
|
|
200
|
+
* shaped for an operator/debug surface, not for serving end users.
|
|
201
|
+
*/
|
|
202
|
+
listChunks(documentId: string, options?: ListChunksOptions): Promise<StoredChunk[]>;
|
|
152
203
|
/**
|
|
153
204
|
* The distinct source document ids currently indexed, optionally narrowed by a metadata `filter` —
|
|
154
205
|
* {@link VectorStore.listDocuments} without the metadata. Use it when you only need the id set (a
|
|
@@ -246,6 +297,25 @@ interface IndexedDocument {
|
|
|
246
297
|
id: string;
|
|
247
298
|
metadata?: Record<string, unknown>;
|
|
248
299
|
}
|
|
300
|
+
/** One stored chunk, read back verbatim — what {@link VectorStore.listChunks} yields. */
|
|
301
|
+
interface StoredChunk {
|
|
302
|
+
/** The chunk id: `${documentId}#<n>`, or the bare document id for a single-chunk document. */
|
|
303
|
+
id: string;
|
|
304
|
+
/** Zero-based position within the document, parsed from the id. A bare id is chunk `0`. */
|
|
305
|
+
index: number;
|
|
306
|
+
/** The chunk text as it was indexed — the exact string the embedding was computed from. */
|
|
307
|
+
text: string;
|
|
308
|
+
metadata?: Record<string, unknown>;
|
|
309
|
+
}
|
|
310
|
+
/** Paging for {@link VectorStore.listChunks}. Omit both to read the whole document. */
|
|
311
|
+
interface ListChunksOptions {
|
|
312
|
+
/** Max chunks to return, applied after ordering by {@link StoredChunk.index}. */
|
|
313
|
+
limit?: number;
|
|
314
|
+
/** Chunks to skip, applied after ordering. */
|
|
315
|
+
offset?: number;
|
|
316
|
+
}
|
|
317
|
+
/** Chunk `n` of `${documentId}#<n>`; a bare document id (single-chunk document) is `0`. */
|
|
318
|
+
declare function chunkIndexOf(chunkId: string): number;
|
|
249
319
|
/** Collapse a chunk id (`${documentId}#<n>`) back to its source document id. */
|
|
250
320
|
declare function documentIdOf(chunkId: string): string;
|
|
251
321
|
|
|
@@ -271,6 +341,16 @@ declare class MemoryVectorStore implements VectorStore {
|
|
|
271
341
|
*/
|
|
272
342
|
updateMetadata(documentId: string, patch: MetadataPatch): Promise<number>;
|
|
273
343
|
listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
|
|
344
|
+
/**
|
|
345
|
+
* The chunks of one document, in document order. See {@link VectorStore.listChunks}.
|
|
346
|
+
*
|
|
347
|
+
* Insertion order is not document order — `upsert` stores whatever the caller passed, and a
|
|
348
|
+
* re-ingest that rewrites chunk 3 alone moves it to the end of the Map. So this sorts by the index
|
|
349
|
+
* parsed from the id, the same thing the SQL and Redis adapters order by, rather than trusting the
|
|
350
|
+
* Map. A reference adapter that returned "usually ordered" would let a consumer ship a bug that
|
|
351
|
+
* only shows up on the store tests are NOT written against.
|
|
352
|
+
*/
|
|
353
|
+
listChunks(documentId: string, options?: ListChunksOptions): Promise<StoredChunk[]>;
|
|
274
354
|
listDocumentIds(filter?: Record<string, unknown>): Promise<string[]>;
|
|
275
355
|
removeMany(documentIds: string[]): Promise<void>;
|
|
276
356
|
/**
|
|
@@ -349,6 +429,19 @@ declare class PgVectorStore implements VectorStore {
|
|
|
349
429
|
*/
|
|
350
430
|
updateMetadata(documentId: string, patch: MetadataPatch): Promise<number>;
|
|
351
431
|
listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
|
|
432
|
+
/**
|
|
433
|
+
* The chunks of one document, in document order. See {@link VectorStore.listChunks}.
|
|
434
|
+
*
|
|
435
|
+
* The order is computed from the id rather than stored: `#12` must sort after `#2`, which a plain
|
|
436
|
+
* `ORDER BY id` gets wrong on every document past ten chunks because it compares text. So the
|
|
437
|
+
* trailing number is extracted and cast to `int`, and a bare id (a single-chunk document, which
|
|
438
|
+
* carries no `#n` suffix) collapses to `0` — the same rule `chunkIndexOf` applies in the other two
|
|
439
|
+
* adapters.
|
|
440
|
+
*
|
|
441
|
+
* `embedding` is left out of the projection: it is the widest column in the table and useless to a
|
|
442
|
+
* caller reading text back.
|
|
443
|
+
*/
|
|
444
|
+
listChunks(documentId: string, options?: ListChunksOptions): Promise<StoredChunk[]>;
|
|
352
445
|
/** {@link listDocuments} without the metadata: one `DISTINCT` over the derived document id. */
|
|
353
446
|
listDocumentIds(filter?: Record<string, unknown>): Promise<string[]>;
|
|
354
447
|
/** N documents in one statement — the set-based form of {@link PgVectorStore.remove}. */
|
|
@@ -525,6 +618,25 @@ declare class RedisVectorStore implements LexicalVectorStore {
|
|
|
525
618
|
* document owns. The glob is escaped so `gone` does not reach into `goner#0`.
|
|
526
619
|
*/
|
|
527
620
|
private chunkKeys;
|
|
621
|
+
/**
|
|
622
|
+
* The chunks of one document, in document order. See {@link VectorStore.listChunks}.
|
|
623
|
+
*
|
|
624
|
+
* Reads the hashes directly (`HMGET text metadata_json`) rather than going through `FT.SEARCH`.
|
|
625
|
+
* Two reasons, and the second is the one that matters: the query language has no way to ask for
|
|
626
|
+
* "every chunk of this document" — `id` is the key name, not an indexed field — and `FT.SEARCH`
|
|
627
|
+
* caps at `LIMIT`, so paging a document through it would mean ordering by a field the index does
|
|
628
|
+
* not hold. `chunkKeys` is the same enumeration {@link RedisVectorStore.remove} and
|
|
629
|
+
* {@link RedisVectorStore.updateMetadata} already use, so "the chunks this returns" and "the chunks
|
|
630
|
+
* a delete would take" cannot drift apart.
|
|
631
|
+
*
|
|
632
|
+
* It inherits that method's cost too: one keyspace `SCAN` per call. Acceptable here because this
|
|
633
|
+
* is an inspection path (one document, on demand) and not something retrieval runs — but it is why
|
|
634
|
+
* this must not be put on a hot path or called in a loop over a corpus.
|
|
635
|
+
*
|
|
636
|
+
* `embedding` is deliberately not fetched: it is by far the largest field on the hash and nothing
|
|
637
|
+
* a caller reading text back can do with it.
|
|
638
|
+
*/
|
|
639
|
+
listChunks(documentId: string, options?: ListChunksOptions): Promise<StoredChunk[]>;
|
|
528
640
|
listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
|
|
529
641
|
/**
|
|
530
642
|
* The id-only half of {@link RedisVectorStore.listDocuments}, served by `FT.SEARCH … NOCONTENT`:
|
|
@@ -843,4 +955,4 @@ interface RetrievalTool {
|
|
|
843
955
|
*/
|
|
844
956
|
declare function createRetrievalTool(retriever: Retriever, options?: RetrievalToolOptions): RetrievalTool;
|
|
845
957
|
|
|
846
|
-
export { type ChunkOptions, type ChunkRecord, EmbeddingRetriever, type EmbeddingRetrieverOptions, FilteredRetriever, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, KeywordRetriever, type KeywordRetrieverOptions, LexicalRetriever, type LexicalVectorStore, MemoryVectorStore, type MetadataPatch, type PgClient, PgVectorStore, type PgVectorStoreOptions, type RedisSearchClient, RedisVectorSchemaMismatchError, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalTool, type RetrievalToolOptions, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, applyMetadataPatch, chunkDocuments, chunkText, createRetrievalTool, documentIdOf, ingestChunks, ingestDocuments, isLexicalVectorStore };
|
|
958
|
+
export { type ChunkOptions, type ChunkRecord, EmbeddingRetriever, type EmbeddingRetrieverOptions, FilteredRetriever, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, KeywordRetriever, type KeywordRetrieverOptions, LexicalRetriever, type LexicalVectorStore, type ListChunksOptions, MemoryVectorStore, type MetadataPatch, type PgClient, PgVectorStore, type PgVectorStoreOptions, type RedisSearchClient, RedisVectorSchemaMismatchError, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalTool, type RetrievalToolOptions, type StoredChunk, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, applyMetadataPatch, chunkDocuments, chunkIndexOf, chunkText, createRetrievalTool, documentIdOf, ingestChunks, ingestDocuments, isLexicalVectorStore };
|
package/dist/index.js
CHANGED
|
@@ -9,6 +9,9 @@ function chunkText(text, options = {}) {
|
|
|
9
9
|
if (normalized.length === 0) {
|
|
10
10
|
return [];
|
|
11
11
|
}
|
|
12
|
+
if (options.separator !== void 0 && options.separator.length > 0) {
|
|
13
|
+
return chunkByRecord(normalized, chunkSize, options.separator);
|
|
14
|
+
}
|
|
12
15
|
if (normalized.length <= chunkSize) {
|
|
13
16
|
return [
|
|
14
17
|
normalized
|
|
@@ -38,6 +41,40 @@ function chunkText(text, options = {}) {
|
|
|
38
41
|
return chunks;
|
|
39
42
|
}
|
|
40
43
|
__name(chunkText, "chunkText");
|
|
44
|
+
function chunkByRecord(text, chunkSize, separator) {
|
|
45
|
+
const chunks = [];
|
|
46
|
+
let current = "";
|
|
47
|
+
const flush = /* @__PURE__ */ __name(() => {
|
|
48
|
+
if (current.length > 0) {
|
|
49
|
+
chunks.push(current);
|
|
50
|
+
current = "";
|
|
51
|
+
}
|
|
52
|
+
}, "flush");
|
|
53
|
+
for (const record of text.split(separator)) {
|
|
54
|
+
const trimmed = record.trim();
|
|
55
|
+
if (trimmed.length === 0) {
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
if (trimmed.length > chunkSize) {
|
|
59
|
+
flush();
|
|
60
|
+
chunks.push(...chunkText(trimmed, {
|
|
61
|
+
chunkSize,
|
|
62
|
+
overlap: 0
|
|
63
|
+
}));
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
const candidate = current.length > 0 ? `${current}${separator}${trimmed}` : trimmed;
|
|
67
|
+
if (candidate.length > chunkSize) {
|
|
68
|
+
flush();
|
|
69
|
+
current = trimmed;
|
|
70
|
+
} else {
|
|
71
|
+
current = candidate;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
flush();
|
|
75
|
+
return chunks;
|
|
76
|
+
}
|
|
77
|
+
__name(chunkByRecord, "chunkByRecord");
|
|
41
78
|
|
|
42
79
|
// src/vector-store.ts
|
|
43
80
|
function isLexicalVectorStore(store) {
|
|
@@ -54,6 +91,11 @@ var UnsafeRemovalError = class extends Error {
|
|
|
54
91
|
this.name = "UnsafeRemovalError";
|
|
55
92
|
}
|
|
56
93
|
};
|
|
94
|
+
function chunkIndexOf(chunkId) {
|
|
95
|
+
const match = /#(\d+)$/.exec(chunkId);
|
|
96
|
+
return match?.[1] === void 0 ? 0 : Number(match[1]);
|
|
97
|
+
}
|
|
98
|
+
__name(chunkIndexOf, "chunkIndexOf");
|
|
57
99
|
function documentIdOf(chunkId) {
|
|
58
100
|
return chunkId.replace(/#\d+$/, "");
|
|
59
101
|
}
|
|
@@ -178,6 +220,34 @@ var MemoryVectorStore = class {
|
|
|
178
220
|
...documents.values()
|
|
179
221
|
];
|
|
180
222
|
}
|
|
223
|
+
/**
|
|
224
|
+
* The chunks of one document, in document order. See {@link VectorStore.listChunks}.
|
|
225
|
+
*
|
|
226
|
+
* Insertion order is not document order — `upsert` stores whatever the caller passed, and a
|
|
227
|
+
* re-ingest that rewrites chunk 3 alone moves it to the end of the Map. So this sorts by the index
|
|
228
|
+
* parsed from the id, the same thing the SQL and Redis adapters order by, rather than trusting the
|
|
229
|
+
* Map. A reference adapter that returned "usually ordered" would let a consumer ship a bug that
|
|
230
|
+
* only shows up on the store tests are NOT written against.
|
|
231
|
+
*/
|
|
232
|
+
async listChunks(documentId, options) {
|
|
233
|
+
const chunks = [];
|
|
234
|
+
for (const record of this.records.values()) {
|
|
235
|
+
if (documentIdOf(record.id) !== documentId) {
|
|
236
|
+
continue;
|
|
237
|
+
}
|
|
238
|
+
chunks.push({
|
|
239
|
+
id: record.id,
|
|
240
|
+
index: chunkIndexOf(record.id),
|
|
241
|
+
text: record.text,
|
|
242
|
+
...record.metadata !== void 0 ? {
|
|
243
|
+
metadata: record.metadata
|
|
244
|
+
} : {}
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
chunks.sort((a, b) => a.index - b.index);
|
|
248
|
+
const offset = options?.offset ?? 0;
|
|
249
|
+
return options?.limit === void 0 ? chunks.slice(offset) : chunks.slice(offset, offset + options.limit);
|
|
250
|
+
}
|
|
181
251
|
async listDocumentIds(filter) {
|
|
182
252
|
if (filterMatchesNothing(filter)) {
|
|
183
253
|
return [];
|
|
@@ -364,6 +434,44 @@ var PgVectorStore = class {
|
|
|
364
434
|
} : {}
|
|
365
435
|
}));
|
|
366
436
|
}
|
|
437
|
+
/**
|
|
438
|
+
* The chunks of one document, in document order. See {@link VectorStore.listChunks}.
|
|
439
|
+
*
|
|
440
|
+
* The order is computed from the id rather than stored: `#12` must sort after `#2`, which a plain
|
|
441
|
+
* `ORDER BY id` gets wrong on every document past ten chunks because it compares text. So the
|
|
442
|
+
* trailing number is extracted and cast to `int`, and a bare id (a single-chunk document, which
|
|
443
|
+
* carries no `#n` suffix) collapses to `0` — the same rule `chunkIndexOf` applies in the other two
|
|
444
|
+
* adapters.
|
|
445
|
+
*
|
|
446
|
+
* `embedding` is left out of the projection: it is the widest column in the table and useless to a
|
|
447
|
+
* caller reading text back.
|
|
448
|
+
*/
|
|
449
|
+
async listChunks(documentId, options) {
|
|
450
|
+
const params = [
|
|
451
|
+
documentId
|
|
452
|
+
];
|
|
453
|
+
let sql = `SELECT id, ${CHUNK_INDEX_FROM_ID} AS chunk_index, text, metadata
|
|
454
|
+
FROM ${this.table}
|
|
455
|
+
WHERE ${DOCUMENT_ID_FROM_CHUNK} = $1
|
|
456
|
+
ORDER BY chunk_index`;
|
|
457
|
+
if (options?.limit !== void 0) {
|
|
458
|
+
params.push(options.limit);
|
|
459
|
+
sql += ` LIMIT $${params.length}`;
|
|
460
|
+
}
|
|
461
|
+
if (options?.offset !== void 0) {
|
|
462
|
+
params.push(options.offset);
|
|
463
|
+
sql += ` OFFSET $${params.length}`;
|
|
464
|
+
}
|
|
465
|
+
const rows = await this.client.query(sql, params);
|
|
466
|
+
return rows.map((row) => ({
|
|
467
|
+
id: row.id,
|
|
468
|
+
index: Number(row.chunk_index),
|
|
469
|
+
text: row.text,
|
|
470
|
+
...row.metadata !== null ? {
|
|
471
|
+
metadata: row.metadata
|
|
472
|
+
} : {}
|
|
473
|
+
}));
|
|
474
|
+
}
|
|
367
475
|
/** {@link listDocuments} without the metadata: one `DISTINCT` over the derived document id. */
|
|
368
476
|
async listDocumentIds(filter) {
|
|
369
477
|
if (filterMatchesNothing(filter)) {
|
|
@@ -474,6 +582,7 @@ function buildWhere(filter, startIndex) {
|
|
|
474
582
|
}
|
|
475
583
|
__name(buildWhere, "buildWhere");
|
|
476
584
|
var DOCUMENT_ID_FROM_CHUNK = "regexp_replace(id, '#[0-9]+$', '')";
|
|
585
|
+
var CHUNK_INDEX_FROM_ID = "COALESCE((substring(id from '#([0-9]+)$'))::int, 0)";
|
|
477
586
|
function toVectorLiteral(embedding) {
|
|
478
587
|
return `[${embedding.join(",")}]`;
|
|
479
588
|
}
|
|
@@ -766,6 +875,58 @@ var RedisVectorStore = class {
|
|
|
766
875
|
...keys
|
|
767
876
|
];
|
|
768
877
|
}
|
|
878
|
+
/**
|
|
879
|
+
* The chunks of one document, in document order. See {@link VectorStore.listChunks}.
|
|
880
|
+
*
|
|
881
|
+
* Reads the hashes directly (`HMGET text metadata_json`) rather than going through `FT.SEARCH`.
|
|
882
|
+
* Two reasons, and the second is the one that matters: the query language has no way to ask for
|
|
883
|
+
* "every chunk of this document" — `id` is the key name, not an indexed field — and `FT.SEARCH`
|
|
884
|
+
* caps at `LIMIT`, so paging a document through it would mean ordering by a field the index does
|
|
885
|
+
* not hold. `chunkKeys` is the same enumeration {@link RedisVectorStore.remove} and
|
|
886
|
+
* {@link RedisVectorStore.updateMetadata} already use, so "the chunks this returns" and "the chunks
|
|
887
|
+
* a delete would take" cannot drift apart.
|
|
888
|
+
*
|
|
889
|
+
* It inherits that method's cost too: one keyspace `SCAN` per call. Acceptable here because this
|
|
890
|
+
* is an inspection path (one document, on demand) and not something retrieval runs — but it is why
|
|
891
|
+
* this must not be put on a hot path or called in a loop over a corpus.
|
|
892
|
+
*
|
|
893
|
+
* `embedding` is deliberately not fetched: it is by far the largest field on the hash and nothing
|
|
894
|
+
* a caller reading text back can do with it.
|
|
895
|
+
*/
|
|
896
|
+
async listChunks(documentId, options) {
|
|
897
|
+
const keys = await this.chunkKeys(documentId);
|
|
898
|
+
const ordered = keys.map((key) => ({
|
|
899
|
+
key,
|
|
900
|
+
id: key.slice(this.prefix.length)
|
|
901
|
+
})).map((entry) => ({
|
|
902
|
+
...entry,
|
|
903
|
+
index: chunkIndexOf(entry.id)
|
|
904
|
+
})).sort((a, b) => a.index - b.index);
|
|
905
|
+
const offset = options?.offset ?? 0;
|
|
906
|
+
const page = options?.limit === void 0 ? ordered.slice(offset) : ordered.slice(offset, offset + options.limit);
|
|
907
|
+
const chunks = [];
|
|
908
|
+
for (const entry of page) {
|
|
909
|
+
const reply = await this.client.sendCommand([
|
|
910
|
+
"HMGET",
|
|
911
|
+
entry.key,
|
|
912
|
+
"text",
|
|
913
|
+
"metadata_json"
|
|
914
|
+
]);
|
|
915
|
+
const fields = Array.isArray(reply) ? reply : [];
|
|
916
|
+
const text = fields[0] === null || fields[0] === void 0 ? void 0 : toStr(fields[0]);
|
|
917
|
+
if (text === void 0) {
|
|
918
|
+
continue;
|
|
919
|
+
}
|
|
920
|
+
const raw = fields[1] === null || fields[1] === void 0 ? void 0 : toStr(fields[1]);
|
|
921
|
+
chunks.push({
|
|
922
|
+
id: entry.id,
|
|
923
|
+
index: entry.index,
|
|
924
|
+
text,
|
|
925
|
+
...parseMetadata(raw)
|
|
926
|
+
});
|
|
927
|
+
}
|
|
928
|
+
return chunks;
|
|
929
|
+
}
|
|
769
930
|
async listDocuments(filter) {
|
|
770
931
|
if (filterMatchesNothing(filter)) {
|
|
771
932
|
return [];
|
|
@@ -1655,6 +1816,7 @@ export {
|
|
|
1655
1816
|
UnsafeRemovalError,
|
|
1656
1817
|
applyMetadataPatch,
|
|
1657
1818
|
chunkDocuments,
|
|
1819
|
+
chunkIndexOf,
|
|
1658
1820
|
chunkText,
|
|
1659
1821
|
createRetrievalTool,
|
|
1660
1822
|
documentIdOf,
|