@dudousxd/nestjs-agent-rag 0.7.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -5,6 +5,31 @@ interface ChunkOptions {
5
5
  chunkSize?: number;
6
6
  /** Characters of overlap carried from the end of one chunk into the next. Default 100. */
7
7
  overlap?: number;
8
+ /**
9
+ * Treat the text as records joined by this string, and never split one across chunks.
10
+ *
11
+ * Default chunking is structure-blind: it breaks at the *latest* paragraph/sentence/word boundary
12
+ * in the window, which for record-shaped text means a cut lands mid-record routinely. That is fine
13
+ * for prose, where the two halves are still sentences. It is not fine for a table flattened to one
14
+ * record per row, because the half that keeps the row identifier and the half that keeps the value
15
+ * end up in different chunks, and neither can answer a question about that row on its own.
16
+ *
17
+ * With a separator set, whole records are packed greedily up to `chunkSize` and a record is never
18
+ * cut — except one longer than `chunkSize`, which has nowhere to go and is character-split as
19
+ * usual, since no chunker can avoid that.
20
+ *
21
+ * `overlap` is IGNORED in this mode, deliberately. Overlap exists to rescue a sentence that a
22
+ * boundary cut in half; a boundary that never falls inside a record has nothing to rescue, and
23
+ * carrying the tail of the previous chunk in would duplicate whole records into neighbouring chunks
24
+ * — paying for the same rows twice at embedding time and letting one row match from two places.
25
+ *
26
+ * Measured on a 200-row spreadsheet flattened to one field-labelled record per row (15 columns,
27
+ * ~800-char budget): blind chunking left 27% of records split, and on questions targeting a field
28
+ * on the far side of such a cut, BM25 answered 54/66 (MRR 0.818) against 66/66 (MRR 1.000) with
29
+ * records kept whole. The dense leg scored 7/66 either way — 200 near-identical rows produce
30
+ * near-identical vectors, so that leg cannot do row lookup at all and chunking does not change it.
31
+ */
32
+ separator?: string;
8
33
  }
9
34
  /**
10
35
  * Splits text into overlapping chunks for embedding. Greedy up to `chunkSize`, but prefers to break
@@ -149,6 +174,32 @@ interface VectorStore {
149
174
  * `size` changed), or {@link VectorStore.remove remove} (orphaned).
150
175
  */
151
176
  listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
177
+ /**
178
+ * Read one document's chunks back — id, text, metadata — ordered by their position in the
179
+ * document, optionally paged with `limit`/`offset`.
180
+ *
181
+ * The gap this closes is that **the text a store holds is otherwise write-only**. Every other way
182
+ * back to it is a ranked query: {@link VectorStore.search} needs an embedding,
183
+ * {@link LexicalVectorStore.searchText} needs a query string, and both return whatever *matched*,
184
+ * in relevance order, capped at `topK`. Neither can answer "show me what was indexed for this
185
+ * document, in order" — the question you ask when retrieval returns the wrong passage and you need
186
+ * to know whether the chunker cut badly or the retriever chose badly. Without this, the only way to
187
+ * see stored text is to reach around the adapter into the backing engine, which is exactly the
188
+ * abstraction break a consumer adopts a store to avoid.
189
+ *
190
+ * Document-scoped rather than filter-scoped, matching {@link VectorStore.remove} and
191
+ * {@link VectorStore.updateMetadata}: ordering is the whole point, and "position within the
192
+ * document" is only meaningful once a document has been picked. Enumerate documents with
193
+ * {@link VectorStore.listDocuments} first when you need to sweep.
194
+ *
195
+ * An unknown `documentId` yields `[]`; it does not throw, matching `remove` and `updateMetadata`.
196
+ *
197
+ * NOT an access-control seam. It takes no metadata `filter` and applies none, so it cannot enforce
198
+ * one — a caller that gates retrieval on a metadata field (tenant, audience, collection) MUST
199
+ * resolve that gate itself before calling, exactly as it would before `remove`. It is deliberately
200
+ * shaped for an operator/debug surface, not for serving end users.
201
+ */
202
+ listChunks(documentId: string, options?: ListChunksOptions): Promise<StoredChunk[]>;
152
203
  /**
153
204
  * The distinct source document ids currently indexed, optionally narrowed by a metadata `filter` —
154
205
  * {@link VectorStore.listDocuments} without the metadata. Use it when you only need the id set (a
@@ -246,6 +297,25 @@ interface IndexedDocument {
246
297
  id: string;
247
298
  metadata?: Record<string, unknown>;
248
299
  }
300
+ /** One stored chunk, read back verbatim — what {@link VectorStore.listChunks} yields. */
301
+ interface StoredChunk {
302
+ /** The chunk id: `${documentId}#<n>`, or the bare document id for a single-chunk document. */
303
+ id: string;
304
+ /** Zero-based position within the document, parsed from the id. A bare id is chunk `0`. */
305
+ index: number;
306
+ /** The chunk text as it was indexed — the exact string the embedding was computed from. */
307
+ text: string;
308
+ metadata?: Record<string, unknown>;
309
+ }
310
+ /** Paging for {@link VectorStore.listChunks}. Omit both to read the whole document. */
311
+ interface ListChunksOptions {
312
+ /** Max chunks to return, applied after ordering by {@link StoredChunk.index}. */
313
+ limit?: number;
314
+ /** Chunks to skip, applied after ordering. */
315
+ offset?: number;
316
+ }
317
+ /** Chunk `n` of `${documentId}#<n>`; a bare document id (single-chunk document) is `0`. */
318
+ declare function chunkIndexOf(chunkId: string): number;
249
319
  /** Collapse a chunk id (`${documentId}#<n>`) back to its source document id. */
250
320
  declare function documentIdOf(chunkId: string): string;
251
321
 
@@ -271,6 +341,16 @@ declare class MemoryVectorStore implements VectorStore {
271
341
  */
272
342
  updateMetadata(documentId: string, patch: MetadataPatch): Promise<number>;
273
343
  listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
344
+ /**
345
+ * The chunks of one document, in document order. See {@link VectorStore.listChunks}.
346
+ *
347
+ * Insertion order is not document order — `upsert` stores whatever the caller passed, and a
348
+ * re-ingest that rewrites chunk 3 alone moves it to the end of the Map. So this sorts by the index
349
+ * parsed from the id, the same thing the SQL and Redis adapters order by, rather than trusting the
350
+ * Map. A reference adapter that returned "usually ordered" would let a consumer ship a bug that
351
+ * only shows up on the store tests are NOT written against.
352
+ */
353
+ listChunks(documentId: string, options?: ListChunksOptions): Promise<StoredChunk[]>;
274
354
  listDocumentIds(filter?: Record<string, unknown>): Promise<string[]>;
275
355
  removeMany(documentIds: string[]): Promise<void>;
276
356
  /**
@@ -349,6 +429,19 @@ declare class PgVectorStore implements VectorStore {
349
429
  */
350
430
  updateMetadata(documentId: string, patch: MetadataPatch): Promise<number>;
351
431
  listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
432
+ /**
433
+ * The chunks of one document, in document order. See {@link VectorStore.listChunks}.
434
+ *
435
+ * The order is computed from the id rather than stored: `#12` must sort after `#2`, which a plain
436
+ * `ORDER BY id` gets wrong on every document past ten chunks because it compares text. So the
437
+ * trailing number is extracted and cast to `int`, and a bare id (a single-chunk document, which
438
+ * carries no `#n` suffix) collapses to `0` — the same rule `chunkIndexOf` applies in the other two
439
+ * adapters.
440
+ *
441
+ * `embedding` is left out of the projection: it is the widest column in the table and useless to a
442
+ * caller reading text back.
443
+ */
444
+ listChunks(documentId: string, options?: ListChunksOptions): Promise<StoredChunk[]>;
352
445
  /** {@link listDocuments} without the metadata: one `DISTINCT` over the derived document id. */
353
446
  listDocumentIds(filter?: Record<string, unknown>): Promise<string[]>;
354
447
  /** N documents in one statement — the set-based form of {@link PgVectorStore.remove}. */
@@ -525,6 +618,25 @@ declare class RedisVectorStore implements LexicalVectorStore {
525
618
  * document owns. The glob is escaped so `gone` does not reach into `goner#0`.
526
619
  */
527
620
  private chunkKeys;
621
+ /**
622
+ * The chunks of one document, in document order. See {@link VectorStore.listChunks}.
623
+ *
624
+ * Reads the hashes directly (`HMGET text metadata_json`) rather than going through `FT.SEARCH`.
625
+ * Two reasons, and the second is the one that matters: the query language has no way to ask for
626
+ * "every chunk of this document" — `id` is the key name, not an indexed field — and `FT.SEARCH`
627
+ * caps at `LIMIT`, so paging a document through it would mean ordering by a field the index does
628
+ * not hold. `chunkKeys` is the same enumeration {@link RedisVectorStore.remove} and
629
+ * {@link RedisVectorStore.updateMetadata} already use, so "the chunks this returns" and "the chunks
630
+ * a delete would take" cannot drift apart.
631
+ *
632
+ * It inherits that method's cost too: one keyspace `SCAN` per call. Acceptable here because this
633
+ * is an inspection path (one document, on demand) and not something retrieval runs — but it is why
634
+ * this must not be put on a hot path or called in a loop over a corpus.
635
+ *
636
+ * `embedding` is deliberately not fetched: it is by far the largest field on the hash and nothing
637
+ * a caller reading text back can do with it.
638
+ */
639
+ listChunks(documentId: string, options?: ListChunksOptions): Promise<StoredChunk[]>;
528
640
  listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
529
641
  /**
530
642
  * The id-only half of {@link RedisVectorStore.listDocuments}, served by `FT.SEARCH … NOCONTENT`:
@@ -843,4 +955,4 @@ interface RetrievalTool {
843
955
  */
844
956
  declare function createRetrievalTool(retriever: Retriever, options?: RetrievalToolOptions): RetrievalTool;
845
957
 
846
- export { type ChunkOptions, type ChunkRecord, EmbeddingRetriever, type EmbeddingRetrieverOptions, FilteredRetriever, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, KeywordRetriever, type KeywordRetrieverOptions, LexicalRetriever, type LexicalVectorStore, MemoryVectorStore, type MetadataPatch, type PgClient, PgVectorStore, type PgVectorStoreOptions, type RedisSearchClient, RedisVectorSchemaMismatchError, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalTool, type RetrievalToolOptions, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, applyMetadataPatch, chunkDocuments, chunkText, createRetrievalTool, documentIdOf, ingestChunks, ingestDocuments, isLexicalVectorStore };
958
+ export { type ChunkOptions, type ChunkRecord, EmbeddingRetriever, type EmbeddingRetrieverOptions, FilteredRetriever, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, KeywordRetriever, type KeywordRetrieverOptions, LexicalRetriever, type LexicalVectorStore, type ListChunksOptions, MemoryVectorStore, type MetadataPatch, type PgClient, PgVectorStore, type PgVectorStoreOptions, type RedisSearchClient, RedisVectorSchemaMismatchError, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalTool, type RetrievalToolOptions, type StoredChunk, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, applyMetadataPatch, chunkDocuments, chunkIndexOf, chunkText, createRetrievalTool, documentIdOf, ingestChunks, ingestDocuments, isLexicalVectorStore };
package/dist/index.d.ts CHANGED
@@ -5,6 +5,31 @@ interface ChunkOptions {
5
5
  chunkSize?: number;
6
6
  /** Characters of overlap carried from the end of one chunk into the next. Default 100. */
7
7
  overlap?: number;
8
+ /**
9
+ * Treat the text as records joined by this string, and never split one across chunks.
10
+ *
11
+ * Default chunking is structure-blind: it breaks at the *latest* paragraph/sentence/word boundary
12
+ * in the window, which for record-shaped text means a cut lands mid-record routinely. That is fine
13
+ * for prose, where the two halves are still sentences. It is not fine for a table flattened to one
14
+ * record per row, because the half that keeps the row identifier and the half that keeps the value
15
+ * end up in different chunks, and neither can answer a question about that row on its own.
16
+ *
17
+ * With a separator set, whole records are packed greedily up to `chunkSize` and a record is never
18
+ * cut — except one longer than `chunkSize`, which has nowhere to go and is character-split as
19
+ * usual, since no chunker can avoid that.
20
+ *
21
+ * `overlap` is IGNORED in this mode, deliberately. Overlap exists to rescue a sentence that a
22
+ * boundary cut in half; a boundary that never falls inside a record has nothing to rescue, and
23
+ * carrying the tail of the previous chunk in would duplicate whole records into neighbouring chunks
24
+ * — paying for the same rows twice at embedding time and letting one row match from two places.
25
+ *
26
+ * Measured on a 200-row spreadsheet flattened to one field-labelled record per row (15 columns,
27
+ * ~800-char budget): blind chunking left 27% of records split, and on questions targeting a field
28
+ * on the far side of such a cut, BM25 answered 54/66 (MRR 0.818) against 66/66 (MRR 1.000) with
29
+ * records kept whole. The dense leg scored 7/66 either way — 200 near-identical rows produce
30
+ * near-identical vectors, so that leg cannot do row lookup at all and chunking does not change it.
31
+ */
32
+ separator?: string;
8
33
  }
9
34
  /**
10
35
  * Splits text into overlapping chunks for embedding. Greedy up to `chunkSize`, but prefers to break
@@ -149,6 +174,32 @@ interface VectorStore {
149
174
  * `size` changed), or {@link VectorStore.remove remove} (orphaned).
150
175
  */
151
176
  listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
177
+ /**
178
+ * Read one document's chunks back — id, text, metadata — ordered by their position in the
179
+ * document, optionally paged with `limit`/`offset`.
180
+ *
181
+ * The gap this closes is that **the text a store holds is otherwise write-only**. Every other way
182
+ * back to it is a ranked query: {@link VectorStore.search} needs an embedding,
183
+ * {@link LexicalVectorStore.searchText} needs a query string, and both return whatever *matched*,
184
+ * in relevance order, capped at `topK`. Neither can answer "show me what was indexed for this
185
+ * document, in order" — the question you ask when retrieval returns the wrong passage and you need
186
+ * to know whether the chunker cut badly or the retriever chose badly. Without this, the only way to
187
+ * see stored text is to reach around the adapter into the backing engine, which is exactly the
188
+ * abstraction break a consumer adopts a store to avoid.
189
+ *
190
+ * Document-scoped rather than filter-scoped, matching {@link VectorStore.remove} and
191
+ * {@link VectorStore.updateMetadata}: ordering is the whole point, and "position within the
192
+ * document" is only meaningful once a document has been picked. Enumerate documents with
193
+ * {@link VectorStore.listDocuments} first when you need to sweep.
194
+ *
195
+ * An unknown `documentId` yields `[]`; it does not throw, matching `remove` and `updateMetadata`.
196
+ *
197
+ * NOT an access-control seam. It takes no metadata `filter` and applies none, so it cannot enforce
198
+ * one — a caller that gates retrieval on a metadata field (tenant, audience, collection) MUST
199
+ * resolve that gate itself before calling, exactly as it would before `remove`. It is deliberately
200
+ * shaped for an operator/debug surface, not for serving end users.
201
+ */
202
+ listChunks(documentId: string, options?: ListChunksOptions): Promise<StoredChunk[]>;
152
203
  /**
153
204
  * The distinct source document ids currently indexed, optionally narrowed by a metadata `filter` —
154
205
  * {@link VectorStore.listDocuments} without the metadata. Use it when you only need the id set (a
@@ -246,6 +297,25 @@ interface IndexedDocument {
246
297
  id: string;
247
298
  metadata?: Record<string, unknown>;
248
299
  }
300
+ /** One stored chunk, read back verbatim — what {@link VectorStore.listChunks} yields. */
301
+ interface StoredChunk {
302
+ /** The chunk id: `${documentId}#<n>`, or the bare document id for a single-chunk document. */
303
+ id: string;
304
+ /** Zero-based position within the document, parsed from the id. A bare id is chunk `0`. */
305
+ index: number;
306
+ /** The chunk text as it was indexed — the exact string the embedding was computed from. */
307
+ text: string;
308
+ metadata?: Record<string, unknown>;
309
+ }
310
+ /** Paging for {@link VectorStore.listChunks}. Omit both to read the whole document. */
311
+ interface ListChunksOptions {
312
+ /** Max chunks to return, applied after ordering by {@link StoredChunk.index}. */
313
+ limit?: number;
314
+ /** Chunks to skip, applied after ordering. */
315
+ offset?: number;
316
+ }
317
+ /** Chunk `n` of `${documentId}#<n>`; a bare document id (single-chunk document) is `0`. */
318
+ declare function chunkIndexOf(chunkId: string): number;
249
319
  /** Collapse a chunk id (`${documentId}#<n>`) back to its source document id. */
250
320
  declare function documentIdOf(chunkId: string): string;
251
321
 
@@ -271,6 +341,16 @@ declare class MemoryVectorStore implements VectorStore {
271
341
  */
272
342
  updateMetadata(documentId: string, patch: MetadataPatch): Promise<number>;
273
343
  listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
344
+ /**
345
+ * The chunks of one document, in document order. See {@link VectorStore.listChunks}.
346
+ *
347
+ * Insertion order is not document order — `upsert` stores whatever the caller passed, and a
348
+ * re-ingest that rewrites chunk 3 alone moves it to the end of the Map. So this sorts by the index
349
+ * parsed from the id, the same thing the SQL and Redis adapters order by, rather than trusting the
350
+ * Map. A reference adapter that returned "usually ordered" would let a consumer ship a bug that
351
+ * only shows up on the store tests are NOT written against.
352
+ */
353
+ listChunks(documentId: string, options?: ListChunksOptions): Promise<StoredChunk[]>;
274
354
  listDocumentIds(filter?: Record<string, unknown>): Promise<string[]>;
275
355
  removeMany(documentIds: string[]): Promise<void>;
276
356
  /**
@@ -349,6 +429,19 @@ declare class PgVectorStore implements VectorStore {
349
429
  */
350
430
  updateMetadata(documentId: string, patch: MetadataPatch): Promise<number>;
351
431
  listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
432
+ /**
433
+ * The chunks of one document, in document order. See {@link VectorStore.listChunks}.
434
+ *
435
+ * The order is computed from the id rather than stored: `#12` must sort after `#2`, which a plain
436
+ * `ORDER BY id` gets wrong on every document past ten chunks because it compares text. So the
437
+ * trailing number is extracted and cast to `int`, and a bare id (a single-chunk document, which
438
+ * carries no `#n` suffix) collapses to `0` — the same rule `chunkIndexOf` applies in the other two
439
+ * adapters.
440
+ *
441
+ * `embedding` is left out of the projection: it is the widest column in the table and useless to a
442
+ * caller reading text back.
443
+ */
444
+ listChunks(documentId: string, options?: ListChunksOptions): Promise<StoredChunk[]>;
352
445
  /** {@link listDocuments} without the metadata: one `DISTINCT` over the derived document id. */
353
446
  listDocumentIds(filter?: Record<string, unknown>): Promise<string[]>;
354
447
  /** N documents in one statement — the set-based form of {@link PgVectorStore.remove}. */
@@ -525,6 +618,25 @@ declare class RedisVectorStore implements LexicalVectorStore {
525
618
  * document owns. The glob is escaped so `gone` does not reach into `goner#0`.
526
619
  */
527
620
  private chunkKeys;
621
+ /**
622
+ * The chunks of one document, in document order. See {@link VectorStore.listChunks}.
623
+ *
624
+ * Reads the hashes directly (`HMGET text metadata_json`) rather than going through `FT.SEARCH`.
625
+ * Two reasons, and the second is the one that matters: the query language has no way to ask for
626
+ * "every chunk of this document" — `id` is the key name, not an indexed field — and `FT.SEARCH`
627
+ * caps at `LIMIT`, so paging a document through it would mean ordering by a field the index does
628
+ * not hold. `chunkKeys` is the same enumeration {@link RedisVectorStore.remove} and
629
+ * {@link RedisVectorStore.updateMetadata} already use, so "the chunks this returns" and "the chunks
630
+ * a delete would take" cannot drift apart.
631
+ *
632
+ * It inherits that method's cost too: one keyspace `SCAN` per call. Acceptable here because this
633
+ * is an inspection path (one document, on demand) and not something retrieval runs — but it is why
634
+ * this must not be put on a hot path or called in a loop over a corpus.
635
+ *
636
+ * `embedding` is deliberately not fetched: it is by far the largest field on the hash and nothing
637
+ * a caller reading text back can do with it.
638
+ */
639
+ listChunks(documentId: string, options?: ListChunksOptions): Promise<StoredChunk[]>;
528
640
  listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
529
641
  /**
530
642
  * The id-only half of {@link RedisVectorStore.listDocuments}, served by `FT.SEARCH … NOCONTENT`:
@@ -843,4 +955,4 @@ interface RetrievalTool {
843
955
  */
844
956
  declare function createRetrievalTool(retriever: Retriever, options?: RetrievalToolOptions): RetrievalTool;
845
957
 
846
- export { type ChunkOptions, type ChunkRecord, EmbeddingRetriever, type EmbeddingRetrieverOptions, FilteredRetriever, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, KeywordRetriever, type KeywordRetrieverOptions, LexicalRetriever, type LexicalVectorStore, MemoryVectorStore, type MetadataPatch, type PgClient, PgVectorStore, type PgVectorStoreOptions, type RedisSearchClient, RedisVectorSchemaMismatchError, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalTool, type RetrievalToolOptions, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, applyMetadataPatch, chunkDocuments, chunkText, createRetrievalTool, documentIdOf, ingestChunks, ingestDocuments, isLexicalVectorStore };
958
+ export { type ChunkOptions, type ChunkRecord, EmbeddingRetriever, type EmbeddingRetrieverOptions, FilteredRetriever, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, KeywordRetriever, type KeywordRetrieverOptions, LexicalRetriever, type LexicalVectorStore, type ListChunksOptions, MemoryVectorStore, type MetadataPatch, type PgClient, PgVectorStore, type PgVectorStoreOptions, type RedisSearchClient, RedisVectorSchemaMismatchError, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalTool, type RetrievalToolOptions, type StoredChunk, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, applyMetadataPatch, chunkDocuments, chunkIndexOf, chunkText, createRetrievalTool, documentIdOf, ingestChunks, ingestDocuments, isLexicalVectorStore };
package/dist/index.js CHANGED
@@ -9,6 +9,9 @@ function chunkText(text, options = {}) {
9
9
  if (normalized.length === 0) {
10
10
  return [];
11
11
  }
12
+ if (options.separator !== void 0 && options.separator.length > 0) {
13
+ return chunkByRecord(normalized, chunkSize, options.separator);
14
+ }
12
15
  if (normalized.length <= chunkSize) {
13
16
  return [
14
17
  normalized
@@ -38,6 +41,40 @@ function chunkText(text, options = {}) {
38
41
  return chunks;
39
42
  }
40
43
  __name(chunkText, "chunkText");
44
+ function chunkByRecord(text, chunkSize, separator) {
45
+ const chunks = [];
46
+ let current = "";
47
+ const flush = /* @__PURE__ */ __name(() => {
48
+ if (current.length > 0) {
49
+ chunks.push(current);
50
+ current = "";
51
+ }
52
+ }, "flush");
53
+ for (const record of text.split(separator)) {
54
+ const trimmed = record.trim();
55
+ if (trimmed.length === 0) {
56
+ continue;
57
+ }
58
+ if (trimmed.length > chunkSize) {
59
+ flush();
60
+ chunks.push(...chunkText(trimmed, {
61
+ chunkSize,
62
+ overlap: 0
63
+ }));
64
+ continue;
65
+ }
66
+ const candidate = current.length > 0 ? `${current}${separator}${trimmed}` : trimmed;
67
+ if (candidate.length > chunkSize) {
68
+ flush();
69
+ current = trimmed;
70
+ } else {
71
+ current = candidate;
72
+ }
73
+ }
74
+ flush();
75
+ return chunks;
76
+ }
77
+ __name(chunkByRecord, "chunkByRecord");
41
78
 
42
79
  // src/vector-store.ts
43
80
  function isLexicalVectorStore(store) {
@@ -54,6 +91,11 @@ var UnsafeRemovalError = class extends Error {
54
91
  this.name = "UnsafeRemovalError";
55
92
  }
56
93
  };
94
+ function chunkIndexOf(chunkId) {
95
+ const match = /#(\d+)$/.exec(chunkId);
96
+ return match?.[1] === void 0 ? 0 : Number(match[1]);
97
+ }
98
+ __name(chunkIndexOf, "chunkIndexOf");
57
99
  function documentIdOf(chunkId) {
58
100
  return chunkId.replace(/#\d+$/, "");
59
101
  }
@@ -178,6 +220,34 @@ var MemoryVectorStore = class {
178
220
  ...documents.values()
179
221
  ];
180
222
  }
223
+ /**
224
+ * The chunks of one document, in document order. See {@link VectorStore.listChunks}.
225
+ *
226
+ * Insertion order is not document order — `upsert` stores whatever the caller passed, and a
227
+ * re-ingest that rewrites chunk 3 alone moves it to the end of the Map. So this sorts by the index
228
+ * parsed from the id, the same thing the SQL and Redis adapters order by, rather than trusting the
229
+ * Map. A reference adapter that returned "usually ordered" would let a consumer ship a bug that
230
+ * only shows up on the store tests are NOT written against.
231
+ */
232
+ async listChunks(documentId, options) {
233
+ const chunks = [];
234
+ for (const record of this.records.values()) {
235
+ if (documentIdOf(record.id) !== documentId) {
236
+ continue;
237
+ }
238
+ chunks.push({
239
+ id: record.id,
240
+ index: chunkIndexOf(record.id),
241
+ text: record.text,
242
+ ...record.metadata !== void 0 ? {
243
+ metadata: record.metadata
244
+ } : {}
245
+ });
246
+ }
247
+ chunks.sort((a, b) => a.index - b.index);
248
+ const offset = options?.offset ?? 0;
249
+ return options?.limit === void 0 ? chunks.slice(offset) : chunks.slice(offset, offset + options.limit);
250
+ }
181
251
  async listDocumentIds(filter) {
182
252
  if (filterMatchesNothing(filter)) {
183
253
  return [];
@@ -364,6 +434,44 @@ var PgVectorStore = class {
364
434
  } : {}
365
435
  }));
366
436
  }
437
+ /**
438
+ * The chunks of one document, in document order. See {@link VectorStore.listChunks}.
439
+ *
440
+ * The order is computed from the id rather than stored: `#12` must sort after `#2`, which a plain
441
+ * `ORDER BY id` gets wrong on every document past ten chunks because it compares text. So the
442
+ * trailing number is extracted and cast to `int`, and a bare id (a single-chunk document, which
443
+ * carries no `#n` suffix) collapses to `0` — the same rule `chunkIndexOf` applies in the other two
444
+ * adapters.
445
+ *
446
+ * `embedding` is left out of the projection: it is the widest column in the table and useless to a
447
+ * caller reading text back.
448
+ */
449
+ async listChunks(documentId, options) {
450
+ const params = [
451
+ documentId
452
+ ];
453
+ let sql = `SELECT id, ${CHUNK_INDEX_FROM_ID} AS chunk_index, text, metadata
454
+ FROM ${this.table}
455
+ WHERE ${DOCUMENT_ID_FROM_CHUNK} = $1
456
+ ORDER BY chunk_index`;
457
+ if (options?.limit !== void 0) {
458
+ params.push(options.limit);
459
+ sql += ` LIMIT $${params.length}`;
460
+ }
461
+ if (options?.offset !== void 0) {
462
+ params.push(options.offset);
463
+ sql += ` OFFSET $${params.length}`;
464
+ }
465
+ const rows = await this.client.query(sql, params);
466
+ return rows.map((row) => ({
467
+ id: row.id,
468
+ index: Number(row.chunk_index),
469
+ text: row.text,
470
+ ...row.metadata !== null ? {
471
+ metadata: row.metadata
472
+ } : {}
473
+ }));
474
+ }
367
475
  /** {@link listDocuments} without the metadata: one `DISTINCT` over the derived document id. */
368
476
  async listDocumentIds(filter) {
369
477
  if (filterMatchesNothing(filter)) {
@@ -474,6 +582,7 @@ function buildWhere(filter, startIndex) {
474
582
  }
475
583
  __name(buildWhere, "buildWhere");
476
584
  var DOCUMENT_ID_FROM_CHUNK = "regexp_replace(id, '#[0-9]+$', '')";
585
+ var CHUNK_INDEX_FROM_ID = "COALESCE((substring(id from '#([0-9]+)$'))::int, 0)";
477
586
  function toVectorLiteral(embedding) {
478
587
  return `[${embedding.join(",")}]`;
479
588
  }
@@ -766,6 +875,58 @@ var RedisVectorStore = class {
766
875
  ...keys
767
876
  ];
768
877
  }
878
+ /**
879
+ * The chunks of one document, in document order. See {@link VectorStore.listChunks}.
880
+ *
881
+ * Reads the hashes directly (`HMGET text metadata_json`) rather than going through `FT.SEARCH`.
882
+ * Two reasons, and the second is the one that matters: the query language has no way to ask for
883
+ * "every chunk of this document" — `id` is the key name, not an indexed field — and `FT.SEARCH`
884
+ * caps at `LIMIT`, so paging a document through it would mean ordering by a field the index does
885
+ * not hold. `chunkKeys` is the same enumeration {@link RedisVectorStore.remove} and
886
+ * {@link RedisVectorStore.updateMetadata} already use, so "the chunks this returns" and "the chunks
887
+ * a delete would take" cannot drift apart.
888
+ *
889
+ * It inherits that method's cost too: one keyspace `SCAN` per call. Acceptable here because this
890
+ * is an inspection path (one document, on demand) and not something retrieval runs — but it is why
891
+ * this must not be put on a hot path or called in a loop over a corpus.
892
+ *
893
+ * `embedding` is deliberately not fetched: it is by far the largest field on the hash and nothing
894
+ * a caller reading text back can do with it.
895
+ */
896
+ async listChunks(documentId, options) {
897
+ const keys = await this.chunkKeys(documentId);
898
+ const ordered = keys.map((key) => ({
899
+ key,
900
+ id: key.slice(this.prefix.length)
901
+ })).map((entry) => ({
902
+ ...entry,
903
+ index: chunkIndexOf(entry.id)
904
+ })).sort((a, b) => a.index - b.index);
905
+ const offset = options?.offset ?? 0;
906
+ const page = options?.limit === void 0 ? ordered.slice(offset) : ordered.slice(offset, offset + options.limit);
907
+ const chunks = [];
908
+ for (const entry of page) {
909
+ const reply = await this.client.sendCommand([
910
+ "HMGET",
911
+ entry.key,
912
+ "text",
913
+ "metadata_json"
914
+ ]);
915
+ const fields = Array.isArray(reply) ? reply : [];
916
+ const text = fields[0] === null || fields[0] === void 0 ? void 0 : toStr(fields[0]);
917
+ if (text === void 0) {
918
+ continue;
919
+ }
920
+ const raw = fields[1] === null || fields[1] === void 0 ? void 0 : toStr(fields[1]);
921
+ chunks.push({
922
+ id: entry.id,
923
+ index: entry.index,
924
+ text,
925
+ ...parseMetadata(raw)
926
+ });
927
+ }
928
+ return chunks;
929
+ }
769
930
  async listDocuments(filter) {
770
931
  if (filterMatchesNothing(filter)) {
771
932
  return [];
@@ -1655,6 +1816,7 @@ export {
1655
1816
  UnsafeRemovalError,
1656
1817
  applyMetadataPatch,
1657
1818
  chunkDocuments,
1819
+ chunkIndexOf,
1658
1820
  chunkText,
1659
1821
  createRetrievalTool,
1660
1822
  documentIdOf,