@dudousxd/nestjs-agent-rag 0.8.0 → 0.9.0

package/README.md

@@ -323,7 +323,8 @@ library upgrade. The interface is open, so it can be added later without changin
 - `HybridRetriever(retrievers, { k?, fetchTopK?, weights? })` — RRF fusion of the two.
 - `MemoryVectorStore` / `PgVectorStore` / `RedisVectorStore` — `VectorStore` adapters:
   `upsert(records)`, `search(embedding, opts)`, `remove(documentId)`,
-  `updateMetadata(documentId, patch)`, `listDocuments(filter?)`.
+  `updateMetadata(documentId, patch)`, `listDocuments(filter?)`,
+  `listChunks(documentId, { limit?, offset? })`.
 - `applyMetadataPatch(metadata, patch)` — the `MetadataPatch` merge semantics, exported so your own
   `VectorStore` can implement `updateMetadata` with exactly the same ones.
 - `isLexicalVectorStore(store)` — type guard for the optional `LexicalVectorStore` capability.
@@ -363,6 +364,33 @@ would mean stamping the document id as an indexed field at write time, and chunk
 change carry no such field — the search would find none of them and `remove` would silently stop
 deleting. The methods above need no new field, so they have no migration.
 
+## Reading chunks back
+
+Everything above enumerates *documents*. The text itself is otherwise write-only: `search` needs an
+embedding, `searchText` needs a query, and both return what **matched**, in relevance order, capped
+at `topK`. Neither answers "show me what was indexed for this document, in order" — the question you
+ask when retrieval returned the wrong passage and you need to know whether the chunker cut badly or
+the retriever chose badly.
+
+```ts
+const chunks = await store.listChunks('handbook.pdf', { limit: 20 });
+// [{ id: 'handbook.pdf#0', index: 0, text: 'Chapter 1 …', metadata: { collection: 'handbook' } }, …]
+```
+
+Ordered by `index` (the `n` of `${documentId}#<n>`, numerically — `#10` after `#2`), so paging with
+`limit`/`offset` walks the document front to back. A document stored under a bare id is chunk `0`. An
+unknown `documentId` yields `[]`.
+
+Document-scoped rather than filter-scoped, matching `remove` and `updateMetadata`: ordering only
+means something once a document is picked, so enumerate with `listDocumentIds(filter)` first when you
+need to sweep. Two things it deliberately is not:
+
+- **not an access-control seam** — it takes no filter and applies none, so a consumer that gates
+  retrieval on a metadata field must resolve that gate itself before calling, exactly as before
+  `remove`. It is shaped for an operator/debug surface, not for serving end users;
+- **not a hot path** — on Redis it costs one keyspace `SCAN`, the same as `remove`. Fine for one
+  document on demand, wrong inside a loop over a corpus.
+
 ### Why these are required and `searchText` is optional
 
 A method is **required on `VectorStore`** when every backend can implement it from what it already

package/dist/index.cjs

@@ -34,6 +34,7 @@ __export(index_exports, {
   UnsafeRemovalError: () => UnsafeRemovalError,
   applyMetadataPatch: () => applyMetadataPatch,
   chunkDocuments: () => chunkDocuments,
+  chunkIndexOf: () => chunkIndexOf,
   chunkText: () => chunkText,
   createRetrievalTool: () => createRetrievalTool,
   documentIdOf: () => documentIdOf,
@@ -133,6 +134,11 @@ var UnsafeRemovalError = class extends Error {
     this.name = "UnsafeRemovalError";
   }
 };
+function chunkIndexOf(chunkId) {
+  const match = /#(\d+)$/.exec(chunkId);
+  return match?.[1] === void 0 ? 0 : Number(match[1]);
+}
+__name(chunkIndexOf, "chunkIndexOf");
 function documentIdOf(chunkId) {
   return chunkId.replace(/#\d+$/, "");
 }
@@ -257,6 +263,34 @@ var MemoryVectorStore = class {
       ...documents.values()
     ];
   }
+  /**
+  * The chunks of one document, in document order. See {@link VectorStore.listChunks}.
+  *
+  * Insertion order is not document order — `upsert` stores whatever the caller passed, and a
+  * re-ingest that rewrites chunk 3 alone moves it to the end of the Map. So this sorts by the index
+  * parsed from the id, the same thing the SQL and Redis adapters order by, rather than trusting the
+  * Map. A reference adapter that returned "usually ordered" would let a consumer ship a bug that
+  * only shows up on the store tests are NOT written against.
+  */
+  async listChunks(documentId, options) {
+    const chunks = [];
+    for (const record of this.records.values()) {
+      if (documentIdOf(record.id) !== documentId) {
+        continue;
+      }
+      chunks.push({
+        id: record.id,
+        index: chunkIndexOf(record.id),
+        text: record.text,
+        ...record.metadata !== void 0 ? {
+          metadata: record.metadata
+        } : {}
+      });
+    }
+    chunks.sort((a, b) => a.index - b.index);
+    const offset = options?.offset ?? 0;
+    return options?.limit === void 0 ? chunks.slice(offset) : chunks.slice(offset, offset + options.limit);
+  }
   async listDocumentIds(filter) {
     if (filterMatchesNothing(filter)) {
       return [];
@@ -443,6 +477,44 @@ var PgVectorStore = class {
       } : {}
     }));
   }
+  /**
+  * The chunks of one document, in document order. See {@link VectorStore.listChunks}.
+  *
+  * The order is computed from the id rather than stored: `#12` must sort after `#2`, which a plain
+  * `ORDER BY id` gets wrong on every document past ten chunks because it compares text. So the
+  * trailing number is extracted and cast to `int`, and a bare id (a single-chunk document, which
+  * carries no `#n` suffix) collapses to `0` — the same rule `chunkIndexOf` applies in the other two
+  * adapters.
+  *
+  * `embedding` is left out of the projection: it is the widest column in the table and useless to a
+  * caller reading text back.
+  */
+  async listChunks(documentId, options) {
+    const params = [
+      documentId
+    ];
+    let sql = `SELECT id, ${CHUNK_INDEX_FROM_ID} AS chunk_index, text, metadata
+       FROM ${this.table}
+       WHERE ${DOCUMENT_ID_FROM_CHUNK} = $1
+       ORDER BY chunk_index`;
+    if (options?.limit !== void 0) {
+      params.push(options.limit);
+      sql += ` LIMIT $${params.length}`;
+    }
+    if (options?.offset !== void 0) {
+      params.push(options.offset);
+      sql += ` OFFSET $${params.length}`;
+    }
+    const rows = await this.client.query(sql, params);
+    return rows.map((row) => ({
+      id: row.id,
+      index: Number(row.chunk_index),
+      text: row.text,
+      ...row.metadata !== null ? {
+        metadata: row.metadata
+      } : {}
+    }));
+  }
   /** {@link listDocuments} without the metadata: one `DISTINCT` over the derived document id. */
   async listDocumentIds(filter) {
     if (filterMatchesNothing(filter)) {
@@ -553,6 +625,7 @@ function buildWhere(filter, startIndex) {
 }
 __name(buildWhere, "buildWhere");
 var DOCUMENT_ID_FROM_CHUNK = "regexp_replace(id, '#[0-9]+$', '')";
+var CHUNK_INDEX_FROM_ID = "COALESCE((substring(id from '#([0-9]+)$'))::int, 0)";
 function toVectorLiteral(embedding) {
   return `[${embedding.join(",")}]`;
 }
@@ -845,6 +918,58 @@ var RedisVectorStore = class {
       ...keys
     ];
   }
+  /**
+  * The chunks of one document, in document order. See {@link VectorStore.listChunks}.
+  *
+  * Reads the hashes directly (`HMGET text metadata_json`) rather than going through `FT.SEARCH`.
+  * Two reasons, and the second is the one that matters: the query language has no way to ask for
+  * "every chunk of this document" — `id` is the key name, not an indexed field — and `FT.SEARCH`
+  * caps at `LIMIT`, so paging a document through it would mean ordering by a field the index does
+  * not hold. `chunkKeys` is the same enumeration {@link RedisVectorStore.remove} and
+  * {@link RedisVectorStore.updateMetadata} already use, so "the chunks this returns" and "the chunks
+  * a delete would take" cannot drift apart.
+  *
+  * It inherits that method's cost too: one keyspace `SCAN` per call. Acceptable here because this
+  * is an inspection path (one document, on demand) and not something retrieval runs — but it is why
+  * this must not be put on a hot path or called in a loop over a corpus.
+  *
+  * `embedding` is deliberately not fetched: it is by far the largest field on the hash and nothing
+  * a caller reading text back can do with it.
+  */
+  async listChunks(documentId, options) {
+    const keys = await this.chunkKeys(documentId);
+    const ordered = keys.map((key) => ({
+      key,
+      id: key.slice(this.prefix.length)
+    })).map((entry) => ({
+      ...entry,
+      index: chunkIndexOf(entry.id)
+    })).sort((a, b) => a.index - b.index);
+    const offset = options?.offset ?? 0;
+    const page = options?.limit === void 0 ? ordered.slice(offset) : ordered.slice(offset, offset + options.limit);
+    const chunks = [];
+    for (const entry of page) {
+      const reply = await this.client.sendCommand([
+        "HMGET",
+        entry.key,
+        "text",
+        "metadata_json"
+      ]);
+      const fields = Array.isArray(reply) ? reply : [];
+      const text = fields[0] === null || fields[0] === void 0 ? void 0 : toStr(fields[0]);
+      if (text === void 0) {
+        continue;
+      }
+      const raw = fields[1] === null || fields[1] === void 0 ? void 0 : toStr(fields[1]);
+      chunks.push({
+        id: entry.id,
+        index: entry.index,
+        text,
+        ...parseMetadata(raw)
+      });
+    }
+    return chunks;
+  }
   async listDocuments(filter) {
     if (filterMatchesNothing(filter)) {
       return [];
@@ -1735,6 +1860,7 @@ __name(createRetrievalTool, "createRetrievalTool");
   UnsafeRemovalError,
   applyMetadataPatch,
   chunkDocuments,
+  chunkIndexOf,
   chunkText,
   createRetrievalTool,
   documentIdOf,