@dudousxd/nestjs-agent-rag 0.8.0 → 0.10.0

package/dist/index.js

@@ -91,6 +91,11 @@ var UnsafeRemovalError = class extends Error {
     this.name = "UnsafeRemovalError";
   }
 };
+function chunkIndexOf(chunkId) {
+  const match = /#(\d+)$/.exec(chunkId);
+  return match?.[1] === void 0 ? 0 : Number(match[1]);
+}
+__name(chunkIndexOf, "chunkIndexOf");
 function documentIdOf(chunkId) {
   return chunkId.replace(/#\d+$/, "");
 }
@@ -160,6 +165,16 @@ var MemoryVectorStore = class {
     __name(this, "MemoryVectorStore");
   }
   records = /* @__PURE__ */ new Map();
+  /**
+  * Telemetry self-description. No `collection`: there is no namespace to name — the corpus IS this
+  * instance, and two `MemoryVectorStore`s in one process are two corpora with nothing to tell them
+  * apart on a dashboard.
+  */
+  describeRetrieval() {
+    return {
+      store: "memory"
+    };
+  }
   async upsert(records) {
     for (const record of records) {
       this.records.set(record.id, record);
@@ -215,6 +230,34 @@ var MemoryVectorStore = class {
       ...documents.values()
     ];
   }
+  /**
+  * The chunks of one document, in document order. See {@link VectorStore.listChunks}.
+  *
+  * Insertion order is not document order — `upsert` stores whatever the caller passed, and a
+  * re-ingest that rewrites chunk 3 alone moves it to the end of the Map. So this sorts by the index
+  * parsed from the id, the same thing the SQL and Redis adapters order by, rather than trusting the
+  * Map. A reference adapter that returned "usually ordered" would let a consumer ship a bug that
+  * only shows up on the store tests are NOT written against.
+  */
+  async listChunks(documentId, options) {
+    const chunks = [];
+    for (const record of this.records.values()) {
+      if (documentIdOf(record.id) !== documentId) {
+        continue;
+      }
+      chunks.push({
+        id: record.id,
+        index: chunkIndexOf(record.id),
+        text: record.text,
+        ...record.metadata !== void 0 ? {
+          metadata: record.metadata
+        } : {}
+      });
+    }
+    chunks.sort((a, b) => a.index - b.index);
+    const offset = options?.offset ?? 0;
+    return options?.limit === void 0 ? chunks.slice(offset) : chunks.slice(offset, offset + options.limit);
+  }
   async listDocumentIds(filter) {
     if (filterMatchesNothing(filter)) {
       return [];
@@ -326,6 +369,13 @@ var PgVectorStore = class {
     this.table = options.table ?? "agent_rag_chunks";
     this.dimensions = options.dimensions ?? 1536;
   }
+  /** Telemetry self-description — the table is the namespace a retrieval was served from. */
+  describeRetrieval() {
+    return {
+      store: "pg",
+      collection: this.table
+    };
+  }
   /** Idempotent DDL: the `vector` extension, the chunks table, and the cosine HNSW index. */
   async ensureSchema() {
     await this.client.query("CREATE EXTENSION IF NOT EXISTS vector");
@@ -401,6 +451,44 @@ var PgVectorStore = class {
       } : {}
     }));
   }
+  /**
+  * The chunks of one document, in document order. See {@link VectorStore.listChunks}.
+  *
+  * The order is computed from the id rather than stored: `#12` must sort after `#2`, which a plain
+  * `ORDER BY id` gets wrong on every document past ten chunks because it compares text. So the
+  * trailing number is extracted and cast to `int`, and a bare id (a single-chunk document, which
+  * carries no `#n` suffix) collapses to `0` — the same rule `chunkIndexOf` applies in the other two
+  * adapters.
+  *
+  * `embedding` is left out of the projection: it is the widest column in the table and useless to a
+  * caller reading text back.
+  */
+  async listChunks(documentId, options) {
+    const params = [
+      documentId
+    ];
+    let sql = `SELECT id, ${CHUNK_INDEX_FROM_ID} AS chunk_index, text, metadata
+       FROM ${this.table}
+       WHERE ${DOCUMENT_ID_FROM_CHUNK} = $1
+       ORDER BY chunk_index`;
+    if (options?.limit !== void 0) {
+      params.push(options.limit);
+      sql += ` LIMIT $${params.length}`;
+    }
+    if (options?.offset !== void 0) {
+      params.push(options.offset);
+      sql += ` OFFSET $${params.length}`;
+    }
+    const rows = await this.client.query(sql, params);
+    return rows.map((row) => ({
+      id: row.id,
+      index: Number(row.chunk_index),
+      text: row.text,
+      ...row.metadata !== null ? {
+        metadata: row.metadata
+      } : {}
+    }));
+  }
   /** {@link listDocuments} without the metadata: one `DISTINCT` over the derived document id. */
   async listDocumentIds(filter) {
     if (filterMatchesNothing(filter)) {
@@ -511,6 +599,7 @@ function buildWhere(filter, startIndex) {
 }
 __name(buildWhere, "buildWhere");
 var DOCUMENT_ID_FROM_CHUNK = "regexp_replace(id, '#[0-9]+$', '')";
+var CHUNK_INDEX_FROM_ID = "COALESCE((substring(id from '#([0-9]+)$'))::int, 0)";
 function toVectorLiteral(embedding) {
   return `[${embedding.join(",")}]`;
 }
@@ -549,6 +638,17 @@ var RedisVectorStore = class {
     this.lexicalScorer = options.lexicalScorer ?? "BM25";
   }
   /**
+  * Telemetry self-description — the RediSearch INDEX, not the key prefix: the index is what both
+  * legs query, and two stores sharing one index (the dense and lexical halves of a hybrid) must
+  * report the same namespace or `describeSharedSource` would see them disagree and drop the store.
+  */
+  describeRetrieval() {
+    return {
+      store: "redis",
+      collection: this.index
+    };
+  }
+  /**
   * Idempotent, and drift-aware: create the HNSW/cosine index if `FT.INFO` says it doesn't exist —
   * and if it DOES exist, compare what's there against this store's configuration instead of
   * assuming they agree. An index that outlives a config change is the normal case (schemas live in
@@ -803,6 +903,58 @@ var RedisVectorStore = class {
       ...keys
     ];
   }
+  /**
+  * The chunks of one document, in document order. See {@link VectorStore.listChunks}.
+  *
+  * Reads the hashes directly (`HMGET text metadata_json`) rather than going through `FT.SEARCH`.
+  * Two reasons, and the second is the one that matters: the query language has no way to ask for
+  * "every chunk of this document" — `id` is the key name, not an indexed field — and `FT.SEARCH`
+  * caps at `LIMIT`, so paging a document through it would mean ordering by a field the index does
+  * not hold. `chunkKeys` is the same enumeration {@link RedisVectorStore.remove} and
+  * {@link RedisVectorStore.updateMetadata} already use, so "the chunks this returns" and "the chunks
+  * a delete would take" cannot drift apart.
+  *
+  * It inherits that method's cost too: one keyspace `SCAN` per call. Acceptable here because this
+  * is an inspection path (one document, on demand) and not something retrieval runs — but it is why
+  * this must not be put on a hot path or called in a loop over a corpus.
+  *
+  * `embedding` is deliberately not fetched: it is by far the largest field on the hash and nothing
+  * a caller reading text back can do with it.
+  */
+  async listChunks(documentId, options) {
+    const keys = await this.chunkKeys(documentId);
+    const ordered = keys.map((key) => ({
+      key,
+      id: key.slice(this.prefix.length)
+    })).map((entry) => ({
+      ...entry,
+      index: chunkIndexOf(entry.id)
+    })).sort((a, b) => a.index - b.index);
+    const offset = options?.offset ?? 0;
+    const page = options?.limit === void 0 ? ordered.slice(offset) : ordered.slice(offset, offset + options.limit);
+    const chunks = [];
+    for (const entry of page) {
+      const reply = await this.client.sendCommand([
+        "HMGET",
+        entry.key,
+        "text",
+        "metadata_json"
+      ]);
+      const fields = Array.isArray(reply) ? reply : [];
+      const text = fields[0] === null || fields[0] === void 0 ? void 0 : toStr(fields[0]);
+      if (text === void 0) {
+        continue;
+      }
+      const raw = fields[1] === null || fields[1] === void 0 ? void 0 : toStr(fields[1]);
+      chunks.push({
+        id: entry.id,
+        index: entry.index,
+        text,
+        ...parseMetadata(raw)
+      });
+    }
+    return chunks;
+  }
   async listDocuments(filter) {
     if (filterMatchesNothing(filter)) {
       return [];
@@ -1325,6 +1477,31 @@ function parseMetadata(raw) {
 }
 __name(parseMetadata, "parseMetadata");
 
+// src/retrieval-descriptor.ts
+function describesRetrieval(value) {
+  return typeof value === "object" && value !== null && typeof Reflect.get(value, "describeRetrieval") === "function";
+}
+__name(describesRetrieval, "describesRetrieval");
+function describeRetrieval(candidate) {
+  return describesRetrieval(candidate) ? candidate.describeRetrieval() : {};
+}
+__name(describeRetrieval, "describeRetrieval");
+function describeSource(base) {
+  const { retriever: _ignored, ...source } = describeRetrieval(base);
+  return source;
+}
+__name(describeSource, "describeSource");
+function describeSharedSource(legs) {
+  const sources = legs.map(describeSource);
+  const [first] = sources;
+  if (first === void 0) {
+    return {};
+  }
+  const agrees = sources.every((source) => source.store === first.store && source.collection === first.collection);
+  return agrees ? first : {};
+}
+__name(describeSharedSource, "describeSharedSource");
+
 // src/embedding-retriever.ts
 var EmbeddingRetriever = class {
   static {
@@ -1338,6 +1515,13 @@ var EmbeddingRetriever = class {
     this.store = store;
     this.options = options;
   }
+  /** Telemetry self-description — the dense strategy, over whatever store it was handed. */
+  describeRetrieval() {
+    return {
+      retriever: "embedding",
+      ...describeSource(this.store)
+    };
+  }
   async retrieve(query, options = {}) {
     const [embedding] = await this.embedder.embed([
       query
@@ -1370,6 +1554,16 @@ var KeywordRetriever = class {
     this.k1 = options.k1 ?? 1.5;
     this.b = options.b ?? 0.75;
   }
+  /**
+  * Telemetry self-description. No `store`: this retriever holds its own copy of the corpus in
+  * process, so attributing its retrievals to whichever vector store happens to hold the same chunks
+  * would credit a backend that was never queried.
+  */
+  describeRetrieval() {
+    return {
+      retriever: "keyword"
+    };
+  }
   /** How many chunks are currently indexed. */
   get size() {
     return this.docs.size;
@@ -1487,6 +1681,13 @@ var LexicalRetriever = class {
   constructor(store) {
     this.store = store;
   }
+  /** Telemetry self-description — the store's own lexical index, so the store is the same one. */
+  describeRetrieval() {
+    return {
+      retriever: "lexical",
+      ...describeSource(this.store)
+    };
+  }
   async retrieve(query, options = {}) {
     return this.store.searchText(query, {
       topK: options.topK ?? 5,
@@ -1508,6 +1709,13 @@ var HybridRetriever = class {
     this.retrievers = retrievers;
     this.options = options;
   }
+  /** Telemetry self-description — the store only when every leg agrees; see `describeSharedSource`. */
+  describeRetrieval() {
+    return {
+      retriever: "hybrid",
+      ...describeSharedSource(this.retrievers)
+    };
+  }
   async retrieve(query, options = {}) {
     const k = this.options.k ?? 60;
     const fetchTopK = this.options.fetchTopK ?? 20;
@@ -1556,6 +1764,13 @@ var RerankingRetriever = class {
     this.reranker = reranker;
     this.options = options;
   }
+  /** Telemetry self-description — the two-stage strategy, over the base's store. */
+  describeRetrieval() {
+    return {
+      retriever: "reranking",
+      ...describeSource(this.base)
+    };
+  }
   async retrieve(query, options = {}) {
     const candidates = await this.base.retrieve(query, {
       topK: this.options.fetchTopK ?? 20,
@@ -1580,6 +1795,13 @@ var FilteredRetriever = class {
     this.base = base;
     this.filter = filter;
   }
+  /** Telemetry self-description — this scope, over the base's store. */
+  describeRetrieval() {
+    return {
+      retriever: "filtered",
+      ...describeSource(this.base)
+    };
+  }
   async retrieve(query, options = {}) {
     return this.base.retrieve(query, {
       ...options,
@@ -1629,6 +1851,108 @@ async function ingestDocuments(documents, options) {
 }
 __name(ingestDocuments, "ingestDocuments");
 
+// src/retrieval-telemetry.ts
+import { AsyncLocalStorage } from "node:async_hooks";
+import { channelName, emit, getChannel } from "@dudousxd/nestjs-diagnostics";
+var RAG_DIAGNOSTIC_LIB = "rag";
+var RAG_RETRIEVAL_EVENT = "retrieval";
+var RAG_RETRIEVAL_CHANNEL = channelName(RAG_DIAGNOSTIC_LIB, RAG_RETRIEVAL_EVENT);
+function emitRagRetrieval(event, durationMs) {
+  emit(RAG_DIAGNOSTIC_LIB, RAG_RETRIEVAL_EVENT, event, {
+    durationMs
+  });
+}
+__name(emitRagRetrieval, "emitRagRetrieval");
+var DEFAULT_TOP_K = 5;
+var inRetrieval = new AsyncLocalStorage();
+function round4(value) {
+  return Math.round(value * 1e4) / 1e4;
+}
+__name(round4, "round4");
+function scoreStats(passages) {
+  const scores = passages.map((passage) => passage.score).filter((score) => Number.isFinite(score));
+  if (scores.length === 0) {
+    return {};
+  }
+  const sum = scores.reduce((total, score) => total + score, 0);
+  return {
+    topScore: round4(Math.max(...scores)),
+    meanScore: round4(sum / scores.length)
+  };
+}
+__name(scoreStats, "scoreStats");
+var InstrumentedRetriever = class InstrumentedRetriever2 {
+  static {
+    __name(this, "InstrumentedRetriever");
+  }
+  base;
+  options;
+  constructor(base, options = {}) {
+    this.base = base;
+    this.options = options;
+  }
+  /** Transparent to detection: wrapping a retriever must not turn it into an `unknown` one. */
+  describeRetrieval() {
+    return {
+      ...describeRetrieval(this.base),
+      ...this.options.describe
+    };
+  }
+  async retrieve(query, options = {}) {
+    if (!getChannel(RAG_DIAGNOSTIC_LIB, RAG_RETRIEVAL_EVENT).hasSubscribers) {
+      return this.base.retrieve(query, options);
+    }
+    if (inRetrieval.getStore() === true) {
+      return this.base.retrieve(query, options);
+    }
+    const descriptor = this.describeRetrieval();
+    const topK = options.topK ?? DEFAULT_TOP_K;
+    const startedAt = Date.now();
+    try {
+      const passages = await inRetrieval.run(true, () => this.base.retrieve(query, options));
+      emitRagRetrieval({
+        retriever: descriptor.retriever ?? "unknown",
+        ...descriptor.store !== void 0 ? {
+          store: descriptor.store
+        } : {},
+        ...this.collectionOf(options.filter, descriptor),
+        topK,
+        chunks: passages.length,
+        zeroHit: passages.length === 0,
+        ...scoreStats(passages),
+        failed: false
+      }, Date.now() - startedAt);
+      return passages;
+    } catch (error) {
+      emitRagRetrieval({
+        retriever: descriptor.retriever ?? "unknown",
+        ...descriptor.store !== void 0 ? {
+          store: descriptor.store
+        } : {},
+        ...this.collectionOf(options.filter, descriptor),
+        topK,
+        chunks: 0,
+        zeroHit: true,
+        failed: true,
+        error: error instanceof Error ? error.message : String(error)
+      }, Date.now() - startedAt);
+      throw error;
+    }
+  }
+  /** The host's collection for this call, else the store namespace, else nothing. */
+  collectionOf(filter, descriptor) {
+    const { collection } = this.options;
+    const resolved = typeof collection === "function" ? collection(filter) : collection ?? descriptor.collection;
+    return resolved !== void 0 ? {
+      collection: resolved
+    } : {};
+  }
+};
+function instrumentRetriever(retriever, options = {}) {
+  return new InstrumentedRetriever(retriever, options);
+}
+__name(instrumentRetriever, "instrumentRetriever");
+
 // src/retrieval-tool.ts
 function isQueryInput(value) {
   return typeof value === "object" && value !== null && "query" in value && typeof value.query === "string";
@@ -1655,6 +1979,8 @@ var querySchema = {
 };
 function createRetrievalTool(retriever, options = {}) {
   const topK = options.topK;
+  const { telemetry = true } = options;
+  const instrumented = telemetry === false ? retriever : instrumentRetriever(retriever, telemetry === true ? {} : telemetry);
   return {
     spec: {
       name: options.name ?? "search_knowledge",
@@ -1667,7 +1993,7 @@ function createRetrievalTool(retriever, options = {}) {
         if (!isQueryInput(input)) {
           throw new Error("retrieval tool expects { query: string }");
         }
-        const passages = await retriever.retrieve(input.query, topK !== void 0 ? {
+        const passages = await instrumented.retrieve(input.query, topK !== void 0 ? {
           topK
         } : {});
         return {
@@ -1686,17 +2012,27 @@ export {
   LexicalRetriever,
   MemoryVectorStore,
   PgVectorStore,
+  RAG_DIAGNOSTIC_LIB,
+  RAG_RETRIEVAL_CHANNEL,
+  RAG_RETRIEVAL_EVENT,
   RedisVectorSchemaMismatchError,
   RedisVectorStore,
   RerankingRetriever,
   UnsafeRemovalError,
   applyMetadataPatch,
   chunkDocuments,
+  chunkIndexOf,
   chunkText,
   createRetrievalTool,
+  describeRetrieval,
+  describeSharedSource,
+  describeSource,
+  describesRetrieval,
   documentIdOf,
+  emitRagRetrieval,
   ingestChunks,
   ingestDocuments,
+  instrumentRetriever,
   isLexicalVectorStore
 };
 //# sourceMappingURL=index.js.map