@dudousxd/nestjs-agent-rag 0.6.0 → 0.8.0

package/dist/index.d.cts

@@ -5,6 +5,31 @@ interface ChunkOptions {
     chunkSize?: number;
     /** Characters of overlap carried from the end of one chunk into the next. Default 100. */
     overlap?: number;
+    /**
+     * Treat the text as records joined by this string, and never split one across chunks.
+     *
+     * Default chunking is structure-blind: it breaks at the *latest* paragraph/sentence/word boundary
+     * in the window, which for record-shaped text means a cut lands mid-record routinely. That is fine
+     * for prose, where the two halves are still sentences. It is not fine for a table flattened to one
+     * record per row, because the half that keeps the row identifier and the half that keeps the value
+     * end up in different chunks, and neither can answer a question about that row on its own.
+     *
+     * With a separator set, whole records are packed greedily up to `chunkSize` and a record is never
+     * cut — except one longer than `chunkSize`, which has nowhere to go and is character-split as
+     * usual, since no chunker can avoid that.
+     *
+     * `overlap` is IGNORED in this mode, deliberately. Overlap exists to rescue a sentence that a
+     * boundary cut in half; a boundary that never falls inside a record has nothing to rescue, and
+     * carrying the tail of the previous chunk in would duplicate whole records into neighbouring chunks
+     * — paying for the same rows twice at embedding time and letting one row match from two places.
+     *
+     * Measured on a 200-row spreadsheet flattened to one field-labelled record per row (15 columns,
+     * ~800-char budget): blind chunking left 27% of records split, and on questions targeting a field
+     * on the far side of such a cut, BM25 answered 54/66 (MRR 0.818) against 66/66 (MRR 1.000) with
+     * records kept whole. The dense leg scored 7/66 either way — 200 near-identical rows produce
+     * near-identical vectors, so that leg cannot do row lookup at all and chunking does not change it.
+     */
+    separator?: string;
 }
 /**
  * Splits text into overlapping chunks for embedding. Greedy up to `chunkSize`, but prefers to break
@@ -595,6 +620,29 @@ declare class RedisVectorStore implements LexicalVectorStore {
     searchText(query: string, options: VectorSearchOptions): Promise<Passage[]>;
 }
 
+interface EmbeddingRetrieverOptions {
+    /**
+     * Drop passages scoring below this, so a query the corpus cannot answer returns nothing.
+     *
+     * A vector search always returns its `topK` nearest neighbours, and "nearest" is not "relevant".
+     * Ask a corpus of maintenance policy about the dining hall menu and it hands back the five least
+     * unrelated paragraphs it has, each with a score, in confident rank order. Nothing in that result
+     * says the answer is absent, so without a floor the consumer either feeds a model five irrelevant
+     * passages or invents a threshold of its own.
+     *
+     * **This is the only place in the package where a threshold is meaningful, which is why it lives
+     * here and not on `HybridRetriever`.** The score here comes straight from the store's vector
+     * search: a calibrated similarity, comparable across queries and monotonic in how well the passage
+     * matches. A fused score is neither — see the score section on `HybridRetriever`. Threshold each
+     * leg, then fuse; never the other way round.
+     *
+     * Pick the number by measuring, not by taste: score questions you know the corpus answers against
+     * questions you know it does not, and put the floor in the gap between the two populations. It is
+     * specific to the embedding model and the distance metric, so it does not survive changing either.
+     * Left unset nothing is dropped, which is the previous behaviour exactly.
+     */
+    minScore?: number;
+}
 /**
  * Bridges an {@link EmbeddingProvider} + {@link VectorStore} into the core {@link Retriever} SPI:
  * embed the query, then vector-search. This is what you wire as the agent's retriever (as a tool
@@ -603,7 +651,8 @@ declare class RedisVectorStore implements LexicalVectorStore {
 declare class EmbeddingRetriever implements Retriever {
     private readonly embedder;
     private readonly store;
-    constructor(embedder: EmbeddingProvider, store: VectorStore);
+    private readonly options;
+    constructor(embedder: EmbeddingProvider, store: VectorStore, options?: EmbeddingRetrieverOptions);
     retrieve(query: string, options?: RetrieveOptions): Promise<Passage[]>;
 }
 
@@ -724,6 +773,36 @@ interface HybridRetrieverOptions {
  * vector search with lexical BM25. RRF scores each passage by `Σ weight / (k + rank)` across the
  * lists it appears in, so it needs no score normalization between incompatible scales (cosine vs
  * BM25). Deduplicates by passage id, so aligned chunk ids from the two retrievers reinforce.
+ *
+ * ## The score on a fused passage is not a similarity. Do not threshold it.
+ *
+ * Read this before using `passage.score` for anything other than the order it already produced.
+ *
+ * Not normalizing between scales is what makes RRF robust, and the price is that the number it
+ * emits is a function of **rank alone** — it never sees a similarity at all. Two consequences that
+ * bite in practice:
+ *
+ * - **It is not comparable to the score on a passage from {@link EmbeddingRetriever}**, even though
+ *   it arrives in the same field, of the same type, from the same `Retriever` interface. There is no
+ *   discriminator; a consumer holding a `Passage[]` cannot tell which kind of number it has. If you
+ *   surface scores to a user or a model, know which retriever produced them.
+ * - **Its range is narrow, fixed, and says nothing about quality.** Every possible score lies in
+ *   `[minWeight / (k + fetchTopK), Σ weights / (k + 1)]` — with the defaults (`k = 60`,
+ *   `fetchTopK = 20`, two unweighted legs) that is `[0.0125, 0.0328]`. A perfect match and the
+ *   least-bad member of a set of terrible matches both come back around `0.016`.
+ *
+ * The second point is why a relevance floor cannot live here. What the fused score actually measures
+ * is how strongly the legs *agree* — and agreement tracks nothing about correctness. When both legs
+ * return the same nearest nothing for an unanswerable query, they reinforce it to the ceiling, the
+ * same ceiling a unanimous correct answer gets. Measured on one corpus, the dense leg alone
+ * separated cleanly (weakest real answer 0.244, loudest absent-topic answer 0.143) while those same
+ * queries after fusion scored 0.032 and 0.033 — inverted. On another corpus the inversion may not
+ * appear; what does not change is that both numbers live in a band roughly one part in fifty wide,
+ * with no room for a threshold to mean anything.
+ *
+ * So: put the floor on the legs, with {@link EmbeddingRetrieverOptions.minScore}, and fuse what
+ * survives. `HybridRetriever` deliberately has no `minScore` of its own; a threshold option here
+ * would be an invitation to apply it to a number that cannot support one.
  */
 declare class HybridRetriever implements Retriever {
     private readonly retrievers;
@@ -789,4 +868,4 @@ interface RetrievalTool {
  */
 declare function createRetrievalTool(retriever: Retriever, options?: RetrievalToolOptions): RetrievalTool;
 
-export { type ChunkOptions, type ChunkRecord, EmbeddingRetriever, FilteredRetriever, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, KeywordRetriever, type KeywordRetrieverOptions, LexicalRetriever, type LexicalVectorStore, MemoryVectorStore, type MetadataPatch, type PgClient, PgVectorStore, type PgVectorStoreOptions, type RedisSearchClient, RedisVectorSchemaMismatchError, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalTool, type RetrievalToolOptions, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, applyMetadataPatch, chunkDocuments, chunkText, createRetrievalTool, documentIdOf, ingestChunks, ingestDocuments, isLexicalVectorStore };
+export { type ChunkOptions, type ChunkRecord, EmbeddingRetriever, type EmbeddingRetrieverOptions, FilteredRetriever, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, KeywordRetriever, type KeywordRetrieverOptions, LexicalRetriever, type LexicalVectorStore, MemoryVectorStore, type MetadataPatch, type PgClient, PgVectorStore, type PgVectorStoreOptions, type RedisSearchClient, RedisVectorSchemaMismatchError, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalTool, type RetrievalToolOptions, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, applyMetadataPatch, chunkDocuments, chunkText, createRetrievalTool, documentIdOf, ingestChunks, ingestDocuments, isLexicalVectorStore };

package/dist/index.d.ts

@@ -5,6 +5,31 @@ interface ChunkOptions {
     chunkSize?: number;
     /** Characters of overlap carried from the end of one chunk into the next. Default 100. */
     overlap?: number;
+    /**
+     * Treat the text as records joined by this string, and never split one across chunks.
+     *
+     * Default chunking is structure-blind: it breaks at the *latest* paragraph/sentence/word boundary
+     * in the window, which for record-shaped text means a cut lands mid-record routinely. That is fine
+     * for prose, where the two halves are still sentences. It is not fine for a table flattened to one
+     * record per row, because the half that keeps the row identifier and the half that keeps the value
+     * end up in different chunks, and neither can answer a question about that row on its own.
+     *
+     * With a separator set, whole records are packed greedily up to `chunkSize` and a record is never
+     * cut — except one longer than `chunkSize`, which has nowhere to go and is character-split as
+     * usual, since no chunker can avoid that.
+     *
+     * `overlap` is IGNORED in this mode, deliberately. Overlap exists to rescue a sentence that a
+     * boundary cut in half; a boundary that never falls inside a record has nothing to rescue, and
+     * carrying the tail of the previous chunk in would duplicate whole records into neighbouring chunks
+     * — paying for the same rows twice at embedding time and letting one row match from two places.
+     *
+     * Measured on a 200-row spreadsheet flattened to one field-labelled record per row (15 columns,
+     * ~800-char budget): blind chunking left 27% of records split, and on questions targeting a field
+     * on the far side of such a cut, BM25 answered 54/66 (MRR 0.818) against 66/66 (MRR 1.000) with
+     * records kept whole. The dense leg scored 7/66 either way — 200 near-identical rows produce
+     * near-identical vectors, so that leg cannot do row lookup at all and chunking does not change it.
+     */
+    separator?: string;
 }
 /**
  * Splits text into overlapping chunks for embedding. Greedy up to `chunkSize`, but prefers to break
@@ -595,6 +620,29 @@ declare class RedisVectorStore implements LexicalVectorStore {
     searchText(query: string, options: VectorSearchOptions): Promise<Passage[]>;
 }
 
+interface EmbeddingRetrieverOptions {
+    /**
+     * Drop passages scoring below this, so a query the corpus cannot answer returns nothing.
+     *
+     * A vector search always returns its `topK` nearest neighbours, and "nearest" is not "relevant".
+     * Ask a corpus of maintenance policy about the dining hall menu and it hands back the five least
+     * unrelated paragraphs it has, each with a score, in confident rank order. Nothing in that result
+     * says the answer is absent, so without a floor the consumer either feeds a model five irrelevant
+     * passages or invents a threshold of its own.
+     *
+     * **This is the only place in the package where a threshold is meaningful, which is why it lives
+     * here and not on `HybridRetriever`.** The score here comes straight from the store's vector
+     * search: a calibrated similarity, comparable across queries and monotonic in how well the passage
+     * matches. A fused score is neither — see the score section on `HybridRetriever`. Threshold each
+     * leg, then fuse; never the other way round.
+     *
+     * Pick the number by measuring, not by taste: score questions you know the corpus answers against
+     * questions you know it does not, and put the floor in the gap between the two populations. It is
+     * specific to the embedding model and the distance metric, so it does not survive changing either.
+     * Left unset nothing is dropped, which is the previous behaviour exactly.
+     */
+    minScore?: number;
+}
 /**
  * Bridges an {@link EmbeddingProvider} + {@link VectorStore} into the core {@link Retriever} SPI:
  * embed the query, then vector-search. This is what you wire as the agent's retriever (as a tool
@@ -603,7 +651,8 @@ declare class RedisVectorStore implements LexicalVectorStore {
 declare class EmbeddingRetriever implements Retriever {
     private readonly embedder;
     private readonly store;
-    constructor(embedder: EmbeddingProvider, store: VectorStore);
+    private readonly options;
+    constructor(embedder: EmbeddingProvider, store: VectorStore, options?: EmbeddingRetrieverOptions);
     retrieve(query: string, options?: RetrieveOptions): Promise<Passage[]>;
 }
 
@@ -724,6 +773,36 @@ interface HybridRetrieverOptions {
  * vector search with lexical BM25. RRF scores each passage by `Σ weight / (k + rank)` across the
  * lists it appears in, so it needs no score normalization between incompatible scales (cosine vs
  * BM25). Deduplicates by passage id, so aligned chunk ids from the two retrievers reinforce.
+ *
+ * ## The score on a fused passage is not a similarity. Do not threshold it.
+ *
+ * Read this before using `passage.score` for anything other than the order it already produced.
+ *
+ * Not normalizing between scales is what makes RRF robust, and the price is that the number it
+ * emits is a function of **rank alone** — it never sees a similarity at all. Two consequences that
+ * bite in practice:
+ *
+ * - **It is not comparable to the score on a passage from {@link EmbeddingRetriever}**, even though
+ *   it arrives in the same field, of the same type, from the same `Retriever` interface. There is no
+ *   discriminator; a consumer holding a `Passage[]` cannot tell which kind of number it has. If you
+ *   surface scores to a user or a model, know which retriever produced them.
+ * - **Its range is narrow, fixed, and says nothing about quality.** Every possible score lies in
+ *   `[minWeight / (k + fetchTopK), Σ weights / (k + 1)]` — with the defaults (`k = 60`,
+ *   `fetchTopK = 20`, two unweighted legs) that is `[0.0125, 0.0328]`. A perfect match and the
+ *   least-bad member of a set of terrible matches both come back around `0.016`.
+ *
+ * The second point is why a relevance floor cannot live here. What the fused score actually measures
+ * is how strongly the legs *agree* — and agreement tracks nothing about correctness. When both legs
+ * return the same nearest nothing for an unanswerable query, they reinforce it to the ceiling, the
+ * same ceiling a unanimous correct answer gets. Measured on one corpus, the dense leg alone
+ * separated cleanly (weakest real answer 0.244, loudest absent-topic answer 0.143) while those same
+ * queries after fusion scored 0.032 and 0.033 — inverted. On another corpus the inversion may not
+ * appear; what does not change is that both numbers live in a band roughly one part in fifty wide,
+ * with no room for a threshold to mean anything.
+ *
+ * So: put the floor on the legs, with {@link EmbeddingRetrieverOptions.minScore}, and fuse what
+ * survives. `HybridRetriever` deliberately has no `minScore` of its own; a threshold option here
+ * would be an invitation to apply it to a number that cannot support one.
  */
 declare class HybridRetriever implements Retriever {
     private readonly retrievers;
@@ -789,4 +868,4 @@ interface RetrievalTool {
  */
 declare function createRetrievalTool(retriever: Retriever, options?: RetrievalToolOptions): RetrievalTool;
 
-export { type ChunkOptions, type ChunkRecord, EmbeddingRetriever, FilteredRetriever, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, KeywordRetriever, type KeywordRetrieverOptions, LexicalRetriever, type LexicalVectorStore, MemoryVectorStore, type MetadataPatch, type PgClient, PgVectorStore, type PgVectorStoreOptions, type RedisSearchClient, RedisVectorSchemaMismatchError, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalTool, type RetrievalToolOptions, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, applyMetadataPatch, chunkDocuments, chunkText, createRetrievalTool, documentIdOf, ingestChunks, ingestDocuments, isLexicalVectorStore };
+export { type ChunkOptions, type ChunkRecord, EmbeddingRetriever, type EmbeddingRetrieverOptions, FilteredRetriever, HybridRetriever, type HybridRetrieverOptions, type IndexedDocument, type IngestChunksOptions, type IngestDocument, type IngestOptions, KeywordRetriever, type KeywordRetrieverOptions, LexicalRetriever, type LexicalVectorStore, MemoryVectorStore, type MetadataPatch, type PgClient, PgVectorStore, type PgVectorStoreOptions, type RedisSearchClient, RedisVectorSchemaMismatchError, RedisVectorStore, type RedisVectorStoreOptions, RerankingRetriever, type RerankingRetrieverOptions, type RetrievalTool, type RetrievalToolOptions, UnsafeRemovalError, type VectorRecord, type VectorSearchOptions, type VectorStore, applyMetadataPatch, chunkDocuments, chunkText, createRetrievalTool, documentIdOf, ingestChunks, ingestDocuments, isLexicalVectorStore };

package/dist/index.js

@@ -9,6 +9,9 @@ function chunkText(text, options = {}) {
   if (normalized.length === 0) {
     return [];
   }
+  if (options.separator !== void 0 && options.separator.length > 0) {
+    return chunkByRecord(normalized, chunkSize, options.separator);
+  }
   if (normalized.length <= chunkSize) {
     return [
       normalized
@@ -38,6 +41,40 @@ function chunkText(text, options = {}) {
   return chunks;
 }
 __name(chunkText, "chunkText");
+function chunkByRecord(text, chunkSize, separator) {
+  const chunks = [];
+  let current = "";
+  const flush = /* @__PURE__ */ __name(() => {
+    if (current.length > 0) {
+      chunks.push(current);
+      current = "";
+    }
+  }, "flush");
+  for (const record of text.split(separator)) {
+    const trimmed = record.trim();
+    if (trimmed.length === 0) {
+      continue;
+    }
+    if (trimmed.length > chunkSize) {
+      flush();
+      chunks.push(...chunkText(trimmed, {
+        chunkSize,
+        overlap: 0
+      }));
+      continue;
+    }
+    const candidate = current.length > 0 ? `${current}${separator}${trimmed}` : trimmed;
+    if (candidate.length > chunkSize) {
+      flush();
+      current = trimmed;
+    } else {
+      current = candidate;
+    }
+  }
+  flush();
+  return chunks;
+}
+__name(chunkByRecord, "chunkByRecord");
 
 // src/vector-store.ts
 function isLexicalVectorStore(store) {
@@ -1295,9 +1332,11 @@ var EmbeddingRetriever = class {
   }
   embedder;
   store;
-  constructor(embedder, store) {
+  options;
+  constructor(embedder, store, options = {}) {
     this.embedder = embedder;
     this.store = store;
+    this.options = options;
   }
   async retrieve(query, options = {}) {
     const [embedding] = await this.embedder.embed([
@@ -1306,12 +1345,14 @@ var EmbeddingRetriever = class {
     if (embedding === void 0) {
       return [];
     }
-    return this.store.search(embedding, {
+    const passages = await this.store.search(embedding, {
       topK: options.topK ?? 5,
       ...options.filter !== void 0 ? {
         filter: options.filter
       } : {}
     });
+    const { minScore } = this.options;
+    return minScore === void 0 ? passages : passages.filter((passage) => passage.score >= minScore);
   }
 };