@dudousxd/nestjs-agent-rag 0.7.0 → 0.8.0

package/dist/index.d.cts

@@ -5,6 +5,31 @@ interface ChunkOptions {
     chunkSize?: number;
     /** Characters of overlap carried from the end of one chunk into the next. Default 100. */
     overlap?: number;
+    /**
+     * Treat the text as records joined by this string, and never split one across chunks.
+     *
+     * Default chunking is structure-blind: it breaks at the *latest* paragraph/sentence/word boundary
+     * in the window, which for record-shaped text means a cut lands mid-record routinely. That is fine
+     * for prose, where the two halves are still sentences. It is not fine for a table flattened to one
+     * record per row, because the half that keeps the row identifier and the half that keeps the value
+     * end up in different chunks, and neither can answer a question about that row on its own.
+     *
+     * With a separator set, whole records are packed greedily up to `chunkSize` and a record is never
+     * cut — except one longer than `chunkSize`, which has nowhere to go and is character-split as
+     * usual, since no chunker can avoid that.
+     *
+     * `overlap` is IGNORED in this mode, deliberately. Overlap exists to rescue a sentence that a
+     * boundary cut in half; a boundary that never falls inside a record has nothing to rescue, and
+     * carrying the tail of the previous chunk in would duplicate whole records into neighbouring chunks
+     * — paying for the same rows twice at embedding time and letting one row match from two places.
+     *
+     * Measured on a 200-row spreadsheet flattened to one field-labelled record per row (15 columns,
+     * ~800-char budget): blind chunking left 27% of records split, and on questions targeting a field
+     * on the far side of such a cut, BM25 answered 54/66 (MRR 0.818) against 66/66 (MRR 1.000) with
+     * records kept whole. The dense leg scored 7/66 either way — 200 near-identical rows produce
+     * near-identical vectors, so that leg cannot do row lookup at all and chunking does not change it.
+     */
+    separator?: string;
 }
 /**
  * Splits text into overlapping chunks for embedding. Greedy up to `chunkSize`, but prefers to break

package/dist/index.d.ts

@@ -5,6 +5,31 @@ interface ChunkOptions {
     chunkSize?: number;
     /** Characters of overlap carried from the end of one chunk into the next. Default 100. */
     overlap?: number;
+    /**
+     * Treat the text as records joined by this string, and never split one across chunks.
+     *
+     * Default chunking is structure-blind: it breaks at the *latest* paragraph/sentence/word boundary
+     * in the window, which for record-shaped text means a cut lands mid-record routinely. That is fine
+     * for prose, where the two halves are still sentences. It is not fine for a table flattened to one
+     * record per row, because the half that keeps the row identifier and the half that keeps the value
+     * end up in different chunks, and neither can answer a question about that row on its own.
+     *
+     * With a separator set, whole records are packed greedily up to `chunkSize` and a record is never
+     * cut — except one longer than `chunkSize`, which has nowhere to go and is character-split as
+     * usual, since no chunker can avoid that.
+     *
+     * `overlap` is IGNORED in this mode, deliberately. Overlap exists to rescue a sentence that a
+     * boundary cut in half; a boundary that never falls inside a record has nothing to rescue, and
+     * carrying the tail of the previous chunk in would duplicate whole records into neighbouring chunks
+     * — paying for the same rows twice at embedding time and letting one row match from two places.
+     *
+     * Measured on a 200-row spreadsheet flattened to one field-labelled record per row (15 columns,
+     * ~800-char budget): blind chunking left 27% of records split, and on questions targeting a field
+     * on the far side of such a cut, BM25 answered 54/66 (MRR 0.818) against 66/66 (MRR 1.000) with
+     * records kept whole. The dense leg scored 7/66 either way — 200 near-identical rows produce
+     * near-identical vectors, so that leg cannot do row lookup at all and chunking does not change it.
+     */
+    separator?: string;
 }
 /**
  * Splits text into overlapping chunks for embedding. Greedy up to `chunkSize`, but prefers to break

package/dist/index.js

@@ -9,6 +9,9 @@ function chunkText(text, options = {}) {
   if (normalized.length === 0) {
     return [];
   }
+  if (options.separator !== void 0 && options.separator.length > 0) {
+    return chunkByRecord(normalized, chunkSize, options.separator);
+  }
   if (normalized.length <= chunkSize) {
     return [
       normalized
@@ -38,6 +41,40 @@ function chunkText(text, options = {}) {
   return chunks;
 }
 __name(chunkText, "chunkText");
+function chunkByRecord(text, chunkSize, separator) {
+  const chunks = [];
+  let current = "";
+  const flush = /* @__PURE__ */ __name(() => {
+    if (current.length > 0) {
+      chunks.push(current);
+      current = "";
+    }
+  }, "flush");
+  for (const record of text.split(separator)) {
+    const trimmed = record.trim();
+    if (trimmed.length === 0) {
+      continue;
+    }
+    if (trimmed.length > chunkSize) {
+      flush();
+      chunks.push(...chunkText(trimmed, {
+        chunkSize,
+        overlap: 0
+      }));
+      continue;
+    }
+    const candidate = current.length > 0 ? `${current}${separator}${trimmed}` : trimmed;
+    if (candidate.length > chunkSize) {
+      flush();
+      current = trimmed;
+    } else {
+      current = candidate;
+    }
+  }
+  flush();
+  return chunks;
+}
+__name(chunkByRecord, "chunkByRecord");
 
 // src/vector-store.ts
 function isLexicalVectorStore(store) {