@fllf/agent-sdk 0.1.2 → 0.1.3

package/dist/rag/chunking/chunker.js

@@ -14,11 +14,21 @@ const DEFAULT_SEPARATORS = [
     '\n',
     '。 ',
     '。', // 中文文档常见句末符号，不能只依赖空格。
+    '！',
+    '？',
+    '；',
+    '，',
+    '、',
     '. ',
     '! ',
     '? ',
     '; ',
     ', ',
+    '.',
+    '!',
+    '?',
+    ';',
+    ',',
     ' ',
 ];
 async function chunkManyDocuments(chunker, documents) {
@@ -104,7 +114,7 @@ function createChunkId(input) {
 function findBestBreak(text, start, hardEnd, minChunkLength) {
     const minEnd = Math.min(start + minChunkLength, hardEnd);
     for (const separator of DEFAULT_SEPARATORS) {
-        const index = text.lastIndexOf(separator, hardEnd);
+        const index = text.lastIndexOf(separator, hardEnd - separator.length);
         if (index >= minEnd) {
             return index + separator.length;
         }
@@ -133,11 +143,25 @@ function trimRange(text, start, end) {
     };
 }
 function findForwardBoundary(text, preferredStart, end) {
-    let index = preferredStart;
-    while (index < end && !/\s/.test(text[index - 1] ?? '')) {
-        index += 1;
+    const searchEnd = Math.min(end, preferredStart + getBoundarySearchWindow(end - preferredStart));
+    for (let index = preferredStart; index < searchEnd; index += 1) {
+        if (isNaturalStartBoundary(text[index - 1])) {
+            return index;
+        }
+    }
+    return preferredStart;
+}
+function getBoundarySearchWindow(overlapLength) {
+    return Math.max(16, Math.ceil(overlapLength / 2));
+}
+function isNaturalStartBoundary(previousCharacter) {
+    if (previousCharacter === undefined) {
+        return true;
+    }
+    if (/\s/.test(previousCharacter)) {
+        return true;
     }
-    return Math.min(index, end);
+    return /[。！？；，、.!?;,]/u.test(previousCharacter);
 }
 function toPositiveInteger(value, fallback) {
     if (value === undefined || !Number.isFinite(value) || value <= 0) {

package/dist/rag/chunking/chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../../src/rag/chunking/chunker.ts"],"names":[],"mappings":";;;AA0DA,gDAOC;AAED,8CA+CC;AAED,kCA2CC;AASD,sCASC;AAjLD,6CAAyC;AAyC5B,QAAA,wBAAwB,GAAG,IAAI,CAAC;AAChC,QAAA,wBAAwB,GAAG,GAAG,CAAC;AAC/B,QAAA,sBAAsB,GAAG,GAAG,CAAC;AAE1C,MAAM,kBAAkB,GAAG;IACvB,MAAM;IACN,IAAI;IACJ,IAAI;IACJ,GAAG,EAAE,sBAAsB;IAC3B,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,GAAG;CACG,CAAC;AAEJ,KAAK,UAAU,kBAAkB,CACpC,OAAgB,EAChB,SAAwB;IAExB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAEvF,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAED,SAAgB,iBAAiB,CAC7B,IAAY,EACZ,UAA4B,EAAE;IAE9B,MAAM,cAAc,GAAG,iBAAiB,CACpC,OAAO,CAAC,cAAc,EACtB,gCAAwB,CAC3B,CAAC;IACF,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAC3B,iBAAiB,CAAC,OAAO,CAAC,cAAc,EAAE,gCAAwB,CAAC,EACnE,cAAc,CACjB,CAAC;IACF,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC1B,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,OAAO,CAAC,aAAa,EAAE,8BAAsB,CAAC,EAAE,CAAC,CAAC,EAC7E,IAAI,CAAC,GAAG,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC,CAAC,CAClC,CAAC;IAEF,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,IAAI,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAEpC,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9D,MAAM,GAAG,GAAG,OAAO,GAAG,IAAI,CAAC,MAAM;YAC7B,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,CAAC;YACrD,CAAC,CAAC,OAAO,CAAC;QACd,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAE5C,IAAI,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;YAC9B,QAAQ,CAAC,IAAI,CAAC;gBACV,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC;gBAC/C,WAAW,EAAE,OAAO,CAAC,KAAK;gBAC1B,SAAS,EAAE,OAAO,CAAC,GAAG;aACzB,CAAC,CAAC;QACP,CAAC;QAED,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACrB,MAAM;QACV,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,aAAa,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QAChE,MAAM,SAAS,GAAG,aAAa,GAAG,CAAC;YAC/B,CAAC,CAAC,mBAAmB,CAAC,IAAI,EAAE,cAAc,EAAE,GAAG,CAAC;YAChD,CAAC,CAAC,GAAG,CAAC;QACV,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED,SAAgB,WAAW,CAAC,KAAuB;IAC/C,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAkB;QAC5B,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM;QACtC,UAAU,EAAE,KAAK,CAAC,UAAU;KAC/B,CAAC;IAEF,eAAe,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAClE,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACxE,eAAe,CAAC,QAAQ,EAAE,iBAAiB,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;IACtF,eAAe,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IAC5D,eAAe,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAExD,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;QAC5C,QAAQ,CAAC,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IACpD,CAAC;IACD,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClE,QAAQ,CAAC,WAAW,GAAG,CAAC,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC;IAClD,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IACvE,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACtB,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED,MAAM,YAAY,GAAiB;QAC/B,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE;QAC7B,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,OAAO;KACV,CAAC;IACF,eAAe,CAAC,YAAY,EAAE,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IAE1D,MAAM,KAAK,GAAU;QACjB,EAAE,EAAE,aAAa,CAAC,YAAY,CAAC;QAC/B,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE;QAC7B,OAAO;QACP,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,QAAQ;KACX,CAAC;IAEF,eAAe,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IAEnD,OAAO,KAAK,CAAC;AACjB,CAAC;AASD,SAAgB,aAAa,CAAC,KAAmB;IAC7C,MAAM,SAAS,GAAG;QACd,KAAK,CAAC,UAAU;QAChB,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC;QACxB,KAAK,CAAC,QAAQ,IAAI,EAAE;QACpB,KAAK,CAAC,OAAO;KAChB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,OAAO,SAAS,IAAA,wBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;AACxF,CAAC;AAED,SAAS,aAAa,CAClB,IAAY,EACZ,KAAa,EACb,OAAe,EACf,cAAsB;IAEtB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,cAAc,EAAE,OAAO,CAAC,CAAC;IAEzD,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACnD,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;YAClB,OAAO,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC;QACpC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,KAAa;IAC/C,IAAI,KAAK,GAAG,KAAK,CAAC;IAClB,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACzD,KAAK,IAAI,CAAC,CAAC;IACf,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,KAAa,EAAE,GAAW;IACvD,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,UAAU,GAAG,GAAG,CAAC;IAErB,OAAO,YAAY,GAAG,UAAU,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACtE,YAAY,IAAI,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,UAAU,GAAG,YAAY,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACxE,UAAU,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,OAAO;QACH,KAAK,EAAE,YAAY;QACnB,GAAG,EAAE,UAAU;KAClB,CAAC;AACN,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY,EAAE,cAAsB,EAAE,GAAW;IAC1E,IAAI,KAAK,GAAG,cAAc,CAAC;IAE3B,OAAO,KAAK,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACtD,KAAK,IAAI,CAAC,CAAC;IACf,CAAC;IAED,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAyB,EAAE,QAAgB;IAClE,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/D,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,YAAY,CACjB,IAAyC,EACzC,KAA0C;IAE1C,IAAI,IAAI,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAC5C,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,OAAO;QACH,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;QACf,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;KACnB,CAAC;AACN,CAAC;AAED,SAAS,eAAe,CACpB,MAAS,EACT,GAAM,EACN,KAAuB;IAEvB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IACxB,CAAC;AACL,CAAC"}
+{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../../src/rag/chunking/chunker.ts"],"names":[],"mappings":";;;AAoEA,gDAOC;AAED,8CA+CC;AAED,kCA2CC;AASD,sCASC;AA3LD,6CAAyC;AAyC5B,QAAA,wBAAwB,GAAG,IAAI,CAAC;AAChC,QAAA,wBAAwB,GAAG,GAAG,CAAC;AAC/B,QAAA,sBAAsB,GAAG,GAAG,CAAC;AAE1C,MAAM,kBAAkB,GAAG;IACvB,MAAM;IACN,IAAI;IACJ,IAAI;IACJ,GAAG,EAAE,sBAAsB;IAC3B,GAAG;IACH,GAAG;IACH,GAAG;IACH,GAAG;IACH,GAAG;IACH,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,GAAG;IACH,GAAG;IACH,GAAG;IACH,GAAG;IACH,GAAG;IACH,GAAG;CACG,CAAC;AAEJ,KAAK,UAAU,kBAAkB,CACpC,OAAgB,EAChB,SAAwB;IAExB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAEvF,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAED,SAAgB,iBAAiB,CAC7B,IAAY,EACZ,UAA4B,EAAE;IAE9B,MAAM,cAAc,GAAG,iBAAiB,CACpC,OAAO,CAAC,cAAc,EACtB,gCAAwB,CAC3B,CAAC;IACF,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAC3B,iBAAiB,CAAC,OAAO,CAAC,cAAc,EAAE,gCAAwB,CAAC,EACnE,cAAc,CACjB,CAAC;IACF,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC1B,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,OAAO,CAAC,aAAa,EAAE,8BAAsB,CAAC,EAAE,CAAC,CAAC,EAC7E,IAAI,CAAC,GAAG,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC,CAAC,CAClC,CAAC;IAEF,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,IAAI,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAEpC,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9D,MAAM,GAAG,GAAG,OAAO,GAAG,IAAI,CAAC,MAAM;YAC7B,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,CAAC;YACrD,CAAC,CAAC,OAAO,CAAC;QACd,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAE5C,IAAI,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;YAC9B,QAAQ,CAAC,IAAI,CAAC;gBACV,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC;gBAC/C,WAAW,EAAE,OAAO,CAAC,KAAK;gBAC1B,SAAS,EAAE,OAAO,CAAC,GAAG;aACzB,CAAC,CAAC;QACP,CAAC;QAED,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACrB,MAAM;QACV,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,aAAa,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QAChE,MAAM,SAAS,GAAG,aAAa,GAAG,CAAC;YAC/B,CAAC,CAAC,mBAAmB,CAAC,IAAI,EAAE,cAAc,EAAE,GAAG,CAAC;YAChD,CAAC,CAAC,GAAG,CAAC;QACV,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED,SAAgB,WAAW,CAAC,KAAuB;IAC/C,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAkB;QAC5B,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM;QACtC,UAAU,EAAE,KAAK,CAAC,UAAU;KAC/B,CAAC;IAEF,eAAe,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAClE,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACxE,eAAe,CAAC,QAAQ,EAAE,iBAAiB,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;IACtF,eAAe,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IAC5D,eAAe,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAExD,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;QAC5C,QAAQ,CAAC,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IACpD,CAAC;IACD,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClE,QAAQ,CAAC,WAAW,GAAG,CAAC,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC;IAClD,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IACvE,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACtB,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED,MAAM,YAAY,GAAiB;QAC/B,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE;QAC7B,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,OAAO;KACV,CAAC;IACF,eAAe,CAAC,YAAY,EAAE,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IAE1D,MAAM,KAAK,GAAU;QACjB,EAAE,EAAE,aAAa,CAAC,YAAY,CAAC;QAC/B,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE;QAC7B,OAAO;QACP,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,QAAQ;KACX,CAAC;IAEF,eAAe,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IAEnD,OAAO,KAAK,CAAC;AACjB,CAAC;AASD,SAAgB,aAAa,CAAC,KAAmB;IAC7C,MAAM,SAAS,GAAG;QACd,KAAK,CAAC,UAAU;QAChB,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC;QACxB,KAAK,CAAC,QAAQ,IAAI,EAAE;QACpB,KAAK,CAAC,OAAO;KAChB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,OAAO,SAAS,IAAA,wBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;AACxF,CAAC;AAED,SAAS,aAAa,CAClB,IAAY,EACZ,KAAa,EACb,OAAe,EACf,cAAsB;IAEtB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,cAAc,EAAE,OAAO,CAAC,CAAC;IAEzD,KAAK,MAAM,SAAS,IAAI,kBAAkB,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;QACtE,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;YAClB,OAAO,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC;QACpC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,KAAa;IAC/C,IAAI,KAAK,GAAG,KAAK,CAAC;IAClB,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACzD,KAAK,IAAI,CAAC,CAAC;IACf,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,KAAa,EAAE,GAAW;IACvD,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,UAAU,GAAG,GAAG,CAAC;IAErB,OAAO,YAAY,GAAG,UAAU,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACtE,YAAY,IAAI,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,UAAU,GAAG,YAAY,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACxE,UAAU,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,OAAO;QACH,KAAK,EAAE,YAAY;QACnB,GAAG,EAAE,UAAU;KAClB,CAAC;AACN,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY,EAAE,cAAsB,EAAE,GAAW;IAC1E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,cAAc,GAAG,uBAAuB,CAAC,GAAG,GAAG,cAAc,CAAC,CAAC,CAAC;IAEhG,KAAK,IAAI,KAAK,GAAG,cAAc,EAAE,KAAK,GAAG,SAAS,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC7D,IAAI,sBAAsB,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1C,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAED,OAAO,cAAc,CAAC;AAC1B,CAAC;AAED,SAAS,uBAAuB,CAAC,aAAqB;IAClD,OAAO,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,sBAAsB,CAAC,iBAAqC;IACjE,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAC/B,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,gBAAgB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;AACpD,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAyB,EAAE,QAAgB;IAClE,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/D,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,YAAY,CACjB,IAAyC,EACzC,KAA0C;IAE1C,IAAI,IAAI,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAC5C,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,OAAO;QACH,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;QACf,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;KACnB,CAAC;AACN,CAAC;AAED,SAAS,eAAe,CACpB,MAAS,EACT,GAAM,EACN,KAAuB;IAEvB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IACxB,CAAC;AACL,CAAC"}

package/dist/rag/chunking/index.d.ts

@@ -2,5 +2,7 @@ export * from './chunker';
 export * from './auto-chunker';
 export * from './markdown-chunker';
 export * from './parent-child';
+export * from './qa-pair-chunker';
 export * from './recursive-chunker';
+export * from './semantic-chunker';
 //# sourceMappingURL=index.d.ts.map

package/dist/rag/chunking/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/rag/chunking/index.ts"],"names":[],"mappings":"AAAA,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,qBAAqB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/rag/chunking/index.ts"],"names":[],"mappings":"AAAA,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC"}

package/dist/rag/chunking/index.js

@@ -18,5 +18,7 @@ __exportStar(require("./chunker"), exports);
 __exportStar(require("./auto-chunker"), exports);
 __exportStar(require("./markdown-chunker"), exports);
 __exportStar(require("./parent-child"), exports);
+__exportStar(require("./qa-pair-chunker"), exports);
 __exportStar(require("./recursive-chunker"), exports);
+__exportStar(require("./semantic-chunker"), exports);
 //# sourceMappingURL=index.js.map

package/dist/rag/chunking/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rag/chunking/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,4CAA0B;AAC1B,iDAA+B;AAC/B,qDAAmC;AACnC,iDAA+B;AAC/B,sDAAoC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rag/chunking/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,4CAA0B;AAC1B,iDAA+B;AAC/B,qDAAmC;AACnC,iDAA+B;AAC/B,oDAAkC;AAClC,sDAAoC;AACpC,qDAAmC"}

package/dist/rag/chunking/markdown-chunker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"markdown-chunker.d.ts","sourceRoot":"","sources":["../../../src/rag/chunking/markdown-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,KAAK,EAEL,WAAW,EACd,MAAM,UAAU,CAAC;AAMlB,OAAO,KAAK,EACR,OAAO,EACP,gBAAgB,EACnB,MAAM,WAAW,CAAC;AAEnB,MAAM,WAAW,sBAAuB,SAAQ,gBAAgB;IAC5D,uBAAuB,CAAC,EAAE,OAAO,CAAC;CACrC;AAOD,qBAAa,eAAgB,YAAW,OAAO;IAC3C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyB;gBAErC,OAAO,GAAE,sBAA2B;IAI1C,KAAK,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAyC9C,SAAS,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAI3D,OAAO,CAAC,iBAAiB;CAY5B"}
+{"version":3,"file":"markdown-chunker.d.ts","sourceRoot":"","sources":["../../../src/rag/chunking/markdown-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,KAAK,EAEL,WAAW,EACd,MAAM,UAAU,CAAC;AAMlB,OAAO,KAAK,EACR,OAAO,EACP,gBAAgB,EACnB,MAAM,WAAW,CAAC;AAInB,MAAM,WAAW,sBAAuB,SAAQ,gBAAgB;IAC5D,uBAAuB,CAAC,EAAE,OAAO,CAAC;CACrC;AAOD,qBAAa,eAAgB,YAAW,OAAO;IAC3C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyB;gBAErC,OAAO,GAAE,sBAA2B;IAI1C,KAAK,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAyC9C,SAAS,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAI3D,OAAO,CAAC,iBAAiB;CAY5B"}

package/dist/rag/chunking/markdown-chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"markdown-chunker.js","sourceRoot":"","sources":["../../../src/rag/chunking/markdown-chunker.ts"],"names":[],"mappings":";;;AAKA,uCAImB;AAenB,MAAa,eAAe;IACP,OAAO,CAAyB;IAEjD,YAAY,UAAkC,EAAE;QAC5C,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAqB;QAC7B,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChE,OAAO,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,MAAM,YAAY,GAAmB,EAAE,CAAC;QACxC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;YAClC,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAC3B,kBAAkB,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;gBACxC,IAAI,IAAI,CAAC,OAAO,CAAC,uBAAuB,KAAK,IAAI,EAAE,CAAC;oBAChD,MAAM,CAAC,IAAI,CAAC,IAAA,qBAAW,EAAC;wBACpB,QAAQ;wBACR,OAAO,EAAE,KAAK,CAAC,IAAI;wBACnB,IAAI,EAAE,MAAM;wBACZ,UAAU;wBACV,WAAW,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;wBACxD,KAAK,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE;qBACxD,CAAC,CAAC,CAAC;oBACJ,UAAU,IAAI,CAAC,CAAC;gBACpB,CAAC;gBACD,SAAS;YACb,CAAC;YAED,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAChE,MAAM,WAAW,GAAG,iBAAiB,CAAC;gBAClC,QAAQ;gBACR,KAAK;gBACL,WAAW;gBACX,cAAc,EAAE,UAAU;gBAC1B,OAAO,EAAE,IAAI,CAAC,OAAO;aACxB,CAAC,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;YAC5B,UAAU,IAAI,WAAW,CAAC,MAAM,CAAC;QACrC,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,SAAwB;QACpC,OAAO,MAAM,IAAA,4BAAkB,EAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IACrD,CAAC;IAEO,iBAAiB,CAAC,QAAqB;QAC3C,OAAO,IAAA,2BAAiB,EAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC;aACnD,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,IAAA,qBAAW,EAAC;YACjC,QAAQ;YACR,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,IAAI,EAAE,MAAM;YACZ,UAAU,EAAE,KAAK;YACjB,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,KAAK,EAAE,EAAE,QAAQ,EAAE,mBAAmB,EAAE;SAC3C,CAAC,CAAC,CAAC;IACZ,CAAC;CACJ;AAhED,0CAgEC;AAUD,SAAS,iBAAiB,CAAC,KAA6B;IACpD,QAAQ,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACvB,KAAK,WAAW;YACZ,OAAO,IAAA,2BAAiB,EAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC;iBACpD,GAAG,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,IAAA,qBAAW,EAAC;gBAClC,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,IAAI,EAAE,MAAM;gBACZ,UAAU,EAAE,KAAK,CAAC,cAAc,GAAG,MAAM;gBACzC,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,KAAK,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE;aAC1D,CAAC,CAAC,CAAC;QACZ,KAAK,OAAO;YACR,OAAO;gBACH,IAAA,qBAAW,EAAC;oBACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ;oBAC7B,IAAI,EAAE,OAAO;oBACb,UAAU,EAAE,KAAK,CAAC,cAAc;oBAChC,WAAW,EAAE,KAAK,CAAC,WAAW;oBAC9B,KAAK,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE;iBACtD,CAAC;aACL,CAAC;QACN,KAAK,MAAM;YACP,OAAO;gBACH,IAAA,qBAAW,EAAC;oBACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,OAAO,EAAE,eAAe,CAAC,KAAK,CAAC,KAAK,CAAC;oBACrC,IAAI,EAAE,MAAM;oBACZ,UAAU,EAAE,KAAK,CAAC,cAAc;oBAChC,WAAW,EAAE,KAAK,CAAC,WAAW;oBAC9B,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC,KAAK,CAAC;iBACtC,CAAC;aACL,CAAC;QACN,KAAK,OAAO,CAAC,CAAC,CAAC;YACX,MAAM,OAAO,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC;iBAC9C,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;iBAClF,IAAI,CAAC,IAAI,CAAC,CAAC;YAChB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,OAAO,EAAE,CAAC;YACd,CAAC;YAED,OAAO;gBACH,IAAA,qBAAW,EAAC;oBACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,OAAO;oBACP,IAAI,EAAE,YAAY;oBAClB,UAAU,EAAE,KAAK,CAAC,cAAc;oBAChC,WAAW,EAAE,KAAK,CAAC,WAAW;oBAC9B,KAAK,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE;iBACtD,CAAC;aACL,CAAC;QACN,CAAC;IACL,CAAC;AACL,CAAC;AAED,SAAS,kBAAkB,CACvB,KAAqB,EACrB,KAAkD;IAElD,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QACnE,KAAK,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC;YACP,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,IAAI,EAAE,KAAK,CAAC,IAAI;SACnB,CAAC,CAAC;IACP,CAAC;AACL,CAAC;AAED,SAAS,eAAe,CAAC,KAA+C;IACpE,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,IAAI,CAAC;IACtB,CAAC;IAED,OAAO,SAAS,KAAK,CAAC,QAAQ,KAAK,KAAK,CAAC,IAAI,UAAU,CAAC;AAC5D,CAAC;AAED,SAAS,eAAe,CAAC,KAA+C;IACpE,MAAM,KAAK,GAA4B;QACnC,SAAS,EAAE,MAAM;QACjB,QAAQ,EAAE,UAAU;KACvB,CAAC;IAEF,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC/B,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;IACpC,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC"}
+{"version":3,"file":"markdown-chunker.js","sourceRoot":"","sources":["../../../src/rag/chunking/markdown-chunker.ts"],"names":[],"mappings":";;;AAKA,uCAImB;AAiBnB,MAAa,eAAe;IACP,OAAO,CAAyB;IAEjD,YAAY,UAAkC,EAAE;QAC5C,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAqB;QAC7B,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChE,OAAO,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,MAAM,YAAY,GAAmB,EAAE,CAAC;QACxC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;YAClC,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAC3B,kBAAkB,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;gBACxC,IAAI,IAAI,CAAC,OAAO,CAAC,uBAAuB,KAAK,IAAI,EAAE,CAAC;oBAChD,MAAM,CAAC,IAAI,CAAC,IAAA,qBAAW,EAAC;wBACpB,QAAQ;wBACR,OAAO,EAAE,KAAK,CAAC,IAAI;wBACnB,IAAI,EAAE,MAAM;wBACZ,UAAU;wBACV,WAAW,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;wBACxD,KAAK,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE;qBACxD,CAAC,CAAC,CAAC;oBACJ,UAAU,IAAI,CAAC,CAAC;gBACpB,CAAC;gBACD,SAAS;YACb,CAAC;YAED,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAChE,MAAM,WAAW,GAAG,iBAAiB,CAAC;gBAClC,QAAQ;gBACR,KAAK;gBACL,WAAW;gBACX,cAAc,EAAE,UAAU;gBAC1B,OAAO,EAAE,IAAI,CAAC,OAAO;aACxB,CAAC,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;YAC5B,UAAU,IAAI,WAAW,CAAC,MAAM,CAAC;QACrC,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,SAAwB;QACpC,OAAO,MAAM,IAAA,4BAAkB,EAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IACrD,CAAC;IAEO,iBAAiB,CAAC,QAAqB;QAC3C,OAAO,IAAA,2BAAiB,EAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC;aACnD,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,IAAA,qBAAW,EAAC;YACjC,QAAQ;YACR,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,IAAI,EAAE,MAAM;YACZ,UAAU,EAAE,KAAK;YACjB,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,KAAK,EAAE,EAAE,QAAQ,EAAE,mBAAmB,EAAE;SAC3C,CAAC,CAAC,CAAC;IACZ,CAAC;CACJ;AAhED,0CAgEC;AAUD,SAAS,iBAAiB,CAAC,KAA6B;IACpD,QAAQ,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACvB,KAAK,WAAW;YACZ,OAAO,IAAA,2BAAiB,EAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC;iBACpD,GAAG,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,IAAA,qBAAW,EAAC;gBAClC,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,IAAI,EAAE,MAAM;gBACZ,UAAU,EAAE,KAAK,CAAC,cAAc,GAAG,MAAM;gBACzC,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,KAAK,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE;aAC1D,CAAC,CAAC,CAAC;QACZ,KAAK,OAAO;YACR,OAAO;gBACH,IAAA,qBAAW,EAAC;oBACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ;oBAC7B,IAAI,EAAE,OAAO;oBACb,UAAU,EAAE,KAAK,CAAC,cAAc;oBAChC,WAAW,EAAE,KAAK,CAAC,WAAW;oBAC9B,KAAK,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE;iBACtD,CAAC;aACL,CAAC;QACN,KAAK,MAAM;YACP,OAAO;gBACH,IAAA,qBAAW,EAAC;oBACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,OAAO,EAAE,eAAe,CAAC,KAAK,CAAC,KAAK,CAAC;oBACrC,IAAI,EAAE,MAAM;oBACZ,UAAU,EAAE,KAAK,CAAC,cAAc;oBAChC,WAAW,EAAE,KAAK,CAAC,WAAW;oBAC9B,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC,KAAK,CAAC;iBACtC,CAAC;aACL,CAAC;QACN,KAAK,OAAO,CAAC,CAAC,CAAC;YACX,MAAM,OAAO,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC;iBAC9C,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;iBAClF,IAAI,CAAC,IAAI,CAAC,CAAC;YAChB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,OAAO,EAAE,CAAC;YACd,CAAC;YAED,OAAO;gBACH,IAAA,qBAAW,EAAC;oBACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,OAAO;oBACP,IAAI,EAAE,YAAY;oBAClB,UAAU,EAAE,KAAK,CAAC,cAAc;oBAChC,WAAW,EAAE,KAAK,CAAC,WAAW;oBAC9B,KAAK,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE;iBACtD,CAAC;aACL,CAAC;QACN,CAAC;IACL,CAAC;AACL,CAAC;AAED,SAAS,kBAAkB,CACvB,KAAqB,EACrB,KAAkD;IAElD,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QACnE,KAAK,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAED,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC;YACP,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,IAAI,EAAE,KAAK,CAAC,IAAI;SACnB,CAAC,CAAC;IACP,CAAC;AACL,CAAC;AAED,SAAS,eAAe,CAAC,KAA+C;IACpE,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,IAAI,CAAC;IACtB,CAAC;IAED,OAAO,SAAS,KAAK,CAAC,QAAQ,KAAK,KAAK,CAAC,IAAI,UAAU,CAAC;AAC5D,CAAC;AAED,SAAS,eAAe,CAAC,KAA+C;IACpE,MAAM,KAAK,GAA4B;QACnC,SAAS,EAAE,MAAM;QACjB,QAAQ,EAAE,UAAU;KACvB,CAAC;IAEF,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC/B,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;IACpC,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC"}

package/dist/rag/chunking/qa-pair-chunker.d.ts

@@ -0,0 +1,23 @@
+import type { Chunk, RawDocument } from '../types';
+import type { Chunker, TextSplitOptions } from './chunker';
+export interface QaPair {
+    question: string;
+    answer: string;
+    content: string;
+    startOffset: number;
+    endOffset: number;
+}
+export interface QaPairChunkerOptions extends TextSplitOptions {
+    minPairs?: number;
+    fallbackChunker?: Chunker;
+}
+export declare class QaPairChunker implements Chunker {
+    private readonly minPairs;
+    private readonly fallbackChunker;
+    constructor(options?: QaPairChunkerOptions);
+    chunk(document: RawDocument): Promise<Chunk[]>;
+    chunkMany(documents: RawDocument[]): Promise<Chunk[]>;
+}
+export declare function hasQaPairs(text: string, minPairs?: number): boolean;
+export declare function extractQaPairs(text: string): QaPair[];
+//# sourceMappingURL=qa-pair-chunker.d.ts.map

package/dist/rag/chunking/qa-pair-chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"qa-pair-chunker.d.ts","sourceRoot":"","sources":["../../../src/rag/chunking/qa-pair-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,KAAK,EACL,WAAW,EACd,MAAM,UAAU,CAAC;AAKlB,OAAO,KAAK,EACR,OAAO,EACP,gBAAgB,EACnB,MAAM,WAAW,CAAC;AAQnB,MAAM,WAAW,MAAM;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,oBAAqB,SAAQ,gBAAgB;IAC1D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC7B;AAgBD,qBAAa,aAAc,YAAW,OAAO;IACzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAU;gBAE9B,OAAO,GAAE,oBAAyB;IAKxC,KAAK,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAoB9C,SAAS,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;CAG9D;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,SAAI,GAAG,OAAO,CAE9D;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAkDrD"}

package/dist/rag/chunking/qa-pair-chunker.js

@@ -0,0 +1,162 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.QaPairChunker = void 0;
+exports.hasQaPairs = hasQaPairs;
+exports.extractQaPairs = extractQaPairs;
+const chunker_1 = require("./chunker");
+const recursive_chunker_1 = require("./recursive-chunker");
+const QUESTION_MARKER_PATTERN = /^(?:#{1,6}\s*)?(?:Q(?:uestion)?|问(?:题)?)(?:\s*\d+)?[：:]\s*(.*)$/iu;
+const ANSWER_MARKER_PATTERN = /^(?:A(?:nswer)?|答(?:案)?)(?:\s*\d+)?[：:]\s*(.*)$/iu;
+class QaPairChunker {
+    minPairs;
+    fallbackChunker;
+    constructor(options = {}) {
+        this.minPairs = toPositiveInteger(options.minPairs, 1);
+        this.fallbackChunker = options.fallbackChunker ?? new recursive_chunker_1.RecursiveChunker(options);
+    }
+    async chunk(document) {
+        const pairs = extractQaPairs(document.content);
+        if (pairs.length < this.minPairs) {
+            return await this.fallbackChunker.chunk(document);
+        }
+        return pairs.map((pair, index) => (0, chunker_1.createChunk)({
+            document,
+            content: pair.content,
+            kind: 'text',
+            chunkIndex: index,
+            startOffset: pair.startOffset,
+            endOffset: pair.endOffset,
+            extra: {
+                strategy: 'qa-pair',
+                pairIndex: index,
+            },
+        }));
+    }
+    async chunkMany(documents) {
+        return await (0, chunker_1.chunkManyDocuments)(this, documents);
+    }
+}
+exports.QaPairChunker = QaPairChunker;
+function hasQaPairs(text, minPairs = 1) {
+    return extractQaPairs(text).length >= toPositiveInteger(minPairs, 1);
+}
+function extractQaPairs(text) {
+    const pairs = [];
+    let current;
+    for (const line of splitLines(text)) {
+        const question = readQuestionMarker(line.text);
+        if (question !== undefined) {
+            const pair = finalizeDraft(current, text);
+            if (pair !== undefined) {
+                pairs.push(pair);
+            }
+            current = {
+                questionLines: question.length > 0 ? [question] : [],
+                answerLines: [],
+                startOffset: line.startOffset,
+                endOffset: line.endOffset,
+                state: 'question',
+            };
+            continue;
+        }
+        const answer = readAnswerMarker(line.text);
+        if (answer !== undefined && current !== undefined) {
+            current.state = 'answer';
+            if (answer.length > 0) {
+                current.answerLines.push(answer);
+            }
+            current.endOffset = line.endOffset;
+            continue;
+        }
+        if (current === undefined) {
+            continue;
+        }
+        if (current.state === 'question') {
+            appendLine(current.questionLines, line.text);
+        }
+        else {
+            current.answerLines.push(line.text);
+            current.endOffset = line.endOffset;
+        }
+    }
+    const pair = finalizeDraft(current, text);
+    if (pair !== undefined) {
+        pairs.push(pair);
+    }
+    return pairs;
+}
+function finalizeDraft(draft, text) {
+    if (draft === undefined) {
+        return undefined;
+    }
+    const question = normalizeMultilineText(draft.questionLines);
+    const answer = normalizeMultilineText(draft.answerLines);
+    if (question.length === 0 || answer.length === 0) {
+        return undefined;
+    }
+    const content = text.slice(draft.startOffset, draft.endOffset).trim();
+    if (content.length === 0) {
+        return undefined;
+    }
+    return {
+        question,
+        answer,
+        content,
+        startOffset: draft.startOffset,
+        endOffset: draft.endOffset,
+    };
+}
+function splitLines(text) {
+    const lines = [];
+    let startOffset = 0;
+    while (startOffset <= text.length) {
+        const newlineIndex = text.indexOf('\n', startOffset);
+        const rawEndOffset = newlineIndex < 0 ? text.length : newlineIndex;
+        const hasCarriageReturn = text[rawEndOffset - 1] === '\r';
+        const endOffset = hasCarriageReturn ? rawEndOffset - 1 : rawEndOffset;
+        lines.push({
+            text: text.slice(startOffset, endOffset),
+            startOffset,
+            endOffset,
+        });
+        if (newlineIndex < 0) {
+            break;
+        }
+        startOffset = newlineIndex + 1;
+    }
+    return lines;
+}
+function readQuestionMarker(line) {
+    return readMarker(line, QUESTION_MARKER_PATTERN);
+}
+function readAnswerMarker(line) {
+    return readMarker(line, ANSWER_MARKER_PATTERN);
+}
+function readMarker(line, pattern) {
+    const match = line.trim().match(pattern);
+    return match?.[1]?.trim();
+}
+function appendLine(lines, line) {
+    if (line.trim().length === 0 && lines.length === 0) {
+        return;
+    }
+    lines.push(line);
+}
+function normalizeMultilineText(lines) {
+    let start = 0;
+    let end = lines.length;
+    while (start < end && lines[start]?.trim().length === 0) {
+        start += 1;
+    }
+    while (end > start && lines[end - 1]?.trim().length === 0) {
+        end -= 1;
+    }
+    return lines.slice(start, end).join('\n').trim();
+}
+function toPositiveInteger(value, fallback) {
+    if (value === undefined || !Number.isFinite(value) || value <= 0) {
+        return fallback;
+    }
+    return Math.floor(value);
+}
+//# sourceMappingURL=qa-pair-chunker.js.map

package/dist/rag/chunking/qa-pair-chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"qa-pair-chunker.js","sourceRoot":"","sources":["../../../src/rag/chunking/qa-pair-chunker.ts"],"names":[],"mappings":";;;AAgFA,gCAEC;AAED,wCAkDC;AAlID,uCAGmB;AAKnB,2DAE6B;AAE7B,MAAM,uBAAuB,GAAG,mEAAmE,CAAC;AACpG,MAAM,qBAAqB,GAAG,mDAAmD,CAAC;AA6BlF,MAAa,aAAa;IACL,QAAQ,CAAS;IACjB,eAAe,CAAU;IAE1C,YAAY,UAAgC,EAAE;QAC1C,IAAI,CAAC,QAAQ,GAAG,iBAAiB,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,IAAI,oCAAgB,CAAC,OAAO,CAAC,CAAC;IACpF,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAqB;QAC7B,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC/C,IAAI,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC/B,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACtD,CAAC;QAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAA,qBAAW,EAAC;YAC1C,QAAQ;YACR,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,MAAM;YACZ,UAAU,EAAE,KAAK;YACjB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,KAAK,EAAE;gBACH,QAAQ,EAAE,SAAS;gBACnB,SAAS,EAAE,KAAK;aACnB;SACJ,CAAC,CAAC,CAAC;IACR,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,SAAwB;QACpC,OAAO,MAAM,IAAA,4BAAkB,EAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IACrD,CAAC;CACJ;AAhCD,sCAgCC;AAED,SAAgB,UAAU,CAAC,IAAY,EAAE,QAAQ,GAAG,CAAC;IACjD,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,iBAAiB,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,SAAgB,cAAc,CAAC,IAAY;IACvC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,OAAgC,CAAC;IAErC,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YAC1C,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBACrB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,CAAC;YAED,OAAO,GAAG;gBACN,aAAa,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE;gBACpD,WAAW,EAAE,EAAE;gBACf,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,KAAK,EAAE,UAAU;aACpB,CAAC;YACF,SAAS;QACb,CAAC;QAED,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,MAAM,KAAK,SAAS,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAChD,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC;YACzB,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;YACD,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;YACnC,SAAS;QACb,CAAC;QAED,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YACxB,SAAS;QACb,CAAC;QAED,IAAI,OAAO,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YAC/B,UAAU,CAAC,OAAO,CAAC,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACJ,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpC,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QACvC,CAAC;IACL,CAAC;IAED,MAAM,IAAI,GAAG,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC1C,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACrB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrB,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,aAAa,CAAC,KAA8B,EAAE,IAAY;IAC/D,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACtB,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,MAAM,QAAQ,GAAG,sBAAsB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,sBAAsB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IACzD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/C,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;IACtE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,OAAO;QACH,QAAQ;QACR,MAAM;QACN,OAAO;QACP,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;KAC7B,CAAC;AACN,CAAC;AAED,SAAS,UAAU,CAAC,IAAY;IAC5B,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,OAAO,WAAW,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChC,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC;QACnE,MAAM,iBAAiB,GAAG,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC;QAC1D,MAAM,SAAS,GAAG,iBAAiB,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC;QAEtE,KAAK,CAAC,IAAI,CAAC;YACP,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC;YACxC,WAAW;YACX,SAAS;SACZ,CAAC,CAAC;QAEH,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACnB,MAAM;QACV,CAAC;QAED,WAAW,GAAG,YAAY,GAAG,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACpC,OAAO,UAAU,CAAC,IAAI,EAAE,uBAAuB,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY;IAClC,OAAO,UAAU,CAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,UAAU,CAAC,IAAY,EAAE,OAAe;IAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAEzC,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;AAC9B,CAAC;AAED,SAAS,UAAU,CAAC,KAAe,EAAE,IAAY;IAC7C,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjD,OAAO;IACX,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACrB,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAe;IAC3C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC;IAEvB,OAAO,KAAK,GAAG,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtD,KAAK,IAAI,CAAC,CAAC;IACf,CAAC;IACD,OAAO,GAAG,GAAG,KAAK,IAAI,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxD,GAAG,IAAI,CAAC,CAAC;IACb,CAAC;IAED,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;AACrD,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAyB,EAAE,QAAgB;IAClE,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/D,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC7B,CAAC"}

package/dist/rag/chunking/semantic-chunker.d.ts

@@ -0,0 +1,19 @@
+import type { Embedder } from '../embeddings';
+import type { Chunk, RawDocument } from '../types';
+import type { Chunker, TextSplitOptions } from './chunker';
+export interface SemanticChunkerOptions extends TextSplitOptions {
+    embedder: Embedder;
+    similarityThreshold?: number;
+    overlapLength?: number;
+}
+export declare class SemanticChunker implements Chunker {
+    private readonly embedder;
+    private readonly similarityThreshold;
+    private readonly maxChunkLength;
+    private readonly minChunkLength;
+    private readonly overlapLength;
+    constructor(options: SemanticChunkerOptions);
+    chunk(document: RawDocument): Promise<Chunk[]>;
+    chunkMany(documents: RawDocument[]): Promise<Chunk[]>;
+}
+//# sourceMappingURL=semantic-chunker.d.ts.map

package/dist/rag/chunking/semantic-chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-chunker.d.ts","sourceRoot":"","sources":["../../../src/rag/chunking/semantic-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAK9C,OAAO,KAAK,EACR,KAAK,EACL,WAAW,EACd,MAAM,UAAU,CAAC;AAQlB,OAAO,KAAK,EACR,OAAO,EACP,gBAAgB,EACnB,MAAM,WAAW,CAAC;AAKnB,MAAM,WAAW,sBAAuB,SAAQ,gBAAgB;IAC5D,QAAQ,EAAE,QAAQ,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,aAAa,CAAC,EAAE,MAAM,CAAC;CAC1B;AAaD,qBAAa,eAAgB,YAAW,OAAO;IAC3C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;gBAE3B,OAAO,EAAE,sBAAsB;IAoBrC,KAAK,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IA2C9C,SAAS,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;CAG9D"}