@truto/truto-jsonata 1.0.50 → 1.0.51

package/dist/cjs/index.cjs

@@ -75968,62 +75968,38 @@ async function encodingForModel(model) {
 }
 
 // node_modules/@langchain/textsplitters/dist/text_splitter.js
-class TextSplitter extends BaseDocumentTransformer {
+var TextSplitter = class extends BaseDocumentTransformer {
+  lc_namespace = [
+    "langchain",
+    "document_transformers",
+    "text_splitters"
+  ];
+  chunkSize = 1000;
+  chunkOverlap = 200;
+  keepSeparator = false;
+  lengthFunction;
   constructor(fields) {
     super(fields);
-    Object.defineProperty(this, "lc_namespace", {
-      enumerable: true,
-      configurable: true,
-      writable: true,
-      value: ["langchain", "document_transformers", "text_splitters"]
-    });
-    Object.defineProperty(this, "chunkSize", {
-      enumerable: true,
-      configurable: true,
-      writable: true,
-      value: 1000
-    });
-    Object.defineProperty(this, "chunkOverlap", {
-      enumerable: true,
-      configurable: true,
-      writable: true,
-      value: 200
-    });
-    Object.defineProperty(this, "keepSeparator", {
-      enumerable: true,
-      configurable: true,
-      writable: true,
-      value: false
-    });
-    Object.defineProperty(this, "lengthFunction", {
-      enumerable: true,
-      configurable: true,
-      writable: true,
-      value: undefined
-    });
     this.chunkSize = fields?.chunkSize ?? this.chunkSize;
     this.chunkOverlap = fields?.chunkOverlap ?? this.chunkOverlap;
     this.keepSeparator = fields?.keepSeparator ?? this.keepSeparator;
     this.lengthFunction = fields?.lengthFunction ?? ((text) => text.length);
-    if (this.chunkOverlap >= this.chunkSize) {
+    if (this.chunkOverlap >= this.chunkSize)
       throw new Error("Cannot have chunkOverlap >= chunkSize");
-    }
   }
   async transformDocuments(documents, chunkHeaderOptions = {}) {
     return this.splitDocuments(documents, chunkHeaderOptions);
   }
   splitOnSeparator(text, separator) {
     let splits;
-    if (separator) {
+    if (separator)
       if (this.keepSeparator) {
         const regexEscapedSeparator = separator.replace(/[/\-\\^$*+?.()|[\]{}]/g, "\\$&");
-        splits = text.split(new RegExp(`(?=${regexEscapedSeparator})`));
-      } else {
+        splits = text.split(/* @__PURE__ */ new RegExp(`(?=${regexEscapedSeparator})`));
+      } else
         splits = text.split(separator);
-      }
-    } else {
+    else
       splits = text.split("");
-    }
     return splits.filter((s2) => s2 !== "");
   }
   async createDocuments(texts, metadatas = [], chunkHeaderOptions = {}) {
@@ -76050,9 +76026,8 @@ class TextSplitter extends BaseDocumentTransformer {
             const numberOfIntermediateNewLines = this.numberOfNewLines(text, indexChunk, indexEndPrevChunk);
             lineCounterIndex -= numberOfIntermediateNewLines;
           }
-          if (appendChunkOverlapHeader) {
+          if (appendChunkOverlapHeader)
             pageContent += chunkOverlapHeader;
-          }
         }
         const newLinesCount = this.numberOfNewLines(chunk2);
         const loc = _metadatas[i3].loc && typeof _metadatas[i3].loc === "object" ? { ..._metadatas[i3].loc } : {};
@@ -76097,15 +76072,13 @@ class TextSplitter extends BaseDocumentTransformer {
     for (const d of splits) {
       const _len = await this.lengthFunction(d);
       if (total + _len + currentDoc.length * separator.length > this.chunkSize) {
-        if (total > this.chunkSize) {
+        if (total > this.chunkSize)
           console.warn(`Created a chunk of size ${total}, +
 which is longer than the specified ${this.chunkSize}`);
-        }
         if (currentDoc.length > 0) {
-          const doc3 = this.joinDocs(currentDoc, separator);
-          if (doc3 !== null) {
-            docs.push(doc3);
-          }
+          const doc$1 = this.joinDocs(currentDoc, separator);
+          if (doc$1 !== null)
+            docs.push(doc$1);
           while (total > this.chunkOverlap || total + _len + currentDoc.length * separator.length > this.chunkSize && total > 0) {
             total -= await this.lengthFunction(currentDoc[0]);
             currentDoc.shift();
@@ -76116,27 +76089,26 @@ which is longer than the specified ${this.chunkSize}`);
       total += _len;
     }
     const doc2 = this.joinDocs(currentDoc, separator);
-    if (doc2 !== null) {
+    if (doc2 !== null)
       docs.push(doc2);
-    }
     return docs;
   }
-}
-class RecursiveCharacterTextSplitter extends TextSplitter {
+};
+var RecursiveCharacterTextSplitter = class RecursiveCharacterTextSplitter2 extends TextSplitter {
   static lc_name() {
     return "RecursiveCharacterTextSplitter";
   }
+  separators = [
+    `
+
+`,
+    `
+`,
+    " ",
+    ""
+  ];
   constructor(fields) {
     super(fields);
-    Object.defineProperty(this, "separators", {
-      enumerable: true,
-      configurable: true,
-      writable: true,
-      value: [`
-
-`, `
-`, " ", ""]
-    });
     this.separators = fields?.separators ?? this.separators;
     this.keepSeparator = fields?.keepSeparator ?? true;
   }
@@ -76159,23 +76131,22 @@ class RecursiveCharacterTextSplitter extends TextSplitter {
     const splits = this.splitOnSeparator(text, separator);
     let goodSplits = [];
     const _separator = this.keepSeparator ? "" : separator;
-    for (const s2 of splits) {
-      if (await this.lengthFunction(s2) < this.chunkSize) {
+    for (const s2 of splits)
+      if (await this.lengthFunction(s2) < this.chunkSize)
         goodSplits.push(s2);
-      } else {
+      else {
         if (goodSplits.length) {
           const mergedText = await this.mergeSplits(goodSplits, _separator);
           finalChunks.push(...mergedText);
           goodSplits = [];
         }
-        if (!newSeparators) {
+        if (!newSeparators)
           finalChunks.push(s2);
-        } else {
+        else {
           const otherInfo = await this._splitText(s2, newSeparators);
           finalChunks.push(...otherInfo);
         }
       }
-    }
     if (goodSplits.length) {
       const mergedText = await this.mergeSplits(goodSplits, _separator);
       finalChunks.push(...mergedText);
@@ -76186,13 +76157,13 @@ class RecursiveCharacterTextSplitter extends TextSplitter {
     return this._splitText(text, this.separators);
   }
   static fromLanguage(language, options3) {
-    return new RecursiveCharacterTextSplitter({
+    return new RecursiveCharacterTextSplitter2({
       ...options3,
-      separators: RecursiveCharacterTextSplitter.getSeparatorsForLanguage(language)
+      separators: RecursiveCharacterTextSplitter2.getSeparatorsForLanguage(language)
     });
   }
   static getSeparatorsForLanguage(language) {
-    if (language === "cpp") {
+    if (language === "cpp")
       return [
         `
 class `,
@@ -76222,7 +76193,7 @@ case `,
         " ",
         ""
       ];
-    } else if (language === "go") {
+    else if (language === "go")
       return [
         `
 func `,
@@ -76248,7 +76219,7 @@ case `,
         " ",
         ""
       ];
-    } else if (language === "java") {
+    else if (language === "java")
       return [
         `
 class `,
@@ -76278,7 +76249,7 @@ case `,
         " ",
         ""
       ];
-    } else if (language === "js") {
+    else if (language === "js")
       return [
         `
 function `,
@@ -76310,7 +76281,7 @@ default `,
         " ",
         ""
       ];
-    } else if (language === "php") {
+    else if (language === "php")
       return [
         `
 function `,
@@ -76336,7 +76307,7 @@ case `,
         " ",
         ""
       ];
-    } else if (language === "proto") {
+    else if (language === "proto")
       return [
         `
 message `,
@@ -76358,7 +76329,7 @@ syntax `,
         " ",
         ""
       ];
-    } else if (language === "python") {
+    else if (language === "python")
       return [
         `
 class `,
@@ -76374,7 +76345,7 @@ def `,
         " ",
         ""
       ];
-    } else if (language === "rst") {
+    else if (language === "rst")
       return [
         `
 ===
@@ -76395,7 +76366,7 @@ def `,
         " ",
         ""
       ];
-    } else if (language === "ruby") {
+    else if (language === "ruby")
       return [
         `
 def `,
@@ -76423,7 +76394,7 @@ rescue `,
         " ",
         ""
       ];
-    } else if (language === "rust") {
+    else if (language === "rust")
       return [
         `
 fn `,
@@ -76451,7 +76422,7 @@ const `,
         " ",
         ""
       ];
-    } else if (language === "scala") {
+    else if (language === "scala")
       return [
         `
 class `,
@@ -76481,7 +76452,7 @@ case `,
         " ",
         ""
       ];
-    } else if (language === "swift") {
+    else if (language === "swift")
       return [
         `
 func `,
@@ -76511,7 +76482,7 @@ case `,
         " ",
         ""
       ];
-    } else if (language === "markdown") {
+    else if (language === "markdown")
       return [
         `
 ## `,
@@ -76547,7 +76518,7 @@ ___
         " ",
         ""
       ];
-    } else if (language === "latex") {
+    else if (language === "latex")
       return [
         `
 \\chapter{`,
@@ -76585,7 +76556,7 @@ ___
         " ",
         ""
       ];
-    } else if (language === "html") {
+    else if (language === "html")
       return [
         "<body>",
         "<div>",
@@ -76616,7 +76587,7 @@ ___
         " ",
         ""
       ];
-    } else if (language === "sol") {
+    else if (language === "sol")
       return [
         `
 pragma `,
@@ -76662,11 +76633,11 @@ assembly `,
         " ",
         ""
       ];
-    } else {
+    else
       throw new Error(`Language ${language} is not supported.`);
-    }
   }
-}
+};
+
 // src/functions/recursiveCharacterTextSplitter.ts
 async function recursiveCharacterTextSplitter(text, options3 = { chunkSize: 200, chunkOverlap: 60 }) {
   let textToSplit;
@@ -76931,4 +76902,4 @@ function trutoJsonata(expression) {
   return registerJsonataExtensions(import_jsonata.default(expression));
 }
 
-//# debugId=650A0F5DE8EB77ED64756E2164756E21
+//# debugId=90F868AC30527B0264756E2164756E21