@atom8n/n8n-nodes-langchain 2.2.0

package/dist/nodes/text_splitters/TextSplitterRecursiveCharacterTextSplitter/TextSplitterRecursiveCharacterTextSplitter.node.js

@@ -0,0 +1,139 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var TextSplitterRecursiveCharacterTextSplitter_node_exports = {};
+__export(TextSplitterRecursiveCharacterTextSplitter_node_exports, {
+  TextSplitterRecursiveCharacterTextSplitter: () => TextSplitterRecursiveCharacterTextSplitter
+});
+module.exports = __toCommonJS(TextSplitterRecursiveCharacterTextSplitter_node_exports);
+var import_textsplitters = require("@langchain/textsplitters");
+var import_n8n_workflow = require("n8n-workflow");
+var import_logWrapper = require("../../../utils/logWrapper");
+var import_sharedFields = require("../../../utils/sharedFields");
+const supportedLanguages = [
+  "cpp",
+  "go",
+  "java",
+  "js",
+  "php",
+  "proto",
+  "python",
+  "rst",
+  "ruby",
+  "rust",
+  "scala",
+  "swift",
+  "markdown",
+  "latex",
+  "html"
+];
+class TextSplitterRecursiveCharacterTextSplitter {
+  constructor() {
+    this.description = {
+      displayName: "Recursive Character Text Splitter",
+      name: "textSplitterRecursiveCharacterTextSplitter",
+      icon: "fa:grip-lines-vertical",
+      iconColor: "black",
+      group: ["transform"],
+      version: 1,
+      description: "Split text into chunks by characters recursively, recommended for most use cases",
+      defaults: {
+        name: "Recursive Character Text Splitter"
+      },
+      codex: {
+        categories: ["AI"],
+        subcategories: {
+          AI: ["Text Splitters"]
+        },
+        resources: {
+          primaryDocumentation: [
+            {
+              url: "https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.textsplitterrecursivecharactertextsplitter/"
+            }
+          ]
+        }
+      },
+      inputs: [],
+      outputs: [import_n8n_workflow.NodeConnectionTypes.AiTextSplitter],
+      outputNames: ["Text Splitter"],
+      properties: [
+        (0, import_sharedFields.getConnectionHintNoticeField)([import_n8n_workflow.NodeConnectionTypes.AiDocument]),
+        {
+          displayName: "Chunk Size",
+          name: "chunkSize",
+          type: "number",
+          default: 1e3
+        },
+        {
+          displayName: "Chunk Overlap",
+          name: "chunkOverlap",
+          type: "number",
+          default: 0
+        },
+        {
+          displayName: "Options",
+          name: "options",
+          placeholder: "Add Option",
+          description: "Additional options to add",
+          type: "collection",
+          default: {},
+          options: [
+            {
+              displayName: "Split Code",
+              name: "splitCode",
+              default: "markdown",
+              type: "options",
+              options: supportedLanguages.map((lang) => ({ name: lang, value: lang }))
+            }
+          ]
+        }
+      ]
+    };
+  }
+  async supplyData(itemIndex) {
+    this.logger.debug("Supply Data for Text Splitter");
+    const chunkSize = this.getNodeParameter("chunkSize", itemIndex);
+    const chunkOverlap = this.getNodeParameter("chunkOverlap", itemIndex);
+    const splitCode = this.getNodeParameter(
+      "options.splitCode",
+      itemIndex,
+      null
+    );
+    const params = {
+      // TODO: These are the default values, should we allow the user to change them?
+      separators: ["\n\n", "\n", " ", ""],
+      chunkSize,
+      chunkOverlap,
+      keepSeparator: false
+    };
+    let splitter;
+    if (splitCode && supportedLanguages.includes(splitCode)) {
+      splitter = import_textsplitters.RecursiveCharacterTextSplitter.fromLanguage(splitCode, params);
+    } else {
+      splitter = new import_textsplitters.RecursiveCharacterTextSplitter(params);
+    }
+    return {
+      response: (0, import_logWrapper.logWrapper)(splitter, this)
+    };
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  TextSplitterRecursiveCharacterTextSplitter
+});
+//# sourceMappingURL=TextSplitterRecursiveCharacterTextSplitter.node.js.map

package/dist/nodes/text_splitters/TextSplitterRecursiveCharacterTextSplitter/TextSplitterRecursiveCharacterTextSplitter.node.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../nodes/text_splitters/TextSplitterRecursiveCharacterTextSplitter/TextSplitterRecursiveCharacterTextSplitter.node.ts"],"sourcesContent":["import type {\n\tRecursiveCharacterTextSplitterParams,\n\tSupportedTextSplitterLanguage,\n} from '@langchain/textsplitters';\nimport { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';\nimport {\n\tNodeConnectionTypes,\n\ttype INodeType,\n\ttype INodeTypeDescription,\n\ttype ISupplyDataFunctions,\n\ttype SupplyData,\n} from 'n8n-workflow';\n\nimport { logWrapper } from '@utils/logWrapper';\nimport { getConnectionHintNoticeField } from '@utils/sharedFields';\n\nconst supportedLanguages: SupportedTextSplitterLanguage[] = [\n\t'cpp',\n\t'go',\n\t'java',\n\t'js',\n\t'php',\n\t'proto',\n\t'python',\n\t'rst',\n\t'ruby',\n\t'rust',\n\t'scala',\n\t'swift',\n\t'markdown',\n\t'latex',\n\t'html',\n];\nexport class TextSplitterRecursiveCharacterTextSplitter implements INodeType {\n\tdescription: INodeTypeDescription = {\n\t\tdisplayName: 'Recursive Character Text Splitter',\n\t\tname: 'textSplitterRecursiveCharacterTextSplitter',\n\t\ticon: 'fa:grip-lines-vertical',\n\t\ticonColor: 'black',\n\t\tgroup: ['transform'],\n\t\tversion: 1,\n\t\tdescription: 'Split text into chunks by characters recursively, recommended for most use cases',\n\t\tdefaults: {\n\t\t\tname: 'Recursive Character Text Splitter',\n\t\t},\n\t\tcodex: {\n\t\t\tcategories: ['AI'],\n\t\t\tsubcategories: {\n\t\t\t\tAI: ['Text Splitters'],\n\t\t\t},\n\t\t\tresources: {\n\t\t\t\tprimaryDocumentation: [\n\t\t\t\t\t{\n\t\t\t\t\t\turl: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.textsplitterrecursivecharactertextsplitter/',\n\t\t\t\t\t},\n\t\t\t\t],\n\t\t\t},\n\t\t},\n\n\t\tinputs: [],\n\n\t\toutputs: [NodeConnectionTypes.AiTextSplitter],\n\t\toutputNames: ['Text Splitter'],\n\t\tproperties: [\n\t\t\tgetConnectionHintNoticeField([NodeConnectionTypes.AiDocument]),\n\t\t\t{\n\t\t\t\tdisplayName: 'Chunk Size',\n\t\t\t\tname: 'chunkSize',\n\t\t\t\ttype: 'number',\n\t\t\t\tdefault: 1000,\n\t\t\t},\n\t\t\t{\n\t\t\t\tdisplayName: 'Chunk Overlap',\n\t\t\t\tname: 'chunkOverlap',\n\t\t\t\ttype: 'number',\n\t\t\t\tdefault: 0,\n\t\t\t},\n\t\t\t{\n\t\t\t\tdisplayName: 'Options',\n\t\t\t\tname: 'options',\n\t\t\t\tplaceholder: 'Add Option',\n\t\t\t\tdescription: 'Additional options to add',\n\t\t\t\ttype: 'collection',\n\t\t\t\tdefault: {},\n\t\t\t\toptions: [\n\t\t\t\t\t{\n\t\t\t\t\t\tdisplayName: 'Split Code',\n\t\t\t\t\t\tname: 'splitCode',\n\t\t\t\t\t\tdefault: 'markdown',\n\t\t\t\t\t\ttype: 'options',\n\t\t\t\t\t\toptions: supportedLanguages.map((lang) => ({ name: lang, value: lang })),\n\t\t\t\t\t},\n\t\t\t\t],\n\t\t\t},\n\t\t],\n\t};\n\n\tasync supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise<SupplyData> {\n\t\tthis.logger.debug('Supply Data for Text Splitter');\n\n\t\tconst chunkSize = this.getNodeParameter('chunkSize', itemIndex) as number;\n\t\tconst chunkOverlap = this.getNodeParameter('chunkOverlap', itemIndex) as number;\n\t\tconst splitCode = this.getNodeParameter(\n\t\t\t'options.splitCode',\n\t\t\titemIndex,\n\t\t\tnull,\n\t\t) as SupportedTextSplitterLanguage | null;\n\t\tconst params: RecursiveCharacterTextSplitterParams = {\n\t\t\t// TODO: These are the default values, should we allow the user to change them?\n\t\t\tseparators: ['\\n\\n', '\\n', ' ', ''],\n\t\t\tchunkSize,\n\t\t\tchunkOverlap,\n\t\t\tkeepSeparator: false,\n\t\t};\n\t\tlet splitter: RecursiveCharacterTextSplitter;\n\n\t\tif (splitCode && supportedLanguages.includes(splitCode)) {\n\t\t\tsplitter = RecursiveCharacterTextSplitter.fromLanguage(splitCode, params);\n\t\t} else {\n\t\t\tsplitter = new RecursiveCharacterTextSplitter(params);\n\t\t}\n\n\t\treturn {\n\t\t\tresponse: logWrapper(splitter, this),\n\t\t};\n\t}\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,2BAA+C;AAC/C,0BAMO;AAEP,wBAA2B;AAC3B,0BAA6C;AAE7C,MAAM,qBAAsD;AAAA,EAC3D;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACD;AACO,MAAM,2CAAgE;AAAA,EAAtE;AACN,uBAAoC;AAAA,MACnC,aAAa;AAAA,MACb,MAAM;AAAA,MACN,MAAM;AAAA,MACN,WAAW;AAAA,MACX,OAAO,CAAC,WAAW;AAAA,MACnB,SAAS;AAAA,MACT,aAAa;AAAA,MACb,UAAU;AAAA,QACT,MAAM;AAAA,MACP;AAAA,MACA,OAAO;AAAA,QACN,YAAY,CAAC,IAAI;AAAA,QACjB,eAAe;AAAA,UACd,IAAI,CAAC,gBAAgB;AAAA,QACtB;AAAA,QACA,WAAW;AAAA,UACV,sBAAsB;AAAA,YACrB;AAAA,cACC,KAAK;AAAA,YACN;AAAA,UACD;AAAA,QACD;AAAA,MACD;AAAA,MAEA,QAAQ,CAAC;AAAA,MAET,SAAS,CAAC,wCAAoB,cAAc;AAAA,MAC5C,aAAa,CAAC,eAAe;AAAA,MAC7B,YAAY;AAAA,YACX,kDAA6B,CAAC,wCAAoB,UAAU,CAAC;AAAA,QAC7D;AAAA,UACC,aAAa;AAAA,UACb,MAAM;AAAA,UACN,MAAM;AAAA,UACN,SAAS;AAAA,QACV;AAAA,QACA;AAAA,UACC,aAAa;AAAA,UACb,MAAM;AAAA,UACN,MAAM;AAAA,UACN,SAAS;AAAA,QACV;AAAA,QACA;AAAA,UACC,aAAa;AAAA,UACb,MAAM;AAAA,UACN,aAAa;AAAA,UACb,aAAa;AAAA,UACb,MAAM;AAAA,UACN,SAAS,CAAC;AAAA,UACV,SAAS;AAAA,YACR;AAAA,cACC,aAAa;AAAA,cACb,MAAM;AAAA,cACN,SAAS;AAAA,cACT,MAAM;AAAA,cACN,SAAS,mBAAmB,IAAI,CAAC,UAAU,EAAE,MAAM,MAAM,OAAO,KAAK,EAAE;AAAA,YACxE;AAAA,UACD;AAAA,QACD;AAAA,MACD;AAAA,IACD;AAAA;AAAA,EAEA,MAAM,WAAuC,WAAwC;AACpF,SAAK,OAAO,MAAM,+BAA+B;AAEjD,UAAM,YAAY,KAAK,iBAAiB,aAAa,SAAS;AAC9D,UAAM,eAAe,KAAK,iBAAiB,gBAAgB,SAAS;AACpE,UAAM,YAAY,KAAK;AAAA,MACtB;AAAA,MACA;AAAA,MACA;AAAA,IACD;AACA,UAAM,SAA+C;AAAA;AAAA,MAEpD,YAAY,CAAC,QAAQ,MAAM,KAAK,EAAE;AAAA,MAClC;AAAA,MACA;AAAA,MACA,eAAe;AAAA,IAChB;AACA,QAAI;AAEJ,QAAI,aAAa,mBAAmB,SAAS,SAAS,GAAG;AACxD,iBAAW,oDAA+B,aAAa,WAAW,MAAM;AAAA,IACzE,OAAO;AACN,iBAAW,IAAI,oDAA+B,MAAM;AAAA,IACrD;AAEA,WAAO;AAAA,MACN,cAAU,8BAAW,UAAU,IAAI;AAAA,IACpC;AAAA,EACD;AACD;","names":[]}

package/dist/nodes/text_splitters/TextSplitterTokenSplitter/TextSplitterTokenSplitter.node.js

@@ -0,0 +1,97 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var TextSplitterTokenSplitter_node_exports = {};
+__export(TextSplitterTokenSplitter_node_exports, {
+  TextSplitterTokenSplitter: () => TextSplitterTokenSplitter
+});
+module.exports = __toCommonJS(TextSplitterTokenSplitter_node_exports);
+var import_n8n_workflow = require("n8n-workflow");
+var import_logWrapper = require("../../../utils/logWrapper");
+var import_sharedFields = require("../../../utils/sharedFields");
+var import_TokenTextSplitter = require("./TokenTextSplitter");
+class TextSplitterTokenSplitter {
+  constructor() {
+    this.description = {
+      displayName: "Token Splitter",
+      name: "textSplitterTokenSplitter",
+      icon: "fa:grip-lines-vertical",
+      iconColor: "black",
+      group: ["transform"],
+      version: 1,
+      description: "Split text into chunks by tokens",
+      defaults: {
+        name: "Token Splitter"
+      },
+      codex: {
+        categories: ["AI"],
+        subcategories: {
+          AI: ["Text Splitters"]
+        },
+        resources: {
+          primaryDocumentation: [
+            {
+              url: "https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.textsplittertokensplitter/"
+            }
+          ]
+        }
+      },
+      inputs: [],
+      outputs: [import_n8n_workflow.NodeConnectionTypes.AiTextSplitter],
+      outputNames: ["Text Splitter"],
+      properties: [
+        (0, import_sharedFields.getConnectionHintNoticeField)([import_n8n_workflow.NodeConnectionTypes.AiDocument]),
+        {
+          displayName: "Chunk Size",
+          name: "chunkSize",
+          type: "number",
+          default: 1e3,
+          description: "Maximum number of tokens per chunk"
+        },
+        {
+          displayName: "Chunk Overlap",
+          name: "chunkOverlap",
+          type: "number",
+          default: 0,
+          description: "Number of tokens shared between consecutive chunks to preserve context"
+        }
+      ]
+    };
+  }
+  async supplyData(itemIndex) {
+    this.logger.debug("Supply Data for Text Splitter");
+    const chunkSize = this.getNodeParameter("chunkSize", itemIndex);
+    const chunkOverlap = this.getNodeParameter("chunkOverlap", itemIndex);
+    const splitter = new import_TokenTextSplitter.TokenTextSplitter({
+      chunkSize,
+      chunkOverlap,
+      allowedSpecial: "all",
+      disallowedSpecial: "all",
+      encodingName: "cl100k_base",
+      keepSeparator: false
+    });
+    return {
+      response: (0, import_logWrapper.logWrapper)(splitter, this)
+    };
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  TextSplitterTokenSplitter
+});
+//# sourceMappingURL=TextSplitterTokenSplitter.node.js.map

package/dist/nodes/text_splitters/TextSplitterTokenSplitter/TextSplitterTokenSplitter.node.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../nodes/text_splitters/TextSplitterTokenSplitter/TextSplitterTokenSplitter.node.ts"],"sourcesContent":["import {\n\tNodeConnectionTypes,\n\ttype INodeType,\n\ttype INodeTypeDescription,\n\ttype ISupplyDataFunctions,\n\ttype SupplyData,\n} from 'n8n-workflow';\n\nimport { logWrapper } from '@utils/logWrapper';\nimport { getConnectionHintNoticeField } from '@utils/sharedFields';\n\nimport { TokenTextSplitter } from './TokenTextSplitter';\n\nexport class TextSplitterTokenSplitter implements INodeType {\n\tdescription: INodeTypeDescription = {\n\t\tdisplayName: 'Token Splitter',\n\t\tname: 'textSplitterTokenSplitter',\n\t\ticon: 'fa:grip-lines-vertical',\n\t\ticonColor: 'black',\n\t\tgroup: ['transform'],\n\t\tversion: 1,\n\t\tdescription: 'Split text into chunks by tokens',\n\t\tdefaults: {\n\t\t\tname: 'Token Splitter',\n\t\t},\n\t\tcodex: {\n\t\t\tcategories: ['AI'],\n\t\t\tsubcategories: {\n\t\t\t\tAI: ['Text Splitters'],\n\t\t\t},\n\t\t\tresources: {\n\t\t\t\tprimaryDocumentation: [\n\t\t\t\t\t{\n\t\t\t\t\t\turl: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.textsplittertokensplitter/',\n\t\t\t\t\t},\n\t\t\t\t],\n\t\t\t},\n\t\t},\n\n\t\tinputs: [],\n\n\t\toutputs: [NodeConnectionTypes.AiTextSplitter],\n\t\toutputNames: ['Text Splitter'],\n\t\tproperties: [\n\t\t\tgetConnectionHintNoticeField([NodeConnectionTypes.AiDocument]),\n\t\t\t{\n\t\t\t\tdisplayName: 'Chunk Size',\n\t\t\t\tname: 'chunkSize',\n\t\t\t\ttype: 'number',\n\t\t\t\tdefault: 1000,\n\t\t\t\tdescription: 'Maximum number of tokens per chunk',\n\t\t\t},\n\t\t\t{\n\t\t\t\tdisplayName: 'Chunk Overlap',\n\t\t\t\tname: 'chunkOverlap',\n\t\t\t\ttype: 'number',\n\t\t\t\tdefault: 0,\n\t\t\t\tdescription: 'Number of tokens shared between consecutive chunks to preserve context',\n\t\t\t},\n\t\t],\n\t};\n\n\tasync supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise<SupplyData> {\n\t\tthis.logger.debug('Supply Data for Text Splitter');\n\n\t\tconst chunkSize = this.getNodeParameter('chunkSize', itemIndex) as number;\n\t\tconst chunkOverlap = this.getNodeParameter('chunkOverlap', itemIndex) as number;\n\n\t\tconst splitter = new TokenTextSplitter({\n\t\t\tchunkSize,\n\t\t\tchunkOverlap,\n\t\t\tallowedSpecial: 'all',\n\t\t\tdisallowedSpecial: 'all',\n\t\t\tencodingName: 'cl100k_base',\n\t\t\tkeepSeparator: false,\n\t\t});\n\n\t\treturn {\n\t\t\tresponse: logWrapper(splitter, this),\n\t\t};\n\t}\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,0BAMO;AAEP,wBAA2B;AAC3B,0BAA6C;AAE7C,+BAAkC;AAE3B,MAAM,0BAA+C;AAAA,EAArD;AACN,uBAAoC;AAAA,MACnC,aAAa;AAAA,MACb,MAAM;AAAA,MACN,MAAM;AAAA,MACN,WAAW;AAAA,MACX,OAAO,CAAC,WAAW;AAAA,MACnB,SAAS;AAAA,MACT,aAAa;AAAA,MACb,UAAU;AAAA,QACT,MAAM;AAAA,MACP;AAAA,MACA,OAAO;AAAA,QACN,YAAY,CAAC,IAAI;AAAA,QACjB,eAAe;AAAA,UACd,IAAI,CAAC,gBAAgB;AAAA,QACtB;AAAA,QACA,WAAW;AAAA,UACV,sBAAsB;AAAA,YACrB;AAAA,cACC,KAAK;AAAA,YACN;AAAA,UACD;AAAA,QACD;AAAA,MACD;AAAA,MAEA,QAAQ,CAAC;AAAA,MAET,SAAS,CAAC,wCAAoB,cAAc;AAAA,MAC5C,aAAa,CAAC,eAAe;AAAA,MAC7B,YAAY;AAAA,YACX,kDAA6B,CAAC,wCAAoB,UAAU,CAAC;AAAA,QAC7D;AAAA,UACC,aAAa;AAAA,UACb,MAAM;AAAA,UACN,MAAM;AAAA,UACN,SAAS;AAAA,UACT,aAAa;AAAA,QACd;AAAA,QACA;AAAA,UACC,aAAa;AAAA,UACb,MAAM;AAAA,UACN,MAAM;AAAA,UACN,SAAS;AAAA,UACT,aAAa;AAAA,QACd;AAAA,MACD;AAAA,IACD;AAAA;AAAA,EAEA,MAAM,WAAuC,WAAwC;AACpF,SAAK,OAAO,MAAM,+BAA+B;AAEjD,UAAM,YAAY,KAAK,iBAAiB,aAAa,SAAS;AAC9D,UAAM,eAAe,KAAK,iBAAiB,gBAAgB,SAAS;AAEpE,UAAM,WAAW,IAAI,2CAAkB;AAAA,MACtC;AAAA,MACA;AAAA,MACA,gBAAgB;AAAA,MAChB,mBAAmB;AAAA,MACnB,cAAc;AAAA,MACd,eAAe;AAAA,IAChB,CAAC;AAED,WAAO;AAAA,MACN,cAAU,8BAAW,UAAU,IAAI;AAAA,IACpC;AAAA,EACD;AACD;","names":[]}

package/dist/nodes/text_splitters/TextSplitterTokenSplitter/TokenTextSplitter.js

@@ -0,0 +1,86 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var TokenTextSplitter_exports = {};
+__export(TokenTextSplitter_exports, {
+  TokenTextSplitter: () => TokenTextSplitter
+});
+module.exports = __toCommonJS(TokenTextSplitter_exports);
+var import_textsplitters = require("@langchain/textsplitters");
+var import_helpers = require("../../../utils/helpers");
+var import_tiktoken = require("../../../utils/tokenizer/tiktoken");
+var import_token_estimator = require("../../../utils/tokenizer/token-estimator");
+class TokenTextSplitter extends import_textsplitters.TextSplitter {
+  static lc_name() {
+    return "TokenTextSplitter";
+  }
+  constructor(fields) {
+    super(fields);
+    this.encodingName = fields?.encodingName ?? "cl100k_base";
+    this.allowedSpecial = fields?.allowedSpecial ?? [];
+    this.disallowedSpecial = fields?.disallowedSpecial ?? "all";
+  }
+  async splitText(text) {
+    try {
+      if (!text || typeof text !== "string") {
+        return [];
+      }
+      if ((0, import_helpers.hasLongSequentialRepeat)(text)) {
+        const splits = (0, import_token_estimator.estimateTextSplitsByTokens)(
+          text,
+          this.chunkSize,
+          this.chunkOverlap,
+          this.encodingName
+        );
+        return splits;
+      }
+      try {
+        this.tokenizer ??= await (0, import_tiktoken.getEncoding)(this.encodingName);
+        const splits = [];
+        const input_ids = this.tokenizer.encode(text, this.allowedSpecial, this.disallowedSpecial);
+        let start_idx = 0;
+        let chunkCount = 0;
+        while (start_idx < input_ids.length) {
+          if (start_idx > 0) {
+            start_idx = Math.max(0, start_idx - this.chunkOverlap);
+          }
+          const end_idx = Math.min(start_idx + this.chunkSize, input_ids.length);
+          const chunk_ids = input_ids.slice(start_idx, end_idx);
+          splits.push(this.tokenizer.decode(chunk_ids));
+          chunkCount++;
+          start_idx = end_idx;
+        }
+        return splits;
+      } catch (tiktokenError) {
+        return (0, import_token_estimator.estimateTextSplitsByTokens)(
+          text,
+          this.chunkSize,
+          this.chunkOverlap,
+          this.encodingName
+        );
+      }
+    } catch (error) {
+      return [];
+    }
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  TokenTextSplitter
+});
+//# sourceMappingURL=TokenTextSplitter.js.map

package/dist/nodes/text_splitters/TextSplitterTokenSplitter/TokenTextSplitter.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../nodes/text_splitters/TextSplitterTokenSplitter/TokenTextSplitter.ts"],"sourcesContent":["import type { TokenTextSplitterParams } from '@langchain/textsplitters';\nimport { TextSplitter } from '@langchain/textsplitters';\nimport { hasLongSequentialRepeat } from '@utils/helpers';\nimport { getEncoding } from '@utils/tokenizer/tiktoken';\nimport { estimateTextSplitsByTokens } from '@utils/tokenizer/token-estimator';\nimport type * as tiktoken from 'js-tiktoken';\n\n/**\n * Implementation of splitter which looks at tokens.\n * This is override of the LangChain TokenTextSplitter\n * to use the n8n tokenizer utility which uses local JSON encodings\n */\nexport class TokenTextSplitter extends TextSplitter implements TokenTextSplitterParams {\n\tstatic lc_name() {\n\t\treturn 'TokenTextSplitter';\n\t}\n\n\tencodingName: tiktoken.TiktokenEncoding;\n\n\tallowedSpecial: 'all' | string[];\n\n\tdisallowedSpecial: 'all' | string[];\n\n\tprivate tokenizer: tiktoken.Tiktoken | undefined;\n\n\tconstructor(fields?: Partial<TokenTextSplitterParams>) {\n\t\tsuper(fields);\n\n\t\tthis.encodingName = fields?.encodingName ?? 'cl100k_base';\n\t\tthis.allowedSpecial = fields?.allowedSpecial ?? [];\n\t\tthis.disallowedSpecial = fields?.disallowedSpecial ?? 'all';\n\t}\n\n\tasync splitText(text: string): Promise<string[]> {\n\t\ttry {\n\t\t\t// Validate input\n\t\t\tif (!text || typeof text !== 'string') {\n\t\t\t\treturn [];\n\t\t\t}\n\n\t\t\t// Check for repetitive content\n\t\t\tif (hasLongSequentialRepeat(text)) {\n\t\t\t\tconst splits = estimateTextSplitsByTokens(\n\t\t\t\t\ttext,\n\t\t\t\t\tthis.chunkSize,\n\t\t\t\t\tthis.chunkOverlap,\n\t\t\t\t\tthis.encodingName,\n\t\t\t\t);\n\t\t\t\treturn splits;\n\t\t\t}\n\n\t\t\t// Use tiktoken for normal text\n\t\t\ttry {\n\t\t\t\tthis.tokenizer ??= await getEncoding(this.encodingName);\n\n\t\t\t\tconst splits: string[] = [];\n\t\t\t\tconst input_ids = this.tokenizer.encode(text, this.allowedSpecial, this.disallowedSpecial);\n\n\t\t\t\tlet start_idx = 0;\n\t\t\t\tlet chunkCount = 0;\n\n\t\t\t\twhile (start_idx < input_ids.length) {\n\t\t\t\t\tif (start_idx > 0) {\n\t\t\t\t\t\tstart_idx = Math.max(0, start_idx - this.chunkOverlap);\n\t\t\t\t\t}\n\t\t\t\t\tconst end_idx = Math.min(start_idx + this.chunkSize, input_ids.length);\n\t\t\t\t\tconst chunk_ids = input_ids.slice(start_idx, end_idx);\n\n\t\t\t\t\tsplits.push(this.tokenizer.decode(chunk_ids));\n\n\t\t\t\t\tchunkCount++;\n\t\t\t\t\tstart_idx = end_idx;\n\t\t\t\t}\n\n\t\t\t\treturn splits;\n\t\t\t} catch (tiktokenError) {\n\t\t\t\t// Fall back to character-based splitting if tiktoken fails\n\t\t\t\treturn estimateTextSplitsByTokens(\n\t\t\t\t\ttext,\n\t\t\t\t\tthis.chunkSize,\n\t\t\t\t\tthis.chunkOverlap,\n\t\t\t\t\tthis.encodingName,\n\t\t\t\t);\n\t\t\t}\n\t\t} catch (error) {\n\t\t\t// Return empty array on complete failure\n\t\t\treturn [];\n\t\t}\n\t}\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AACA,2BAA6B;AAC7B,qBAAwC;AACxC,sBAA4B;AAC5B,6BAA2C;AAQpC,MAAM,0BAA0B,kCAAgD;AAAA,EACtF,OAAO,UAAU;AAChB,WAAO;AAAA,EACR;AAAA,EAUA,YAAY,QAA2C;AACtD,UAAM,MAAM;AAEZ,SAAK,eAAe,QAAQ,gBAAgB;AAC5C,SAAK,iBAAiB,QAAQ,kBAAkB,CAAC;AACjD,SAAK,oBAAoB,QAAQ,qBAAqB;AAAA,EACvD;AAAA,EAEA,MAAM,UAAU,MAAiC;AAChD,QAAI;AAEH,UAAI,CAAC,QAAQ,OAAO,SAAS,UAAU;AACtC,eAAO,CAAC;AAAA,MACT;AAGA,cAAI,wCAAwB,IAAI,GAAG;AAClC,cAAM,aAAS;AAAA,UACd;AAAA,UACA,KAAK;AAAA,UACL,KAAK;AAAA,UACL,KAAK;AAAA,QACN;AACA,eAAO;AAAA,MACR;AAGA,UAAI;AACH,aAAK,cAAc,UAAM,6BAAY,KAAK,YAAY;AAEtD,cAAM,SAAmB,CAAC;AAC1B,cAAM,YAAY,KAAK,UAAU,OAAO,MAAM,KAAK,gBAAgB,KAAK,iBAAiB;AAEzF,YAAI,YAAY;AAChB,YAAI,aAAa;AAEjB,eAAO,YAAY,UAAU,QAAQ;AACpC,cAAI,YAAY,GAAG;AAClB,wBAAY,KAAK,IAAI,GAAG,YAAY,KAAK,YAAY;AAAA,UACtD;AACA,gBAAM,UAAU,KAAK,IAAI,YAAY,KAAK,WAAW,UAAU,MAAM;AACrE,gBAAM,YAAY,UAAU,MAAM,WAAW,OAAO;AAEpD,iBAAO,KAAK,KAAK,UAAU,OAAO,SAAS,CAAC;AAE5C;AACA,sBAAY;AAAA,QACb;AAEA,eAAO;AAAA,MACR,SAAS,eAAe;AAEvB,mBAAO;AAAA,UACN;AAAA,UACA,KAAK;AAAA,UACL,KAAK;AAAA,UACL,KAAK;AAAA,QACN;AAAA,MACD;AAAA,IACD,SAAS,OAAO;AAEf,aAAO,CAAC;AAAA,IACT;AAAA,EACD;AACD;","names":[]}

package/dist/nodes/tools/ToolCalculator/ToolCalculator.node.js

@@ -0,0 +1,94 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var ToolCalculator_node_exports = {};
+__export(ToolCalculator_node_exports, {
+  ToolCalculator: () => ToolCalculator
+});
+module.exports = __toCommonJS(ToolCalculator_node_exports);
+var import_calculator = require("@langchain/community/tools/calculator");
+var import_n8n_workflow = require("n8n-workflow");
+var import_logWrapper = require("../../../utils/logWrapper");
+var import_sharedFields = require("../../../utils/sharedFields");
+function getTool(ctx) {
+  const calculator = new import_calculator.Calculator();
+  calculator.name = ctx.getNode().name;
+  return calculator;
+}
+class ToolCalculator {
+  constructor() {
+    this.description = {
+      displayName: "Calculator",
+      name: "toolCalculator",
+      icon: "fa:calculator",
+      iconColor: "black",
+      group: ["transform"],
+      version: 1,
+      description: "Make it easier for AI agents to perform arithmetic",
+      defaults: {
+        name: "Calculator"
+      },
+      codex: {
+        categories: ["AI"],
+        subcategories: {
+          AI: ["Tools"],
+          Tools: ["Other Tools"]
+        },
+        resources: {
+          primaryDocumentation: [
+            {
+              url: "https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.toolcalculator/"
+            }
+          ]
+        }
+      },
+      inputs: [],
+      outputs: [import_n8n_workflow.NodeConnectionTypes.AiTool],
+      outputNames: ["Tool"],
+      properties: [(0, import_sharedFields.getConnectionHintNoticeField)([import_n8n_workflow.NodeConnectionTypes.AiAgent])]
+    };
+  }
+  async supplyData() {
+    return {
+      response: (0, import_logWrapper.logWrapper)(getTool(this), this)
+    };
+  }
+  async execute() {
+    const calculator = getTool(this);
+    const input = this.getInputData();
+    const response = [];
+    for (let i = 0; i < input.length; i++) {
+      const inputItem = input[i];
+      const result = await calculator.invoke(inputItem.json);
+      response.push({
+        json: {
+          response: result
+        },
+        pairedItem: {
+          item: i
+        }
+      });
+    }
+    return [response];
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  ToolCalculator
+});
+//# sourceMappingURL=ToolCalculator.node.js.map

package/dist/nodes/tools/ToolCalculator/ToolCalculator.node.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../nodes/tools/ToolCalculator/ToolCalculator.node.ts"],"sourcesContent":["import { Calculator } from '@langchain/community/tools/calculator';\nimport {\n\ttype IExecuteFunctions,\n\ttype INodeExecutionData,\n\tNodeConnectionTypes,\n\ttype INodeType,\n\ttype INodeTypeDescription,\n\ttype ISupplyDataFunctions,\n\ttype SupplyData,\n} from 'n8n-workflow';\n\nimport { logWrapper } from '@utils/logWrapper';\nimport { getConnectionHintNoticeField } from '@utils/sharedFields';\n\nfunction getTool(ctx: ISupplyDataFunctions | IExecuteFunctions): Calculator {\n\tconst calculator = new Calculator();\n\tcalculator.name = ctx.getNode().name;\n\treturn calculator;\n}\n\nexport class ToolCalculator implements INodeType {\n\tdescription: INodeTypeDescription = {\n\t\tdisplayName: 'Calculator',\n\t\tname: 'toolCalculator',\n\t\ticon: 'fa:calculator',\n\t\ticonColor: 'black',\n\t\tgroup: ['transform'],\n\t\tversion: 1,\n\t\tdescription: 'Make it easier for AI agents to perform arithmetic',\n\t\tdefaults: {\n\t\t\tname: 'Calculator',\n\t\t},\n\t\tcodex: {\n\t\t\tcategories: ['AI'],\n\t\t\tsubcategories: {\n\t\t\t\tAI: ['Tools'],\n\t\t\t\tTools: ['Other Tools'],\n\t\t\t},\n\t\t\tresources: {\n\t\t\t\tprimaryDocumentation: [\n\t\t\t\t\t{\n\t\t\t\t\t\turl: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.toolcalculator/',\n\t\t\t\t\t},\n\t\t\t\t],\n\t\t\t},\n\t\t},\n\n\t\tinputs: [],\n\n\t\toutputs: [NodeConnectionTypes.AiTool],\n\t\toutputNames: ['Tool'],\n\t\tproperties: [getConnectionHintNoticeField([NodeConnectionTypes.AiAgent])],\n\t};\n\n\tasync supplyData(this: ISupplyDataFunctions): Promise<SupplyData> {\n\t\treturn {\n\t\t\tresponse: logWrapper(getTool(this), this),\n\t\t};\n\t}\n\n\tasync execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {\n\t\tconst calculator = getTool(this);\n\t\tconst input = this.getInputData();\n\t\tconst response: INodeExecutionData[] = [];\n\t\tfor (let i = 0; i < input.length; i++) {\n\t\t\tconst inputItem = input[i];\n\t\t\tconst result = await calculator.invoke(inputItem.json);\n\t\t\tresponse.push({\n\t\t\t\tjson: {\n\t\t\t\t\tresponse: result,\n\t\t\t\t},\n\t\t\t\tpairedItem: {\n\t\t\t\t\titem: i,\n\t\t\t\t},\n\t\t\t});\n\t\t}\n\n\t\treturn [response];\n\t}\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,wBAA2B;AAC3B,0BAQO;AAEP,wBAA2B;AAC3B,0BAA6C;AAE7C,SAAS,QAAQ,KAA2D;AAC3E,QAAM,aAAa,IAAI,6BAAW;AAClC,aAAW,OAAO,IAAI,QAAQ,EAAE;AAChC,SAAO;AACR;AAEO,MAAM,eAAoC;AAAA,EAA1C;AACN,uBAAoC;AAAA,MACnC,aAAa;AAAA,MACb,MAAM;AAAA,MACN,MAAM;AAAA,MACN,WAAW;AAAA,MACX,OAAO,CAAC,WAAW;AAAA,MACnB,SAAS;AAAA,MACT,aAAa;AAAA,MACb,UAAU;AAAA,QACT,MAAM;AAAA,MACP;AAAA,MACA,OAAO;AAAA,QACN,YAAY,CAAC,IAAI;AAAA,QACjB,eAAe;AAAA,UACd,IAAI,CAAC,OAAO;AAAA,UACZ,OAAO,CAAC,aAAa;AAAA,QACtB;AAAA,QACA,WAAW;AAAA,UACV,sBAAsB;AAAA,YACrB;AAAA,cACC,KAAK;AAAA,YACN;AAAA,UACD;AAAA,QACD;AAAA,MACD;AAAA,MAEA,QAAQ,CAAC;AAAA,MAET,SAAS,CAAC,wCAAoB,MAAM;AAAA,MACpC,aAAa,CAAC,MAAM;AAAA,MACpB,YAAY,KAAC,kDAA6B,CAAC,wCAAoB,OAAO,CAAC,CAAC;AAAA,IACzE;AAAA;AAAA,EAEA,MAAM,aAA4D;AACjE,WAAO;AAAA,MACN,cAAU,8BAAW,QAAQ,IAAI,GAAG,IAAI;AAAA,IACzC;AAAA,EACD;AAAA,EAEA,MAAM,UAAkE;AACvE,UAAM,aAAa,QAAQ,IAAI;AAC/B,UAAM,QAAQ,KAAK,aAAa;AAChC,UAAM,WAAiC,CAAC;AACxC,aAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACtC,YAAM,YAAY,MAAM,CAAC;AACzB,YAAM,SAAS,MAAM,WAAW,OAAO,UAAU,IAAI;AACrD,eAAS,KAAK;AAAA,QACb,MAAM;AAAA,UACL,UAAU;AAAA,QACX;AAAA,QACA,YAAY;AAAA,UACX,MAAM;AAAA,QACP;AAAA,MACD,CAAC;AAAA,IACF;AAEA,WAAO,CAAC,QAAQ;AAAA,EACjB;AACD;","names":[]}