@n8n/n8n-nodes-langchain 1.108.0 → 1.109.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (12) hide show
  1. package/dist/nodes/text_splitters/TextSplitterCharacterTextSplitter/TextSplitterCharacterTextSplitter.node.js +4 -2
  2. package/dist/nodes/text_splitters/TextSplitterCharacterTextSplitter/TextSplitterCharacterTextSplitter.node.js.map +1 -1
  3. package/dist/nodes/text_splitters/TextSplitterTokenSplitter/TextSplitterTokenSplitter.node.js +4 -2
  4. package/dist/nodes/text_splitters/TextSplitterTokenSplitter/TextSplitterTokenSplitter.node.js.map +1 -1
  5. package/dist/nodes/text_splitters/TextSplitterTokenSplitter/TokenTextSplitter.js +1 -1
  6. package/dist/nodes/text_splitters/TextSplitterTokenSplitter/TokenTextSplitter.js.map +1 -1
  7. package/dist/types/nodes.json +2 -2
  8. package/dist/utils/tokenizer/tiktoken.js +26 -22
  9. package/dist/utils/tokenizer/tiktoken.js.map +1 -1
  10. package/dist/utils/tokenizer/token-estimator.js +1 -1
  11. package/dist/utils/tokenizer/token-estimator.js.map +1 -1
  12. package/package.json +6 -6
package/dist/nodes/text_splitters/TextSplitterCharacterTextSplitter/TextSplitterCharacterTextSplitter.node.js CHANGED
@@ -66,13 +66,15 @@ class TextSplitterCharacterTextSplitter {
66
66
  displayName: "Chunk Size",
67
67
  name: "chunkSize",
68
68
  type: "number",
69
- default: 1e3
69
+ default: 1e3,
70
+ description: "Maximum number of characters per chunk"
70
71
  },
71
72
  {
72
73
  displayName: "Chunk Overlap",
73
74
  name: "chunkOverlap",
74
75
  type: "number",
75
- default: 0
76
+ default: 0,
77
+ description: "Number of characters shared between consecutive chunks to preserve context"
76
78
  }
77
79
  ]
78
80
  };
package/dist/nodes/text_splitters/TextSplitterCharacterTextSplitter/TextSplitterCharacterTextSplitter.node.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../../../nodes/text_splitters/TextSplitterCharacterTextSplitter/TextSplitterCharacterTextSplitter.node.ts"],"sourcesContent":["import type { CharacterTextSplitterParams } from '@langchain/textsplitters';\nimport { CharacterTextSplitter } from '@langchain/textsplitters';\nimport {\n\tNodeConnectionTypes,\n\ttype INodeType,\n\ttype INodeTypeDescription,\n\ttype ISupplyDataFunctions,\n\ttype SupplyData,\n} from 'n8n-workflow';\n\nimport { logWrapper } from '@utils/logWrapper';\nimport { getConnectionHintNoticeField } from '@utils/sharedFields';\n\nexport class TextSplitterCharacterTextSplitter implements INodeType {\n\tdescription: INodeTypeDescription = {\n\t\tdisplayName: 'Character Text Splitter',\n\t\tname: 'textSplitterCharacterTextSplitter',\n\t\ticon: 'fa:grip-lines-vertical',\n\t\ticonColor: 'black',\n\t\tgroup: ['transform'],\n\t\tversion: 1,\n\t\tdescription: 'Split text into chunks by characters',\n\t\tdefaults: {\n\t\t\tname: 'Character Text Splitter',\n\t\t},\n\t\tcodex: {\n\t\t\tcategories: ['AI'],\n\t\t\tsubcategories: {\n\t\t\t\tAI: ['Text Splitters'],\n\t\t\t},\n\t\t\tresources: {\n\t\t\t\tprimaryDocumentation: [\n\t\t\t\t\t{\n\t\t\t\t\t\turl: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.textsplittercharactertextsplitter/',\n\t\t\t\t\t},\n\t\t\t\t],\n\t\t\t},\n\t\t},\n\n\t\tinputs: [],\n\n\t\toutputs: [NodeConnectionTypes.AiTextSplitter],\n\t\toutputNames: ['Text Splitter'],\n\t\tproperties: [\n\t\t\tgetConnectionHintNoticeField([NodeConnectionTypes.AiDocument]),\n\t\t\t{\n\t\t\t\tdisplayName: 'Separator',\n\t\t\t\tname: 'separator',\n\t\t\t\ttype: 'string',\n\t\t\t\tdefault: '',\n\t\t\t},\n\t\t\t{\n\t\t\t\tdisplayName: 'Chunk Size',\n\t\t\t\tname: 'chunkSize',\n\t\t\t\ttype: 'number',\n\t\t\t\tdefault: 1000,\n\t\t\t},\n\t\t\t{\n\t\t\t\tdisplayName: 'Chunk Overlap',\n\t\t\t\tname: 'chunkOverlap',\n\t\t\t\ttype: 'number',\n\t\t\t\tdefault: 0,\n\t\t\t},\n\t\t],\n\t};\n\n\tasync supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise<SupplyData> {\n\t\tthis.logger.debug('Supply Data for Text Splitter');\n\n\t\tconst separator = this.getNodeParameter('separator', itemIndex) as string;\n\t\tconst chunkSize = this.getNodeParameter('chunkSize', itemIndex) as number;\n\t\tconst chunkOverlap = this.getNodeParameter('chunkOverlap', itemIndex) as number;\n\n\t\tconst params: CharacterTextSplitterParams = {\n\t\t\tseparator,\n\t\t\tchunkSize,\n\t\t\tchunkOverlap,\n\t\t\tkeepSeparator: false,\n\t\t};\n\n\t\tconst splitter = new CharacterTextSplitter(params);\n\n\t\treturn {\n\t\t\tresponse: logWrapper(splitter, this),\n\t\t};\n\t}\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AACA,2BAAsC;AACtC,0BAMO;AAEP,wBAA2B;AAC3B,0BAA6C;AAEtC,MAAM,kCAAuD;AAAA,EAA7D;AACN,uBAAoC;AAAA,MACnC,aAAa;AAAA,MACb,MAAM;AAAA,MACN,MAAM;AAAA,MACN,WAAW;AAAA,MACX,OAAO,CAAC,WAAW;AAAA,MACnB,SAAS;AAAA,MACT,aAAa;AAAA,MACb,UAAU;AAAA,QACT,MAAM;AAAA,MACP;AAAA,MACA,OAAO;AAAA,QACN,YAAY,CAAC,IAAI;AAAA,QACjB,eAAe;AAAA,UACd,IAAI,CAAC,gBAAgB;AAAA,QACtB;AAAA,QACA,WAAW;AAAA,UACV,sBAAsB;AAAA,YACrB;AAAA,cACC,KAAK;AAAA,YACN;AAAA,UACD;AAAA,QACD;AAAA,MACD;AAAA,MAEA,QAAQ,CAAC;AAAA,MAET,SAAS,CAAC,wCAAoB,cAAc;AAAA,MAC5C,aAAa,CAAC,eAAe;AAAA,MAC7B,YAAY;AAAA,YACX,kDAA6B,CAAC,wCAAoB,UAAU,CAAC;AAAA,QAC7D;AAAA,UACC,aAAa;AAAA,UACb,MAAM;AAAA,UACN,MAAM;AAAA,UACN,SAAS;AAAA,QACV;AAAA,QACA;AAAA,UACC,aAAa;AAAA,UACb,MAAM;AAAA,UACN,MAAM;AAAA,UACN,SAAS;AAAA,QACV;AAAA,QACA;AAAA,UACC,aAAa;AAAA,UACb,MAAM;AAAA,UACN,MAAM;AAAA,UACN,SAAS;AAAA,QACV;AAAA,MACD;AAAA,IACD;AAAA;AAAA,EAEA,MAAM,WAAuC,WAAwC;AACpF,SAAK,OAAO,MAAM,+BAA+B;AAEjD,UAAM,YAAY,KAAK,iBAAiB,aAAa,SAAS;AAC9D,UAAM,YAAY,KAAK,iBAAiB,aAAa,SAAS;AAC9D,UAAM,eAAe,KAAK,iBAAiB,gBAAgB,SAAS;AAEpE,UAAM,SAAsC;AAAA,MAC3C;AAAA,MACA;AAAA,MACA;AAAA,MACA,eAAe;AAAA,IAChB;AAEA,UAAM,WAAW,IAAI,2CAAsB,MAAM;AAEjD,WAAO;AAAA,MACN,cAAU,8BAAW,UAAU,IAAI;AAAA,IACpC;AAAA,EACD;AACD;","names":[]}
1
+ {"version":3,"sources":["../../../../nodes/text_splitters/TextSplitterCharacterTextSplitter/TextSplitterCharacterTextSplitter.node.ts"],"sourcesContent":["import type { CharacterTextSplitterParams } from '@langchain/textsplitters';\nimport { CharacterTextSplitter } from '@langchain/textsplitters';\nimport {\n\tNodeConnectionTypes,\n\ttype INodeType,\n\ttype INodeTypeDescription,\n\ttype ISupplyDataFunctions,\n\ttype SupplyData,\n} from 'n8n-workflow';\n\nimport { logWrapper } from '@utils/logWrapper';\nimport { getConnectionHintNoticeField } from '@utils/sharedFields';\n\nexport class TextSplitterCharacterTextSplitter implements INodeType {\n\tdescription: INodeTypeDescription = {\n\t\tdisplayName: 'Character Text Splitter',\n\t\tname: 'textSplitterCharacterTextSplitter',\n\t\ticon: 'fa:grip-lines-vertical',\n\t\ticonColor: 'black',\n\t\tgroup: ['transform'],\n\t\tversion: 1,\n\t\tdescription: 'Split text into chunks by characters',\n\t\tdefaults: {\n\t\t\tname: 'Character Text Splitter',\n\t\t},\n\t\tcodex: {\n\t\t\tcategories: ['AI'],\n\t\t\tsubcategories: {\n\t\t\t\tAI: ['Text Splitters'],\n\t\t\t},\n\t\t\tresources: {\n\t\t\t\tprimaryDocumentation: [\n\t\t\t\t\t{\n\t\t\t\t\t\turl: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.textsplittercharactertextsplitter/',\n\t\t\t\t\t},\n\t\t\t\t],\n\t\t\t},\n\t\t},\n\n\t\tinputs: [],\n\n\t\toutputs: [NodeConnectionTypes.AiTextSplitter],\n\t\toutputNames: ['Text Splitter'],\n\t\tproperties: [\n\t\t\tgetConnectionHintNoticeField([NodeConnectionTypes.AiDocument]),\n\t\t\t{\n\t\t\t\tdisplayName: 'Separator',\n\t\t\t\tname: 'separator',\n\t\t\t\ttype: 'string',\n\t\t\t\tdefault: '',\n\t\t\t},\n\t\t\t{\n\t\t\t\tdisplayName: 'Chunk Size',\n\t\t\t\tname: 'chunkSize',\n\t\t\t\ttype: 'number',\n\t\t\t\tdefault: 1000,\n\t\t\t\tdescription: 'Maximum number of characters per chunk',\n\t\t\t},\n\t\t\t{\n\t\t\t\tdisplayName: 'Chunk Overlap',\n\t\t\t\tname: 'chunkOverlap',\n\t\t\t\ttype: 'number',\n\t\t\t\tdefault: 0,\n\t\t\t\tdescription: 'Number of characters shared between consecutive chunks to preserve context',\n\t\t\t},\n\t\t],\n\t};\n\n\tasync supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise<SupplyData> {\n\t\tthis.logger.debug('Supply Data for Text Splitter');\n\n\t\tconst separator = this.getNodeParameter('separator', itemIndex) as string;\n\t\tconst chunkSize = this.getNodeParameter('chunkSize', itemIndex) as number;\n\t\tconst chunkOverlap = this.getNodeParameter('chunkOverlap', itemIndex) as number;\n\n\t\tconst params: CharacterTextSplitterParams = {\n\t\t\tseparator,\n\t\t\tchunkSize,\n\t\t\tchunkOverlap,\n\t\t\tkeepSeparator: false,\n\t\t};\n\n\t\tconst splitter = new CharacterTextSplitter(params);\n\n\t\treturn {\n\t\t\tresponse: logWrapper(splitter, this),\n\t\t};\n\t}\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AACA,2BAAsC;AACtC,0BAMO;AAEP,wBAA2B;AAC3B,0BAA6C;AAEtC,MAAM,kCAAuD;AAAA,EAA7D;AACN,uBAAoC;AAAA,MACnC,aAAa;AAAA,MACb,MAAM;AAAA,MACN,MAAM;AAAA,MACN,WAAW;AAAA,MACX,OAAO,CAAC,WAAW;AAAA,MACnB,SAAS;AAAA,MACT,aAAa;AAAA,MACb,UAAU;AAAA,QACT,MAAM;AAAA,MACP;AAAA,MACA,OAAO;AAAA,QACN,YAAY,CAAC,IAAI;AAAA,QACjB,eAAe;AAAA,UACd,IAAI,CAAC,gBAAgB;AAAA,QACtB;AAAA,QACA,WAAW;AAAA,UACV,sBAAsB;AAAA,YACrB;AAAA,cACC,KAAK;AAAA,YACN;AAAA,UACD;AAAA,QACD;AAAA,MACD;AAAA,MAEA,QAAQ,CAAC;AAAA,MAET,SAAS,CAAC,wCAAoB,cAAc;AAAA,MAC5C,aAAa,CAAC,eAAe;AAAA,MAC7B,YAAY;AAAA,YACX,kDAA6B,CAAC,wCAAoB,UAAU,CAAC;AAAA,QAC7D;AAAA,UACC,aAAa;AAAA,UACb,MAAM;AAAA,UACN,MAAM;AAAA,UACN,SAAS;AAAA,QACV;AAAA,QACA;AAAA,UACC,aAAa;AAAA,UACb,MAAM;AAAA,UACN,MAAM;AAAA,UACN,SAAS;AAAA,UACT,aAAa;AAAA,QACd;AAAA,QACA;AAAA,UACC,aAAa;AAAA,UACb,MAAM;AAAA,UACN,MAAM;AAAA,UACN,SAAS;AAAA,UACT,aAAa;AAAA,QACd;AAAA,MACD;AAAA,IACD;AAAA;AAAA,EAEA,MAAM,WAAuC,WAAwC;AACpF,SAAK,OAAO,MAAM,+BAA+B;AAEjD,UAAM,YAAY,KAAK,iBAAiB,aAAa,SAAS;AAC9D,UAAM,YAAY,KAAK,iBAAiB,aAAa,SAAS;AAC9D,UAAM,eAAe,KAAK,iBAAiB,gBAAgB,SAAS;AAEpE,UAAM,SAAsC;AAAA,MAC3C;AAAA,MACA;AAAA,MACA;AAAA,MACA,eAAe;AAAA,IAChB;AAEA,UAAM,WAAW,IAAI,2CAAsB,MAAM;AAEjD,WAAO;AAAA,MACN,cAAU,8BAAW,UAAU,IAAI;AAAA,IACpC;AAAA,EACD;AACD;","names":[]}
package/dist/nodes/text_splitters/TextSplitterTokenSplitter/TextSplitterTokenSplitter.node.js CHANGED
@@ -60,13 +60,15 @@ class TextSplitterTokenSplitter {
60
60
  displayName: "Chunk Size",
61
61
  name: "chunkSize",
62
62
  type: "number",
63
- default: 1e3
63
+ default: 1e3,
64
+ description: "Maximum number of tokens per chunk"
64
65
  },
65
66
  {
66
67
  displayName: "Chunk Overlap",
67
68
  name: "chunkOverlap",
68
69
  type: "number",
69
- default: 0
70
+ default: 0,
71
+ description: "Number of tokens shared between consecutive chunks to preserve context"
70
72
  }
71
73
  ]
72
74
  };
package/dist/nodes/text_splitters/TextSplitterTokenSplitter/TextSplitterTokenSplitter.node.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../../../nodes/text_splitters/TextSplitterTokenSplitter/TextSplitterTokenSplitter.node.ts"],"sourcesContent":["import {\n\tNodeConnectionTypes,\n\ttype INodeType,\n\ttype INodeTypeDescription,\n\ttype ISupplyDataFunctions,\n\ttype SupplyData,\n} from 'n8n-workflow';\n\nimport { logWrapper } from '@utils/logWrapper';\nimport { getConnectionHintNoticeField } from '@utils/sharedFields';\n\nimport { TokenTextSplitter } from './TokenTextSplitter';\n\nexport class TextSplitterTokenSplitter implements INodeType {\n\tdescription: INodeTypeDescription = {\n\t\tdisplayName: 'Token Splitter',\n\t\tname: 'textSplitterTokenSplitter',\n\t\ticon: 'fa:grip-lines-vertical',\n\t\ticonColor: 'black',\n\t\tgroup: ['transform'],\n\t\tversion: 1,\n\t\tdescription: 'Split text into chunks by tokens',\n\t\tdefaults: {\n\t\t\tname: 'Token Splitter',\n\t\t},\n\t\tcodex: {\n\t\t\tcategories: ['AI'],\n\t\t\tsubcategories: {\n\t\t\t\tAI: ['Text Splitters'],\n\t\t\t},\n\t\t\tresources: {\n\t\t\t\tprimaryDocumentation: [\n\t\t\t\t\t{\n\t\t\t\t\t\turl: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.textsplittertokensplitter/',\n\t\t\t\t\t},\n\t\t\t\t],\n\t\t\t},\n\t\t},\n\n\t\tinputs: [],\n\n\t\toutputs: [NodeConnectionTypes.AiTextSplitter],\n\t\toutputNames: ['Text Splitter'],\n\t\tproperties: [\n\t\t\tgetConnectionHintNoticeField([NodeConnectionTypes.AiDocument]),\n\t\t\t{\n\t\t\t\tdisplayName: 'Chunk Size',\n\t\t\t\tname: 'chunkSize',\n\t\t\t\ttype: 'number',\n\t\t\t\tdefault: 1000,\n\t\t\t},\n\t\t\t{\n\t\t\t\tdisplayName: 'Chunk Overlap',\n\t\t\t\tname: 'chunkOverlap',\n\t\t\t\ttype: 'number',\n\t\t\t\tdefault: 0,\n\t\t\t},\n\t\t],\n\t};\n\n\tasync supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise<SupplyData> {\n\t\tthis.logger.debug('Supply Data for Text Splitter');\n\n\t\tconst chunkSize = this.getNodeParameter('chunkSize', itemIndex) as number;\n\t\tconst chunkOverlap = this.getNodeParameter('chunkOverlap', itemIndex) as number;\n\n\t\tconst splitter = new TokenTextSplitter({\n\t\t\tchunkSize,\n\t\t\tchunkOverlap,\n\t\t\tallowedSpecial: 'all',\n\t\t\tdisallowedSpecial: 'all',\n\t\t\tencodingName: 'cl100k_base',\n\t\t\tkeepSeparator: false,\n\t\t});\n\n\t\treturn {\n\t\t\tresponse: logWrapper(splitter, this),\n\t\t};\n\t}\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,0BAMO;AAEP,wBAA2B;AAC3B,0BAA6C;AAE7C,+BAAkC;AAE3B,MAAM,0BAA+C;AAAA,EAArD;AACN,uBAAoC;AAAA,MACnC,aAAa;AAAA,MACb,MAAM;AAAA,MACN,MAAM;AAAA,MACN,WAAW;AAAA,MACX,OAAO,CAAC,WAAW;AAAA,MACnB,SAAS;AAAA,MACT,aAAa;AAAA,MACb,UAAU;AAAA,QACT,MAAM;AAAA,MACP;AAAA,MACA,OAAO;AAAA,QACN,YAAY,CAAC,IAAI;AAAA,QACjB,eAAe;AAAA,UACd,IAAI,CAAC,gBAAgB;AAAA,QACtB;AAAA,QACA,WAAW;AAAA,UACV,sBAAsB;AAAA,YACrB;AAAA,cACC,KAAK;AAAA,YACN;AAAA,UACD;AAAA,QACD;AAAA,MACD;AAAA,MAEA,QAAQ,CAAC;AAAA,MAET,SAAS,CAAC,wCAAoB,cAAc;AAAA,MAC5C,aAAa,CAAC,eAAe;AAAA,MAC7B,YAAY;AAAA,YACX,kDAA6B,CAAC,wCAAoB,UAAU,CAAC;AAAA,QAC7D;AAAA,UACC,aAAa;AAAA,UACb,MAAM;AAAA,UACN,MAAM;AAAA,UACN,SAAS;AAAA,QACV;AAAA,QACA;AAAA,UACC,aAAa;AAAA,UACb,MAAM;AAAA,UACN,MAAM;AAAA,UACN,SAAS;AAAA,QACV;AAAA,MACD;AAAA,IACD;AAAA;AAAA,EAEA,MAAM,WAAuC,WAAwC;AACpF,SAAK,OAAO,MAAM,+BAA+B;AAEjD,UAAM,YAAY,KAAK,iBAAiB,aAAa,SAAS;AAC9D,UAAM,eAAe,KAAK,iBAAiB,gBAAgB,SAAS;AAEpE,UAAM,WAAW,IAAI,2CAAkB;AAAA,MACtC;AAAA,MACA;AAAA,MACA,gBAAgB;AAAA,MAChB,mBAAmB;AAAA,MACnB,cAAc;AAAA,MACd,eAAe;AAAA,IAChB,CAAC;AAED,WAAO;AAAA,MACN,cAAU,8BAAW,UAAU,IAAI;AAAA,IACpC;AAAA,EACD;AACD;","names":[]}
1
+ {"version":3,"sources":["../../../../nodes/text_splitters/TextSplitterTokenSplitter/TextSplitterTokenSplitter.node.ts"],"sourcesContent":["import {\n\tNodeConnectionTypes,\n\ttype INodeType,\n\ttype INodeTypeDescription,\n\ttype ISupplyDataFunctions,\n\ttype SupplyData,\n} from 'n8n-workflow';\n\nimport { logWrapper } from '@utils/logWrapper';\nimport { getConnectionHintNoticeField } from '@utils/sharedFields';\n\nimport { TokenTextSplitter } from './TokenTextSplitter';\n\nexport class TextSplitterTokenSplitter implements INodeType {\n\tdescription: INodeTypeDescription = {\n\t\tdisplayName: 'Token Splitter',\n\t\tname: 'textSplitterTokenSplitter',\n\t\ticon: 'fa:grip-lines-vertical',\n\t\ticonColor: 'black',\n\t\tgroup: ['transform'],\n\t\tversion: 1,\n\t\tdescription: 'Split text into chunks by tokens',\n\t\tdefaults: {\n\t\t\tname: 'Token Splitter',\n\t\t},\n\t\tcodex: {\n\t\t\tcategories: ['AI'],\n\t\t\tsubcategories: {\n\t\t\t\tAI: ['Text Splitters'],\n\t\t\t},\n\t\t\tresources: {\n\t\t\t\tprimaryDocumentation: [\n\t\t\t\t\t{\n\t\t\t\t\t\turl: 'https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.textsplittertokensplitter/',\n\t\t\t\t\t},\n\t\t\t\t],\n\t\t\t},\n\t\t},\n\n\t\tinputs: [],\n\n\t\toutputs: [NodeConnectionTypes.AiTextSplitter],\n\t\toutputNames: ['Text Splitter'],\n\t\tproperties: [\n\t\t\tgetConnectionHintNoticeField([NodeConnectionTypes.AiDocument]),\n\t\t\t{\n\t\t\t\tdisplayName: 'Chunk Size',\n\t\t\t\tname: 'chunkSize',\n\t\t\t\ttype: 'number',\n\t\t\t\tdefault: 1000,\n\t\t\t\tdescription: 'Maximum number of tokens per chunk',\n\t\t\t},\n\t\t\t{\n\t\t\t\tdisplayName: 'Chunk Overlap',\n\t\t\t\tname: 'chunkOverlap',\n\t\t\t\ttype: 'number',\n\t\t\t\tdefault: 0,\n\t\t\t\tdescription: 'Number of tokens shared between consecutive chunks to preserve context',\n\t\t\t},\n\t\t],\n\t};\n\n\tasync supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise<SupplyData> {\n\t\tthis.logger.debug('Supply Data for Text Splitter');\n\n\t\tconst chunkSize = this.getNodeParameter('chunkSize', itemIndex) as number;\n\t\tconst chunkOverlap = this.getNodeParameter('chunkOverlap', itemIndex) as number;\n\n\t\tconst splitter = new TokenTextSplitter({\n\t\t\tchunkSize,\n\t\t\tchunkOverlap,\n\t\t\tallowedSpecial: 'all',\n\t\t\tdisallowedSpecial: 'all',\n\t\t\tencodingName: 'cl100k_base',\n\t\t\tkeepSeparator: false,\n\t\t});\n\n\t\treturn {\n\t\t\tresponse: logWrapper(splitter, this),\n\t\t};\n\t}\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,0BAMO;AAEP,wBAA2B;AAC3B,0BAA6C;AAE7C,+BAAkC;AAE3B,MAAM,0BAA+C;AAAA,EAArD;AACN,uBAAoC;AAAA,MACnC,aAAa;AAAA,MACb,MAAM;AAAA,MACN,MAAM;AAAA,MACN,WAAW;AAAA,MACX,OAAO,CAAC,WAAW;AAAA,MACnB,SAAS;AAAA,MACT,aAAa;AAAA,MACb,UAAU;AAAA,QACT,MAAM;AAAA,MACP;AAAA,MACA,OAAO;AAAA,QACN,YAAY,CAAC,IAAI;AAAA,QACjB,eAAe;AAAA,UACd,IAAI,CAAC,gBAAgB;AAAA,QACtB;AAAA,QACA,WAAW;AAAA,UACV,sBAAsB;AAAA,YACrB;AAAA,cACC,KAAK;AAAA,YACN;AAAA,UACD;AAAA,QACD;AAAA,MACD;AAAA,MAEA,QAAQ,CAAC;AAAA,MAET,SAAS,CAAC,wCAAoB,cAAc;AAAA,MAC5C,aAAa,CAAC,eAAe;AAAA,MAC7B,YAAY;AAAA,YACX,kDAA6B,CAAC,wCAAoB,UAAU,CAAC;AAAA,QAC7D;AAAA,UACC,aAAa;AAAA,UACb,MAAM;AAAA,UACN,MAAM;AAAA,UACN,SAAS;AAAA,UACT,aAAa;AAAA,QACd;AAAA,QACA;AAAA,UACC,aAAa;AAAA,UACb,MAAM;AAAA,UACN,MAAM;AAAA,UACN,SAAS;AAAA,UACT,aAAa;AAAA,QACd;AAAA,MACD;AAAA,IACD;AAAA;AAAA,EAEA,MAAM,WAAuC,WAAwC;AACpF,SAAK,OAAO,MAAM,+BAA+B;AAEjD,UAAM,YAAY,KAAK,iBAAiB,aAAa,SAAS;AAC9D,UAAM,eAAe,KAAK,iBAAiB,gBAAgB,SAAS;AAEpE,UAAM,WAAW,IAAI,2CAAkB;AAAA,MACtC;AAAA,MACA;AAAA,MACA,gBAAgB;AAAA,MAChB,mBAAmB;AAAA,MACnB,cAAc;AAAA,MACd,eAAe;AAAA,IAChB,CAAC;AAED,WAAO;AAAA,MACN,cAAU,8BAAW,UAAU,IAAI;AAAA,IACpC;AAAA,EACD;AACD;","names":[]}
package/dist/nodes/text_splitters/TextSplitterTokenSplitter/TokenTextSplitter.js CHANGED
@@ -50,7 +50,7 @@ class TokenTextSplitter extends import_textsplitters.TextSplitter {
50
50
  return splits;
51
51
  }
52
52
  try {
53
- this.tokenizer ??= (0, import_tiktoken.getEncoding)(this.encodingName);
53
+ this.tokenizer ??= await (0, import_tiktoken.getEncoding)(this.encodingName);
54
54
  const splits = [];
55
55
  const input_ids = this.tokenizer.encode(text, this.allowedSpecial, this.disallowedSpecial);
56
56
  let start_idx = 0;
package/dist/nodes/text_splitters/TextSplitterTokenSplitter/TokenTextSplitter.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../../../nodes/text_splitters/TextSplitterTokenSplitter/TokenTextSplitter.ts"],"sourcesContent":["import type { TokenTextSplitterParams } from '@langchain/textsplitters';\nimport { TextSplitter } from '@langchain/textsplitters';\nimport { hasLongSequentialRepeat } from '@utils/helpers';\nimport { getEncoding } from '@utils/tokenizer/tiktoken';\nimport { estimateTextSplitsByTokens } from '@utils/tokenizer/token-estimator';\nimport type * as tiktoken from 'js-tiktoken';\n\n/**\n * Implementation of splitter which looks at tokens.\n * This is override of the LangChain TokenTextSplitter\n * to use the n8n tokenizer utility which uses local JSON encodings\n */\nexport class TokenTextSplitter extends TextSplitter implements TokenTextSplitterParams {\n\tstatic lc_name() {\n\t\treturn 'TokenTextSplitter';\n\t}\n\n\tencodingName: tiktoken.TiktokenEncoding;\n\n\tallowedSpecial: 'all' | string[];\n\n\tdisallowedSpecial: 'all' | string[];\n\n\tprivate tokenizer: tiktoken.Tiktoken | undefined;\n\n\tconstructor(fields?: Partial<TokenTextSplitterParams>) {\n\t\tsuper(fields);\n\n\t\tthis.encodingName = fields?.encodingName ?? 'cl100k_base';\n\t\tthis.allowedSpecial = fields?.allowedSpecial ?? [];\n\t\tthis.disallowedSpecial = fields?.disallowedSpecial ?? 'all';\n\t}\n\n\tasync splitText(text: string): Promise<string[]> {\n\t\ttry {\n\t\t\t// Validate input\n\t\t\tif (!text || typeof text !== 'string') {\n\t\t\t\treturn [];\n\t\t\t}\n\n\t\t\t// Check for repetitive content\n\t\t\tif (hasLongSequentialRepeat(text)) {\n\t\t\t\tconst splits = estimateTextSplitsByTokens(\n\t\t\t\t\ttext,\n\t\t\t\t\tthis.chunkSize,\n\t\t\t\t\tthis.chunkOverlap,\n\t\t\t\t\tthis.encodingName,\n\t\t\t\t);\n\t\t\t\treturn splits;\n\t\t\t}\n\n\t\t\t// Use tiktoken for normal text\n\t\t\ttry {\n\t\t\t\tthis.tokenizer ??= getEncoding(this.encodingName);\n\n\t\t\t\tconst splits: string[] = [];\n\t\t\t\tconst input_ids = this.tokenizer.encode(text, this.allowedSpecial, this.disallowedSpecial);\n\n\t\t\t\tlet start_idx = 0;\n\t\t\t\tlet chunkCount = 0;\n\n\t\t\t\twhile (start_idx < input_ids.length) {\n\t\t\t\t\tif (start_idx > 0) {\n\t\t\t\t\t\tstart_idx = Math.max(0, start_idx - this.chunkOverlap);\n\t\t\t\t\t}\n\t\t\t\t\tconst end_idx = Math.min(start_idx + this.chunkSize, input_ids.length);\n\t\t\t\t\tconst chunk_ids = input_ids.slice(start_idx, end_idx);\n\n\t\t\t\t\tsplits.push(this.tokenizer.decode(chunk_ids));\n\n\t\t\t\t\tchunkCount++;\n\t\t\t\t\tstart_idx = end_idx;\n\t\t\t\t}\n\n\t\t\t\treturn splits;\n\t\t\t} catch (tiktokenError) {\n\t\t\t\t// Fall back to character-based splitting if tiktoken fails\n\t\t\t\treturn estimateTextSplitsByTokens(\n\t\t\t\t\ttext,\n\t\t\t\t\tthis.chunkSize,\n\t\t\t\t\tthis.chunkOverlap,\n\t\t\t\t\tthis.encodingName,\n\t\t\t\t);\n\t\t\t}\n\t\t} catch (error) {\n\t\t\t// Return empty array on complete failure\n\t\t\treturn [];\n\t\t}\n\t}\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AACA,2BAA6B;AAC7B,qBAAwC;AACxC,sBAA4B;AAC5B,6BAA2C;AAQpC,MAAM,0BAA0B,kCAAgD;AAAA,EACtF,OAAO,UAAU;AAChB,WAAO;AAAA,EACR;AAAA,EAUA,YAAY,QAA2C;AACtD,UAAM,MAAM;AAEZ,SAAK,eAAe,QAAQ,gBAAgB;AAC5C,SAAK,iBAAiB,QAAQ,kBAAkB,CAAC;AACjD,SAAK,oBAAoB,QAAQ,qBAAqB;AAAA,EACvD;AAAA,EAEA,MAAM,UAAU,MAAiC;AAChD,QAAI;AAEH,UAAI,CAAC,QAAQ,OAAO,SAAS,UAAU;AACtC,eAAO,CAAC;AAAA,MACT;AAGA,cAAI,wCAAwB,IAAI,GAAG;AAClC,cAAM,aAAS;AAAA,UACd;AAAA,UACA,KAAK;AAAA,UACL,KAAK;AAAA,UACL,KAAK;AAAA,QACN;AACA,eAAO;AAAA,MACR;AAGA,UAAI;AACH,aAAK,kBAAc,6BAAY,KAAK,YAAY;AAEhD,cAAM,SAAmB,CAAC;AAC1B,cAAM,YAAY,KAAK,UAAU,OAAO,MAAM,KAAK,gBAAgB,KAAK,iBAAiB;AAEzF,YAAI,YAAY;AAChB,YAAI,aAAa;AAEjB,eAAO,YAAY,UAAU,QAAQ;AACpC,cAAI,YAAY,GAAG;AAClB,wBAAY,KAAK,IAAI,GAAG,YAAY,KAAK,YAAY;AAAA,UACtD;AACA,gBAAM,UAAU,KAAK,IAAI,YAAY,KAAK,WAAW,UAAU,MAAM;AACrE,gBAAM,YAAY,UAAU,MAAM,WAAW,OAAO;AAEpD,iBAAO,KAAK,KAAK,UAAU,OAAO,SAAS,CAAC;AAE5C;AACA,sBAAY;AAAA,QACb;AAEA,eAAO;AAAA,MACR,SAAS,eAAe;AAEvB,mBAAO;AAAA,UACN;AAAA,UACA,KAAK;AAAA,UACL,KAAK;AAAA,UACL,KAAK;AAAA,QACN;AAAA,MACD;AAAA,IACD,SAAS,OAAO;AAEf,aAAO,CAAC;AAAA,IACT;AAAA,EACD;AACD;","names":[]}
1
+ {"version":3,"sources":["../../../../nodes/text_splitters/TextSplitterTokenSplitter/TokenTextSplitter.ts"],"sourcesContent":["import type { TokenTextSplitterParams } from '@langchain/textsplitters';\nimport { TextSplitter } from '@langchain/textsplitters';\nimport { hasLongSequentialRepeat } from '@utils/helpers';\nimport { getEncoding } from '@utils/tokenizer/tiktoken';\nimport { estimateTextSplitsByTokens } from '@utils/tokenizer/token-estimator';\nimport type * as tiktoken from 'js-tiktoken';\n\n/**\n * Implementation of splitter which looks at tokens.\n * This is override of the LangChain TokenTextSplitter\n * to use the n8n tokenizer utility which uses local JSON encodings\n */\nexport class TokenTextSplitter extends TextSplitter implements TokenTextSplitterParams {\n\tstatic lc_name() {\n\t\treturn 'TokenTextSplitter';\n\t}\n\n\tencodingName: tiktoken.TiktokenEncoding;\n\n\tallowedSpecial: 'all' | string[];\n\n\tdisallowedSpecial: 'all' | string[];\n\n\tprivate tokenizer: tiktoken.Tiktoken | undefined;\n\n\tconstructor(fields?: Partial<TokenTextSplitterParams>) {\n\t\tsuper(fields);\n\n\t\tthis.encodingName = fields?.encodingName ?? 'cl100k_base';\n\t\tthis.allowedSpecial = fields?.allowedSpecial ?? [];\n\t\tthis.disallowedSpecial = fields?.disallowedSpecial ?? 'all';\n\t}\n\n\tasync splitText(text: string): Promise<string[]> {\n\t\ttry {\n\t\t\t// Validate input\n\t\t\tif (!text || typeof text !== 'string') {\n\t\t\t\treturn [];\n\t\t\t}\n\n\t\t\t// Check for repetitive content\n\t\t\tif (hasLongSequentialRepeat(text)) {\n\t\t\t\tconst splits = estimateTextSplitsByTokens(\n\t\t\t\t\ttext,\n\t\t\t\t\tthis.chunkSize,\n\t\t\t\t\tthis.chunkOverlap,\n\t\t\t\t\tthis.encodingName,\n\t\t\t\t);\n\t\t\t\treturn splits;\n\t\t\t}\n\n\t\t\t// Use tiktoken for normal text\n\t\t\ttry {\n\t\t\t\tthis.tokenizer ??= await getEncoding(this.encodingName);\n\n\t\t\t\tconst splits: string[] = [];\n\t\t\t\tconst input_ids = this.tokenizer.encode(text, this.allowedSpecial, this.disallowedSpecial);\n\n\t\t\t\tlet start_idx = 0;\n\t\t\t\tlet chunkCount = 0;\n\n\t\t\t\twhile (start_idx < input_ids.length) {\n\t\t\t\t\tif (start_idx > 0) {\n\t\t\t\t\t\tstart_idx = Math.max(0, start_idx - this.chunkOverlap);\n\t\t\t\t\t}\n\t\t\t\t\tconst end_idx = Math.min(start_idx + this.chunkSize, input_ids.length);\n\t\t\t\t\tconst chunk_ids = input_ids.slice(start_idx, end_idx);\n\n\t\t\t\t\tsplits.push(this.tokenizer.decode(chunk_ids));\n\n\t\t\t\t\tchunkCount++;\n\t\t\t\t\tstart_idx = end_idx;\n\t\t\t\t}\n\n\t\t\t\treturn splits;\n\t\t\t} catch (tiktokenError) {\n\t\t\t\t// Fall back to character-based splitting if tiktoken fails\n\t\t\t\treturn estimateTextSplitsByTokens(\n\t\t\t\t\ttext,\n\t\t\t\t\tthis.chunkSize,\n\t\t\t\t\tthis.chunkOverlap,\n\t\t\t\t\tthis.encodingName,\n\t\t\t\t);\n\t\t\t}\n\t\t} catch (error) {\n\t\t\t// Return empty array on complete failure\n\t\t\treturn [];\n\t\t}\n\t}\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AACA,2BAA6B;AAC7B,qBAAwC;AACxC,sBAA4B;AAC5B,6BAA2C;AAQpC,MAAM,0BAA0B,kCAAgD;AAAA,EACtF,OAAO,UAAU;AAChB,WAAO;AAAA,EACR;AAAA,EAUA,YAAY,QAA2C;AACtD,UAAM,MAAM;AAEZ,SAAK,eAAe,QAAQ,gBAAgB;AAC5C,SAAK,iBAAiB,QAAQ,kBAAkB,CAAC;AACjD,SAAK,oBAAoB,QAAQ,qBAAqB;AAAA,EACvD;AAAA,EAEA,MAAM,UAAU,MAAiC;AAChD,QAAI;AAEH,UAAI,CAAC,QAAQ,OAAO,SAAS,UAAU;AACtC,eAAO,CAAC;AAAA,MACT;AAGA,cAAI,wCAAwB,IAAI,GAAG;AAClC,cAAM,aAAS;AAAA,UACd;AAAA,UACA,KAAK;AAAA,UACL,KAAK;AAAA,UACL,KAAK;AAAA,QACN;AACA,eAAO;AAAA,MACR;AAGA,UAAI;AACH,aAAK,cAAc,UAAM,6BAAY,KAAK,YAAY;AAEtD,cAAM,SAAmB,CAAC;AAC1B,cAAM,YAAY,KAAK,UAAU,OAAO,MAAM,KAAK,gBAAgB,KAAK,iBAAiB;AAEzF,YAAI,YAAY;AAChB,YAAI,aAAa;AAEjB,eAAO,YAAY,UAAU,QAAQ;AACpC,cAAI,YAAY,GAAG;AAClB,wBAAY,KAAK,IAAI,GAAG,YAAY,KAAK,YAAY;AAAA,UACtD;AACA,gBAAM,UAAU,KAAK,IAAI,YAAY,KAAK,WAAW,UAAU,MAAM;AACrE,gBAAM,YAAY,UAAU,MAAM,WAAW,OAAO;AAEpD,iBAAO,KAAK,KAAK,UAAU,OAAO,SAAS,CAAC;AAE5C;AACA,sBAAY;AAAA,QACb;AAEA,eAAO;AAAA,MACR,SAAS,eAAe;AAEvB,mBAAO;AAAA,UACN;AAAA,UACA,KAAK;AAAA,UACL,KAAK;AAAA,UACL,KAAK;AAAA,QACN;AAAA,MACD;AAAA,IACD,SAAS,OAAO;AAEf,aAAO,CAAC;AAAA,IACT;AAAA,EACD;AACD;","names":[]}
package/dist/types/nodes.json CHANGED
@@ -64,9 +64,9 @@
64
64
  {"displayName":"Vector Store Retriever","name":"retrieverVectorStore","icon":"fa:box-open","iconColor":"black","group":["transform"],"version":1,"description":"Use a Vector Store as Retriever","defaults":{"name":"Vector Store Retriever"},"codex":{"categories":["AI"],"subcategories":{"AI":["Retrievers"]},"resources":{"primaryDocumentation":[{"url":"https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.retrievervectorstore/"}]}},"inputs":[{"displayName":"Vector Store","maxConnections":1,"type":"ai_vectorStore","required":true}],"outputs":["ai_retriever"],"outputNames":["Retriever"],"properties":[{"displayName":"Limit","name":"topK","type":"number","default":4,"description":"The maximum number of results to return"}]},
65
65
  {"displayName":"MultiQuery Retriever","name":"retrieverMultiQuery","icon":"fa:box-open","iconColor":"black","group":["transform"],"version":1,"description":"Automates prompt tuning, generates diverse queries and expands document pool for enhanced retrieval.","defaults":{"name":"MultiQuery Retriever"},"codex":{"categories":["AI"],"subcategories":{"AI":["Retrievers"]},"resources":{"primaryDocumentation":[{"url":"https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.retrievermultiquery/"}]}},"inputs":[{"displayName":"Model","maxConnections":1,"type":"ai_languageModel","required":true},{"displayName":"Retriever","maxConnections":1,"type":"ai_retriever","required":true}],"outputs":[{"displayName":"Retriever","maxConnections":1,"type":"ai_retriever"}],"properties":[{"displayName":"Options","name":"options","placeholder":"Add Option","description":"Additional options to add","type":"collection","default":{},"options":[{"displayName":"Query Count","name":"queryCount","default":3,"typeOptions":{"minValue":1},"description":"Number of different versions of the given question to generate","type":"number"}]}]},
66
66
  {"displayName":"Workflow Retriever","name":"retrieverWorkflow","icon":"fa:box-open","iconColor":"black","group":["transform"],"version":[1,1.1],"description":"Use an n8n Workflow as Retriever","defaults":{"name":"Workflow Retriever"},"codex":{"categories":["AI"],"subcategories":{"AI":["Retrievers"]},"resources":{"primaryDocumentation":[{"url":"https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.retrieverworkflow/"}]}},"inputs":[],"outputs":[{"displayName":"Retriever","maxConnections":1,"type":"ai_retriever"}],"properties":[{"displayName":"The workflow will receive \"query\" as input and the output of the last node will be returned and converted to Documents","name":"executeNotice","type":"notice","default":""},{"displayName":"Source","name":"source","type":"options","options":[{"name":"Database","value":"database","description":"Load the workflow from the database by ID"},{"name":"Parameter","value":"parameter","description":"Load the workflow from a parameter"}],"default":"database","description":"Where to get the workflow to execute from"},{"displayName":"Workflow ID","name":"workflowId","type":"string","displayOptions":{"show":{"source":["database"],"@version":[{"_cnd":{"eq":1}}]}},"default":"","required":true,"description":"The workflow to execute"},{"displayName":"Workflow","name":"workflowId","type":"workflowSelector","displayOptions":{"show":{"source":["database"],"@version":[{"_cnd":{"gte":1.1}}]}},"default":"","required":true},{"displayName":"Workflow JSON","name":"workflowJson","type":"json","typeOptions":{"rows":10},"displayOptions":{"show":{"source":["parameter"]}},"default":"\n\n\n","required":true,"description":"The workflow JSON code to execute"},{"displayName":"Workflow Values","name":"fields","placeholder":"Add Value","type":"fixedCollection","description":"Set the values which should be made available in the workflow","typeOptions":{"multipleValues":true,"sortable":true},"default":{},"options":[{"name":"values","displayName":"Values","values":[{"displayName":"Name","name":"name","type":"string","default":"","placeholder":"e.g. fieldName","description":"Name of the field to set the value of. Supports dot-notation. Example: data.person[0].name.","requiresDataPath":"single"},{"displayName":"Type","name":"type","type":"options","description":"The field value type","options":[{"name":"String","value":"stringValue"},{"name":"Number","value":"numberValue"},{"name":"Boolean","value":"booleanValue"},{"name":"Array","value":"arrayValue"},{"name":"Object","value":"objectValue"}],"default":"stringValue"},{"displayName":"Value","name":"stringValue","type":"string","default":"","displayOptions":{"show":{"type":["stringValue"]}},"validateType":"string","ignoreValidationDuringExecution":true},{"displayName":"Value","name":"numberValue","type":"string","default":"","displayOptions":{"show":{"type":["numberValue"]}},"validateType":"number","ignoreValidationDuringExecution":true},{"displayName":"Value","name":"booleanValue","type":"options","default":"true","options":[{"name":"True","value":"true"},{"name":"False","value":"false"}],"displayOptions":{"show":{"type":["booleanValue"]}},"validateType":"boolean","ignoreValidationDuringExecution":true},{"displayName":"Value","name":"arrayValue","type":"string","default":"","placeholder":"e.g. [ arrayItem1, arrayItem2, arrayItem3 ]","displayOptions":{"show":{"type":["arrayValue"]}},"validateType":"array","ignoreValidationDuringExecution":true},{"displayName":"Value","name":"objectValue","type":"json","default":"={}","typeOptions":{"rows":2},"displayOptions":{"show":{"type":["objectValue"]}},"validateType":"object","ignoreValidationDuringExecution":true}]}]}]},
67
- {"displayName":"Character Text Splitter","name":"textSplitterCharacterTextSplitter","icon":"fa:grip-lines-vertical","iconColor":"black","group":["transform"],"version":1,"description":"Split text into chunks by characters","defaults":{"name":"Character Text Splitter"},"codex":{"categories":["AI"],"subcategories":{"AI":["Text Splitters"]},"resources":{"primaryDocumentation":[{"url":"https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.textsplittercharactertextsplitter/"}]}},"inputs":[],"outputs":["ai_textSplitter"],"outputNames":["Text Splitter"],"properties":[{"displayName":"This node must be connected to a document loader. <a data-action='openSelectiveNodeCreator' data-action-parameter-connectiontype='ai_document'>Insert one</a>","name":"notice","type":"notice","default":"","typeOptions":{"containerClass":"ndv-connection-hint-notice"}},{"displayName":"Separator","name":"separator","type":"string","default":""},{"displayName":"Chunk Size","name":"chunkSize","type":"number","default":1000},{"displayName":"Chunk Overlap","name":"chunkOverlap","type":"number","default":0}]},
67
+ {"displayName":"Character Text Splitter","name":"textSplitterCharacterTextSplitter","icon":"fa:grip-lines-vertical","iconColor":"black","group":["transform"],"version":1,"description":"Split text into chunks by characters","defaults":{"name":"Character Text Splitter"},"codex":{"categories":["AI"],"subcategories":{"AI":["Text Splitters"]},"resources":{"primaryDocumentation":[{"url":"https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.textsplittercharactertextsplitter/"}]}},"inputs":[],"outputs":["ai_textSplitter"],"outputNames":["Text Splitter"],"properties":[{"displayName":"This node must be connected to a document loader. <a data-action='openSelectiveNodeCreator' data-action-parameter-connectiontype='ai_document'>Insert one</a>","name":"notice","type":"notice","default":"","typeOptions":{"containerClass":"ndv-connection-hint-notice"}},{"displayName":"Separator","name":"separator","type":"string","default":""},{"displayName":"Chunk Size","name":"chunkSize","type":"number","default":1000,"description":"Maximum number of characters per chunk"},{"displayName":"Chunk Overlap","name":"chunkOverlap","type":"number","default":0,"description":"Number of characters shared between consecutive chunks to preserve context"}]},
68
68
  {"displayName":"Recursive Character Text Splitter","name":"textSplitterRecursiveCharacterTextSplitter","icon":"fa:grip-lines-vertical","iconColor":"black","group":["transform"],"version":1,"description":"Split text into chunks by characters recursively, recommended for most use cases","defaults":{"name":"Recursive Character Text Splitter"},"codex":{"categories":["AI"],"subcategories":{"AI":["Text Splitters"]},"resources":{"primaryDocumentation":[{"url":"https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.textsplitterrecursivecharactertextsplitter/"}]}},"inputs":[],"outputs":["ai_textSplitter"],"outputNames":["Text Splitter"],"properties":[{"displayName":"This node must be connected to a document loader. <a data-action='openSelectiveNodeCreator' data-action-parameter-connectiontype='ai_document'>Insert one</a>","name":"notice","type":"notice","default":"","typeOptions":{"containerClass":"ndv-connection-hint-notice"}},{"displayName":"Chunk Size","name":"chunkSize","type":"number","default":1000},{"displayName":"Chunk Overlap","name":"chunkOverlap","type":"number","default":0},{"displayName":"Options","name":"options","placeholder":"Add Option","description":"Additional options to add","type":"collection","default":{},"options":[{"displayName":"Split Code","name":"splitCode","default":"markdown","type":"options","options":[{"name":"cpp","value":"cpp"},{"name":"go","value":"go"},{"name":"java","value":"java"},{"name":"js","value":"js"},{"name":"php","value":"php"},{"name":"proto","value":"proto"},{"name":"python","value":"python"},{"name":"rst","value":"rst"},{"name":"ruby","value":"ruby"},{"name":"rust","value":"rust"},{"name":"scala","value":"scala"},{"name":"swift","value":"swift"},{"name":"markdown","value":"markdown"},{"name":"latex","value":"latex"},{"name":"html","value":"html"}]}]}]},
69
- {"displayName":"Token Splitter","name":"textSplitterTokenSplitter","icon":"fa:grip-lines-vertical","iconColor":"black","group":["transform"],"version":1,"description":"Split text into chunks by tokens","defaults":{"name":"Token Splitter"},"codex":{"categories":["AI"],"subcategories":{"AI":["Text Splitters"]},"resources":{"primaryDocumentation":[{"url":"https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.textsplittertokensplitter/"}]}},"inputs":[],"outputs":["ai_textSplitter"],"outputNames":["Text Splitter"],"properties":[{"displayName":"This node must be connected to a document loader. <a data-action='openSelectiveNodeCreator' data-action-parameter-connectiontype='ai_document'>Insert one</a>","name":"notice","type":"notice","default":"","typeOptions":{"containerClass":"ndv-connection-hint-notice"}},{"displayName":"Chunk Size","name":"chunkSize","type":"number","default":1000},{"displayName":"Chunk Overlap","name":"chunkOverlap","type":"number","default":0}]},
69
+ {"displayName":"Token Splitter","name":"textSplitterTokenSplitter","icon":"fa:grip-lines-vertical","iconColor":"black","group":["transform"],"version":1,"description":"Split text into chunks by tokens","defaults":{"name":"Token Splitter"},"codex":{"categories":["AI"],"subcategories":{"AI":["Text Splitters"]},"resources":{"primaryDocumentation":[{"url":"https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.textsplittertokensplitter/"}]}},"inputs":[],"outputs":["ai_textSplitter"],"outputNames":["Text Splitter"],"properties":[{"displayName":"This node must be connected to a document loader. <a data-action='openSelectiveNodeCreator' data-action-parameter-connectiontype='ai_document'>Insert one</a>","name":"notice","type":"notice","default":"","typeOptions":{"containerClass":"ndv-connection-hint-notice"}},{"displayName":"Chunk Size","name":"chunkSize","type":"number","default":1000,"description":"Maximum number of tokens per chunk"},{"displayName":"Chunk Overlap","name":"chunkOverlap","type":"number","default":0,"description":"Number of tokens shared between consecutive chunks to preserve context"}]},
70
70
  {"displayName":"Calculator","name":"toolCalculator","icon":"fa:calculator","iconColor":"black","group":["transform"],"version":1,"description":"Make it easier for AI agents to perform arithmetic","defaults":{"name":"Calculator"},"codex":{"categories":["AI"],"subcategories":{"AI":["Tools"],"Tools":["Other Tools"]},"resources":{"primaryDocumentation":[{"url":"https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.toolcalculator/"}]}},"inputs":[],"outputs":["ai_tool"],"outputNames":["Tool"],"properties":[{"displayName":"This node must be connected to an AI agent. <a data-action='openSelectiveNodeCreator' data-action-parameter-creatorview='AI'>Insert one</a>","name":"notice","type":"notice","default":"","typeOptions":{"containerClass":"ndv-connection-hint-notice"}}]},
71
71
  {"displayName":"Code Tool","name":"toolCode","icon":"fa:code","iconColor":"black","group":["transform"],"version":[1,1.1,1.2,1.3],"description":"Write a tool in JS or Python","defaults":{"name":"Code Tool"},"codex":{"categories":["AI"],"subcategories":{"AI":["Tools"],"Tools":["Recommended Tools"]},"resources":{"primaryDocumentation":[{"url":"https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.toolcode/"}]}},"inputs":[],"outputs":["ai_tool"],"outputNames":["Tool"],"properties":[{"displayName":"This node must be connected to an AI agent. <a data-action='openSelectiveNodeCreator' data-action-parameter-creatorview='AI'>Insert one</a>","name":"notice","type":"notice","default":"","typeOptions":{"containerClass":"ndv-connection-hint-notice"}},{"displayName":"See an example of a conversational agent with custom tool written in JavaScript <a href=\"/templates/1963\" target=\"_blank\">here</a>.","name":"noticeTemplateExample","type":"notice","default":""},{"displayName":"Name","name":"name","type":"string","default":"","placeholder":"My_Tool","displayOptions":{"show":{"@version":[1]}}},{"displayName":"Name","name":"name","type":"string","default":"","placeholder":"e.g. My_Tool","validateType":"string-alphanumeric","description":"The name of the function to be called, could contain letters, numbers, and underscores only","displayOptions":{"show":{"@version":[1.1]}}},{"displayName":"Description","name":"description","type":"string","default":"","placeholder":"Call this tool to get a random color. The input should be a string with comma separted names of colors to exclude.","typeOptions":{"rows":3}},{"displayName":"Language","name":"language","type":"options","noDataExpression":true,"options":[{"name":"JavaScript","value":"javaScript"},{"name":"Python (Beta)","value":"python"}],"default":"javaScript"},{"displayName":"JavaScript","name":"jsCode","type":"string","displayOptions":{"show":{"language":["javaScript"]}},"typeOptions":{"editor":"jsEditor"},"default":"// Example: convert the incoming query to uppercase and return it\nreturn query.toUpperCase()","hint":"You can access the input the tool receives via the input property \"query\". The returned value should be a single string.","description":"E.g. Converts any text to uppercase","noDataExpression":true},{"displayName":"Python","name":"pythonCode","type":"string","displayOptions":{"show":{"language":["python"]}},"typeOptions":{"editor":"codeNodeEditor","editorLanguage":"python"},"default":"# Example: convert the incoming query to uppercase and return it\nreturn query.upper()","hint":"You can access the input the tool receives via the input property \"query\". The returned value should be a single string.","description":"E.g. Converts any text to uppercase","noDataExpression":true},{"displayName":"Specify Input Schema","name":"specifyInputSchema","type":"boolean","description":"Whether to specify the schema for the function. This would require the LLM to provide the input in the correct format and would validate it against the schema.","noDataExpression":true,"default":false},{"displayName":"Schema Type","name":"schemaType","type":"options","noDataExpression":true,"options":[{"name":"Generate From JSON Example","value":"fromJson","description":"Generate a schema from an example JSON object"},{"name":"Define using JSON Schema","value":"manual","description":"Define the JSON schema manually"}],"default":"fromJson","description":"How to specify the schema for the function","displayOptions":{"show":{"specifyInputSchema":[true]}}},{"displayName":"JSON Example","name":"jsonSchemaExample","type":"json","default":"{\n\t\"some_input\": \"some_value\"\n}","noDataExpression":true,"typeOptions":{"rows":10},"displayOptions":{"show":{"specifyInputSchema":[true],"schemaType":["fromJson"]}},"description":"Example JSON object to use to generate the schema"},{"displayName":"All properties will be required. To make them optional, use the 'JSON Schema' schema type instead","name":"notice","type":"notice","default":"","displayOptions":{"show":{"specifyInputSchema":[true],"@version":[{"_cnd":{"gte":1.3}}],"schemaType":["fromJson"]}}},{"displayName":"Input Schema","name":"inputSchema","type":"json","default":"{\n\"type\": \"object\",\n\"properties\": {\n\t\"some_input\": {\n\t\t\"type\": \"string\",\n\t\t\"description\": \"Some input to the function\"\n\t\t}\n\t}\n}","noDataExpression":false,"typeOptions":{"rows":10},"displayOptions":{"show":{"specifyInputSchema":[true],"schemaType":["manual"]}},"description":"Schema to use for the function","hint":"Use <a target=\"_blank\" href=\"https://json-schema.org/\">JSON Schema</a> format (<a target=\"_blank\" href=\"https://json-schema.org/learn/miscellaneous-examples.html\">examples</a>). $refs syntax is currently not supported."}]},
72
72
  {"displayName":"HTTP Request Tool","name":"toolHttpRequest","group":["output"],"version":[1,1.1],"description":"Makes an HTTP request and returns the response data","subtitle":"={{ $parameter.toolDescription }}","defaults":{"name":"HTTP Request"},"credentials":[],"codex":{"categories":["AI"],"subcategories":{"AI":["Tools"],"Tools":["Recommended Tools"]},"resources":{"primaryDocumentation":[{"url":"https://docs.n8n.io/integrations/builtin/cluster-nodes/sub-nodes/n8n-nodes-langchain.toolhttprequest/"}]}},"hidden":true,"inputs":[],"outputs":["ai_tool"],"outputNames":["Tool"],"properties":[{"displayName":"This node must be connected to an AI agent. <a data-action='openSelectiveNodeCreator' data-action-parameter-creatorview='AI'>Insert one</a>","name":"notice","type":"notice","default":"","typeOptions":{"containerClass":"ndv-connection-hint-notice"}},{"displayName":"Description","name":"toolDescription","type":"string","description":"Explain to LLM what this tool does, better description would allow LLM to produce expected result","placeholder":"e.g. Get the current weather in the requested city","default":"","typeOptions":{"rows":3}},{"displayName":"Method","name":"method","type":"options","options":[{"name":"DELETE","value":"DELETE"},{"name":"GET","value":"GET"},{"name":"PATCH","value":"PATCH"},{"name":"POST","value":"POST"},{"name":"PUT","value":"PUT"}],"default":"GET"},{"displayName":"Tip: You can use a {placeholder} for any part of the request to be filled by the model. Provide more context about them in the placeholders section","name":"placeholderNotice","type":"notice","default":""},{"displayName":"URL","name":"url","type":"string","default":"","required":true,"placeholder":"e.g. http://www.example.com/{path}"},{"displayName":"Authentication","name":"authentication","description":"Select the type of authentication to use if needed, authentication would be done by n8n and your credentials will not be shared with the LLM","noDataExpression":true,"type":"options","options":[{"name":"None","value":"none"},{"name":"Predefined Credential Type","value":"predefinedCredentialType","description":"We've already implemented auth for many services so that you don't have to set it up manually"},{"name":"Generic Credential Type","value":"genericCredentialType","description":"Fully customizable. Choose between basic, header, OAuth2, etc."}],"default":"none"},{"displayName":"Credential Type","name":"nodeCredentialType","type":"credentialsSelect","noDataExpression":true,"required":true,"default":"","credentialTypes":["extends:oAuth2Api","extends:oAuth1Api","has:authenticate"],"displayOptions":{"show":{"authentication":["predefinedCredentialType"]}}},{"displayName":"Make sure you have specified the scope(s) for the Service Account in the credential","name":"googleApiWarning","type":"notice","default":"","displayOptions":{"show":{"nodeCredentialType":["googleApi"]}}},{"displayName":"Generic Auth Type","name":"genericAuthType","type":"credentialsSelect","required":true,"default":"","credentialTypes":["has:genericAuth"],"displayOptions":{"show":{"authentication":["genericCredentialType"]}}},{"displayName":"Send Query Parameters","name":"sendQuery","type":"boolean","default":false,"noDataExpression":true,"description":"Whether the request has query params or not"},{"displayName":"Specify Query Parameters","name":"specifyQuery","type":"options","options":[{"name":"Using Fields Below","value":"keypair"},{"name":"Using JSON Below","value":"json"},{"name":"Let Model Specify Entire Body","value":"model"}],"default":"keypair","displayOptions":{"show":{"sendQuery":[true]}}},{"displayName":"Query Parameters","name":"parametersQuery","type":"fixedCollection","typeOptions":{"multipleValues":true},"placeholder":"Add Parameter","default":{"values":[{"name":""}]},"options":[{"name":"values","displayName":"Values","values":[{"displayName":"Name","name":"name","type":"string","default":""},{"displayName":"Value Provided","name":"valueProvider","type":"options","options":[{"name":"By Model (and is required)","value":"modelRequired"},{"name":"By Model (but is optional)","value":"modelOptional"},{"name":"Using Field Below","value":"fieldValue"}],"default":"modelRequired"},{"displayName":"Value","name":"value","type":"string","default":"","hint":"Use a {placeholder} for any data to be filled in by the model","displayOptions":{"show":{"valueProvider":["fieldValue"]}}}]}],"displayOptions":{"show":{"sendQuery":[true],"specifyQuery":["keypair"]}}},{"displayName":"JSON","name":"jsonQuery","type":"string","typeOptions":{"rows":5},"hint":"Use a {placeholder} for any data to be filled in by the model","default":"","displayOptions":{"show":{"sendQuery":[true],"specifyQuery":["json"]}}},{"displayName":"Send Headers","name":"sendHeaders","type":"boolean","default":false,"noDataExpression":true,"description":"Whether the request has headers or not"},{"displayName":"Specify Headers","name":"specifyHeaders","type":"options","options":[{"name":"Using Fields Below","value":"keypair"},{"name":"Using JSON Below","value":"json"},{"name":"Let Model Specify Entire Body","value":"model"}],"default":"keypair","displayOptions":{"show":{"sendHeaders":[true]}}},{"displayName":"Header Parameters","name":"parametersHeaders","type":"fixedCollection","typeOptions":{"multipleValues":true},"placeholder":"Add Parameter","default":{"values":[{"name":""}]},"options":[{"name":"values","displayName":"Values","values":[{"displayName":"Name","name":"name","type":"string","default":""},{"displayName":"Value Provided","name":"valueProvider","type":"options","options":[{"name":"By Model (and is required)","value":"modelRequired"},{"name":"By Model (but is optional)","value":"modelOptional"},{"name":"Using Field Below","value":"fieldValue"}],"default":"modelRequired"},{"displayName":"Value","name":"value","type":"string","default":"","hint":"Use a {placeholder} for any data to be filled in by the model","displayOptions":{"show":{"valueProvider":["fieldValue"]}}}]}],"displayOptions":{"show":{"sendHeaders":[true],"specifyHeaders":["keypair"]}}},{"displayName":"JSON","name":"jsonHeaders","type":"string","typeOptions":{"rows":5},"hint":"Use a {placeholder} for any data to be filled in by the model","default":"","displayOptions":{"show":{"sendHeaders":[true],"specifyHeaders":["json"]}}},{"displayName":"Send Body","name":"sendBody","type":"boolean","default":false,"noDataExpression":true,"description":"Whether the request has body or not"},{"displayName":"Specify Body","name":"specifyBody","type":"options","options":[{"name":"Using Fields Below","value":"keypair"},{"name":"Using JSON Below","value":"json"},{"name":"Let Model Specify Entire Body","value":"model"}],"default":"keypair","displayOptions":{"show":{"sendBody":[true]}}},{"displayName":"Body Parameters","name":"parametersBody","type":"fixedCollection","typeOptions":{"multipleValues":true},"placeholder":"Add Parameter","default":{"values":[{"name":""}]},"options":[{"name":"values","displayName":"Values","values":[{"displayName":"Name","name":"name","type":"string","default":""},{"displayName":"Value Provided","name":"valueProvider","type":"options","options":[{"name":"By Model (and is required)","value":"modelRequired"},{"name":"By Model (but is optional)","value":"modelOptional"},{"name":"Using Field Below","value":"fieldValue"}],"default":"modelRequired"},{"displayName":"Value","name":"value","type":"string","default":"","hint":"Use a {placeholder} for any data to be filled in by the model","displayOptions":{"show":{"valueProvider":["fieldValue"]}}}]}],"displayOptions":{"show":{"sendBody":[true],"specifyBody":["keypair"]}}},{"displayName":"JSON","name":"jsonBody","type":"string","typeOptions":{"rows":5},"hint":"Use a {placeholder} for any data to be filled in by the model","default":"","displayOptions":{"show":{"sendBody":[true],"specifyBody":["json"]}}},{"displayName":"Placeholder Definitions","name":"placeholderDefinitions","type":"fixedCollection","typeOptions":{"multipleValues":true},"placeholder":"Add Definition","default":[],"options":[{"name":"values","displayName":"Values","values":[{"displayName":"Placeholder Name","name":"name","type":"string","default":""},{"displayName":"Description","name":"description","type":"string","default":""},{"displayName":"Type","name":"type","type":"options","options":[{"name":"Not Specified (Default)","value":"not specified"},{"name":"String","value":"string"},{"name":"Number","value":"number"},{"name":"Boolean","value":"boolean"},{"name":"JSON","value":"json"}],"default":"not specified"}]}]},{"displayName":"Optimize Response","name":"optimizeResponse","type":"boolean","default":false,"noDataExpression":true,"description":"Whether the optimize the tool response to reduce amount of data passed to the LLM that could lead to better result and reduce cost"},{"displayName":"Expected Response Type","name":"responseType","type":"options","displayOptions":{"show":{"optimizeResponse":[true]}},"options":[{"name":"JSON","value":"json"},{"name":"HTML","value":"html"},{"name":"Text","value":"text"}],"default":"json"},{"displayName":"Field Containing Data","name":"dataField","type":"string","default":"","placeholder":"e.g. records","description":"Specify the name of the field in the response containing the data","hint":"leave blank to use whole response","requiresDataPath":"single","displayOptions":{"show":{"optimizeResponse":[true],"responseType":["json"]}}},{"displayName":"Include Fields","name":"fieldsToInclude","type":"options","description":"What fields response object should include","default":"all","displayOptions":{"show":{"optimizeResponse":[true],"responseType":["json"]}},"options":[{"name":"All","value":"all","description":"Include all fields"},{"name":"Selected","value":"selected","description":"Include only fields specified below"},{"name":"Except","value":"except","description":"Exclude fields specified below"}]},{"displayName":"Fields","name":"fields","type":"string","default":"","placeholder":"e.g. field1,field2","description":"Comma-separated list of the field names. Supports dot notation. You can drag the selected fields from the input panel.","requiresDataPath":"multiple","displayOptions":{"show":{"optimizeResponse":[true],"responseType":["json"]},"hide":{"fieldsToInclude":["all"]}}},{"displayName":"Selector (CSS)","name":"cssSelector","type":"string","description":"Select specific element(e.g. body) or multiple elements(e.g. div) of chosen type in the response HTML.","placeholder":"e.g. body","default":"body","displayOptions":{"show":{"optimizeResponse":[true],"responseType":["html"]}}},{"displayName":"Return Only Content","name":"onlyContent","type":"boolean","default":false,"description":"Whether to return only content of html elements, stripping html tags and attributes","hint":"Uses less tokens and may be easier for model to understand","displayOptions":{"show":{"optimizeResponse":[true],"responseType":["html"]}}},{"displayName":"Elements To Omit","name":"elementsToOmit","type":"string","displayOptions":{"show":{"optimizeResponse":[true],"responseType":["html"],"onlyContent":[true]}},"default":"","placeholder":"e.g. img, .className, #ItemId","description":"Comma-separated list of selectors that would be excluded when extracting content"},{"displayName":"Truncate Response","name":"truncateResponse","type":"boolean","default":false,"hint":"Helps save tokens","displayOptions":{"show":{"optimizeResponse":[true],"responseType":["text","html"]}}},{"displayName":"Max Response Characters","name":"maxLength","type":"number","default":1000,"typeOptions":{"minValue":1},"displayOptions":{"show":{"optimizeResponse":[true],"responseType":["text","html"],"truncateResponse":[true]}}}],"iconUrl":{"light":"icons/@n8n/n8n-nodes-langchain/dist/nodes/tools/ToolHttpRequest/httprequest.svg","dark":"icons/@n8n/n8n-nodes-langchain/dist/nodes/tools/ToolHttpRequest/httprequest.dark.svg"}},
package/dist/utils/tokenizer/tiktoken.js CHANGED
@@ -22,36 +22,40 @@ __export(tiktoken_exports, {
22
22
  getEncoding: () => getEncoding
23
23
  });
24
24
  module.exports = __toCommonJS(tiktoken_exports);
25
- var import_fs = require("fs");
25
+ var import_promises = require("fs/promises");
26
26
  var import_lite = require("js-tiktoken/lite");
27
27
  var import_n8n_workflow = require("n8n-workflow");
28
28
  var import_path = require("path");
29
29
  const cache = {};
30
- const loadJSONFile = (filename) => {
30
+ const loadJSONFile = async (filename) => {
31
31
  const filePath = (0, import_path.join)(__dirname, filename);
32
- const content = (0, import_fs.readFileSync)(filePath, "utf-8");
33
- return (0, import_n8n_workflow.jsonParse)(content);
32
+ const content = await (0, import_promises.readFile)(filePath, "utf-8");
33
+ return await (0, import_n8n_workflow.jsonParse)(content);
34
34
  };
35
- function getEncoding(encoding) {
36
- if (cache[encoding]) {
37
- return cache[encoding];
35
+ async function getEncoding(encoding) {
36
+ if (!(encoding in cache)) {
37
+ cache[encoding] = (async () => {
38
+ let jsonData;
39
+ switch (encoding) {
40
+ case "o200k_base":
41
+ jsonData = await loadJSONFile("./o200k_base.json");
42
+ break;
43
+ case "cl100k_base":
44
+ jsonData = await loadJSONFile("./cl100k_base.json");
45
+ break;
46
+ default:
47
+ jsonData = await loadJSONFile("./cl100k_base.json");
48
+ }
49
+ return new import_lite.Tiktoken(jsonData);
50
+ })().catch((error) => {
51
+ delete cache[encoding];
52
+ throw error;
53
+ });
38
54
  }
39
- let jsonData;
40
- switch (encoding) {
41
- case "o200k_base":
42
- jsonData = loadJSONFile("./o200k_base.json");
43
- break;
44
- case "cl100k_base":
45
- jsonData = loadJSONFile("./cl100k_base.json");
46
- break;
47
- default:
48
- jsonData = loadJSONFile("./cl100k_base.json");
49
- }
50
- cache[encoding] = new import_lite.Tiktoken(jsonData);
51
- return cache[encoding];
55
+ return await cache[encoding];
52
56
  }
53
- function encodingForModel(model) {
54
- return getEncoding((0, import_lite.getEncodingNameForModel)(model));
57
+ async function encodingForModel(model) {
58
+ return await getEncoding((0, import_lite.getEncodingNameForModel)(model));
55
59
  }
56
60
  // Annotate the CommonJS export names for ESM import in node:
57
61
  0 && (module.exports = {
package/dist/utils/tokenizer/tiktoken.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../../utils/tokenizer/tiktoken.ts"],"sourcesContent":["import { readFileSync } from 'fs';\nimport type { TiktokenBPE, TiktokenEncoding, TiktokenModel } from 'js-tiktoken/lite';\nimport { Tiktoken, getEncodingNameForModel } from 'js-tiktoken/lite';\nimport { jsonParse } from 'n8n-workflow';\nimport { join } from 'path';\n\nconst cache: Record<string, Tiktoken> = {};\n\nconst loadJSONFile = (filename: string): TiktokenBPE => {\n\tconst filePath = join(__dirname, filename);\n\tconst content = readFileSync(filePath, 'utf-8');\n\treturn jsonParse(content);\n};\n\nexport function getEncoding(encoding: TiktokenEncoding): Tiktoken {\n\tif (cache[encoding]) {\n\t\treturn cache[encoding];\n\t}\n\n\tlet jsonData: TiktokenBPE;\n\n\tswitch (encoding) {\n\t\tcase 'o200k_base':\n\t\t\tjsonData = loadJSONFile('./o200k_base.json');\n\t\t\tbreak;\n\t\tcase 'cl100k_base':\n\t\t\tjsonData = loadJSONFile('./cl100k_base.json');\n\t\t\tbreak;\n\t\tdefault:\n\t\t\t// Fall back to cl100k_base for unsupported encodings\n\t\t\tjsonData = loadJSONFile('./cl100k_base.json');\n\t}\n\n\tcache[encoding] = new Tiktoken(jsonData);\n\treturn cache[encoding];\n}\n\nexport function encodingForModel(model: TiktokenModel): Tiktoken {\n\treturn getEncoding(getEncodingNameForModel(model));\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,gBAA6B;AAE7B,kBAAkD;AAClD,0BAA0B;AAC1B,kBAAqB;AAErB,MAAM,QAAkC,CAAC;AAEzC,MAAM,eAAe,CAAC,aAAkC;AACvD,QAAM,eAAW,kBAAK,WAAW,QAAQ;AACzC,QAAM,cAAU,wBAAa,UAAU,OAAO;AAC9C,aAAO,+BAAU,OAAO;AACzB;AAEO,SAAS,YAAY,UAAsC;AACjE,MAAI,MAAM,QAAQ,GAAG;AACpB,WAAO,MAAM,QAAQ;AAAA,EACtB;AAEA,MAAI;AAEJ,UAAQ,UAAU;AAAA,IACjB,KAAK;AACJ,iBAAW,aAAa,mBAAmB;AAC3C;AAAA,IACD,KAAK;AACJ,iBAAW,aAAa,oBAAoB;AAC5C;AAAA,IACD;AAEC,iBAAW,aAAa,oBAAoB;AAAA,EAC9C;AAEA,QAAM,QAAQ,IAAI,IAAI,qBAAS,QAAQ;AACvC,SAAO,MAAM,QAAQ;AACtB;AAEO,SAAS,iBAAiB,OAAgC;AAChE,SAAO,gBAAY,qCAAwB,KAAK,CAAC;AAClD;","names":[]}
1
+ {"version":3,"sources":["../../../utils/tokenizer/tiktoken.ts"],"sourcesContent":["import { readFile } from 'fs/promises';\nimport type { TiktokenBPE, TiktokenEncoding, TiktokenModel } from 'js-tiktoken/lite';\nimport { Tiktoken, getEncodingNameForModel } from 'js-tiktoken/lite';\nimport { jsonParse } from 'n8n-workflow';\nimport { join } from 'path';\n\nconst cache: Record<string, Promise<Tiktoken>> = {};\n\nconst loadJSONFile = async (filename: string): Promise<TiktokenBPE> => {\n\tconst filePath = join(__dirname, filename);\n\tconst content = await readFile(filePath, 'utf-8');\n\treturn await jsonParse(content);\n};\n\nexport async function getEncoding(encoding: TiktokenEncoding): Promise<Tiktoken> {\n\tif (!(encoding in cache)) {\n\t\t// Create and cache the promise for loading this encoding\n\t\tcache[encoding] = (async () => {\n\t\t\tlet jsonData: TiktokenBPE;\n\n\t\t\tswitch (encoding) {\n\t\t\t\tcase 'o200k_base':\n\t\t\t\t\tjsonData = await loadJSONFile('./o200k_base.json');\n\t\t\t\t\tbreak;\n\t\t\t\tcase 'cl100k_base':\n\t\t\t\t\tjsonData = await loadJSONFile('./cl100k_base.json');\n\t\t\t\t\tbreak;\n\t\t\t\tdefault:\n\t\t\t\t\t// Fall back to cl100k_base for unsupported encodings\n\t\t\t\t\tjsonData = await loadJSONFile('./cl100k_base.json');\n\t\t\t}\n\n\t\t\treturn new Tiktoken(jsonData);\n\t\t})().catch((error) => {\n\t\t\tdelete cache[encoding];\n\t\t\tthrow error;\n\t\t});\n\t}\n\n\treturn await cache[encoding];\n}\n\nexport async function encodingForModel(model: TiktokenModel): Promise<Tiktoken> {\n\treturn await getEncoding(getEncodingNameForModel(model));\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,sBAAyB;AAEzB,kBAAkD;AAClD,0BAA0B;AAC1B,kBAAqB;AAErB,MAAM,QAA2C,CAAC;AAElD,MAAM,eAAe,OAAO,aAA2C;AACtE,QAAM,eAAW,kBAAK,WAAW,QAAQ;AACzC,QAAM,UAAU,UAAM,0BAAS,UAAU,OAAO;AAChD,SAAO,UAAM,+BAAU,OAAO;AAC/B;AAEA,eAAsB,YAAY,UAA+C;AAChF,MAAI,EAAE,YAAY,QAAQ;AAEzB,UAAM,QAAQ,KAAK,YAAY;AAC9B,UAAI;AAEJ,cAAQ,UAAU;AAAA,QACjB,KAAK;AACJ,qBAAW,MAAM,aAAa,mBAAmB;AACjD;AAAA,QACD,KAAK;AACJ,qBAAW,MAAM,aAAa,oBAAoB;AAClD;AAAA,QACD;AAEC,qBAAW,MAAM,aAAa,oBAAoB;AAAA,MACpD;AAEA,aAAO,IAAI,qBAAS,QAAQ;AAAA,IAC7B,GAAG,EAAE,MAAM,CAAC,UAAU;AACrB,aAAO,MAAM,QAAQ;AACrB,YAAM;AAAA,IACP,CAAC;AAAA,EACF;AAEA,SAAO,MAAM,MAAM,QAAQ;AAC5B;AAEA,eAAsB,iBAAiB,OAAyC;AAC/E,SAAO,MAAM,gBAAY,qCAAwB,KAAK,CAAC;AACxD;","names":[]}
package/dist/utils/tokenizer/token-estimator.js CHANGED
@@ -82,7 +82,7 @@ async function estimateTokensFromStringList(list, model) {
82
82
  if (!Array.isArray(list)) {
83
83
  return 0;
84
84
  }
85
- const encoder = (0, import_tiktoken.encodingForModel)(model);
85
+ const encoder = await (0, import_tiktoken.encodingForModel)(model);
86
86
  const encodedListLength = await Promise.all(
87
87
  list.map(async (text) => {
88
88
  try {
package/dist/utils/tokenizer/token-estimator.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../../../utils/tokenizer/token-estimator.ts"],"sourcesContent":["/**\n * Token estimation utilities for handling text without using tiktoken.\n * This is used as a fallback when tiktoken would be too slow (e.g., with repetitive content).\n */\n\nimport type { TiktokenModel } from 'js-tiktoken';\n\nimport { encodingForModel } from './tiktoken';\nimport { hasLongSequentialRepeat } from '../helpers';\n\n/**\n * Model-specific average characters per token ratios.\n * These are approximate values based on typical English text.\n */\nconst MODEL_CHAR_PER_TOKEN_RATIOS: Record<string, number> = {\n\t'gpt-4o': 3.8,\n\t'gpt-4': 4.0,\n\t'gpt-3.5-turbo': 4.0,\n\tcl100k_base: 4.0,\n\to200k_base: 3.5,\n\tp50k_base: 4.2,\n\tr50k_base: 4.2,\n};\n\n/**\n * Estimates the number of tokens in a text based on character count.\n * This is much faster than tiktoken but less accurate.\n *\n * @param text The text to estimate tokens for\n * @param model The model or encoding name (optional)\n * @returns Estimated number of tokens\n */\nexport function estimateTokensByCharCount(text: string, model: string = 'cl100k_base'): number {\n\ttry {\n\t\t// Validate input\n\t\tif (!text || typeof text !== 'string' || text.length === 0) {\n\t\t\treturn 0;\n\t\t}\n\n\t\t// Get the ratio for the specific model, or use default\n\t\tconst charsPerToken = MODEL_CHAR_PER_TOKEN_RATIOS[model] || 4.0;\n\n\t\t// Validate ratio\n\t\tif (!Number.isFinite(charsPerToken) || charsPerToken <= 0) {\n\t\t\t// Fallback to default ratio\n\t\t\tconst estimatedTokens = Math.ceil(text.length / 4.0);\n\t\t\treturn estimatedTokens;\n\t\t}\n\n\t\t// Calculate estimated tokens\n\t\tconst estimatedTokens = Math.ceil(text.length / charsPerToken);\n\n\t\treturn estimatedTokens;\n\t} catch (error) {\n\t\t// Return conservative estimate on error\n\t\treturn Math.ceil((text?.length || 0) / 4.0);\n\t}\n}\n\n/**\n * Estimates tokens for text splitting purposes.\n * Returns chunk boundaries based on character positions rather than token positions.\n *\n * @param text The text to split\n * @param chunkSize Target chunk size in tokens\n * @param chunkOverlap Overlap between chunks in tokens\n * @param model The model or encoding name (optional)\n * @returns Array of text chunks\n */\nexport function estimateTextSplitsByTokens(\n\ttext: string,\n\tchunkSize: number,\n\tchunkOverlap: number,\n\tmodel: string = 'cl100k_base',\n): string[] {\n\ttry {\n\t\t// Validate inputs\n\t\tif (!text || typeof text !== 'string' || text.length === 0) {\n\t\t\treturn [];\n\t\t}\n\n\t\t// Validate numeric parameters\n\t\tif (!Number.isFinite(chunkSize) || chunkSize <= 0) {\n\t\t\t// Return whole text as single chunk if invalid chunk size\n\t\t\treturn [text];\n\t\t}\n\n\t\t// Ensure overlap is valid and less than chunk size\n\t\tconst validOverlap =\n\t\t\tNumber.isFinite(chunkOverlap) && chunkOverlap >= 0\n\t\t\t\t? Math.min(chunkOverlap, chunkSize - 1)\n\t\t\t\t: 0;\n\n\t\tconst charsPerToken = MODEL_CHAR_PER_TOKEN_RATIOS[model] || 4.0;\n\t\tconst chunkSizeInChars = Math.floor(chunkSize * charsPerToken);\n\t\tconst overlapInChars = Math.floor(validOverlap * charsPerToken);\n\n\t\tconst chunks: string[] = [];\n\t\tlet start = 0;\n\n\t\twhile (start < text.length) {\n\t\t\tconst end = Math.min(start + chunkSizeInChars, text.length);\n\t\t\tchunks.push(text.slice(start, end));\n\n\t\t\tif (end >= text.length) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\t// Move to next chunk with overlap\n\t\t\tstart = Math.max(end - overlapInChars, start + 1);\n\t\t}\n\n\t\treturn chunks;\n\t} catch (error) {\n\t\t// Return text as single chunk on error\n\t\treturn text ? [text] : [];\n\t}\n}\n\n/**\n * Estimates the total number of tokens for a list of strings.\n * Uses tiktoken for normal text but falls back to character-based estimation\n * for repetitive content or on errors.\n *\n * @param list Array of strings to estimate tokens for\n * @param model The model or encoding name to use for estimation\n * @returns Total estimated number of tokens across all strings\n */\nexport async function estimateTokensFromStringList(\n\tlist: string[],\n\tmodel: TiktokenModel,\n): Promise<number> {\n\ttry {\n\t\t// Validate input\n\t\tif (!Array.isArray(list)) {\n\t\t\treturn 0;\n\t\t}\n\n\t\tconst encoder = encodingForModel(model);\n\t\tconst encodedListLength = await Promise.all(\n\t\t\tlist.map(async (text) => {\n\t\t\t\ttry {\n\t\t\t\t\t// Handle null/undefined text\n\t\t\t\t\tif (!text || typeof text !== 'string') {\n\t\t\t\t\t\treturn 0;\n\t\t\t\t\t}\n\n\t\t\t\t\t// Check for repetitive content\n\t\t\t\t\tif (hasLongSequentialRepeat(text)) {\n\t\t\t\t\t\tconst estimatedTokens = estimateTokensByCharCount(text, model);\n\t\t\t\t\t\treturn estimatedTokens;\n\t\t\t\t\t}\n\n\t\t\t\t\t// Use tiktoken for normal text\n\t\t\t\t\ttry {\n\t\t\t\t\t\tconst tokens = encoder.encode(text);\n\t\t\t\t\t\treturn tokens.length;\n\t\t\t\t\t} catch (encodingError) {\n\t\t\t\t\t\t// Fall back to estimation if tiktoken fails\n\t\t\t\t\t\treturn estimateTokensByCharCount(text, model);\n\t\t\t\t\t}\n\t\t\t\t} catch (itemError) {\n\t\t\t\t\t// Return 0 for individual item errors\n\t\t\t\t\treturn 0;\n\t\t\t\t}\n\t\t\t}),\n\t\t);\n\n\t\tconst totalTokens = encodedListLength.reduce((acc, curr) => acc + curr, 0);\n\n\t\treturn totalTokens;\n\t} catch (error) {\n\t\t// Return 0 on complete failure\n\t\treturn 0;\n\t}\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAOA,sBAAiC;AACjC,qBAAwC;AAMxC,MAAM,8BAAsD;AAAA,EAC3D,UAAU;AAAA,EACV,SAAS;AAAA,EACT,iBAAiB;AAAA,EACjB,aAAa;AAAA,EACb,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,WAAW;AACZ;AAUO,SAAS,0BAA0B,MAAc,QAAgB,eAAuB;AAC9F,MAAI;AAEH,QAAI,CAAC,QAAQ,OAAO,SAAS,YAAY,KAAK,WAAW,GAAG;AAC3D,aAAO;AAAA,IACR;AAGA,UAAM,gBAAgB,4BAA4B,KAAK,KAAK;AAG5D,QAAI,CAAC,OAAO,SAAS,aAAa,KAAK,iBAAiB,GAAG;AAE1D,YAAMA,mBAAkB,KAAK,KAAK,KAAK,SAAS,CAAG;AACnD,aAAOA;AAAA,IACR;AAGA,UAAM,kBAAkB,KAAK,KAAK,KAAK,SAAS,aAAa;AAE7D,WAAO;AAAA,EACR,SAAS,OAAO;AAEf,WAAO,KAAK,MAAM,MAAM,UAAU,KAAK,CAAG;AAAA,EAC3C;AACD;AAYO,SAAS,2BACf,MACA,WACA,cACA,QAAgB,eACL;AACX,MAAI;AAEH,QAAI,CAAC,QAAQ,OAAO,SAAS,YAAY,KAAK,WAAW,GAAG;AAC3D,aAAO,CAAC;AAAA,IACT;AAGA,QAAI,CAAC,OAAO,SAAS,SAAS,KAAK,aAAa,GAAG;AAElD,aAAO,CAAC,IAAI;AAAA,IACb;AAGA,UAAM,eACL,OAAO,SAAS,YAAY,KAAK,gBAAgB,IAC9C,KAAK,IAAI,cAAc,YAAY,CAAC,IACpC;AAEJ,UAAM,gBAAgB,4BAA4B,KAAK,KAAK;AAC5D,UAAM,mBAAmB,KAAK,MAAM,YAAY,aAAa;AAC7D,UAAM,iBAAiB,KAAK,MAAM,eAAe,aAAa;AAE9D,UAAM,SAAmB,CAAC;AAC1B,QAAI,QAAQ;AAEZ,WAAO,QAAQ,KAAK,QAAQ;AAC3B,YAAM,MAAM,KAAK,IAAI,QAAQ,kBAAkB,KAAK,MAAM;AAC1D,aAAO,KAAK,KAAK,MAAM,OAAO,GAAG,CAAC;AAElC,UAAI,OAAO,KAAK,QAAQ;AACvB;AAAA,MACD;AAGA,cAAQ,KAAK,IAAI,MAAM,gBAAgB,QAAQ,CAAC;AAAA,IACjD;AAEA,WAAO;AAAA,EACR,SAAS,OAAO;AAEf,WAAO,OAAO,CAAC,IAAI,IAAI,CAAC;AAAA,EACzB;AACD;AAWA,eAAsB,6BACrB,MACA,OACkB;AAClB,MAAI;AAEH,QAAI,CAAC,MAAM,QAAQ,IAAI,GAAG;AACzB,aAAO;AAAA,IACR;AAEA,UAAM,cAAU,kCAAiB,KAAK;AACtC,UAAM,oBAAoB,MAAM,QAAQ;AAAA,MACvC,KAAK,IAAI,OAAO,SAAS;AACxB,YAAI;AAEH,cAAI,CAAC,QAAQ,OAAO,SAAS,UAAU;AACtC,mBAAO;AAAA,UACR;AAGA,kBAAI,wCAAwB,IAAI,GAAG;AAClC,kBAAM,kBAAkB,0BAA0B,MAAM,KAAK;AAC7D,mBAAO;AAAA,UACR;AAGA,cAAI;AACH,kBAAM,SAAS,QAAQ,OAAO,IAAI;AAClC,mBAAO,OAAO;AAAA,UACf,SAAS,eAAe;AAEvB,mBAAO,0BAA0B,MAAM,KAAK;AAAA,UAC7C;AAAA,QACD,SAAS,WAAW;AAEnB,iBAAO;AAAA,QACR;AAAA,MACD,CAAC;AAAA,IACF;AAEA,UAAM,cAAc,kBAAkB,OAAO,CAAC,KAAK,SAAS,MAAM,MAAM,CAAC;AAEzE,WAAO;AAAA,EACR,SAAS,OAAO;AAEf,WAAO;AAAA,EACR;AACD;","names":["estimatedTokens"]}
1
+ {"version":3,"sources":["../../../utils/tokenizer/token-estimator.ts"],"sourcesContent":["/**\n * Token estimation utilities for handling text without using tiktoken.\n * This is used as a fallback when tiktoken would be too slow (e.g., with repetitive content).\n */\n\nimport type { TiktokenModel } from 'js-tiktoken';\n\nimport { encodingForModel } from './tiktoken';\nimport { hasLongSequentialRepeat } from '../helpers';\n\n/**\n * Model-specific average characters per token ratios.\n * These are approximate values based on typical English text.\n */\nconst MODEL_CHAR_PER_TOKEN_RATIOS: Record<string, number> = {\n\t'gpt-4o': 3.8,\n\t'gpt-4': 4.0,\n\t'gpt-3.5-turbo': 4.0,\n\tcl100k_base: 4.0,\n\to200k_base: 3.5,\n\tp50k_base: 4.2,\n\tr50k_base: 4.2,\n};\n\n/**\n * Estimates the number of tokens in a text based on character count.\n * This is much faster than tiktoken but less accurate.\n *\n * @param text The text to estimate tokens for\n * @param model The model or encoding name (optional)\n * @returns Estimated number of tokens\n */\nexport function estimateTokensByCharCount(text: string, model: string = 'cl100k_base'): number {\n\ttry {\n\t\t// Validate input\n\t\tif (!text || typeof text !== 'string' || text.length === 0) {\n\t\t\treturn 0;\n\t\t}\n\n\t\t// Get the ratio for the specific model, or use default\n\t\tconst charsPerToken = MODEL_CHAR_PER_TOKEN_RATIOS[model] || 4.0;\n\n\t\t// Validate ratio\n\t\tif (!Number.isFinite(charsPerToken) || charsPerToken <= 0) {\n\t\t\t// Fallback to default ratio\n\t\t\tconst estimatedTokens = Math.ceil(text.length / 4.0);\n\t\t\treturn estimatedTokens;\n\t\t}\n\n\t\t// Calculate estimated tokens\n\t\tconst estimatedTokens = Math.ceil(text.length / charsPerToken);\n\n\t\treturn estimatedTokens;\n\t} catch (error) {\n\t\t// Return conservative estimate on error\n\t\treturn Math.ceil((text?.length || 0) / 4.0);\n\t}\n}\n\n/**\n * Estimates tokens for text splitting purposes.\n * Returns chunk boundaries based on character positions rather than token positions.\n *\n * @param text The text to split\n * @param chunkSize Target chunk size in tokens\n * @param chunkOverlap Overlap between chunks in tokens\n * @param model The model or encoding name (optional)\n * @returns Array of text chunks\n */\nexport function estimateTextSplitsByTokens(\n\ttext: string,\n\tchunkSize: number,\n\tchunkOverlap: number,\n\tmodel: string = 'cl100k_base',\n): string[] {\n\ttry {\n\t\t// Validate inputs\n\t\tif (!text || typeof text !== 'string' || text.length === 0) {\n\t\t\treturn [];\n\t\t}\n\n\t\t// Validate numeric parameters\n\t\tif (!Number.isFinite(chunkSize) || chunkSize <= 0) {\n\t\t\t// Return whole text as single chunk if invalid chunk size\n\t\t\treturn [text];\n\t\t}\n\n\t\t// Ensure overlap is valid and less than chunk size\n\t\tconst validOverlap =\n\t\t\tNumber.isFinite(chunkOverlap) && chunkOverlap >= 0\n\t\t\t\t? Math.min(chunkOverlap, chunkSize - 1)\n\t\t\t\t: 0;\n\n\t\tconst charsPerToken = MODEL_CHAR_PER_TOKEN_RATIOS[model] || 4.0;\n\t\tconst chunkSizeInChars = Math.floor(chunkSize * charsPerToken);\n\t\tconst overlapInChars = Math.floor(validOverlap * charsPerToken);\n\n\t\tconst chunks: string[] = [];\n\t\tlet start = 0;\n\n\t\twhile (start < text.length) {\n\t\t\tconst end = Math.min(start + chunkSizeInChars, text.length);\n\t\t\tchunks.push(text.slice(start, end));\n\n\t\t\tif (end >= text.length) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\t// Move to next chunk with overlap\n\t\t\tstart = Math.max(end - overlapInChars, start + 1);\n\t\t}\n\n\t\treturn chunks;\n\t} catch (error) {\n\t\t// Return text as single chunk on error\n\t\treturn text ? [text] : [];\n\t}\n}\n\n/**\n * Estimates the total number of tokens for a list of strings.\n * Uses tiktoken for normal text but falls back to character-based estimation\n * for repetitive content or on errors.\n *\n * @param list Array of strings to estimate tokens for\n * @param model The model or encoding name to use for estimation\n * @returns Total estimated number of tokens across all strings\n */\nexport async function estimateTokensFromStringList(\n\tlist: string[],\n\tmodel: TiktokenModel,\n): Promise<number> {\n\ttry {\n\t\t// Validate input\n\t\tif (!Array.isArray(list)) {\n\t\t\treturn 0;\n\t\t}\n\n\t\tconst encoder = await encodingForModel(model);\n\t\tconst encodedListLength = await Promise.all(\n\t\t\tlist.map(async (text) => {\n\t\t\t\ttry {\n\t\t\t\t\t// Handle null/undefined text\n\t\t\t\t\tif (!text || typeof text !== 'string') {\n\t\t\t\t\t\treturn 0;\n\t\t\t\t\t}\n\n\t\t\t\t\t// Check for repetitive content\n\t\t\t\t\tif (hasLongSequentialRepeat(text)) {\n\t\t\t\t\t\tconst estimatedTokens = estimateTokensByCharCount(text, model);\n\t\t\t\t\t\treturn estimatedTokens;\n\t\t\t\t\t}\n\n\t\t\t\t\t// Use tiktoken for normal text\n\t\t\t\t\ttry {\n\t\t\t\t\t\tconst tokens = encoder.encode(text);\n\t\t\t\t\t\treturn tokens.length;\n\t\t\t\t\t} catch (encodingError) {\n\t\t\t\t\t\t// Fall back to estimation if tiktoken fails\n\t\t\t\t\t\treturn estimateTokensByCharCount(text, model);\n\t\t\t\t\t}\n\t\t\t\t} catch (itemError) {\n\t\t\t\t\t// Return 0 for individual item errors\n\t\t\t\t\treturn 0;\n\t\t\t\t}\n\t\t\t}),\n\t\t);\n\n\t\tconst totalTokens = encodedListLength.reduce((acc, curr) => acc + curr, 0);\n\n\t\treturn totalTokens;\n\t} catch (error) {\n\t\t// Return 0 on complete failure\n\t\treturn 0;\n\t}\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAOA,sBAAiC;AACjC,qBAAwC;AAMxC,MAAM,8BAAsD;AAAA,EAC3D,UAAU;AAAA,EACV,SAAS;AAAA,EACT,iBAAiB;AAAA,EACjB,aAAa;AAAA,EACb,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,WAAW;AACZ;AAUO,SAAS,0BAA0B,MAAc,QAAgB,eAAuB;AAC9F,MAAI;AAEH,QAAI,CAAC,QAAQ,OAAO,SAAS,YAAY,KAAK,WAAW,GAAG;AAC3D,aAAO;AAAA,IACR;AAGA,UAAM,gBAAgB,4BAA4B,KAAK,KAAK;AAG5D,QAAI,CAAC,OAAO,SAAS,aAAa,KAAK,iBAAiB,GAAG;AAE1D,YAAMA,mBAAkB,KAAK,KAAK,KAAK,SAAS,CAAG;AACnD,aAAOA;AAAA,IACR;AAGA,UAAM,kBAAkB,KAAK,KAAK,KAAK,SAAS,aAAa;AAE7D,WAAO;AAAA,EACR,SAAS,OAAO;AAEf,WAAO,KAAK,MAAM,MAAM,UAAU,KAAK,CAAG;AAAA,EAC3C;AACD;AAYO,SAAS,2BACf,MACA,WACA,cACA,QAAgB,eACL;AACX,MAAI;AAEH,QAAI,CAAC,QAAQ,OAAO,SAAS,YAAY,KAAK,WAAW,GAAG;AAC3D,aAAO,CAAC;AAAA,IACT;AAGA,QAAI,CAAC,OAAO,SAAS,SAAS,KAAK,aAAa,GAAG;AAElD,aAAO,CAAC,IAAI;AAAA,IACb;AAGA,UAAM,eACL,OAAO,SAAS,YAAY,KAAK,gBAAgB,IAC9C,KAAK,IAAI,cAAc,YAAY,CAAC,IACpC;AAEJ,UAAM,gBAAgB,4BAA4B,KAAK,KAAK;AAC5D,UAAM,mBAAmB,KAAK,MAAM,YAAY,aAAa;AAC7D,UAAM,iBAAiB,KAAK,MAAM,eAAe,aAAa;AAE9D,UAAM,SAAmB,CAAC;AAC1B,QAAI,QAAQ;AAEZ,WAAO,QAAQ,KAAK,QAAQ;AAC3B,YAAM,MAAM,KAAK,IAAI,QAAQ,kBAAkB,KAAK,MAAM;AAC1D,aAAO,KAAK,KAAK,MAAM,OAAO,GAAG,CAAC;AAElC,UAAI,OAAO,KAAK,QAAQ;AACvB;AAAA,MACD;AAGA,cAAQ,KAAK,IAAI,MAAM,gBAAgB,QAAQ,CAAC;AAAA,IACjD;AAEA,WAAO;AAAA,EACR,SAAS,OAAO;AAEf,WAAO,OAAO,CAAC,IAAI,IAAI,CAAC;AAAA,EACzB;AACD;AAWA,eAAsB,6BACrB,MACA,OACkB;AAClB,MAAI;AAEH,QAAI,CAAC,MAAM,QAAQ,IAAI,GAAG;AACzB,aAAO;AAAA,IACR;AAEA,UAAM,UAAU,UAAM,kCAAiB,KAAK;AAC5C,UAAM,oBAAoB,MAAM,QAAQ;AAAA,MACvC,KAAK,IAAI,OAAO,SAAS;AACxB,YAAI;AAEH,cAAI,CAAC,QAAQ,OAAO,SAAS,UAAU;AACtC,mBAAO;AAAA,UACR;AAGA,kBAAI,wCAAwB,IAAI,GAAG;AAClC,kBAAM,kBAAkB,0BAA0B,MAAM,KAAK;AAC7D,mBAAO;AAAA,UACR;AAGA,cAAI;AACH,kBAAM,SAAS,QAAQ,OAAO,IAAI;AAClC,mBAAO,OAAO;AAAA,UACf,SAAS,eAAe;AAEvB,mBAAO,0BAA0B,MAAM,KAAK;AAAA,UAC7C;AAAA,QACD,SAAS,WAAW;AAEnB,iBAAO;AAAA,QACR;AAAA,MACD,CAAC;AAAA,IACF;AAEA,UAAM,cAAc,kBAAkB,OAAO,CAAC,KAAK,SAAS,MAAM,MAAM,CAAC;AAEzE,WAAO;AAAA,EACR,SAAS,OAAO;AAEf,WAAO;AAAA,EACR;AACD;","names":["estimatedTokens"]}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@n8n/n8n-nodes-langchain",
3
- "version": "1.108.0",
3
+ "version": "1.109.0",
4
4
  "description": "",
5
5
  "main": "index.js",
6
6
  "files": [
@@ -148,7 +148,7 @@
148
148
  "fast-glob": "3.2.12",
149
149
  "jest-mock-extended": "^3.0.4",
150
150
  "tsup": "^8.5.0",
151
- "n8n-core": "1.108.0"
151
+ "n8n-core": "1.109.0"
152
152
  },
153
153
  "dependencies": {
154
154
  "@aws-sdk/client-sso-oidc": "3.808.0",
@@ -215,12 +215,12 @@
215
215
  "weaviate-client": "3.6.2",
216
216
  "zod": "3.25.67",
217
217
  "zod-to-json-schema": "3.23.3",
218
- "@n8n/client-oauth2": "0.28.0",
219
- "@n8n/errors": "^0.4.0",
218
+ "@n8n/client-oauth2": "0.29.0",
220
219
  "@n8n/json-schema-to-zod": "1.5.0",
220
+ "@n8n/errors": "^0.5.0",
221
221
  "@n8n/typescript-config": "1.3.0",
222
- "n8n-nodes-base": "1.107.0",
223
- "n8n-workflow": "1.106.0"
222
+ "n8n-workflow": "1.107.0",
223
+ "n8n-nodes-base": "1.108.0"
224
224
  },
225
225
  "license": "SEE LICENSE IN LICENSE.md",
226
226
  "homepage": "https://n8n.io",