@n8n/ai-utilities 0.2.0

package/dist/utils/tokenizer/tiktoken.d.ts

@@ -0,0 +1,4 @@
+import type { TiktokenEncoding, TiktokenModel } from 'js-tiktoken/lite';
+import { Tiktoken } from 'js-tiktoken/lite';
+export declare function getEncoding(encoding: TiktokenEncoding): Promise<Tiktoken>;
+export declare function encodingForModel(model: TiktokenModel): Promise<Tiktoken>;

package/dist/utils/tokenizer/tiktoken.js

@@ -0,0 +1,40 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getEncoding = getEncoding;
+exports.encodingForModel = encodingForModel;
+const promises_1 = require("fs/promises");
+const lite_1 = require("js-tiktoken/lite");
+const n8n_workflow_1 = require("n8n-workflow");
+const path_1 = require("path");
+const cache = {};
+const loadJSONFile = async (filename) => {
+    const filePath = (0, path_1.join)(__dirname, filename);
+    const content = await (0, promises_1.readFile)(filePath, 'utf-8');
+    return await (0, n8n_workflow_1.jsonParse)(content);
+};
+async function getEncoding(encoding) {
+    if (!(encoding in cache)) {
+        cache[encoding] = (async () => {
+            let jsonData;
+            switch (encoding) {
+                case 'o200k_base':
+                    jsonData = await loadJSONFile('./o200k_base.json');
+                    break;
+                case 'cl100k_base':
+                    jsonData = await loadJSONFile('./cl100k_base.json');
+                    break;
+                default:
+                    jsonData = await loadJSONFile('./cl100k_base.json');
+            }
+            return new lite_1.Tiktoken(jsonData);
+        })().catch((error) => {
+            delete cache[encoding];
+            throw error;
+        });
+    }
+    return await cache[encoding];
+}
+async function encodingForModel(model) {
+    return await getEncoding((0, lite_1.getEncodingNameForModel)(model));
+}
+//# sourceMappingURL=tiktoken.js.map

package/dist/utils/tokenizer/tiktoken.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tiktoken.js","sourceRoot":"","sources":["../../../src/utils/tokenizer/tiktoken.ts"],"names":[],"mappings":";;AAcA,kCA0BC;AAED,4CAEC;AA5CD,0CAAuC;AAEvC,2CAAqE;AACrE,+CAAyC;AACzC,+BAA4B;AAE5B,MAAM,KAAK,GAAsC,EAAE,CAAC;AAEpD,MAAM,YAAY,GAAG,KAAK,EAAE,QAAgB,EAAwB,EAAE;IACrE,MAAM,QAAQ,GAAG,IAAA,WAAI,EAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAC3C,MAAM,OAAO,GAAG,MAAM,IAAA,mBAAQ,EAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAClD,OAAO,MAAM,IAAA,wBAAS,EAAC,OAAO,CAAC,CAAC;AACjC,CAAC,CAAC;AAEK,KAAK,UAAU,WAAW,CAAC,QAA0B;IAC3D,IAAI,CAAC,CAAC,QAAQ,IAAI,KAAK,CAAC,EAAE,CAAC;QAE1B,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;YAC7B,IAAI,QAAqB,CAAC;YAE1B,QAAQ,QAAQ,EAAE,CAAC;gBAClB,KAAK,YAAY;oBAChB,QAAQ,GAAG,MAAM,YAAY,CAAC,mBAAmB,CAAC,CAAC;oBACnD,MAAM;gBACP,KAAK,aAAa;oBACjB,QAAQ,GAAG,MAAM,YAAY,CAAC,oBAAoB,CAAC,CAAC;oBACpD,MAAM;gBACP;oBAEC,QAAQ,GAAG,MAAM,YAAY,CAAC,oBAAoB,CAAC,CAAC;YACtD,CAAC;YAED,OAAO,IAAI,eAAQ,CAAC,QAAQ,CAAC,CAAC;QAC/B,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACpB,OAAO,KAAK,CAAC,QAAQ,CAAC,CAAC;YACvB,MAAM,KAAK,CAAC;QACb,CAAC,CAAC,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;AAC9B,CAAC;AAEM,KAAK,UAAU,gBAAgB,CAAC,KAAoB;IAC1D,OAAO,MAAM,WAAW,CAAC,IAAA,8BAAuB,EAAC,KAAK,CAAC,CAAC,CAAC;AAC1D,CAAC"}

package/dist/utils/tokenizer/token-estimator.d.ts

@@ -0,0 +1,4 @@
+import type { TiktokenModel } from 'js-tiktoken';
+export declare function estimateTokensByCharCount(text: string, model?: string): number;
+export declare function estimateTextSplitsByTokens(text: string, chunkSize: number, chunkOverlap: number, model?: string): string[];
+export declare function estimateTokensFromStringList(list: string[], model: TiktokenModel): Promise<number>;

package/dist/utils/tokenizer/token-estimator.js

@@ -0,0 +1,98 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.estimateTokensByCharCount = estimateTokensByCharCount;
+exports.estimateTextSplitsByTokens = estimateTextSplitsByTokens;
+exports.estimateTokensFromStringList = estimateTokensFromStringList;
+const tiktoken_1 = require("./tiktoken");
+const helpers_1 = require("../helpers");
+const MODEL_CHAR_PER_TOKEN_RATIOS = {
+    'gpt-4o': 3.8,
+    'gpt-4': 4.0,
+    'gpt-3.5-turbo': 4.0,
+    cl100k_base: 4.0,
+    o200k_base: 3.5,
+    p50k_base: 4.2,
+    r50k_base: 4.2,
+};
+function estimateTokensByCharCount(text, model = 'cl100k_base') {
+    try {
+        if (!text || typeof text !== 'string' || text.length === 0) {
+            return 0;
+        }
+        const charsPerToken = MODEL_CHAR_PER_TOKEN_RATIOS[model] || 4.0;
+        if (!Number.isFinite(charsPerToken) || charsPerToken <= 0) {
+            const estimatedTokens = Math.ceil(text.length / 4.0);
+            return estimatedTokens;
+        }
+        const estimatedTokens = Math.ceil(text.length / charsPerToken);
+        return estimatedTokens;
+    }
+    catch (error) {
+        return Math.ceil((text?.length || 0) / 4.0);
+    }
+}
+function estimateTextSplitsByTokens(text, chunkSize, chunkOverlap, model = 'cl100k_base') {
+    try {
+        if (!text || typeof text !== 'string' || text.length === 0) {
+            return [];
+        }
+        if (!Number.isFinite(chunkSize) || chunkSize <= 0) {
+            return [text];
+        }
+        const validOverlap = Number.isFinite(chunkOverlap) && chunkOverlap >= 0
+            ? Math.min(chunkOverlap, chunkSize - 1)
+            : 0;
+        const charsPerToken = MODEL_CHAR_PER_TOKEN_RATIOS[model] || 4.0;
+        const chunkSizeInChars = Math.floor(chunkSize * charsPerToken);
+        const overlapInChars = Math.floor(validOverlap * charsPerToken);
+        const chunks = [];
+        let start = 0;
+        while (start < text.length) {
+            const end = Math.min(start + chunkSizeInChars, text.length);
+            chunks.push(text.slice(start, end));
+            if (end >= text.length) {
+                break;
+            }
+            start = Math.max(end - overlapInChars, start + 1);
+        }
+        return chunks;
+    }
+    catch (error) {
+        return text ? [text] : [];
+    }
+}
+async function estimateTokensFromStringList(list, model) {
+    try {
+        if (!Array.isArray(list)) {
+            return 0;
+        }
+        const encoder = await (0, tiktoken_1.encodingForModel)(model);
+        const encodedListLength = await Promise.all(list.map(async (text) => {
+            try {
+                if (!text || typeof text !== 'string') {
+                    return 0;
+                }
+                if ((0, helpers_1.hasLongSequentialRepeat)(text)) {
+                    const estimatedTokens = estimateTokensByCharCount(text, model);
+                    return estimatedTokens;
+                }
+                try {
+                    const tokens = encoder.encode(text);
+                    return tokens.length;
+                }
+                catch (encodingError) {
+                    return estimateTokensByCharCount(text, model);
+                }
+            }
+            catch (itemError) {
+                return 0;
+            }
+        }));
+        const totalTokens = encodedListLength.reduce((acc, curr) => acc + curr, 0);
+        return totalTokens;
+    }
+    catch (error) {
+        return 0;
+    }
+}
+//# sourceMappingURL=token-estimator.js.map

package/dist/utils/tokenizer/token-estimator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-estimator.js","sourceRoot":"","sources":["../../../src/utils/tokenizer/token-estimator.ts"],"names":[],"mappings":";;AAgCA,8DAyBC;AAYD,gEAgDC;AAWD,oEA+CC;AAxKD,yCAA8C;AAC9C,wCAAqD;AAMrD,MAAM,2BAA2B,GAA2B;IAC3D,QAAQ,EAAE,GAAG;IACb,OAAO,EAAE,GAAG;IACZ,eAAe,EAAE,GAAG;IACpB,WAAW,EAAE,GAAG;IAChB,UAAU,EAAE,GAAG;IACf,SAAS,EAAE,GAAG;IACd,SAAS,EAAE,GAAG;CACd,CAAC;AAUF,SAAgB,yBAAyB,CAAC,IAAY,EAAE,QAAgB,aAAa;IACpF,IAAI,CAAC;QAEJ,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5D,OAAO,CAAC,CAAC;QACV,CAAC;QAGD,MAAM,aAAa,GAAG,2BAA2B,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC;QAGhE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,aAAa,IAAI,CAAC,EAAE,CAAC;YAE3D,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;YACrD,OAAO,eAAe,CAAC;QACxB,CAAC;QAGD,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC;QAE/D,OAAO,eAAe,CAAC;IACxB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAEhB,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;IAC7C,CAAC;AACF,CAAC;AAYD,SAAgB,0BAA0B,CACzC,IAAY,EACZ,SAAiB,EACjB,YAAoB,EACpB,QAAgB,aAAa;IAE7B,IAAI,CAAC;QAEJ,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5D,OAAO,EAAE,CAAC;QACX,CAAC;QAGD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;YAEnD,OAAO,CAAC,IAAI,CAAC,CAAC;QACf,CAAC;QAGD,MAAM,YAAY,GACjB,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,YAAY,IAAI,CAAC;YACjD,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,SAAS,GAAG,CAAC,CAAC;YACvC,CAAC,CAAC,CAAC,CAAC;QAEN,MAAM,aAAa,GAAG,2BAA2B,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC;QAChE,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,aAAa,CAAC,CAAC;QAC/D,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,aAAa,CAAC,CAAC;QAEhE,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAC5D,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YAEpC,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACxB,MAAM;YACP,CAAC;YAGD,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,cAAc,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QACnD,CAAC;QAED,OAAO,MAAM,CAAC;IACf,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAEhB,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3B,CAAC;AACF,CAAC;AAWM,KAAK,UAAU,4BAA4B,CACjD,IAAc,EACd,KAAoB;IAEpB,IAAI,CAAC;QAEJ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,CAAC;QACV,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAA,2BAAgB,EAAC,KAAK,CAAC,CAAC;QAC9C,MAAM,iBAAiB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC1C,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACvB,IAAI,CAAC;gBAEJ,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;oBACvC,OAAO,CAAC,CAAC;gBACV,CAAC;gBAGD,IAAI,IAAA,iCAAuB,EAAC,IAAI,CAAC,EAAE,CAAC;oBACnC,MAAM,eAAe,GAAG,yBAAyB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;oBAC/D,OAAO,eAAe,CAAC;gBACxB,CAAC;gBAGD,IAAI,CAAC;oBACJ,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBACpC,OAAO,MAAM,CAAC,MAAM,CAAC;gBACtB,CAAC;gBAAC,OAAO,aAAa,EAAE,CAAC;oBAExB,OAAO,yBAAyB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAC/C,CAAC;YACF,CAAC;YAAC,OAAO,SAAS,EAAE,CAAC;gBAEpB,OAAO,CAAC,CAAC;YACV,CAAC;QACF,CAAC,CAAC,CACF,CAAC;QAEF,MAAM,WAAW,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC;QAE3E,OAAO,WAAW,CAAC;IACpB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAEhB,OAAO,CAAC,CAAC;IACV,CAAC;AACF,CAAC"}

package/package.json

@@ -0,0 +1,51 @@
+{
+  "name": "@n8n/ai-utilities",
+  "version": "0.2.0",
+  "description": "Utilities for building AI nodes in n8n",
+  "main": "dist/index.js",
+  "module": "src/index.ts",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "devDependencies": {
+    "@types/json-schema": "^7.0.15",
+    "jest-mock-extended": "^3.0.4",
+    "@types/mime-types": "3.0.1",
+    "tsx": "^4.19.3"
+  },
+  "dependencies": {
+    "zod": "3.25.67",
+    "zod-to-json-schema": "3.23.3",
+    "@langchain/core": "1.1.8",
+    "@langchain/classic": "1.0.5",
+    "@langchain/community": "1.0.5",
+    "@langchain/textsplitters": "1.0.1",
+    "@langchain/openai": "1.1.3",
+    "langchain": "1.2.3",
+    "tmp-promise": "3.0.3",
+    "js-tiktoken": "1.0.12",
+    "https-proxy-agent": "7.0.6",
+    "proxy-from-env": "^1.1.0",
+    "undici": "^6.21.0",
+    "n8n-workflow": "2.8.0",
+    "@n8n/config": "2.7.0",
+    "@n8n/typescript-config": "1.3.0"
+  },
+  "scripts": {
+    "example:run": "tsx examples/run.ts",
+    "clean": "rimraf dist .turbo",
+    "dev": "pnpm run watch",
+    "typecheck": "tsc --noEmit",
+    "copy-tokenizer-json": "node scripts/copy-tokenizer-json.js .",
+    "build": "tsc --build tsconfig.build.json && tsc-alias -p tsconfig.build.json && pnpm copy-tokenizer-json",
+    "format": "biome format --write .",
+    "format:check": "biome ci .",
+    "lint": "eslint . --quiet",
+    "lint:fix": "eslint . --fix",
+    "watch": "tsc-watch -p tsconfig.build.json",
+    "test": "jest",
+    "test:unit": "jest",
+    "test:dev": "jest --watch"
+  }
+}