@n8n/ai-utilities 0.2.0

package/dist/utils/n8n-binary-loader.js

@@ -0,0 +1,159 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.N8nBinaryLoader = void 0;
+const json_1 = require("@langchain/classic/document_loaders/fs/json");
+const text_1 = require("@langchain/classic/document_loaders/fs/text");
+const csv_1 = require("@langchain/community/document_loaders/fs/csv");
+const docx_1 = require("@langchain/community/document_loaders/fs/docx");
+const epub_1 = require("@langchain/community/document_loaders/fs/epub");
+const pdf_1 = require("@langchain/community/document_loaders/fs/pdf");
+const fs_1 = require("fs");
+const n8n_workflow_1 = require("n8n-workflow");
+const promises_1 = require("stream/promises");
+const tmp_promise_1 = require("tmp-promise");
+const helpers_1 = require("./helpers");
+const SUPPORTED_MIME_TYPES = {
+    auto: ['*/*'],
+    pdfLoader: ['application/pdf'],
+    csvLoader: ['text/csv'],
+    epubLoader: ['application/epub+zip'],
+    docxLoader: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
+    textLoader: ['text/plain', 'text/mdx', 'text/md', 'text/markdown'],
+    jsonLoader: ['application/json'],
+};
+class N8nBinaryLoader {
+    constructor(context, optionsPrefix = '', binaryDataKey = '', textSplitter) {
+        this.context = context;
+        this.optionsPrefix = optionsPrefix;
+        this.binaryDataKey = binaryDataKey;
+        this.textSplitter = textSplitter;
+    }
+    async processAll(items) {
+        const docs = [];
+        if (!items)
+            return [];
+        for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
+            const processedDocuments = await this.processItem(items[itemIndex], itemIndex);
+            docs.push(...processedDocuments);
+        }
+        return docs;
+    }
+    async validateMimeType(mimeType, selectedLoader) {
+        if (selectedLoader !== 'auto' && !SUPPORTED_MIME_TYPES[selectedLoader].includes(mimeType)) {
+            const neededLoader = Object.keys(SUPPORTED_MIME_TYPES).find((loader) => SUPPORTED_MIME_TYPES[loader].includes(mimeType));
+            throw new n8n_workflow_1.NodeOperationError(this.context.getNode(), `Mime type doesn't match selected loader. Please select under "Loader Type": ${neededLoader}`);
+        }
+        if (!Object.values(SUPPORTED_MIME_TYPES).flat().includes(mimeType)) {
+            throw new n8n_workflow_1.NodeOperationError(this.context.getNode(), `Unsupported mime type: ${mimeType}`);
+        }
+        if (!SUPPORTED_MIME_TYPES[selectedLoader].includes(mimeType) &&
+            selectedLoader !== 'textLoader' &&
+            selectedLoader !== 'auto') {
+            throw new n8n_workflow_1.NodeOperationError(this.context.getNode(), `Unsupported mime type: ${mimeType} for selected loader: ${selectedLoader}`);
+        }
+    }
+    async getFilePathOrBlob(binaryData, mimeType) {
+        if (binaryData.id) {
+            const binaryBuffer = await this.context.helpers.binaryToBuffer(await this.context.helpers.getBinaryStream(binaryData.id));
+            return new Blob([binaryBuffer], {
+                type: mimeType,
+            });
+        }
+        else {
+            return new Blob([Buffer.from(binaryData.data, n8n_workflow_1.BINARY_ENCODING)], {
+                type: mimeType,
+            });
+        }
+    }
+    async getLoader(mimeType, filePathOrBlob, itemIndex) {
+        switch (mimeType) {
+            case 'application/pdf':
+                const splitPages = this.context.getNodeParameter(`${this.optionsPrefix}splitPages`, itemIndex, false);
+                return new pdf_1.PDFLoader(filePathOrBlob, { splitPages });
+            case 'text/csv':
+                const column = this.context.getNodeParameter(`${this.optionsPrefix}column`, itemIndex, null);
+                const separator = this.context.getNodeParameter(`${this.optionsPrefix}separator`, itemIndex, ',');
+                return new csv_1.CSVLoader(filePathOrBlob, { column: column ?? undefined, separator });
+            case 'application/epub+zip':
+                let filePath;
+                if (filePathOrBlob instanceof Blob) {
+                    const tmpFileData = await (0, tmp_promise_1.file)({ prefix: 'epub-loader-' });
+                    const bufferData = await filePathOrBlob.arrayBuffer();
+                    await (0, promises_1.pipeline)([new Uint8Array(bufferData)], (0, fs_1.createWriteStream)(tmpFileData.path));
+                    return new epub_1.EPubLoader(tmpFileData.path);
+                }
+                else {
+                    filePath = filePathOrBlob;
+                }
+                return new epub_1.EPubLoader(filePath);
+            case 'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
+                return new docx_1.DocxLoader(filePathOrBlob);
+            case 'text/plain':
+                return new text_1.TextLoader(filePathOrBlob);
+            case 'application/json':
+                const pointers = this.context.getNodeParameter(`${this.optionsPrefix}pointers`, itemIndex, '');
+                const pointersArray = pointers.split(',').map((pointer) => pointer.trim());
+                return new json_1.JSONLoader(filePathOrBlob, pointersArray);
+            default:
+                return new text_1.TextLoader(filePathOrBlob);
+        }
+    }
+    async loadDocuments(loader) {
+        return this.textSplitter
+            ? await this.textSplitter.splitDocuments(await loader.load())
+            : await loader.load();
+    }
+    async cleanupTmpFileIfNeeded(cleanupTmpFile) {
+        if (cleanupTmpFile) {
+            await cleanupTmpFile();
+        }
+    }
+    async processItem(item, itemIndex) {
+        const docs = [];
+        const binaryMode = this.context.getNodeParameter('binaryMode', itemIndex, 'allInputData');
+        if (binaryMode === 'allInputData') {
+            const binaryData = this.context.getInputData();
+            for (const data of binaryData) {
+                if (data.binary) {
+                    const binaryDataKeys = Object.keys(data.binary);
+                    for (const fileKey of binaryDataKeys) {
+                        const processedDocuments = await this.processItemByKey(item, itemIndex, fileKey);
+                        docs.push(...processedDocuments);
+                    }
+                }
+            }
+        }
+        else {
+            const processedDocuments = await this.processItemByKey(item, itemIndex, this.binaryDataKey);
+            docs.push(...processedDocuments);
+        }
+        return docs;
+    }
+    async processItemByKey(item, itemIndex, binaryKey) {
+        const selectedLoader = this.context.getNodeParameter('loader', itemIndex, 'auto');
+        const docs = [];
+        const metadata = (0, helpers_1.getMetadataFiltersValues)(this.context, itemIndex);
+        if (!item)
+            return [];
+        const binaryData = this.context.helpers.assertBinaryData(itemIndex, binaryKey);
+        const { mimeType } = binaryData;
+        await this.validateMimeType(mimeType, selectedLoader);
+        const filePathOrBlob = await this.getFilePathOrBlob(binaryData, mimeType);
+        const cleanupTmpFile = undefined;
+        const loader = await this.getLoader(mimeType, filePathOrBlob, itemIndex);
+        const loadedDoc = await this.loadDocuments(loader);
+        docs.push(...loadedDoc);
+        if (metadata) {
+            docs.forEach((document) => {
+                document.metadata = {
+                    ...document.metadata,
+                    ...metadata,
+                };
+            });
+        }
+        await this.cleanupTmpFileIfNeeded(cleanupTmpFile);
+        return docs;
+    }
+}
+exports.N8nBinaryLoader = N8nBinaryLoader;
+//# sourceMappingURL=n8n-binary-loader.js.map

package/dist/utils/n8n-binary-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"n8n-binary-loader.js","sourceRoot":"","sources":["../../src/utils/n8n-binary-loader.ts"],"names":[],"mappings":";;;AAAA,sEAAyE;AACzE,sEAAyE;AACzE,sEAAyE;AACzE,wEAA2E;AAC3E,wEAA2E;AAC3E,sEAAyE;AAGzE,2BAAuC;AAOvC,+CAAmE;AACnE,8CAA2C;AAC3C,6CAAoE;AAEpE,uCAAqD;AAErD,MAAM,oBAAoB,GAAG;IAC5B,IAAI,EAAE,CAAC,KAAK,CAAC;IACb,SAAS,EAAE,CAAC,iBAAiB,CAAC;IAC9B,SAAS,EAAE,CAAC,UAAU,CAAC;IACvB,UAAU,EAAE,CAAC,sBAAsB,CAAC;IACpC,UAAU,EAAE,CAAC,yEAAyE,CAAC;IACvF,UAAU,EAAE,CAAC,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,eAAe,CAAC;IAClE,UAAU,EAAE,CAAC,kBAAkB,CAAC;CAChC,CAAC;AAEF,MAAa,eAAe;IAC3B,YACS,OAAiD,EACjD,gBAAgB,EAAE,EAClB,gBAAgB,EAAE,EAClB,YAA2B;QAH3B,YAAO,GAAP,OAAO,CAA0C;QACjD,kBAAa,GAAb,aAAa,CAAK;QAClB,kBAAa,GAAb,aAAa,CAAK;QAClB,iBAAY,GAAZ,YAAY,CAAe;IACjC,CAAC;IAEJ,KAAK,CAAC,UAAU,CAAC,KAA4B;QAC5C,MAAM,IAAI,GAAe,EAAE,CAAC;QAE5B,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAC;QAEtB,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;YAC/D,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,SAAS,CAAC,CAAC;YAE/E,IAAI,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,CAAC;QAClC,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC7B,QAAgB,EAChB,cAAiD;QAGjD,IAAI,cAAc,KAAK,MAAM,IAAI,CAAC,oBAAoB,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3F,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CACtE,oBAAoB,CAAC,MAA2C,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CACpF,CAAC;YAEF,MAAM,IAAI,iCAAkB,CAC3B,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EACtB,+EAA+E,YAAY,EAAE,CAC7F,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YACpE,MAAM,IAAI,iCAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,0BAA0B,QAAQ,EAAE,CAAC,CAAC;QAC5F,CAAC;QAED,IACC,CAAC,oBAAoB,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACxD,cAAc,KAAK,YAAY;YAC/B,cAAc,KAAK,MAAM,EACxB,CAAC;YACF,MAAM,IAAI,iCAAkB,CAC3B,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EACtB,0BAA0B,QAAQ,yBAAyB,cAAc,EAAE,CAC3E,CAAC;QACH,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC9B,UAAuB,EACvB,QAAgB;QAEhB,IAAI,UAAU,CAAC,EAAE,EAAE,CAAC;YACnB,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,CAC7D,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,UAAU,CAAC,EAAE,CAAC,CACzD,CAAC;YACF,OAAO,IAAI,IAAI,CAAC,CAAC,YAAwB,CAAC,EAAE;gBAC3C,IAAI,EAAE,QAAQ;aACd,CAAC,CAAC;QACJ,CAAC;aAAM,CAAC;YACP,OAAO,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,8BAAe,CAAC,CAAC,EAAE;gBAChE,IAAI,EAAE,QAAQ;aACd,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,SAAS,CACtB,QAAgB,EAChB,cAA6B,EAC7B,SAAiB;QAEjB,QAAQ,QAAQ,EAAE,CAAC;YAClB,KAAK,iBAAiB;gBACrB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAC/C,GAAG,IAAI,CAAC,aAAa,YAAY,EACjC,SAAS,EACT,KAAK,CACM,CAAC;gBACb,OAAO,IAAI,eAAS,CAAC,cAAc,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;YACtD,KAAK,UAAU;gBACd,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAC3C,GAAG,IAAI,CAAC,aAAa,QAAQ,EAC7B,SAAS,EACT,IAAI,CACM,CAAC;gBACZ,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAC9C,GAAG,IAAI,CAAC,aAAa,WAAW,EAChC,SAAS,EACT,GAAG,CACO,CAAC;gBACZ,OAAO,IAAI,eAAS,CAAC,cAAc,EAAE,EAAE,MAAM,EAAE,MAAM,IAAI,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;YAClF,KAAK,sBAAsB;gBAE1B,IAAI,QAAgB,CAAC;gBACrB,IAAI,cAAc,YAAY,IAAI,EAAE,CAAC;oBACpC,MAAM,WAAW,GAAG,MAAM,IAAA,kBAAO,EAAC,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC,CAAC;oBAC9D,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,WAAW,EAAE,CAAC;oBACtD,MAAM,IAAA,mBAAQ,EAAC,CAAC,IAAI,UAAU,CAAC,UAAU,CAAC,CAAC,EAAE,IAAA,sBAAiB,EAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;oBAClF,OAAO,IAAI,iBAAU,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBACzC,CAAC;qBAAM,CAAC;oBACP,QAAQ,GAAG,cAAc,CAAC;gBAC3B,CAAC;gBACD,OAAO,IAAI,iBAAU,CAAC,QAAQ,CAAC,CAAC;YACjC,KAAK,yEAAyE;gBAC7E,OAAO,IAAI,iBAAU,CAAC,cAAc,CAAC,CAAC;YACvC,KAAK,YAAY;gBAChB,OAAO,IAAI,iBAAU,CAAC,cAAc,CAAC,CAAC;YACvC,KAAK,kBAAkB;gBACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAC7C,GAAG,IAAI,CAAC,aAAa,UAAU,EAC/B,SAAS,EACT,EAAE,CACQ,CAAC;gBACZ,MAAM,aAAa,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3E,OAAO,IAAI,iBAAU,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC;YACtD;gBACC,OAAO,IAAI,iBAAU,CAAC,cAAc,CAAC,CAAC;QACxC,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,aAAa,CAC1B,MAAiF;QAEjF,OAAO,IAAI,CAAC,YAAY;YACvB,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAC7D,CAAC,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,sBAAsB,CACnC,cAAsD;QAEtD,IAAI,cAAc,EAAE,CAAC;YACpB,MAAM,cAAc,EAAE,CAAC;QACxB,CAAC;IACF,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,IAAwB,EAAE,SAAiB;QAC5D,MAAM,IAAI,GAAe,EAAE,CAAC;QAC5B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,YAAY,EAAE,SAAS,EAAE,cAAc,CAAC,CAAC;QAC1F,IAAI,UAAU,KAAK,cAAc,EAAE,CAAC;YACnC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;YAE/C,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;gBAC/B,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;oBACjB,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBAEhD,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;wBACtC,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;wBACjF,IAAI,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,CAAC;oBAClC,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;aAAM,CAAC;YACP,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;YAC5F,IAAI,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,CAAC;QAClC,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC;IAED,KAAK,CAAC,gBAAgB,CACrB,IAAwB,EACxB,SAAiB,EACjB,SAAiB;QAEjB,MAAM,cAAc,GAAsC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CACtF,QAAQ,EACR,SAAS,EACT,MAAM,CAC+B,CAAC;QAEvC,MAAM,IAAI,GAAe,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,IAAA,kCAAwB,EAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAEnE,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QAErB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC/E,MAAM,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC;QAEhC,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;QAEtD,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAC1E,MAAM,cAAc,GAA2C,SAAS,CAAC;QACzE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,cAAc,EAAE,SAAS,CAAC,CAAC;QACzE,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QAEnD,IAAI,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;QAExB,IAAI,QAAQ,EAAE,CAAC;YACd,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;gBACzB,QAAQ,CAAC,QAAQ,GAAG;oBACnB,GAAG,QAAQ,CAAC,QAAQ;oBACpB,GAAG,QAAQ;iBACX,CAAC;YACH,CAAC,CAAC,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,CAAC,sBAAsB,CAAC,cAAc,CAAC,CAAC;QAElD,OAAO,IAAI,CAAC;IACb,CAAC;CACD;AA/MD,0CA+MC"}

package/dist/utils/n8n-json-loader.d.ts

@@ -0,0 +1,11 @@
+import type { Document } from '@langchain/core/documents';
+import type { TextSplitter } from '@langchain/textsplitters';
+import { type IExecuteFunctions, type INodeExecutionData, type ISupplyDataFunctions } from 'n8n-workflow';
+export declare class N8nJsonLoader {
+    private context;
+    private optionsPrefix;
+    private textSplitter?;
+    constructor(context: IExecuteFunctions | ISupplyDataFunctions, optionsPrefix?: string, textSplitter?: TextSplitter | undefined);
+    processAll(items?: INodeExecutionData[]): Promise<Document[]>;
+    processItem(item: INodeExecutionData, itemIndex: number): Promise<Document[]>;
+}

package/dist/utils/n8n-json-loader.js

@@ -0,0 +1,66 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.N8nJsonLoader = void 0;
+const json_1 = require("@langchain/classic/document_loaders/fs/json");
+const text_1 = require("@langchain/classic/document_loaders/fs/text");
+const n8n_workflow_1 = require("n8n-workflow");
+const helpers_1 = require("./helpers");
+class N8nJsonLoader {
+    constructor(context, optionsPrefix = '', textSplitter) {
+        this.context = context;
+        this.optionsPrefix = optionsPrefix;
+        this.textSplitter = textSplitter;
+    }
+    async processAll(items) {
+        const docs = [];
+        if (!items)
+            return [];
+        for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
+            const processedDocuments = await this.processItem(items[itemIndex], itemIndex);
+            docs.push(...processedDocuments);
+        }
+        return docs;
+    }
+    async processItem(item, itemIndex) {
+        const mode = this.context.getNodeParameter('jsonMode', itemIndex, 'allInputData');
+        const pointers = this.context.getNodeParameter(`${this.optionsPrefix}pointers`, itemIndex, '');
+        const pointersArray = pointers.split(',').map((pointer) => pointer.trim());
+        const metadata = (0, helpers_1.getMetadataFiltersValues)(this.context, itemIndex) ?? [];
+        if (!item)
+            return [];
+        let documentLoader = null;
+        if (mode === 'allInputData') {
+            const itemString = JSON.stringify(item.json);
+            const itemBlob = new Blob([itemString], { type: 'application/json' });
+            documentLoader = new json_1.JSONLoader(itemBlob, pointersArray);
+        }
+        if (mode === 'expressionData') {
+            const dataString = this.context.getNodeParameter('jsonData', itemIndex);
+            if (typeof dataString === 'object') {
+                const itemBlob = new Blob([JSON.stringify(dataString)], { type: 'application/json' });
+                documentLoader = new json_1.JSONLoader(itemBlob, pointersArray);
+            }
+            if (typeof dataString === 'string') {
+                const itemBlob = new Blob([dataString], { type: 'text/plain' });
+                documentLoader = new text_1.TextLoader(itemBlob);
+            }
+        }
+        if (documentLoader === null) {
+            throw new n8n_workflow_1.NodeOperationError(this.context.getNode(), 'Document loader is not initialized');
+        }
+        const docs = this.textSplitter
+            ? await this.textSplitter.splitDocuments(await documentLoader.load())
+            : await documentLoader.load();
+        if (metadata) {
+            docs.forEach((doc) => {
+                doc.metadata = {
+                    ...doc.metadata,
+                    ...metadata,
+                };
+            });
+        }
+        return docs;
+    }
+}
+exports.N8nJsonLoader = N8nJsonLoader;
+//# sourceMappingURL=n8n-json-loader.js.map

package/dist/utils/n8n-json-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"n8n-json-loader.js","sourceRoot":"","sources":["../../src/utils/n8n-json-loader.ts"],"names":[],"mappings":";;;AAAA,sEAAyE;AACzE,sEAAyE;AAGzE,+CAKsB;AAEtB,uCAAqD;AAErD,MAAa,aAAa;IACzB,YACS,OAAiD,EACjD,gBAAgB,EAAE,EAClB,YAA2B;QAF3B,YAAO,GAAP,OAAO,CAA0C;QACjD,kBAAa,GAAb,aAAa,CAAK;QAClB,iBAAY,GAAZ,YAAY,CAAe;IACjC,CAAC;IAEJ,KAAK,CAAC,UAAU,CAAC,KAA4B;QAC5C,MAAM,IAAI,GAAe,EAAE,CAAC;QAE5B,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAC;QAEtB,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;YAC/D,MAAM,kBAAkB,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,SAAS,CAAC,CAAC;YAE/E,IAAI,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,CAAC;QAClC,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,IAAwB,EAAE,SAAiB;QAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,UAAU,EAAE,SAAS,EAAE,cAAc,CAE7D,CAAC;QAEpB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAC7C,GAAG,IAAI,CAAC,aAAa,UAAU,EAC/B,SAAS,EACT,EAAE,CACQ,CAAC;QACZ,MAAM,aAAa,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3E,MAAM,QAAQ,GAAG,IAAA,kCAAwB,EAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC;QAEzE,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QAErB,IAAI,cAAc,GAAmC,IAAI,CAAC;QAE1D,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;YAC7B,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,CAAC,UAAU,CAAC,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAAC,CAAC;YACtE,cAAc,GAAG,IAAI,iBAAU,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,IAAI,KAAK,gBAAgB,EAAE,CAAC;YAC/B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,UAAU,EAAE,SAAS,CAAoB,CAAC;YAC3F,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;gBACpC,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAAC,CAAC;gBACtF,cAAc,GAAG,IAAI,iBAAU,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;YAC1D,CAAC;YAED,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;gBACpC,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,CAAC,UAAU,CAAC,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC;gBAChE,cAAc,GAAG,IAAI,iBAAU,CAAC,QAAQ,CAAC,CAAC;YAC3C,CAAC;QACF,CAAC;QAED,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;YAE7B,MAAM,IAAI,iCAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,oCAAoC,CAAC,CAAC;QAC5F,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY;YAC7B,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;YACrE,CAAC,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,CAAC;QAE/B,IAAI,QAAQ,EAAE,CAAC;YACd,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;gBACpB,GAAG,CAAC,QAAQ,GAAG;oBACd,GAAG,GAAG,CAAC,QAAQ;oBACf,GAAG,QAAQ;iBACX,CAAC;YACH,CAAC,CAAC,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;CACD;AA5ED,sCA4EC"}

package/dist/utils/n8n-llm-tracing.d.ts

@@ -0,0 +1,46 @@
+import { BaseCallbackHandler } from '@langchain/core/callbacks/base';
+import type { SerializedFields } from '@langchain/core/dist/load/map_keys';
+import type { Serialized, SerializedNotImplemented, SerializedSecret } from '@langchain/core/load/serializable';
+import type { BaseMessage } from '@langchain/core/messages';
+import type { LLMResult } from '@langchain/core/outputs';
+import type { IDataObject, ISupplyDataFunctions } from 'n8n-workflow';
+import { NodeError } from 'n8n-workflow';
+type TokensUsageParser = (result: LLMResult) => {
+    completionTokens: number;
+    promptTokens: number;
+    totalTokens: number;
+};
+type RunDetail = {
+    index: number;
+    messages: BaseMessage[] | string[] | string;
+    options: SerializedSecret | SerializedNotImplemented | SerializedFields;
+};
+export declare class N8nLlmTracing extends BaseCallbackHandler {
+    #private;
+    private executionFunctions;
+    name: string;
+    awaitHandlers: boolean;
+    connectionType: "ai_languageModel";
+    promptTokensEstimate: number;
+    completionTokensEstimate: number;
+    runsMap: Record<string, RunDetail>;
+    options: {
+        tokensUsageParser: (result: LLMResult) => {
+            completionTokens: number;
+            promptTokens: number;
+            totalTokens: number;
+        };
+        errorDescriptionMapper: (error: NodeError) => string | null | undefined;
+    };
+    constructor(executionFunctions: ISupplyDataFunctions, options?: {
+        tokensUsageParser?: TokensUsageParser;
+        errorDescriptionMapper?: (error: NodeError) => string;
+    });
+    estimateTokensFromGeneration(generations: LLMResult['generations']): Promise<number>;
+    estimateTokensFromStringList(list: string[]): Promise<number>;
+    handleLLMEnd(output: LLMResult, runId: string): Promise<void>;
+    handleLLMStart(llm: Serialized, prompts: string[], runId: string): Promise<void>;
+    handleLLMError(error: IDataObject | Error, runId: string, parentRunId?: string): Promise<void>;
+    setParentRunIndex(runIndex: number): void;
+}
+export {};

package/dist/utils/n8n-llm-tracing.js

@@ -0,0 +1,157 @@
+"use strict";
+var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
+    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
+};
+var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
+    if (kind === "m") throw new TypeError("Private method is not writable");
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
+    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+var _N8nLlmTracing_parentRunIndex;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.N8nLlmTracing = void 0;
+const base_1 = require("@langchain/core/callbacks/base");
+const base_2 = require("@langchain/core/language_models/base");
+const pick_1 = __importDefault(require("lodash/pick"));
+const n8n_workflow_1 = require("n8n-workflow");
+const log_ai_event_1 = require("./log-ai-event");
+const token_estimator_1 = require("./tokenizer/token-estimator");
+const TIKTOKEN_ESTIMATE_MODEL = 'gpt-4o';
+class N8nLlmTracing extends base_1.BaseCallbackHandler {
+    constructor(executionFunctions, options) {
+        super();
+        this.executionFunctions = executionFunctions;
+        this.name = 'N8nLlmTracing';
+        this.awaitHandlers = true;
+        this.connectionType = n8n_workflow_1.NodeConnectionTypes.AiLanguageModel;
+        this.promptTokensEstimate = 0;
+        this.completionTokensEstimate = 0;
+        _N8nLlmTracing_parentRunIndex.set(this, void 0);
+        this.runsMap = {};
+        this.options = {
+            tokensUsageParser: (result) => {
+                const completionTokens = result?.llmOutput?.tokenUsage?.completionTokens ?? 0;
+                const promptTokens = result?.llmOutput?.tokenUsage?.promptTokens ?? 0;
+                return {
+                    completionTokens,
+                    promptTokens,
+                    totalTokens: completionTokens + promptTokens,
+                };
+            },
+            errorDescriptionMapper: (error) => error.description,
+        };
+        this.options = { ...this.options, ...options };
+    }
+    async estimateTokensFromGeneration(generations) {
+        const messages = generations.flatMap((gen) => gen.map((g) => g.text));
+        return await this.estimateTokensFromStringList(messages);
+    }
+    async estimateTokensFromStringList(list) {
+        const embeddingModel = (0, base_2.getModelNameForTiktoken)(TIKTOKEN_ESTIMATE_MODEL);
+        return await (0, token_estimator_1.estimateTokensFromStringList)(list, embeddingModel);
+    }
+    async handleLLMEnd(output, runId) {
+        const runDetails = this.runsMap[runId] ?? { index: Object.keys(this.runsMap).length };
+        output.generations = output.generations.map((gen) => gen.map((g) => (0, pick_1.default)(g, ['text', 'generationInfo'])));
+        const tokenUsageEstimate = {
+            completionTokens: 0,
+            promptTokens: 0,
+            totalTokens: 0,
+        };
+        const tokenUsage = this.options.tokensUsageParser(output);
+        if (output.generations.length > 0) {
+            tokenUsageEstimate.completionTokens = await this.estimateTokensFromGeneration(output.generations);
+            tokenUsageEstimate.promptTokens = this.promptTokensEstimate;
+            tokenUsageEstimate.totalTokens =
+                tokenUsageEstimate.completionTokens + this.promptTokensEstimate;
+        }
+        const response = {
+            response: { generations: output.generations },
+        };
+        if (tokenUsage.completionTokens > 0) {
+            response.tokenUsage = tokenUsage;
+        }
+        else {
+            response.tokenUsageEstimate = tokenUsageEstimate;
+        }
+        const parsedMessages = typeof runDetails.messages === 'string'
+            ? runDetails.messages
+            : runDetails.messages.map((message) => {
+                if (typeof message === 'string')
+                    return message;
+                if (typeof message?.toJSON === 'function')
+                    return message.toJSON();
+                return message;
+            });
+        const sourceNodeRunIndex = __classPrivateFieldGet(this, _N8nLlmTracing_parentRunIndex, "f") !== undefined ? __classPrivateFieldGet(this, _N8nLlmTracing_parentRunIndex, "f") + runDetails.index : undefined;
+        this.executionFunctions.addOutputData(this.connectionType, runDetails.index, [[{ json: { ...response } }]], undefined, sourceNodeRunIndex);
+        (0, log_ai_event_1.logAiEvent)(this.executionFunctions, 'ai-llm-generated-output', {
+            messages: parsedMessages,
+            options: runDetails.options,
+            response,
+        });
+    }
+    async handleLLMStart(llm, prompts, runId) {
+        const estimatedTokens = await this.estimateTokensFromStringList(prompts);
+        const sourceNodeRunIndex = __classPrivateFieldGet(this, _N8nLlmTracing_parentRunIndex, "f") !== undefined
+            ? __classPrivateFieldGet(this, _N8nLlmTracing_parentRunIndex, "f") + this.executionFunctions.getNextRunIndex()
+            : undefined;
+        const options = llm.type === 'constructor' ? llm.kwargs : llm;
+        const { index } = this.executionFunctions.addInputData(this.connectionType, [
+            [
+                {
+                    json: {
+                        messages: prompts,
+                        estimatedTokens,
+                        options,
+                    },
+                },
+            ],
+        ], sourceNodeRunIndex);
+        this.runsMap[runId] = {
+            index,
+            options,
+            messages: prompts,
+        };
+        this.promptTokensEstimate = estimatedTokens;
+    }
+    async handleLLMError(error, runId, parentRunId) {
+        const runDetails = this.runsMap[runId] ?? { index: Object.keys(this.runsMap).length };
+        if (typeof error === 'object' && error?.hasOwnProperty('headers')) {
+            const errorWithHeaders = error;
+            Object.keys(errorWithHeaders.headers).forEach((key) => {
+                if (!key.startsWith('x-')) {
+                    delete errorWithHeaders.headers[key];
+                }
+            });
+        }
+        if (error instanceof n8n_workflow_1.NodeError) {
+            if (this.options.errorDescriptionMapper) {
+                error.description = this.options.errorDescriptionMapper(error);
+            }
+            this.executionFunctions.addOutputData(this.connectionType, runDetails.index, error);
+        }
+        else {
+            this.executionFunctions.addOutputData(this.connectionType, runDetails.index, new n8n_workflow_1.NodeOperationError(this.executionFunctions.getNode(), error, {
+                functionality: 'configuration-node',
+            }));
+        }
+        (0, log_ai_event_1.logAiEvent)(this.executionFunctions, 'ai-llm-errored', {
+            error: Object.keys(error).length === 0 ? error.toString() : error,
+            runId,
+            parentRunId,
+        });
+    }
+    setParentRunIndex(runIndex) {
+        __classPrivateFieldSet(this, _N8nLlmTracing_parentRunIndex, runIndex, "f");
+    }
+}
+exports.N8nLlmTracing = N8nLlmTracing;
+_N8nLlmTracing_parentRunIndex = new WeakMap();
+//# sourceMappingURL=n8n-llm-tracing.js.map

package/dist/utils/n8n-llm-tracing.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"n8n-llm-tracing.js","sourceRoot":"","sources":["../../src/utils/n8n-llm-tracing.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;AAAA,yDAAqE;AAErE,+DAA+E;AAQ/E,uDAA+B;AAE/B,+CAAkF;AAElF,iDAA4C;AAC5C,iEAA2E;AAc3E,MAAM,uBAAuB,GAAG,QAAQ,CAAC;AACzC,MAAa,aAAc,SAAQ,0BAAmB;IAsCrD,YACS,kBAAwC,EAChD,OAGC;QAED,KAAK,EAAE,CAAC;QANA,uBAAkB,GAAlB,kBAAkB,CAAsB;QAtCjD,SAAI,GAAG,eAAe,CAAC;QAIvB,kBAAa,GAAG,IAAI,CAAC;QAErB,mBAAc,GAAG,kCAAmB,CAAC,eAAe,CAAC;QAErD,yBAAoB,GAAG,CAAC,CAAC;QAEzB,6BAAwB,GAAG,CAAC,CAAC;QAE7B,gDAAyB;QAQzB,YAAO,GAA8B,EAAE,CAAC;QAExC,YAAO,GAAG;YAET,iBAAiB,EAAE,CAAC,MAAiB,EAAE,EAAE;gBACxC,MAAM,gBAAgB,GAAI,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,gBAA2B,IAAI,CAAC,CAAC;gBAC1F,MAAM,YAAY,GAAI,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,YAAuB,IAAI,CAAC,CAAC;gBAElF,OAAO;oBACN,gBAAgB;oBAChB,YAAY;oBACZ,WAAW,EAAE,gBAAgB,GAAG,YAAY;iBAC5C,CAAC;YACH,CAAC;YACD,sBAAsB,EAAE,CAAC,KAAgB,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW;SAC/D,CAAC;QAUD,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,4BAA4B,CAAC,WAAqC;QACvE,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACtE,OAAO,MAAM,IAAI,CAAC,4BAA4B,CAAC,QAAQ,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,4BAA4B,CAAC,IAAc;QAChD,MAAM,cAAc,GAAG,IAAA,8BAAuB,EAAC,uBAAuB,CAAC,CAAC;QACxE,OAAO,MAAM,IAAA,8CAA4B,EAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAiB,EAAE,KAAa;QAGlD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;QAEtF,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACnD,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,cAAI,EAAC,CAAC,EAAE,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC,CAAC,CACnD,CAAC;QAEF,MAAM,kBAAkB,GAAG;YAC1B,gBAAgB,EAAE,CAAC;YACnB,YAAY,EAAE,CAAC;YACf,WAAW,EAAE,CAAC;SACd,CAAC;QACF,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAE1D,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,kBAAkB,CAAC,gBAAgB,GAAG,MAAM,IAAI,CAAC,4BAA4B,CAC5E,MAAM,CAAC,WAAW,CAClB,CAAC;YAEF,kBAAkB,CAAC,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAC;YAC5D,kBAAkB,CAAC,WAAW;gBAC7B,kBAAkB,CAAC,gBAAgB,GAAG,IAAI,CAAC,oBAAoB,CAAC;QAClE,CAAC;QACD,MAAM,QAAQ,GAIV;YACH,QAAQ,EAAE,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE;SAC7C,CAAC;QAGF,IAAI,UAAU,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC;YACrC,QAAQ,CAAC,UAAU,GAAG,UAAU,CAAC;QAClC,CAAC;aAAM,CAAC;YACP,QAAQ,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAClD,CAAC;QAED,MAAM,cAAc,GACnB,OAAO,UAAU,CAAC,QAAQ,KAAK,QAAQ;YACtC,CAAC,CAAC,UAAU,CAAC,QAAQ;YACrB,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;gBACpC,IAAI,OAAO,OAAO,KAAK,QAAQ;oBAAE,OAAO,OAAO,CAAC;gBAChD,IAAI,OAAO,OAAO,EAAE,MAAM,KAAK,UAAU;oBAAE,OAAO,OAAO,CAAC,MAAM,EAAE,CAAC;gBAEnE,OAAO,OAAO,CAAC;YAChB,CAAC,CAAC,CAAC;QAEN,MAAM,kBAAkB,GACvB,uBAAA,IAAI,qCAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,uBAAA,IAAI,qCAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QAE1F,IAAI,CAAC,kBAAkB,CAAC,aAAa,CACpC,IAAI,CAAC,cAAc,EACnB,UAAU,CAAC,KAAK,EAChB,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,GAAG,QAAQ,EAAE,EAAE,CAAC,CAAC,EAC7B,SAAS,EACT,kBAAkB,CAClB,CAAC;QAEF,IAAA,yBAAU,EAAC,IAAI,CAAC,kBAAkB,EAAE,yBAAyB,EAAE;YAC9D,QAAQ,EAAE,cAAc;YACxB,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,QAAQ;SACR,CAAC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,GAAe,EAAE,OAAiB,EAAE,KAAa;QACrE,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,4BAA4B,CAAC,OAAO,CAAC,CAAC;QACzE,MAAM,kBAAkB,GACvB,uBAAA,IAAI,qCAAgB,KAAK,SAAS;YACjC,CAAC,CAAC,uBAAA,IAAI,qCAAgB,GAAG,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE;YAClE,CAAC,CAAC,SAAS,CAAC;QAEd,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,KAAK,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC;QAC9D,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,CACrD,IAAI,CAAC,cAAc,EACnB;YACC;gBACC;oBACC,IAAI,EAAE;wBACL,QAAQ,EAAE,OAAO;wBACjB,eAAe;wBACf,OAAO;qBACP;iBACD;aACD;SACD,EACD,kBAAkB,CAClB,CAAC;QAGF,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG;YACrB,KAAK;YACL,OAAO;YACP,QAAQ,EAAE,OAAO;SACjB,CAAC;QACF,IAAI,CAAC,oBAAoB,GAAG,eAAe,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,KAA0B,EAAE,KAAa,EAAE,WAAoB;QACnF,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;QAItF,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,EAAE,cAAc,CAAC,SAAS,CAAC,EAAE,CAAC;YACnE,MAAM,gBAAgB,GAAG,KAA6C,CAAC;YAEvE,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;gBACrD,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC3B,OAAO,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACtC,CAAC;YACF,CAAC,CAAC,CAAC;QACJ,CAAC;QAED,IAAI,KAAK,YAAY,wBAAS,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;gBACzC,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC;YAChE,CAAC;YAED,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,IAAI,CAAC,cAAc,EAAE,UAAU,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACrF,CAAC;aAAM,CAAC;YAEP,IAAI,CAAC,kBAAkB,CAAC,aAAa,CACpC,IAAI,CAAC,cAAc,EACnB,UAAU,CAAC,KAAK,EAChB,IAAI,iCAAkB,CAAC,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,EAAE,KAAmB,EAAE;gBAC9E,aAAa,EAAE,oBAAoB;aACnC,CAAC,CACF,CAAC;QACH,CAAC;QAED,IAAA,yBAAU,EAAC,IAAI,CAAC,kBAAkB,EAAE,gBAAgB,EAAE;YAErD,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,KAAK;YACjE,KAAK;YACL,WAAW;SACX,CAAC,CAAC;IACJ,CAAC;IAGD,iBAAiB,CAAC,QAAgB;QACjC,uBAAA,IAAI,iCAAmB,QAAQ,MAAA,CAAC;IACjC,CAAC;CACD;AA5MD,sCA4MC"}

package/dist/utils/sse.d.ts

@@ -0,0 +1,8 @@
+export interface ServerSentEventMessage {
+    comment?: string;
+    event?: string;
+    data?: string;
+    id?: string | number;
+    retry?: number;
+}
+export declare function parseSSEStream(body: ReadableStream<Uint8Array>): AsyncIterable<ServerSentEventMessage>;

package/dist/utils/sse.js

@@ -0,0 +1,107 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseSSEStream = parseSSEStream;
+async function* parseSSEStream(body) {
+    const reader = body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = '';
+    let currentEvent = {};
+    let dataLines = [];
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done) {
+                buffer += decoder.decode();
+                if (buffer !== '') {
+                    const event = processLine(buffer);
+                    if (event) {
+                        yield event;
+                    }
+                }
+                if (hasEventContent()) {
+                    yield finalizeEvent();
+                }
+                break;
+            }
+            buffer += decoder.decode(value, { stream: true });
+            const lines = buffer.split(/\r\n|\r|\n/);
+            buffer = lines.pop() ?? '';
+            for (const line of lines) {
+                const event = processLine(line);
+                if (event) {
+                    yield event;
+                }
+            }
+        }
+    }
+    finally {
+        reader.releaseLock();
+    }
+    function processLine(line) {
+        if (line === '') {
+            if (hasEventContent()) {
+                const event = finalizeEvent();
+                currentEvent = {};
+                dataLines = [];
+                return event;
+            }
+            return null;
+        }
+        if (line.startsWith(':')) {
+            currentEvent.comment = line.slice(1).trimStart();
+            return null;
+        }
+        const colonIndex = line.indexOf(':');
+        if (colonIndex === -1) {
+            processField(line, '');
+            return null;
+        }
+        const fieldName = line.slice(0, colonIndex);
+        let fieldValue = line.slice(colonIndex + 1);
+        if (fieldValue.startsWith(' ')) {
+            fieldValue = fieldValue.slice(1);
+        }
+        processField(fieldName, fieldValue);
+        return null;
+    }
+    function processField(name, value) {
+        switch (name) {
+            case 'event': {
+                currentEvent.event = value;
+                break;
+            }
+            case 'data': {
+                dataLines.push(value);
+                break;
+            }
+            case 'id': {
+                if (value) {
+                    const numId = Number(value);
+                    currentEvent.id = Number.isNaN(numId) ? value : numId;
+                }
+                break;
+            }
+            case 'retry': {
+                const retryValue = Number.parseInt(value, 10);
+                if (!Number.isNaN(retryValue) && retryValue >= 0) {
+                    currentEvent.retry = retryValue;
+                }
+                break;
+            }
+            default: {
+                break;
+            }
+        }
+    }
+    function hasEventContent() {
+        return !!(currentEvent.event || dataLines.length > 0 || currentEvent.id || currentEvent.retry);
+    }
+    function finalizeEvent() {
+        const finalData = dataLines.join('\n');
+        return {
+            ...currentEvent,
+            data: finalData ? finalData : undefined,
+        };
+    }
+}
+//# sourceMappingURL=sse.js.map

package/dist/utils/sse.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sse.js","sourceRoot":"","sources":["../../src/utils/sse.ts"],"names":[],"mappings":";;AA6BA,wCA2IC;AA3IM,KAAK,SAAS,CAAC,CAAC,cAAc,CACpC,IAAgC;IAEhC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;IAChC,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAClC,IAAI,MAAM,GAAG,EAAE,CAAC;IAGhB,IAAI,YAAY,GAA2B,EAAE,CAAC;IAC9C,IAAI,SAAS,GAAa,EAAE,CAAC;IAE7B,IAAI,CAAC;QACJ,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAE5C,IAAI,IAAI,EAAE,CAAC;gBAEV,MAAM,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBAG3B,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;oBACnB,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;oBAClC,IAAI,KAAK,EAAE,CAAC;wBACX,MAAM,KAAK,CAAC;oBACb,CAAC;gBACF,CAAC;gBAED,IAAI,eAAe,EAAE,EAAE,CAAC;oBACvB,MAAM,aAAa,EAAE,CAAC;gBACvB,CAAC;gBACD,MAAM;YACP,CAAC;YAGD,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YAIlD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YAEzC,MAAM,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;YAE3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBAC1B,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;gBAChC,IAAI,KAAK,EAAE,CAAC;oBACX,MAAM,KAAK,CAAC;gBACb,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;YAAS,CAAC;QACV,MAAM,CAAC,WAAW,EAAE,CAAC;IACtB,CAAC;IAED,SAAS,WAAW,CAAC,IAAY;QAEhC,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;YACjB,IAAI,eAAe,EAAE,EAAE,CAAC;gBACvB,MAAM,KAAK,GAAG,aAAa,EAAE,CAAC;gBAE9B,YAAY,GAAG,EAAE,CAAC;gBAClB,SAAS,GAAG,EAAE,CAAC;gBACf,OAAO,KAAK,CAAC;YACd,CAAC;YACD,OAAO,IAAI,CAAC;QACb,CAAC;QAGD,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,YAAY,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;YACjD,OAAO,IAAI,CAAC;QACb,CAAC;QAGD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;YAEvB,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACvB,OAAO,IAAI,CAAC;QACb,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QAE5C,IAAI,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;QAC5C,IAAI,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAChC,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;QAED,YAAY,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QACpC,OAAO,IAAI,CAAC;IACb,CAAC;IAED,SAAS,YAAY,CAAC,IAAY,EAAE,KAAa;QAChD,QAAQ,IAAI,EAAE,CAAC;YACd,KAAK,OAAO,CAAC,CAAC,CAAC;gBACd,YAAY,CAAC,KAAK,GAAG,KAAK,CAAC;gBAC3B,MAAM;YACP,CAAC;YACD,KAAK,MAAM,CAAC,CAAC,CAAC;gBAEb,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACtB,MAAM;YACP,CAAC;YACD,KAAK,IAAI,CAAC,CAAC,CAAC;gBAEX,IAAI,KAAK,EAAE,CAAC;oBAEX,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;oBAC5B,YAAY,CAAC,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;gBACvD,CAAC;gBACD,MAAM;YACP,CAAC;YACD,KAAK,OAAO,CAAC,CAAC,CAAC;gBAEd,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAC9C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;oBAClD,YAAY,CAAC,KAAK,GAAG,UAAU,CAAC;gBACjC,CAAC;gBACD,MAAM;YACP,CAAC;YACD,OAAO,CAAC,CAAC,CAAC;gBAET,MAAM;YACP,CAAC;QACF,CAAC;IACF,CAAC;IAED,SAAS,eAAe;QAEvB,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,EAAE,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;IAChG,CAAC;IAED,SAAS,aAAa;QAErB,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvC,OAAO;YACN,GAAG,YAAY;YACf,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;SACvC,CAAC;IACH,CAAC;AACF,CAAC"}