@n8n/ai-utilities 0.4.2 → 0.5.1

package/dist/esm/utils/fromai-tool-factory.d.ts

@@ -0,0 +1,14 @@
+import type { CallbackManagerForToolRun } from '@langchain/core/callbacks/manager';
+import { DynamicStructuredTool, DynamicTool } from '@langchain/core/tools';
+import type { FromAIArgument, IDataObject, INode, INodeParameters } from 'n8n-workflow';
+import { z } from 'zod';
+export type ToolFunc = (query: string | IDataObject, runManager?: CallbackManagerForToolRun) => Promise<string | IDataObject | IDataObject[]>;
+export interface CreateToolOptions {
+    name: string;
+    description: string;
+    func: ToolFunc;
+    extraArgs?: FromAIArgument[];
+}
+export declare function extractFromAIParameters(nodeParameters: INodeParameters): FromAIArgument[];
+export declare function createZodSchemaFromArgs(args: FromAIArgument[]): z.ZodObject<z.ZodRawShape>;
+export declare function createToolFromNode(node: INode, options: CreateToolOptions): DynamicStructuredTool | DynamicTool;

package/dist/esm/utils/fromai-tool-factory.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractFromAIParameters = extractFromAIParameters;
+exports.createZodSchemaFromArgs = createZodSchemaFromArgs;
+exports.createToolFromNode = createToolFromNode;
+const tools_1 = require("@langchain/core/tools");
+const n8n_workflow_1 = require("n8n-workflow");
+const zod_1 = require("zod");
+function extractFromAIParameters(nodeParameters) {
+    const collectedArguments = [];
+    (0, n8n_workflow_1.traverseNodeParameters)(nodeParameters, collectedArguments);
+    const uniqueArgsMap = new Map();
+    for (const arg of collectedArguments) {
+        uniqueArgsMap.set(arg.key, arg);
+    }
+    return Array.from(uniqueArgsMap.values());
+}
+function createZodSchemaFromArgs(args) {
+    const schemaObj = args.reduce((acc, placeholder) => {
+        acc[placeholder.key] = (0, n8n_workflow_1.generateZodSchema)(placeholder);
+        return acc;
+    }, {});
+    return zod_1.z.object(schemaObj).required();
+}
+function createToolFromNode(node, options) {
+    const { name, description, func, extraArgs = [] } = options;
+    const collectedArguments = extractFromAIParameters(node.parameters);
+    if (collectedArguments.length === 0 && extraArgs.length === 0) {
+        return new tools_1.DynamicTool({ name, description, func });
+    }
+    const allArguments = [...collectedArguments, ...extraArgs];
+    const schema = createZodSchemaFromArgs(allArguments);
+    return new tools_1.DynamicStructuredTool({ schema, name, description, func });
+}
+//# sourceMappingURL=fromai-tool-factory.js.map

package/dist/esm/utils/fromai-tool-factory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fromai-tool-factory.js","sourceRoot":"","sources":["../../../src/utils/fromai-tool-factory.ts"],"names":[],"mappings":";;AAyBA,0DAUC;AAKD,0DAOC;AASD,gDAkBC;AAzED,iDAA2E;AAE3E,+CAAyE;AACzE,6BAAwB;AAqBxB,SAAgB,uBAAuB,CAAC,cAA+B;IACtE,MAAM,kBAAkB,GAAqB,EAAE,CAAC;IAChD,IAAA,qCAAsB,EAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;IAE3D,MAAM,aAAa,GAAG,IAAI,GAAG,EAA0B,CAAC;IACxD,KAAK,MAAM,GAAG,IAAI,kBAAkB,EAAE,CAAC;QACtC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACjC,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC;AAC3C,CAAC;AAKD,SAAgB,uBAAuB,CAAC,IAAsB;IAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAiC,EAAE,WAAW,EAAE,EAAE;QAChF,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,IAAA,gCAAiB,EAAC,WAAW,CAAC,CAAC;QACtD,OAAO,GAAG,CAAC;IACZ,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,OAAO,OAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,CAAC;AACvC,CAAC;AASD,SAAgB,kBAAkB,CACjC,IAAW,EACX,OAA0B;IAE1B,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;IAE5D,MAAM,kBAAkB,GAAG,uBAAuB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAGpE,IAAI,kBAAkB,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/D,OAAO,IAAI,mBAAW,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;IACrD,CAAC;IAGD,MAAM,YAAY,GAAG,CAAC,GAAG,kBAAkB,EAAE,GAAG,SAAS,CAAC,CAAC;IAC3D,MAAM,MAAM,GAAG,uBAAuB,CAAC,YAAY,CAAC,CAAC;IAErD,OAAO,IAAI,6BAAqB,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;AACvE,CAAC"}

package/dist/esm/utils/shared-fields.d.ts

@@ -0,0 +1,8 @@
+import { NodeConnectionTypes } from 'n8n-workflow';
+import type { IDisplayOptions, INodeProperties } from 'n8n-workflow';
+export declare const metadataFilterField: INodeProperties;
+export declare function getTemplateNoticeField(templateId: number): INodeProperties;
+export declare function getBatchingOptionFields(displayOptions: IDisplayOptions | undefined, defaultBatchSize?: number): INodeProperties;
+type AllowedConnectionTypes = typeof NodeConnectionTypes.AiAgent | typeof NodeConnectionTypes.AiChain | typeof NodeConnectionTypes.AiDocument | typeof NodeConnectionTypes.AiVectorStore | typeof NodeConnectionTypes.AiRetriever;
+export declare function getConnectionHintNoticeField(connectionTypes: AllowedConnectionTypes[]): INodeProperties;
+export {};

package/dist/esm/utils/shared-fields.js

@@ -0,0 +1,149 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.metadataFilterField = void 0;
+exports.getTemplateNoticeField = getTemplateNoticeField;
+exports.getBatchingOptionFields = getBatchingOptionFields;
+exports.getConnectionHintNoticeField = getConnectionHintNoticeField;
+const n8n_workflow_1 = require("n8n-workflow");
+exports.metadataFilterField = {
+    displayName: 'Metadata Filter',
+    name: 'metadata',
+    type: 'fixedCollection',
+    description: 'Metadata to filter the document by',
+    typeOptions: {
+        multipleValues: true,
+    },
+    default: {},
+    placeholder: 'Add filter field',
+    options: [
+        {
+            name: 'metadataValues',
+            displayName: 'Fields to Set',
+            values: [
+                {
+                    displayName: 'Name',
+                    name: 'name',
+                    type: 'string',
+                    default: '',
+                    required: true,
+                },
+                {
+                    displayName: 'Value',
+                    name: 'value',
+                    type: 'string',
+                    default: '',
+                },
+            ],
+        },
+    ],
+};
+function getTemplateNoticeField(templateId) {
+    return {
+        displayName: `Save time with an <a href="/templates/${templateId}" target="_blank">example</a> of how this node works`,
+        name: 'notice',
+        type: 'notice',
+        default: '',
+    };
+}
+function getBatchingOptionFields(displayOptions, defaultBatchSize = 5) {
+    return {
+        displayName: 'Batch Processing',
+        name: 'batching',
+        type: 'collection',
+        placeholder: 'Add Batch Processing Option',
+        description: 'Batch processing options for rate limiting',
+        default: {},
+        options: [
+            {
+                displayName: 'Batch Size',
+                name: 'batchSize',
+                default: defaultBatchSize,
+                type: 'number',
+                description: 'How many items to process in parallel. This is useful for rate limiting, but might impact the log output ordering.',
+            },
+            {
+                displayName: 'Delay Between Batches',
+                name: 'delayBetweenBatches',
+                default: 0,
+                type: 'number',
+                description: 'Delay in milliseconds between batches. This is useful for rate limiting.',
+            },
+        ],
+        displayOptions,
+    };
+}
+const connectionsString = {
+    [n8n_workflow_1.NodeConnectionTypes.AiAgent]: {
+        connection: '',
+        locale: 'AI Agent',
+    },
+    [n8n_workflow_1.NodeConnectionTypes.AiChain]: {
+        connection: '',
+        locale: 'AI Chain',
+    },
+    [n8n_workflow_1.NodeConnectionTypes.AiDocument]: {
+        connection: n8n_workflow_1.NodeConnectionTypes.AiDocument,
+        locale: 'Document Loader',
+    },
+    [n8n_workflow_1.NodeConnectionTypes.AiVectorStore]: {
+        connection: n8n_workflow_1.NodeConnectionTypes.AiVectorStore,
+        locale: 'Vector Store',
+    },
+    [n8n_workflow_1.NodeConnectionTypes.AiRetriever]: {
+        connection: n8n_workflow_1.NodeConnectionTypes.AiRetriever,
+        locale: 'Vector Store Retriever',
+    },
+};
+function determineArticle(nextWord) {
+    const vowels = /^[aeiouAEIOU]/;
+    return vowels.test(nextWord) ? 'an' : 'a';
+}
+const getConnectionParameterString = (connectionType) => {
+    if (connectionType === '')
+        return "data-action-parameter-creatorview='AI'";
+    return `data-action-parameter-connectiontype='${connectionType}'`;
+};
+const getAhref = (connectionType) => `<a class="test" data-action='openSelectiveNodeCreator'${getConnectionParameterString(connectionType.connection)}'>${connectionType.locale}</a>`;
+function getConnectionHintNoticeField(connectionTypes) {
+    const groupedConnections = new Map();
+    connectionTypes.forEach((connectionType) => {
+        const connectionString = connectionsString[connectionType].connection;
+        const localeString = connectionsString[connectionType].locale;
+        if (!groupedConnections.has(connectionString)) {
+            groupedConnections.set(connectionString, [localeString]);
+            return;
+        }
+        groupedConnections.get(connectionString)?.push(localeString);
+    });
+    let displayName;
+    if (groupedConnections.size === 1) {
+        const [[connection, locales]] = Array.from(groupedConnections);
+        displayName = `This node must be connected to ${determineArticle(locales[0])} ${locales[0]
+            .toLowerCase()
+            .replace(/^ai /, 'AI ')}. <a data-action='openSelectiveNodeCreator' ${getConnectionParameterString(connection)}>Insert one</a>`;
+    }
+    else {
+        const ahrefs = Array.from(groupedConnections, ([connection, locales]) => {
+            const locale = locales.length > 1
+                ? locales
+                    .map((localeString, index, { length }) => {
+                    return ((index === 0 ? `${determineArticle(localeString)} ` : '') +
+                        (index < length - 1 ? `${localeString} or ` : localeString));
+                })
+                    .join('')
+                : `${determineArticle(locales[0])} ${locales[0]}`;
+            return getAhref({ connection, locale });
+        });
+        displayName = `This node needs to be connected to ${ahrefs.join(' or ')}.`;
+    }
+    return {
+        displayName,
+        name: 'notice',
+        type: 'notice',
+        default: '',
+        typeOptions: {
+            containerClass: 'ndv-connection-hint-notice',
+        },
+    };
+}
+//# sourceMappingURL=shared-fields.js.map

package/dist/esm/utils/shared-fields.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"shared-fields.js","sourceRoot":"","sources":["../../../src/utils/shared-fields.ts"],"names":[],"mappings":";;;AAoCA,wDAOC;AAED,0DA8BC;AAiDD,oEA8DC;AA1LD,+CAAmD;AAGtC,QAAA,mBAAmB,GAAoB;IACnD,WAAW,EAAE,iBAAiB;IAC9B,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,iBAAiB;IACvB,WAAW,EAAE,oCAAoC;IACjD,WAAW,EAAE;QACZ,cAAc,EAAE,IAAI;KACpB;IACD,OAAO,EAAE,EAAE;IACX,WAAW,EAAE,kBAAkB;IAC/B,OAAO,EAAE;QACR;YACC,IAAI,EAAE,gBAAgB;YACtB,WAAW,EAAE,eAAe;YAC5B,MAAM,EAAE;gBACP;oBACC,WAAW,EAAE,MAAM;oBACnB,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE,EAAE;oBACX,QAAQ,EAAE,IAAI;iBACd;gBACD;oBACC,WAAW,EAAE,OAAO;oBACpB,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE,EAAE;iBACX;aACD;SACD;KACD;CACD,CAAC;AAEF,SAAgB,sBAAsB,CAAC,UAAkB;IACxD,OAAO;QACN,WAAW,EAAE,yCAAyC,UAAU,sDAAsD;QACtH,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,EAAE;KACX,CAAC;AACH,CAAC;AAED,SAAgB,uBAAuB,CACtC,cAA2C,EAC3C,mBAA2B,CAAC;IAE5B,OAAO;QACN,WAAW,EAAE,kBAAkB;QAC/B,IAAI,EAAE,UAAU;QAChB,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,6BAA6B;QAC1C,WAAW,EAAE,4CAA4C;QACzD,OAAO,EAAE,EAAE;QACX,OAAO,EAAE;YACR;gBACC,WAAW,EAAE,YAAY;gBACzB,IAAI,EAAE,WAAW;gBACjB,OAAO,EAAE,gBAAgB;gBACzB,IAAI,EAAE,QAAQ;gBACd,WAAW,EACV,oHAAoH;aACrH;YACD;gBACC,WAAW,EAAE,uBAAuB;gBACpC,IAAI,EAAE,qBAAqB;gBAC3B,OAAO,EAAE,CAAC;gBACV,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,0EAA0E;aACvF;SACD;QACD,cAAc;KACd,CAAC;AACH,CAAC;AAED,MAAM,iBAAiB,GAAG;IACzB,CAAC,kCAAmB,CAAC,OAAO,CAAC,EAAE;QAE9B,UAAU,EAAE,EAAE;QACd,MAAM,EAAE,UAAU;KAClB;IACD,CAAC,kCAAmB,CAAC,OAAO,CAAC,EAAE;QAE9B,UAAU,EAAE,EAAE;QACd,MAAM,EAAE,UAAU;KAClB;IACD,CAAC,kCAAmB,CAAC,UAAU,CAAC,EAAE;QACjC,UAAU,EAAE,kCAAmB,CAAC,UAAU;QAC1C,MAAM,EAAE,iBAAiB;KACzB;IACD,CAAC,kCAAmB,CAAC,aAAa,CAAC,EAAE;QACpC,UAAU,EAAE,kCAAmB,CAAC,aAAa;QAC7C,MAAM,EAAE,cAAc;KACtB;IACD,CAAC,kCAAmB,CAAC,WAAW,CAAC,EAAE;QAClC,UAAU,EAAE,kCAAmB,CAAC,WAAW;QAC3C,MAAM,EAAE,wBAAwB;KAChC;CACD,CAAC;AASF,SAAS,gBAAgB,CAAC,QAAgB;IAEzC,MAAM,MAAM,GAAG,eAAe,CAAC;IAC/B,OAAO,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;AAC3C,CAAC;AACD,MAAM,4BAA4B,GAAG,CAAC,cAAsB,EAAE,EAAE;IAC/D,IAAI,cAAc,KAAK,EAAE;QAAE,OAAO,wCAAwC,CAAC;IAE3E,OAAO,yCAAyC,cAAc,GAAG,CAAC;AACnE,CAAC,CAAC;AACF,MAAM,QAAQ,GAAG,CAAC,cAAsD,EAAE,EAAE,CAC3E,yDAAyD,4BAA4B,CACpF,cAAc,CAAC,UAAU,CACzB,KAAK,cAAc,CAAC,MAAM,MAAM,CAAC;AAEnC,SAAgB,4BAA4B,CAC3C,eAAyC;IAEzC,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAoB,CAAC;IAIvD,eAAe,CAAC,OAAO,CAAC,CAAC,cAAc,EAAE,EAAE;QAC1C,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,cAAc,CAAC,CAAC,UAAU,CAAC;QACtE,MAAM,YAAY,GAAG,iBAAiB,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC;QAE9D,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAC/C,kBAAkB,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;YACzD,OAAO;QACR,CAAC;QAED,kBAAkB,CAAC,GAAG,CAAC,gBAAgB,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,IAAI,WAAW,CAAC;IAEhB,IAAI,kBAAkB,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACnC,MAAM,CAAC,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAE/D,WAAW,GAAG,kCAAkC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC;aACxF,WAAW,EAAE;aACb,OAAO,CACP,MAAM,EACN,KAAK,CACL,+CAA+C,4BAA4B,CAC5E,UAAU,CACV,iBAAiB,CAAC;IACpB,CAAC;SAAM,CAAC;QACP,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC,UAAU,EAAE,OAAO,CAAC,EAAE,EAAE;YAGvE,MAAM,MAAM,GACX,OAAO,CAAC,MAAM,GAAG,CAAC;gBACjB,CAAC,CAAC,OAAO;qBACN,GAAG,CAAC,CAAC,YAAY,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;oBACxC,OAAO,CACN,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,gBAAgB,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;wBACzD,CAAC,KAAK,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,YAAY,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAC3D,CAAC;gBACH,CAAC,CAAC;qBACD,IAAI,CAAC,EAAE,CAAC;gBACX,CAAC,CAAC,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YACpD,OAAO,QAAQ,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QAEH,WAAW,GAAG,sCAAsC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC;IAC5E,CAAC;IAED,OAAO;QACN,WAAW;QACX,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,EAAE;QACX,WAAW,EAAE;YACZ,cAAc,EAAE,4BAA4B;SAC5C;KACD,CAAC;AACH,CAAC"}

package/dist/esm/utils/vector-store/MemoryManager/MemoryCalculator.d.ts

@@ -0,0 +1,7 @@
+import type { MemoryVectorStore } from '@langchain/classic/vectorstores/memory';
+import type { Document } from '@langchain/core/documents';
+import type { IMemoryCalculator } from './types';
+export declare class MemoryCalculator implements IMemoryCalculator {
+    estimateBatchSize(documents: Document[]): number;
+    calculateVectorStoreSize(vectorStore: MemoryVectorStore): number;
+}

package/dist/esm/utils/vector-store/MemoryManager/MemoryCalculator.js

@@ -0,0 +1,50 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MemoryCalculator = void 0;
+const FLOAT_SIZE_BYTES = 8;
+const CHAR_SIZE_BYTES = 2;
+const VECTOR_OVERHEAD_BYTES = 200;
+const EMBEDDING_DIMENSIONS = 1536;
+const EMBEDDING_SIZE_BYTES = EMBEDDING_DIMENSIONS * FLOAT_SIZE_BYTES;
+const AVG_METADATA_SIZE_BYTES = 100;
+class MemoryCalculator {
+    estimateBatchSize(documents) {
+        if (documents.length === 0)
+            return 0;
+        let totalContentSize = 0;
+        let totalMetadataSize = 0;
+        for (const doc of documents) {
+            if (doc.pageContent) {
+                totalContentSize += doc.pageContent.length * CHAR_SIZE_BYTES;
+            }
+            if (doc.metadata) {
+                const metadataKeys = Object.keys(doc.metadata).length;
+                if (metadataKeys > 0) {
+                    totalMetadataSize += metadataKeys * AVG_METADATA_SIZE_BYTES;
+                }
+            }
+        }
+        const embeddingSize = documents.length * EMBEDDING_SIZE_BYTES;
+        const overhead = documents.length * VECTOR_OVERHEAD_BYTES;
+        const calculatedSize = totalContentSize + totalMetadataSize + embeddingSize + overhead;
+        return Math.ceil(calculatedSize);
+    }
+    calculateVectorStoreSize(vectorStore) {
+        if (!vectorStore.memoryVectors || vectorStore.memoryVectors.length === 0) {
+            return 0;
+        }
+        let storeSize = 0;
+        for (const vector of vectorStore.memoryVectors) {
+            storeSize += vector.embedding.length * FLOAT_SIZE_BYTES;
+            storeSize += vector.content ? vector.content.length * CHAR_SIZE_BYTES : 0;
+            if (vector.metadata) {
+                const metadataStr = JSON.stringify(vector.metadata);
+                storeSize += metadataStr.length * CHAR_SIZE_BYTES;
+            }
+            storeSize += VECTOR_OVERHEAD_BYTES;
+        }
+        return Math.ceil(storeSize);
+    }
+}
+exports.MemoryCalculator = MemoryCalculator;
+//# sourceMappingURL=MemoryCalculator.js.map

package/dist/esm/utils/vector-store/MemoryManager/MemoryCalculator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MemoryCalculator.js","sourceRoot":"","sources":["../../../../../src/utils/vector-store/MemoryManager/MemoryCalculator.ts"],"names":[],"mappings":";;;AAMA,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAC3B,MAAM,eAAe,GAAG,CAAC,CAAC;AAC1B,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAClC,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAClC,MAAM,oBAAoB,GAAG,oBAAoB,GAAG,gBAAgB,CAAC;AACrE,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAKpC,MAAa,gBAAgB;IAI5B,iBAAiB,CAAC,SAAqB;QACtC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAErC,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAG1B,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC7B,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;gBACrB,gBAAgB,IAAI,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,eAAe,CAAC;YAC9D,CAAC;YAGD,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;gBAElB,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;gBACtD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;oBAGtB,iBAAiB,IAAI,YAAY,GAAG,uBAAuB,CAAC;gBAC7D,CAAC;YACF,CAAC;QACF,CAAC;QAID,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,GAAG,oBAAoB,CAAC;QAG9D,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,GAAG,qBAAqB,CAAC;QAG1D,MAAM,cAAc,GAAG,gBAAgB,GAAG,iBAAiB,GAAG,aAAa,GAAG,QAAQ,CAAC;QAEvF,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAClC,CAAC;IAKD,wBAAwB,CAAC,WAA8B;QACtD,IAAI,CAAC,WAAW,CAAC,aAAa,IAAI,WAAW,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1E,OAAO,CAAC,CAAC;QACV,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,CAAC;QAGlB,KAAK,MAAM,MAAM,IAAI,WAAW,CAAC,aAAa,EAAE,CAAC;YAEhD,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,gBAAgB,CAAC;YAGxD,SAAS,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;YAG1E,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;gBAErB,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBACpD,SAAS,IAAI,WAAW,CAAC,MAAM,GAAG,eAAe,CAAC;YACnD,CAAC;YAGD,SAAS,IAAI,qBAAqB,CAAC;QACpC,CAAC;QAED,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC7B,CAAC;CACD;AAxED,4CAwEC"}

package/dist/esm/utils/vector-store/MemoryManager/MemoryVectorStoreManager.d.ts

@@ -0,0 +1,34 @@
+import { MemoryVectorStore } from '@langchain/classic/vectorstores/memory';
+import type { Document } from '@langchain/core/documents';
+import type { Embeddings } from '@langchain/core/embeddings';
+import type { OpenAIEmbeddings, AzureOpenAIEmbeddings } from '@langchain/openai';
+import type { Logger } from 'n8n-workflow';
+import { MemoryCalculator } from './MemoryCalculator';
+import { StoreCleanupService } from './StoreCleanupService';
+import type { VectorStoreMetadata, VectorStoreStats } from './types';
+export declare class MemoryVectorStoreManager {
+    protected embeddings: Embeddings | OpenAIEmbeddings | AzureOpenAIEmbeddings;
+    protected logger: Logger;
+    private static instance;
+    protected vectorStoreBuffer: Map<string, MemoryVectorStore>;
+    protected storeMetadata: Map<string, VectorStoreMetadata>;
+    protected memoryUsageBytes: number;
+    protected memoryCalculator: MemoryCalculator;
+    protected cleanupService: StoreCleanupService;
+    protected static logger: Logger;
+    protected maxMemorySizeBytes: number;
+    protected inactiveTtlMs: number;
+    protected ttlCleanupIntervalId: NodeJS.Timeout | null;
+    protected constructor(embeddings: Embeddings | OpenAIEmbeddings | AzureOpenAIEmbeddings, logger: Logger);
+    static getInstance(embeddings: Embeddings | OpenAIEmbeddings | AzureOpenAIEmbeddings, logger: Logger): MemoryVectorStoreManager;
+    private setupTtlCleanup;
+    private handleCleanup;
+    getMemoryKeysList(): string[];
+    getVectorStore(memoryKey: string): Promise<MemoryVectorStore>;
+    protected clearStoreMetadata(memoryKey: string): void;
+    getMemoryUsage(): number;
+    getMemoryUsageFormatted(): string;
+    recalculateMemoryUsage(): void;
+    addDocuments(memoryKey: string, documents: Document[], clearStore?: boolean): Promise<void>;
+    getStats(): VectorStoreStats;
+}

package/dist/esm/utils/vector-store/MemoryManager/MemoryVectorStoreManager.js

@@ -0,0 +1,176 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MemoryVectorStoreManager = void 0;
+const memory_1 = require("@langchain/classic/vectorstores/memory");
+const config_1 = require("./config");
+const MemoryCalculator_1 = require("./MemoryCalculator");
+const StoreCleanupService_1 = require("./StoreCleanupService");
+class MemoryVectorStoreManager {
+    constructor(embeddings, logger) {
+        this.embeddings = embeddings;
+        this.logger = logger;
+        this.memoryUsageBytes = 0;
+        this.ttlCleanupIntervalId = null;
+        this.vectorStoreBuffer = new Map();
+        this.storeMetadata = new Map();
+        this.logger = logger;
+        const config = (0, config_1.getConfig)();
+        this.maxMemorySizeBytes = (0, config_1.mbToBytes)(config.maxMemoryMB);
+        this.inactiveTtlMs = (0, config_1.hoursToMs)(config.ttlHours);
+        this.memoryCalculator = new MemoryCalculator_1.MemoryCalculator();
+        this.cleanupService = new StoreCleanupService_1.StoreCleanupService(this.maxMemorySizeBytes, this.inactiveTtlMs, this.vectorStoreBuffer, this.storeMetadata, this.handleCleanup.bind(this));
+        this.setupTtlCleanup();
+    }
+    static getInstance(embeddings, logger) {
+        if (!MemoryVectorStoreManager.instance) {
+            MemoryVectorStoreManager.instance = new MemoryVectorStoreManager(embeddings, logger);
+        }
+        else {
+            MemoryVectorStoreManager.instance.embeddings = embeddings;
+            MemoryVectorStoreManager.instance.vectorStoreBuffer.forEach((vectorStoreInstance) => {
+                vectorStoreInstance.embeddings = embeddings;
+            });
+        }
+        return MemoryVectorStoreManager.instance;
+    }
+    setupTtlCleanup() {
+        if (this.inactiveTtlMs <= 0) {
+            return;
+        }
+        const CLEANUP_INTERVAL_MS = 60 * 60 * 1000;
+        if (this.ttlCleanupIntervalId) {
+            clearInterval(this.ttlCleanupIntervalId);
+        }
+        this.ttlCleanupIntervalId = setInterval(() => {
+            this.cleanupService.cleanupInactiveStores();
+        }, CLEANUP_INTERVAL_MS);
+    }
+    handleCleanup(removedKeys, freedBytes, reason) {
+        this.memoryUsageBytes -= freedBytes;
+        if (reason === 'ttl') {
+            const ttlHours = Math.round(this.inactiveTtlMs / (60 * 60 * 1000));
+            this.logger.info(`TTL cleanup: removed ${removedKeys.length} inactive vector stores (${ttlHours}h TTL) to free ${Math.round(freedBytes / (1024 * 1024))}MB of memory`);
+        }
+        else {
+            this.logger.info(`Memory cleanup: removed ${removedKeys.length} oldest vector stores to free ${Math.round(freedBytes / (1024 * 1024))}MB of memory`);
+        }
+    }
+    getMemoryKeysList() {
+        return Array.from(this.vectorStoreBuffer.keys());
+    }
+    async getVectorStore(memoryKey) {
+        let vectorStoreInstance = this.vectorStoreBuffer.get(memoryKey);
+        if (!vectorStoreInstance) {
+            vectorStoreInstance = await memory_1.MemoryVectorStore.fromExistingIndex(this.embeddings);
+            this.vectorStoreBuffer.set(memoryKey, vectorStoreInstance);
+            this.storeMetadata.set(memoryKey, {
+                size: 0,
+                createdAt: new Date(),
+                lastAccessed: new Date(),
+            });
+        }
+        else {
+            const metadata = this.storeMetadata.get(memoryKey);
+            if (metadata) {
+                metadata.lastAccessed = new Date();
+            }
+        }
+        return vectorStoreInstance;
+    }
+    clearStoreMetadata(memoryKey) {
+        const metadata = this.storeMetadata.get(memoryKey);
+        if (metadata) {
+            this.memoryUsageBytes -= metadata.size;
+            metadata.size = 0;
+            metadata.lastAccessed = new Date();
+        }
+    }
+    getMemoryUsage() {
+        return this.memoryUsageBytes;
+    }
+    getMemoryUsageFormatted() {
+        return `${Math.round(this.memoryUsageBytes / (1024 * 1024))}MB`;
+    }
+    recalculateMemoryUsage() {
+        this.memoryUsageBytes = 0;
+        for (const [key, vectorStore] of this.vectorStoreBuffer.entries()) {
+            const storeSize = this.memoryCalculator.calculateVectorStoreSize(vectorStore);
+            const metadata = this.storeMetadata.get(key);
+            if (metadata) {
+                metadata.size = storeSize;
+                this.memoryUsageBytes += storeSize;
+            }
+        }
+        this.logger.debug(`Recalculated vector store memory: ${this.getMemoryUsageFormatted()}`);
+    }
+    async addDocuments(memoryKey, documents, clearStore) {
+        if (clearStore) {
+            this.clearStoreMetadata(memoryKey);
+            this.vectorStoreBuffer.delete(memoryKey);
+        }
+        const estimatedAddedSize = this.memoryCalculator.estimateBatchSize(documents);
+        this.cleanupService.cleanupOldestStores(estimatedAddedSize);
+        const vectorStoreInstance = await this.getVectorStore(memoryKey);
+        const vectorCountBefore = vectorStoreInstance.memoryVectors?.length || 0;
+        await vectorStoreInstance.addDocuments(documents);
+        const metadata = this.storeMetadata.get(memoryKey);
+        if (metadata) {
+            metadata.size += estimatedAddedSize;
+            metadata.lastAccessed = new Date();
+            this.memoryUsageBytes += estimatedAddedSize;
+        }
+        const vectorCount = vectorStoreInstance.memoryVectors?.length || 0;
+        if ((vectorCount > 0 && vectorCount % 100 === 0) ||
+            documents.length > 20 ||
+            (vectorCountBefore === 0 && vectorCount > 0)) {
+            this.recalculateMemoryUsage();
+        }
+        const maxMemoryMB = this.maxMemorySizeBytes > 0
+            ? (this.maxMemorySizeBytes / (1024 * 1024)).toFixed(0)
+            : 'unlimited';
+        this.logger.debug(`Vector store memory: ${this.getMemoryUsageFormatted()}/${maxMemoryMB}MB (${vectorCount} vectors in ${this.vectorStoreBuffer.size} stores)`);
+    }
+    getStats() {
+        const now = Date.now();
+        let inactiveStoreCount = 0;
+        this.recalculateMemoryUsage();
+        const stats = {
+            totalSizeBytes: this.memoryUsageBytes,
+            totalSizeMB: Math.round((this.memoryUsageBytes / (1024 * 1024)) * 100) / 100,
+            percentOfLimit: this.maxMemorySizeBytes > 0
+                ? Math.round((this.memoryUsageBytes / this.maxMemorySizeBytes) * 100)
+                : 0,
+            maxMemoryMB: this.maxMemorySizeBytes > 0 ? this.maxMemorySizeBytes / (1024 * 1024) : -1,
+            storeCount: this.vectorStoreBuffer.size,
+            inactiveStoreCount: 0,
+            ttlHours: this.inactiveTtlMs > 0 ? this.inactiveTtlMs / (60 * 60 * 1000) : -1,
+            stores: {},
+        };
+        for (const [key, metadata] of this.storeMetadata.entries()) {
+            const store = this.vectorStoreBuffer.get(key);
+            if (store) {
+                const lastAccessedTime = metadata.lastAccessed.getTime();
+                const inactiveTimeMs = now - lastAccessedTime;
+                const isInactive = this.cleanupService.isStoreInactive(metadata);
+                if (isInactive) {
+                    inactiveStoreCount++;
+                }
+                stats.stores[key] = {
+                    sizeBytes: metadata.size,
+                    sizeMB: Math.round((metadata.size / (1024 * 1024)) * 100) / 100,
+                    percentOfTotal: Math.round((metadata.size / this.memoryUsageBytes) * 100) || 0,
+                    vectors: store.memoryVectors?.length || 0,
+                    createdAt: metadata.createdAt.toISOString(),
+                    lastAccessed: metadata.lastAccessed.toISOString(),
+                    inactive: isInactive,
+                    inactiveForHours: Math.round(inactiveTimeMs / (60 * 60 * 1000)),
+                };
+            }
+        }
+        stats.inactiveStoreCount = inactiveStoreCount;
+        return stats;
+    }
+}
+exports.MemoryVectorStoreManager = MemoryVectorStoreManager;
+MemoryVectorStoreManager.instance = null;
+//# sourceMappingURL=MemoryVectorStoreManager.js.map

package/dist/esm/utils/vector-store/MemoryManager/MemoryVectorStoreManager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MemoryVectorStoreManager.js","sourceRoot":"","sources":["../../../../../src/utils/vector-store/MemoryManager/MemoryVectorStoreManager.ts"],"names":[],"mappings":";;;AAAA,mEAA2E;AAM3E,qCAA2D;AAC3D,yDAAsD;AACtD,+DAA4D;AAM5D,MAAa,wBAAwB;IAyBpC,YACW,UAAiE,EACjE,MAAc;QADd,eAAU,GAAV,UAAU,CAAuD;QACjE,WAAM,GAAN,MAAM,CAAQ;QAnBf,qBAAgB,GAAW,CAAC,CAAC;QAe7B,yBAAoB,GAA0B,IAAI,CAAC;QAO5D,IAAI,CAAC,iBAAiB,GAAG,IAAI,GAAG,EAAE,CAAC;QACnC,IAAI,CAAC,aAAa,GAAG,IAAI,GAAG,EAAE,CAAC;QAC/B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,MAAM,GAAG,IAAA,kBAAS,GAAE,CAAC;QAC3B,IAAI,CAAC,kBAAkB,GAAG,IAAA,kBAAS,EAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACxD,IAAI,CAAC,aAAa,GAAG,IAAA,kBAAS,EAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAGhD,IAAI,CAAC,gBAAgB,GAAG,IAAI,mCAAgB,EAAE,CAAC;QAC/C,IAAI,CAAC,cAAc,GAAG,IAAI,yCAAmB,CAC5C,IAAI,CAAC,kBAAkB,EACvB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,iBAAiB,EACtB,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAC7B,CAAC;QAEF,IAAI,CAAC,eAAe,EAAE,CAAC;IACxB,CAAC;IAKD,MAAM,CAAC,WAAW,CACjB,UAAiE,EACjE,MAAc;QAEd,IAAI,CAAC,wBAAwB,CAAC,QAAQ,EAAE,CAAC;YACxC,wBAAwB,CAAC,QAAQ,GAAG,IAAI,wBAAwB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACtF,CAAC;aAAM,CAAC;YAIP,wBAAwB,CAAC,QAAQ,CAAC,UAAU,GAAG,UAAU,CAAC;YAC1D,wBAAwB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,mBAAmB,EAAE,EAAE;gBACnF,mBAAmB,CAAC,UAAU,GAAG,UAAU,CAAC;YAC7C,CAAC,CAAC,CAAC;QACJ,CAAC;QAED,OAAO,wBAAwB,CAAC,QAAQ,CAAC;IAC1C,CAAC;IAKO,eAAe;QAEtB,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,EAAE,CAAC;YAC7B,OAAO;QACR,CAAC;QAGD,MAAM,mBAAmB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QAG3C,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC/B,aAAa,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAC1C,CAAC;QAGD,IAAI,CAAC,oBAAoB,GAAG,WAAW,CAAC,GAAG,EAAE;YAC5C,IAAI,CAAC,cAAc,CAAC,qBAAqB,EAAE,CAAC;QAC7C,CAAC,EAAE,mBAAmB,CAAC,CAAC;IACzB,CAAC;IAKO,aAAa,CAAC,WAAqB,EAAE,UAAkB,EAAE,MAAwB;QAExF,IAAI,CAAC,gBAAgB,IAAI,UAAU,CAAC;QAGpC,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;YACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;YACnE,IAAI,CAAC,MAAM,CAAC,IAAI,CACf,wBAAwB,WAAW,CAAC,MAAM,4BAA4B,QAAQ,kBAAkB,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,cAAc,CACpJ,CAAC;QACH,CAAC;aAAM,CAAC;YACP,IAAI,CAAC,MAAM,CAAC,IAAI,CACf,2BAA2B,WAAW,CAAC,MAAM,iCAAiC,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,cAAc,CAClI,CAAC;QACH,CAAC;IACF,CAAC;IAED,iBAAiB;QAChB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC;IAClD,CAAC;IAKD,KAAK,CAAC,cAAc,CAAC,SAAiB;QACrC,IAAI,mBAAmB,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAEhE,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC1B,mBAAmB,GAAG,MAAM,0BAAiB,CAAC,iBAAiB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACjF,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,SAAS,EAAE,mBAAmB,CAAC,CAAC;YAE3D,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE;gBACjC,IAAI,EAAE,CAAC;gBACP,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,YAAY,EAAE,IAAI,IAAI,EAAE;aACxB,CAAC,CAAC;QACJ,CAAC;aAAM,CAAC;YACP,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACnD,IAAI,QAAQ,EAAE,CAAC;gBACd,QAAQ,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;YACpC,CAAC;QACF,CAAC;QAED,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAKS,kBAAkB,CAAC,SAAiB;QAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACnD,IAAI,QAAQ,EAAE,CAAC;YACd,IAAI,CAAC,gBAAgB,IAAI,QAAQ,CAAC,IAAI,CAAC;YACvC,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAC;YAClB,QAAQ,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;QACpC,CAAC;IACF,CAAC;IAKD,cAAc;QACb,OAAO,IAAI,CAAC,gBAAgB,CAAC;IAC9B,CAAC;IAKD,uBAAuB;QACtB,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,gBAAgB,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC;IACjE,CAAC;IAMD,sBAAsB;QACrB,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;QAG1B,KAAK,MAAM,CAAC,GAAG,EAAE,WAAW,CAAC,IAAI,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,EAAE,CAAC;YACnE,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,wBAAwB,CAAC,WAAW,CAAC,CAAC;YAG9E,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,QAAQ,EAAE,CAAC;gBACd,QAAQ,CAAC,IAAI,GAAG,SAAS,CAAC;gBAC1B,IAAI,CAAC,gBAAgB,IAAI,SAAS,CAAC;YACpC,CAAC;QACF,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,IAAI,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAC;IAC1F,CAAC;IAKD,KAAK,CAAC,YAAY,CACjB,SAAiB,EACjB,SAAqB,EACrB,UAAoB;QAEpB,IAAI,UAAU,EAAE,CAAC;YAChB,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;YACnC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC1C,CAAC;QAGD,MAAM,kBAAkB,GAAG,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAG9E,IAAI,CAAC,cAAc,CAAC,mBAAmB,CAAC,kBAAkB,CAAC,CAAC;QAE5D,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QAGjE,MAAM,iBAAiB,GAAG,mBAAmB,CAAC,aAAa,EAAE,MAAM,IAAI,CAAC,CAAC;QAEzE,MAAM,mBAAmB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAGlD,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACnD,IAAI,QAAQ,EAAE,CAAC;YACd,QAAQ,CAAC,IAAI,IAAI,kBAAkB,CAAC;YACpC,QAAQ,CAAC,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC;YACnC,IAAI,CAAC,gBAAgB,IAAI,kBAAkB,CAAC;QAC7C,CAAC;QAGD,MAAM,WAAW,GAAG,mBAAmB,CAAC,aAAa,EAAE,MAAM,IAAI,CAAC,CAAC;QAGnE,IACC,CAAC,WAAW,GAAG,CAAC,IAAI,WAAW,GAAG,GAAG,KAAK,CAAC,CAAC;YAC5C,SAAS,CAAC,MAAM,GAAG,EAAE;YACrB,CAAC,iBAAiB,KAAK,CAAC,IAAI,WAAW,GAAG,CAAC,CAAC,EAC3C,CAAC;YACF,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC/B,CAAC;QAGD,MAAM,WAAW,GAChB,IAAI,CAAC,kBAAkB,GAAG,CAAC;YAC1B,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACtD,CAAC,CAAC,WAAW,CAAC;QAEhB,IAAI,CAAC,MAAM,CAAC,KAAK,CAChB,wBAAwB,IAAI,CAAC,uBAAuB,EAAE,IAAI,WAAW,OAAO,WAAW,eAAe,IAAI,CAAC,iBAAiB,CAAC,IAAI,UAAU,CAC3I,CAAC;IACH,CAAC;IAKD,QAAQ;QACP,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,kBAAkB,GAAG,CAAC,CAAC;QAG3B,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAE9B,MAAM,KAAK,GAAqB;YAC/B,cAAc,EAAE,IAAI,CAAC,gBAAgB;YACrC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,gBAAgB,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG;YAC5E,cAAc,EACb,IAAI,CAAC,kBAAkB,GAAG,CAAC;gBAC1B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,GAAG,CAAC;gBACrE,CAAC,CAAC,CAAC;YACL,WAAW,EAAE,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACvF,UAAU,EAAE,IAAI,CAAC,iBAAiB,CAAC,IAAI;YACvC,kBAAkB,EAAE,CAAC;YACrB,QAAQ,EAAE,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7E,MAAM,EAAE,EAAE;SACV,CAAC;QAGF,KAAK,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;YAC5D,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAE9C,IAAI,KAAK,EAAE,CAAC;gBACX,MAAM,gBAAgB,GAAG,QAAQ,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;gBACzD,MAAM,cAAc,GAAG,GAAG,GAAG,gBAAgB,CAAC;gBAC9C,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;gBAEjE,IAAI,UAAU,EAAE,CAAC;oBAChB,kBAAkB,EAAE,CAAC;gBACtB,CAAC;gBAED,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG;oBACnB,SAAS,EAAE,QAAQ,CAAC,IAAI;oBACxB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG;oBAC/D,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC;oBAC9E,OAAO,EAAE,KAAK,CAAC,aAAa,EAAE,MAAM,IAAI,CAAC;oBACzC,SAAS,EAAE,QAAQ,CAAC,SAAS,CAAC,WAAW,EAAE;oBAC3C,YAAY,EAAE,QAAQ,CAAC,YAAY,CAAC,WAAW,EAAE;oBACjD,QAAQ,EAAE,UAAU;oBACpB,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;iBAC/D,CAAC;YACH,CAAC;QACF,CAAC;QAED,KAAK,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE9C,OAAO,KAAK,CAAC;IACd,CAAC;;AA/SF,4DAgTC;AA/Se,iCAAQ,GAAoC,IAAI,AAAxC,CAAyC"}

package/dist/esm/utils/vector-store/MemoryManager/StoreCleanupService.d.ts

@@ -0,0 +1,16 @@
+import type { MemoryVectorStore } from '@langchain/classic/vectorstores/memory';
+import type { VectorStoreMetadata, IStoreCleanupService } from './types';
+export declare class StoreCleanupService implements IStoreCleanupService {
+    private readonly maxMemorySizeBytes;
+    private readonly inactiveTtlMs;
+    private readonly vectorStores;
+    private readonly storeMetadata;
+    private readonly onCleanup;
+    private oldestStoreKeys;
+    private lastSortTime;
+    private readonly CACHE_TTL_MS;
+    constructor(maxMemorySizeBytes: number, inactiveTtlMs: number, vectorStores: Map<string, MemoryVectorStore>, storeMetadata: Map<string, VectorStoreMetadata>, onCleanup: (removedKeys: string[], freedBytes: number, reason: 'ttl' | 'memory') => void);
+    isStoreInactive(metadata: VectorStoreMetadata): boolean;
+    cleanupInactiveStores(): void;
+    cleanupOldestStores(requiredBytes: number): void;
+}

package/dist/esm/utils/vector-store/MemoryManager/StoreCleanupService.js

@@ -0,0 +1,95 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StoreCleanupService = void 0;
+class StoreCleanupService {
+    constructor(maxMemorySizeBytes, inactiveTtlMs, vectorStores, storeMetadata, onCleanup) {
+        this.maxMemorySizeBytes = maxMemorySizeBytes;
+        this.inactiveTtlMs = inactiveTtlMs;
+        this.vectorStores = vectorStores;
+        this.storeMetadata = storeMetadata;
+        this.onCleanup = onCleanup;
+        this.oldestStoreKeys = [];
+        this.lastSortTime = 0;
+        this.CACHE_TTL_MS = 5000;
+    }
+    isStoreInactive(metadata) {
+        if (this.inactiveTtlMs <= 0) {
+            return false;
+        }
+        const now = Date.now();
+        const lastAccessedTime = metadata.lastAccessed.getTime();
+        return now - lastAccessedTime > this.inactiveTtlMs;
+    }
+    cleanupInactiveStores() {
+        if (this.inactiveTtlMs <= 0) {
+            return;
+        }
+        let freedBytes = 0;
+        const removedStores = [];
+        for (const [key, metadata] of this.storeMetadata.entries()) {
+            if (this.isStoreInactive(metadata)) {
+                this.vectorStores.delete(key);
+                freedBytes += metadata.size;
+                removedStores.push(key);
+            }
+        }
+        for (const key of removedStores) {
+            this.storeMetadata.delete(key);
+        }
+        if (removedStores.length > 0) {
+            this.oldestStoreKeys = [];
+            this.onCleanup(removedStores, freedBytes, 'ttl');
+        }
+    }
+    cleanupOldestStores(requiredBytes) {
+        if (this.maxMemorySizeBytes <= 0) {
+            return;
+        }
+        let currentMemoryUsage = 0;
+        for (const metadata of this.storeMetadata.values()) {
+            currentMemoryUsage += metadata.size;
+        }
+        this.cleanupInactiveStores();
+        currentMemoryUsage = 0;
+        for (const metadata of this.storeMetadata.values()) {
+            currentMemoryUsage += metadata.size;
+        }
+        if (currentMemoryUsage + requiredBytes <= this.maxMemorySizeBytes) {
+            return;
+        }
+        const now = Date.now();
+        if (this.oldestStoreKeys.length === 0 || now - this.lastSortTime > this.CACHE_TTL_MS) {
+            const stores = [];
+            for (const [key, metadata] of this.storeMetadata.entries()) {
+                stores.push([key, metadata.createdAt.getTime()]);
+            }
+            stores.sort((a, b) => a[1] - b[1]);
+            this.oldestStoreKeys = stores.map(([key]) => key);
+            this.lastSortTime = now;
+        }
+        let freedBytes = 0;
+        const removedStores = [];
+        for (const key of this.oldestStoreKeys) {
+            if (!this.storeMetadata.has(key))
+                continue;
+            if (currentMemoryUsage - freedBytes + requiredBytes <= this.maxMemorySizeBytes) {
+                break;
+            }
+            const metadata = this.storeMetadata.get(key);
+            if (metadata) {
+                this.vectorStores.delete(key);
+                freedBytes += metadata.size;
+                removedStores.push(key);
+            }
+        }
+        for (const key of removedStores) {
+            this.storeMetadata.delete(key);
+        }
+        if (removedStores.length > 0) {
+            this.oldestStoreKeys = this.oldestStoreKeys.filter((key) => !removedStores.includes(key));
+            this.onCleanup(removedStores, freedBytes, 'memory');
+        }
+    }
+}
+exports.StoreCleanupService = StoreCleanupService;
+//# sourceMappingURL=StoreCleanupService.js.map

package/dist/esm/utils/vector-store/MemoryManager/StoreCleanupService.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"StoreCleanupService.js","sourceRoot":"","sources":["../../../../../src/utils/vector-store/MemoryManager/StoreCleanupService.ts"],"names":[],"mappings":";;;AAOA,MAAa,mBAAmB;IAQ/B,YACkB,kBAA0B,EAC1B,aAAqB,EACrB,YAA4C,EAC5C,aAA+C,EAC/C,SAIR;QARQ,uBAAkB,GAAlB,kBAAkB,CAAQ;QAC1B,kBAAa,GAAb,aAAa,CAAQ;QACrB,iBAAY,GAAZ,YAAY,CAAgC;QAC5C,kBAAa,GAAb,aAAa,CAAkC;QAC/C,cAAS,GAAT,SAAS,CAIjB;QAfF,oBAAe,GAAa,EAAE,CAAC;QAE/B,iBAAY,GAAG,CAAC,CAAC;QAER,iBAAY,GAAG,IAAI,CAAC;IAYlC,CAAC;IAKJ,eAAe,CAAC,QAA6B;QAE5C,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,EAAE,CAAC;YAC7B,OAAO,KAAK,CAAC;QACd,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,gBAAgB,GAAG,QAAQ,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;QACzD,OAAO,GAAG,GAAG,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC;IACpD,CAAC;IAKD,qBAAqB;QAEpB,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,EAAE,CAAC;YAC7B,OAAO;QACR,CAAC;QAED,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,aAAa,GAAa,EAAE,CAAC;QAGnC,KAAK,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;YAC5D,IAAI,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAEpC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC9B,UAAU,IAAI,QAAQ,CAAC,IAAI,CAAC;gBAC5B,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACzB,CAAC;QACF,CAAC;QAGD,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAChC,CAAC;QAGD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;YAC1B,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;QAClD,CAAC;IACF,CAAC;IAKD,mBAAmB,CAAC,aAAqB;QAExC,IAAI,IAAI,CAAC,kBAAkB,IAAI,CAAC,EAAE,CAAC;YAClC,OAAO;QACR,CAAC;QAGD,IAAI,kBAAkB,GAAG,CAAC,CAAC;QAC3B,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC;YACpD,kBAAkB,IAAI,QAAQ,CAAC,IAAI,CAAC;QACrC,CAAC;QAGD,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAG7B,kBAAkB,GAAG,CAAC,CAAC;QACvB,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC;YACpD,kBAAkB,IAAI,QAAQ,CAAC,IAAI,CAAC;QACrC,CAAC;QAGD,IAAI,kBAAkB,GAAG,aAAa,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACnE,OAAO;QACR,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAGvB,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,KAAK,CAAC,IAAI,GAAG,GAAG,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;YAEtF,MAAM,MAAM,GAA4B,EAAE,CAAC;YAE3C,KAAK,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;gBAC5D,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,QAAQ,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAClD,CAAC;YAGD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAGnC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;YAClD,IAAI,CAAC,YAAY,GAAG,GAAG,CAAC;QACzB,CAAC;QAED,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,aAAa,GAAa,EAAE,CAAC;QAGnC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YAExC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAG3C,IAAI,kBAAkB,GAAG,UAAU,GAAG,aAAa,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBAChF,MAAM;YACP,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,QAAQ,EAAE,CAAC;gBACd,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC9B,UAAU,IAAI,QAAQ,CAAC,IAAI,CAAC;gBAC5B,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACzB,CAAC;QACF,CAAC;QAGD,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAChC,CAAC;QAGD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAE9B,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1F,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;QACrD,CAAC;IACF,CAAC;CACD;AArJD,kDAqJC"}

package/dist/esm/utils/vector-store/MemoryManager/config.d.ts

@@ -0,0 +1,4 @@
+import type { MemoryVectorStoreConfig } from './types';
+export declare function getConfig(): MemoryVectorStoreConfig;
+export declare function mbToBytes(mb: number): number;
+export declare function hoursToMs(hours: number): number;