@limetech/n8n-nodes-lime 3.10.1 → 3.11.0-dev.1

package/nodes/lime-crm/resources/data/index.ts

@@ -24,6 +24,7 @@ const moduleHandler = new N8NOperationModuleHandler([
     operations.bulkCreateManyObjects,
     operations.bulkUpdateManyObjects,
     operations.bulkCreateOrUpdateManyObjects,
+    operations.iterateCsv,
 ]);
 
 /**
@@ -140,6 +141,9 @@ export async function dataOperations(
                 );
             return result && { json: result };
         }
+        case 'iterateCsv': {
+            return await operations.iterateCsv.execute.call(this, i);
+        }
     }
 
     throw new NodeOperationError(

package/nodes/lime-crm/resources/data/operations/index.ts

@@ -34,3 +34,7 @@ export * as bulkUpdateManyObjects from './bulkUpdateManyObjects.operation';
  * @group Resources
  */
 export * as bulkCreateOrUpdateManyObjects from './bulkCreateOrUpdateManyObjects.operation';
+/**
+ * @group Resources
+ */
+export * as iterateCsv from './iterateCsv';

package/nodes/lime-crm/resources/data/operations/iterateCsv.ts

@@ -0,0 +1,377 @@
+import { parse, type Parser } from 'csv-parse';
+import {
+    IDataObject,
+    IExecuteFunctions,
+    INodeExecutionData,
+    INodeParameterResourceLocator,
+    INodeProperties,
+    NodeOperationError,
+} from 'n8n-workflow';
+import { Readable } from 'node:stream';
+import { DATA_RESOURCE } from '../../../models';
+
+export const description = {
+    name: 'Process CSV in Batches',
+    value: 'iterateCsv',
+    description:
+        'Stream a CSV file from a binary field and run a sub-workflow for ' +
+        'every N rows. Only one batch is held in memory at a time, so very ' +
+        'large files can be processed without crashing the instance.',
+    action: 'Process a CSV file in batches via a sub-workflow',
+};
+
+export const properties: INodeProperties[] = [
+    {
+        displayName: 'Input Binary Field',
+        name: 'binaryPropertyName',
+        type: 'string',
+        default: 'data',
+        required: true,
+        description:
+            'Name of the binary field that contains the CSV file. ' +
+            'The file can come from any previous node (HTTP Request, ' +
+            'Read/Write Files from Disk, a form upload, etc.).',
+        displayOptions: {
+            show: {
+                resource: [DATA_RESOURCE],
+                operation: ['iterateCsv'],
+            },
+        },
+    },
+    {
+        displayName: 'Sub-Workflow',
+        name: 'workflowId',
+        type: 'workflowSelector',
+        default: '',
+        required: true,
+        description:
+            'The sub-workflow to run for each batch. It receives the batch ' +
+            'rows as input items (one item per row).',
+        displayOptions: {
+            show: {
+                resource: [DATA_RESOURCE],
+                operation: ['iterateCsv'],
+            },
+        },
+    },
+    {
+        displayName: 'Batch Size',
+        name: 'batchSize',
+        type: 'number',
+        default: 100,
+        required: true,
+        typeOptions: {
+            minValue: 1,
+        },
+        description: 'Number of rows to send to the sub-workflow per run',
+        displayOptions: {
+            show: {
+                resource: [DATA_RESOURCE],
+                operation: ['iterateCsv'],
+            },
+        },
+    },
+    {
+        displayName: 'Options',
+        name: 'options',
+        type: 'collection',
+        placeholder: 'Add Option',
+        default: {},
+        displayOptions: {
+            show: {
+                resource: [DATA_RESOURCE],
+                operation: ['iterateCsv'],
+            },
+        },
+        options: [
+            {
+                displayName: 'Delimiter',
+                name: 'delimiter',
+                type: 'string',
+                default: ',',
+                description: 'Character that separates columns',
+            },
+            {
+                displayName: 'Header Row',
+                name: 'header',
+                type: 'boolean',
+                default: true,
+                description:
+                    'Whether the first row holds column names. When enabled, ' +
+                    'each row is an object keyed by those names; otherwise rows ' +
+                    'are positional arrays.',
+            },
+            {
+                displayName: 'Max Rows',
+                name: 'maxRows',
+                type: 'number',
+                default: 0,
+                typeOptions: {
+                    minValue: 0,
+                },
+                description:
+                    'Maximum number of rows to read across all batches. Set to ' +
+                    '0 to read the whole file.',
+            },
+            {
+                displayName: 'Skip Empty Lines',
+                name: 'skipEmptyLines',
+                type: 'boolean',
+                default: true,
+                description:
+                    'Whether to ignore empty lines instead of emitting them',
+            },
+            {
+                displayName: 'Trim Values',
+                name: 'trim',
+                type: 'boolean',
+                default: false,
+                description:
+                    'Whether to strip surrounding whitespace from each value',
+            },
+            {
+                displayName: 'Wait for Sub-Workflow',
+                name: 'waitForSubWorkflow',
+                type: 'boolean',
+                default: true,
+                description:
+                    'Whether to wait for each batch sub-workflow to finish ' +
+                    'before reading the next batch. Keep enabled to bound ' +
+                    'memory; disable to fire batches without waiting (faster, ' +
+                    'but batches may queue up and grow memory).',
+            },
+        ],
+    },
+];
+
+interface ProcessCsvOptions {
+    delimiter?: string;
+    header?: boolean;
+    maxRows?: number;
+    skipEmptyLines?: boolean;
+    trim?: boolean;
+    waitForSubWorkflow?: boolean;
+}
+
+/**
+ * Resolve the CSV bytes as a readable stream.
+ *
+ * Prefers streaming from disk; falls back to the in-memory buffer for the
+ * legacy `default` binary-data mode where there is no file id.
+ *
+ * @param context - The n8n execution context
+ * @param i - The index of the current input item
+ * @param binaryPropertyName - Name of the binary field holding the CSV
+ */
+async function resolveInputStream(
+    context: IExecuteFunctions,
+    i: number,
+    binaryPropertyName: string
+): Promise<Readable> {
+    const binaryData = context.helpers.assertBinaryData(i, binaryPropertyName);
+
+    if (binaryData.id) {
+        return await context.helpers.getBinaryStream(binaryData.id);
+    }
+
+    const buffer = await context.helpers.getBinaryDataBuffer(
+        i,
+        binaryPropertyName
+    );
+    return Readable.from(buffer);
+}
+
+interface RunBatchesConfig {
+    inputStream: Readable;
+    parser: Parser;
+    workflowId: string;
+    batchSize: number;
+    maxRows: number;
+    header: boolean;
+    waitForSubWorkflow: boolean;
+    itemIndex: number;
+}
+
+async function runCsvBatches(
+    context: IExecuteFunctions,
+    config: RunBatchesConfig
+): Promise<INodeExecutionData[]> {
+    const {
+        inputStream,
+        parser,
+        workflowId,
+        batchSize,
+        maxRows,
+        header,
+        waitForSubWorkflow,
+        itemIndex,
+    } = config;
+
+    const summaries: INodeExecutionData[] = [];
+    let currentBatch: IDataObject[] = [];
+    let rowsRead = 0;
+
+    const runBatch = async () => {
+        if (currentBatch.length === 0) return;
+
+        const batchIndex = summaries.length;
+        const rowCount = currentBatch.length;
+        const inputItems: INodeExecutionData[] = currentBatch.map((row) => ({
+            json: row,
+            pairedItem: { item: itemIndex },
+        }));
+
+        // Release the rows before awaiting so they can be collected while the
+        // sub-workflow runs.
+        currentBatch = [];
+
+        const result = await context.executeWorkflow(
+            { id: workflowId },
+            inputItems,
+            undefined,
+            { doNotWaitToFinish: !waitForSubWorkflow }
+        );
+
+        summaries.push({
+            json: { batchIndex, rowCount, executionId: result.executionId },
+            pairedItem: { item: itemIndex },
+        });
+    };
+
+    try {
+        inputStream.pipe(parser);
+
+        for await (const record of parser) {
+            currentBatch.push(
+                header ? (record as IDataObject) : { row: record }
+            );
+            rowsRead += 1;
+
+            if (currentBatch.length >= batchSize) {
+                await runBatch();
+            }
+
+            if (maxRows > 0 && rowsRead >= maxRows) {
+                break;
+            }
+        }
+
+        // Process the final, partially filled batch.
+        await runBatch();
+    } catch (error) {
+        if (error instanceof NodeOperationError) throw error;
+        throw new NodeOperationError(
+            context.getNode(),
+            'Failed to process the CSV file',
+            {
+                itemIndex,
+                description: error.message,
+            }
+        );
+    } finally {
+        // Stop reading early (e.g. when Max Rows is hit) and release the
+        // underlying file handle / network socket.
+        parser.destroy();
+        inputStream.destroy();
+    }
+
+    return summaries;
+}
+
+/**
+ * Execute the "Process CSV in Batches" operation for Lime CRM.
+ *
+ * @remarks
+ * The CSV is read as a stream (from disk via
+ * {@link IExecuteFunctions.helpers.getBinaryStream} when the binary field is
+ * filesystem/database backed, otherwise from the in-memory buffer). Rows are
+ * grouped into batches of `batchSize`; for each full batch the configured
+ * sub-workflow is executed with the batch rows as input items, and the batch
+ * is then discarded before more rows are read.
+ *
+ * Because we `await` each sub-workflow run inside the parse loop, the stream is
+ * paused (backpressure) while a batch is processed, so at most one batch of
+ * rows is ever held in memory. This is what lets large files be processed
+ * without exhausting memory — unlike returning all rows/batches as node output.
+ *
+ * The node itself returns only small per-batch summaries, not the row data.
+ *
+ * @param i - The index of the current input item being processed
+ *
+ * @returns One summary {@link INodeExecutionData} per batch
+ *
+ * @public
+ */
+export async function execute(
+    this: IExecuteFunctions,
+    i: number
+): Promise<INodeExecutionData[]> {
+    const binaryPropertyName = this.getNodeParameter(
+        'binaryPropertyName',
+        i,
+        'data'
+    ) as string;
+    // `workflowSelector` stores a resource-locator object ({ mode, value }).
+    // Read the `value` rather than relying on extractValue, matching the
+    // built-in Execute Workflow node.
+    const workflowIdParam = this.getNodeParameter('workflowId', i, '') as
+        | INodeParameterResourceLocator
+        | string;
+    const workflowId = (
+        typeof workflowIdParam === 'object' && workflowIdParam !== null
+            ? workflowIdParam.value
+            : workflowIdParam
+    ) as string;
+    const batchSize = this.getNodeParameter('batchSize', i, 100) as number;
+    const options = this.getNodeParameter(
+        'options',
+        i,
+        {}
+    ) as ProcessCsvOptions;
+
+    if (batchSize < 1) {
+        throw new NodeOperationError(
+            this.getNode(),
+            'Batch Size must be at least 1',
+            { itemIndex: i }
+        );
+    }
+
+    if (!workflowId) {
+        throw new NodeOperationError(
+            this.getNode(),
+            'No sub-workflow selected',
+            { itemIndex: i }
+        );
+    }
+
+    const {
+        delimiter = ',',
+        header = true,
+        maxRows = 0,
+        skipEmptyLines = true,
+        trim = false,
+        waitForSubWorkflow = true,
+    } = options;
+
+    const inputStream = await resolveInputStream(this, i, binaryPropertyName);
+    const parser = parse({
+        bom: true,
+        columns: header,
+        delimiter,
+        trim,
+        skip_empty_lines: skipEmptyLines,
+        relax_column_count: true,
+    });
+
+    return await runCsvBatches(this, {
+        inputStream,
+        parser,
+        workflowId,
+        batchSize,
+        maxRows,
+        header,
+        waitForSubWorkflow,
+        itemIndex: i,
+    });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@limetech/n8n-nodes-lime",
-  "version": "3.10.1",
+  "version": "3.11.0-dev.1",
   "description": "n8n node to connect to Lime CRM",
   "author": {
     "name": "Lime Technologies",
@@ -67,5 +67,8 @@
   },
   "publishConfig": {
     "access": "public"
+  },
+  "dependencies": {
+    "csv-parse": "^7.0.0"
   }
 }