@fachkraftfreund/n8n-nodes-supabase 1.3.0 → 1.3.1

package/dist/nodes/Supabase/SupabaseCsvExport.node.js

@@ -4,6 +4,9 @@ exports.SupabaseCsvExport = void 0;
 const n8n_workflow_1 = require("n8n-workflow");
 const supabaseClient_1 = require("./utils/supabaseClient");
 const supabaseClient_2 = require("./utils/supabaseClient");
+function escapeRegExp(s) {
+    return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
 function escapeCsvField(value, delimiter, quoteChar) {
     if (value === null || value === undefined)
         return '';
@@ -12,31 +15,28 @@ function escapeCsvField(value, delimiter, quoteChar) {
         str.includes(quoteChar) ||
         str.includes('\n') ||
         str.includes('\r')) {
-        return quoteChar + str.replace(new RegExp(escapeRegExp(quoteChar), 'g'), quoteChar + quoteChar) + quoteChar;
+        const escaped = quoteChar + str.replace(new RegExp(escapeRegExp(quoteChar), 'g'), quoteChar + quoteChar) + quoteChar;
+        return escaped;
     }
     return str;
 }
-function escapeRegExp(s) {
-    return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-}
-function generateCsv(rows, options) {
-    if (rows.length === 0)
-        return '';
-    const { delimiter, quoteChar, includeHeaders } = options;
-    const headerSet = new Set();
+function discoverHeaders(rows) {
+    const set = new Set();
     for (const row of rows) {
         for (const key of Object.keys(row))
-            headerSet.add(key);
-    }
-    const headers = [...headerSet];
-    const lines = [];
-    if (includeHeaders) {
-        lines.push(headers.map((h) => escapeCsvField(h, delimiter, quoteChar)).join(delimiter));
+            set.add(key);
     }
-    for (const row of rows) {
-        lines.push(headers.map((h) => escapeCsvField(row[h], delimiter, quoteChar)).join(delimiter));
+    return [...set];
+}
+function batchToCsvBuffer(rows, headers, delimiter, quoteChar) {
+    const lines = new Array(rows.length);
+    for (let i = 0; i < rows.length; i++) {
+        const row = rows[i];
+        lines[i] = headers
+            .map((h) => escapeCsvField(row[h], delimiter, quoteChar))
+            .join(delimiter);
     }
-    return lines.join('\n');
+    return Buffer.from(lines.join('\n'), 'utf-8');
 }
 function buildSelectQuery(supabase, table, selectFields, filters, sort) {
     let query = supabase.from(table).select(selectFields);
@@ -75,33 +75,36 @@ function parseFilters(context, itemIndex) {
         throw new Error('Invalid advanced filters JSON');
     }
 }
-async function fetchAllRows(supabase, table, selectFields, filters, sort, hostUrl, returnAll, limit) {
+const BATCH_SIZE = 1000;
+async function* fetchBatches(supabase, table, selectFields, filters, sort, hostUrl, returnAll, limit, preferOffset) {
     const overhead = (0, supabaseClient_2.estimateUrlOverhead)(hostUrl, table, selectFields, filters, sort);
     const maxInChars = Math.max(500, supabaseClient_2.MAX_SAFE_URL_LENGTH - overhead);
     const maxItems = (0, supabaseClient_2.computeMaxIdsPerChunk)(selectFields);
     const filterChunks = (0, supabaseClient_2.expandChunkedFilters)(filters, maxInChars, maxItems);
-    const allRows = [];
-    if (returnAll) {
-        const batchSize = 1000;
-        const hasIdColumn = selectFields === '*' || selectFields.split(',').some((f) => f.trim() === 'id');
-        for (const chunkFilters of filterChunks) {
+    let totalYielded = 0;
+    const maxRows = returnAll ? Infinity : limit;
+    for (const chunkFilters of filterChunks) {
+        if (totalYielded >= maxRows)
+            break;
+        if (returnAll) {
+            const useKeyset = !preferOffset &&
+                (selectFields === '*' || selectFields.split(',').some((f) => f.trim() === 'id'));
             let hasMore = true;
-            if (hasIdColumn) {
+            if (useKeyset) {
                 let lastId = null;
                 while (hasMore) {
                     let query = buildSelectQuery(supabase, table, selectFields, chunkFilters, []);
-                    if (lastId !== null) {
+                    if (lastId !== null)
                         query = query.gt('id', lastId);
-                    }
-                    query = query.order('id', { ascending: true }).limit(batchSize);
+                    query = query.order('id', { ascending: true }).limit(BATCH_SIZE);
                     const { data, error } = await query;
                     if (error)
                         throw new Error((0, supabaseClient_2.formatSupabaseError)(error));
                     if (Array.isArray(data) && data.length > 0) {
-                        for (const row of data)
-                            allRows.push(row);
+                        yield data;
+                        totalYielded += data.length;
                         lastId = data[data.length - 1].id;
-                        hasMore = data.length === batchSize;
+                        hasMore = data.length === BATCH_SIZE;
                     }
                     else {
                         hasMore = false;
@@ -112,58 +115,35 @@ async function fetchAllRows(supabase, table, selectFields, filters, sort, hostUr
                 let offset = 0;
                 while (hasMore) {
                     const query = buildSelectQuery(supabase, table, selectFields, chunkFilters, sort);
-                    const { data, error } = await query.range(offset, offset + batchSize - 1);
+                    const { data, error } = await query.range(offset, offset + BATCH_SIZE - 1);
                     if (error)
                         throw new Error((0, supabaseClient_2.formatSupabaseError)(error));
                     if (Array.isArray(data) && data.length > 0) {
-                        for (const row of data)
-                            allRows.push(row);
-                        hasMore = data.length === batchSize;
+                        yield data;
+                        totalYielded += data.length;
+                        hasMore = data.length === BATCH_SIZE;
                     }
                     else {
                         hasMore = false;
                     }
-                    offset += batchSize;
+                    offset += BATCH_SIZE;
                 }
             }
         }
-        if (hasIdColumn && sort.length > 0) {
-            allRows.sort((a, b) => {
-                var _a, _b;
-                for (const s of sort) {
-                    const aVal = ((_a = a[s.column]) !== null && _a !== void 0 ? _a : null);
-                    const bVal = ((_b = b[s.column]) !== null && _b !== void 0 ? _b : null);
-                    if (aVal === bVal)
-                        continue;
-                    if (aVal === null)
-                        return 1;
-                    if (bVal === null)
-                        return -1;
-                    if (aVal < bVal)
-                        return s.ascending ? -1 : 1;
-                    if (aVal > bVal)
-                        return s.ascending ? 1 : -1;
-                }
-                return 0;
-            });
-        }
-    }
-    else {
-        for (const chunkFilters of filterChunks) {
+        else {
+            const remaining = maxRows - totalYielded;
+            if (remaining <= 0)
+                break;
             const query = buildSelectQuery(supabase, table, selectFields, chunkFilters, sort);
-            const { data, error } = await query.limit(limit);
+            const { data, error } = await query.limit(remaining);
             if (error)
                 throw new Error((0, supabaseClient_2.formatSupabaseError)(error));
-            if (Array.isArray(data)) {
-                for (const row of data)
-                    allRows.push(row);
+            if (Array.isArray(data) && data.length > 0) {
+                yield data;
+                totalYielded += data.length;
             }
         }
-        if (allRows.length > limit) {
-            allRows.length = limit;
-        }
     }
-    return allRows;
 }
 class SupabaseCsvExport {
     constructor() {
@@ -366,7 +346,9 @@ class SupabaseCsvExport {
                     name: 'enableTransform',
                     type: 'boolean',
                     default: false,
-                    description: 'Whether to apply a JavaScript transform before generating the CSV',
+                    description: 'Whether to apply a JavaScript transform before generating the CSV. ' +
+                        'The transform runs per batch (~1000 rows) so it stays memory-efficient ' +
+                        'even for very large exports. Use .filter() and .map() to shape your data.',
                 },
                 {
                     displayName: 'Transform Parameters',
@@ -564,6 +546,15 @@ class SupabaseCsvExport {
         const limit = returnAll ? 0 : this.getNodeParameter('limit', 0, 100);
         const filters = parseFilters(this, 0);
         const sort = this.getNodeParameter('sort.sortField', 0, []);
+        const enableTransform = this.getNodeParameter('enableTransform', 0, false);
+        const idColumn = this.getNodeParameter('idColumn', 0, 'id');
+        const csvOpts = this.getNodeParameter('csvOptions', 0, {});
+        const csvOptions = {
+            delimiter: csvOpts.delimiter || ',',
+            quoteChar: csvOpts.quoteChar || '"',
+            includeHeaders: csvOpts.includeHeaders !== false,
+            fileName: csvOpts.fileName || 'export.csv',
+        };
         const joins = this.getNodeParameter('joins.join', 0, []);
         let selectWithJoins = returnFields;
         for (const j of joins) {
@@ -573,66 +564,96 @@ class SupabaseCsvExport {
             const hint = j.joinType === 'inner' ? `${j.table}!inner` : j.table;
             selectWithJoins += `,${hint}(${cols})`;
         }
-        let rows;
+        const { delimiter, quoteChar } = csvOptions;
+        const csvChunks = [];
+        const ids = [];
+        let rowCount = 0;
+        let headers = null;
         try {
-            rows = await fetchAllRows(supabase, table, selectWithJoins, filters, sort, credentials.host, returnAll, limit);
-        }
-        catch (error) {
-            const msg = error instanceof Error ? error.message : 'Unknown error';
-            throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Failed to fetch data: ${msg}`);
-        }
-        const enableTransform = this.getNodeParameter('enableTransform', 0, false);
-        if (enableTransform) {
-            const paramEntries = this.getNodeParameter('transformParams.param', 0, []);
-            const params = {};
-            for (const entry of paramEntries) {
-                if (entry.name) {
-                    params[entry.name] = entry.value;
+            let transformFn = null;
+            let params = {};
+            if (enableTransform) {
+                const paramEntries = this.getNodeParameter('transformParams.param', 0, []);
+                for (const entry of paramEntries) {
+                    if (entry.name)
+                        params[entry.name] = entry.value;
+                }
+                const code = this.getNodeParameter('transformCode', 0, 'return rows;');
+                try {
+                    transformFn = new Function('rows', 'params', code);
+                }
+                catch (error) {
+                    const msg = error instanceof Error ? error.message : 'Unknown error';
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Transform code syntax error: ${msg}`);
                 }
             }
-            const code = this.getNodeParameter('transformCode', 0, 'return rows;');
-            try {
-                const transformFn = new Function('rows', 'params', code);
-                const result = transformFn(rows, params);
-                if (!Array.isArray(result)) {
-                    throw new Error('Transform code must return an array. Got: ' + typeof result);
+            const preferOffset = sort.length > 0;
+            for await (const batch of fetchBatches(supabase, table, selectWithJoins, filters, sort, credentials.host, returnAll, limit, preferOffset)) {
+                let rows = batch;
+                if (transformFn) {
+                    try {
+                        const result = transformFn(batch, params);
+                        if (!Array.isArray(result)) {
+                            throw new Error('Transform code must return an array. Got: ' + typeof result);
+                        }
+                        rows = result;
+                    }
+                    catch (error) {
+                        if (error instanceof n8n_workflow_1.NodeOperationError)
+                            throw error;
+                        const msg = error instanceof Error ? error.message : 'Unknown error';
+                        throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Transform code error: ${msg}`);
+                    }
                 }
-                rows = result;
+                if (rows.length === 0)
+                    continue;
+                if (headers === null) {
+                    headers = discoverHeaders(rows);
+                    if (csvOptions.includeHeaders) {
+                        const headerLine = headers
+                            .map((h) => escapeCsvField(h, delimiter, quoteChar))
+                            .join(delimiter);
+                        csvChunks.push(Buffer.from(headerLine + '\n', 'utf-8'));
+                    }
+                }
+                for (const row of rows) {
+                    if (row[idColumn] != null)
+                        ids.push(row[idColumn]);
+                }
+                const buf = batchToCsvBuffer(rows, headers, delimiter, quoteChar);
+                csvChunks.push(buf);
+                csvChunks.push(Buffer.from('\n', 'utf-8'));
+                rowCount += rows.length;
             }
-            catch (error) {
-                const msg = error instanceof Error ? error.message : 'Unknown error';
-                throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Transform code error: ${msg}`);
+            if (csvChunks.length > 0) {
+                const last = csvChunks[csvChunks.length - 1];
+                if (last.length === 1 && last[0] === 0x0a) {
+                    csvChunks.pop();
+                }
             }
         }
-        const idColumn = this.getNodeParameter('idColumn', 0, 'id');
-        const ids = [];
-        for (const row of rows) {
-            if (row[idColumn] !== undefined && row[idColumn] !== null) {
-                ids.push(row[idColumn]);
-            }
+        catch (error) {
+            const msg = error instanceof Error ? error.message : 'Unknown error';
+            if (error instanceof n8n_workflow_1.NodeOperationError)
+                throw error;
+            throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Export failed: ${msg}`);
         }
-        const csvOpts = this.getNodeParameter('csvOptions', 0, {});
-        const csvOptions = {
-            delimiter: csvOpts.delimiter || ',',
-            quoteChar: csvOpts.quoteChar || '"',
-            includeHeaders: csvOpts.includeHeaders !== false,
-            fileName: csvOpts.fileName || 'export.csv',
-        };
-        const csvContent = generateCsv(rows, csvOptions);
-        const csvBuffer = Buffer.from(csvContent, 'utf-8');
+        const csvBuffer = Buffer.concat(csvChunks);
+        csvChunks.length = 0;
         const binaryData = await this.helpers.prepareBinaryData(csvBuffer, csvOptions.fileName, 'text/csv');
-        const returnItem = {
-            json: {
-                table,
-                rowCount: rows.length,
-                ids,
-                fileName: csvOptions.fileName,
-            },
-            binary: {
-                data: binaryData,
-            },
-        };
-        return [[returnItem]];
+        return [[
+                {
+                    json: {
+                        table,
+                        rowCount,
+                        ids,
+                        fileName: csvOptions.fileName,
+                    },
+                    binary: {
+                        data: binaryData,
+                    },
+                },
+            ]];
     }
 }
 exports.SupabaseCsvExport = SupabaseCsvExport;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fachkraftfreund/n8n-nodes-supabase",
-  "version": "1.3.0",
+  "version": "1.3.1",
   "description": "Comprehensive n8n community node for Supabase with database and storage operations",
   "keywords": [
     "n8n-community-node-package",