n8n-nodes-script-runner 1.7.0 → 1.9.0

package/dist/nodes/LinkedInFetcher/LinkedInFetcher.node.js

@@ -6,71 +6,156 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.LinkedInFetcher = void 0;
 const n8n_workflow_1 = require("n8n-workflow");
 const axios_1 = __importDefault(require("axios"));
-function randomDelay(min = 45, max = 90) {
-    return Math.floor(Math.random() * (max - min + 1)) + min;
+function randomDelayMs(minSeconds = 45, maxSeconds = 90) {
+    return Math.floor(Math.random() * (maxSeconds - minSeconds + 1) + minSeconds) * 1000;
 }
-// Fetch function with cursor loop + total limit + error tracking + optional start cursor
-async function fetchLinkedIn(accountId, url, totalLimit, apiKey, minDelay, maxDelay, cursor = null, items = []) {
-    const limit = 100;
-    let last_page_coursor = cursor;
-    // STOP if we already collected enough
-    if (items.length >= totalLimit) {
-        return { items: items.slice(0, totalLimit), lastFailedCursor: null, error: null };
+async function fetchLinkedIn(accountId, url, totalLimit, apiKey, minDelaySec, maxDelaySec, startCursor = null, collected = [], seenIds = new Set(), consecutiveEmptyPages = 0, pageCount = 0) {
+    const MAX_PAGES = 25;
+    const MAX_CONSECUTIVE_EMPTY = 1;
+    const PAGE_SIZE = 100;
+    // Store API metadata for return values
+    let apiEstimatedTotal = null;
+    let apiNextCursor = null;
+    if (collected.length >= totalLimit || pageCount >= MAX_PAGES) {
+        this.logger?.info?.(`Limit reached: ${collected.length} items | ${pageCount} pages`);
+        return {
+            items: collected,
+            nextCursor: apiNextCursor,
+            lastFailedCursor: null,
+            estimatedTotal: apiEstimatedTotal,
+            error: null,
+            totalFetched: collected.length,
+        };
     }
     try {
+        const queryParams = new URLSearchParams({
+            limit: PAGE_SIZE.toString(),
+            account_id: accountId,
+        });
         const config = {
             method: 'POST',
-            url: `https://api15.unipile.com:14554/api/v1/linkedin/search?limit=${limit}&account_id=${accountId}${cursor ? `&cursor=${cursor}` : ''}`,
+            url: `https://api15.unipile.com:14554/api/v1/linkedin/search?${queryParams.toString()}`,
             headers: {
                 'X-API-KEY': apiKey,
-                'accept': 'application/json',
-                'Content-Type': 'application/json'
+                'Accept': 'application/json',
+                'Content-Type': 'application/json',
             },
-            data: { url }
+            data: startCursor ? { cursor: startCursor } : { url },
+            timeout: 60000, // 60 seconds
         };
+        this.logger.info(`Fetching page ${pageCount + 1} | cursor: ${startCursor ?? 'initial'} | ` +
+            `collected: ${collected.length}/${totalLimit}`);
         const response = await (0, axios_1.default)(config);
         const data = response.data;
-        // Add items (don't exceed totalLimit)
-        if (data.items?.length) {
-            items.push(...data.items);
+        const beforeLength = collected.length;
+        // Deduplicate and collect
+        if (Array.isArray(data.items)) {
+            for (const item of data.items) {
+                if (item?.id && typeof item.id === 'string' && !seenIds.has(item.id)) {
+                    seenIds.add(item.id);
+                    collected.push(item);
+                    if (collected.length >= totalLimit)
+                        break;
+                }
+            }
         }
-        if (!data.cursor) {
-            return { items: items.slice(0, totalLimit), lastFailedCursor: last_page_coursor, error: null };
+        // Early return if page_count is zero (no more pages available)
+        if (data.paging?.page_count === 0) {
+            const estimatedTotal = data.paging?.total_count ?? null;
+            if (estimatedTotal !== null)
+                apiEstimatedTotal = estimatedTotal;
+            this.logger.info(`page_count is 0, no more pages. Total collected: ${collected.length}`);
+            return {
+                items: collected,
+                nextCursor: null,
+                lastFailedCursor: null,
+                estimatedTotal: apiEstimatedTotal ?? estimatedTotal,
+                error: null,
+                totalFetched: collected.length,
+            };
+        }
+        const addedThisPage = collected.length - beforeLength;
+        const nextCursor = data.cursor ?? null;
+        const estimatedTotal = data.paging?.total_count ?? null;
+        // Store for final return
+        if (estimatedTotal !== null)
+            apiEstimatedTotal = estimatedTotal;
+        if (nextCursor !== null)
+            apiNextCursor = nextCursor;
+        if (!nextCursor) {
+            this.logger.info(`No more cursor. Total collected: ${collected.length}`);
+            return {
+                items: collected,
+                nextCursor: apiNextCursor,
+                lastFailedCursor: null,
+                estimatedTotal: apiEstimatedTotal ?? estimatedTotal,
+                error: null,
+                totalFetched: collected.length,
+            };
+        }
+        if (addedThisPage === 0) {
+            consecutiveEmptyPages++;
+            if (consecutiveEmptyPages >= MAX_CONSECUTIVE_EMPTY) {
+                this.logger.warn(`Stopping: ${consecutiveEmptyPages} consecutive empty pages`);
+                return {
+                    items: collected,
+                    nextCursor: null,
+                    lastFailedCursor: startCursor,
+                    estimatedTotal,
+                    error: { message: 'Too many consecutive empty pages from API' },
+                    totalFetched: collected.length,
+                };
+            }
         }
-        // Continue ONLY if cursor exists and we still need more items
-        if (data.cursor && items.length < totalLimit) {
-            const delay = randomDelay(minDelay, maxDelay);
-            console.log(`Fetched ${items.length} items, waiting ${delay}s before next request...`);
-            last_page_coursor = data.cursor;
-            console.log('Next cursor:', data.cursor);
-            await new Promise(res => setTimeout(res, delay * 1000));
-            return fetchLinkedIn(accountId, url, totalLimit, apiKey, minDelay, maxDelay, data.cursor, items);
+        else {
+            consecutiveEmptyPages = 0;
+        }
+        // Continue only if needed
+        if (collected.length < totalLimit) {
+            const delayMs = randomDelayMs(minDelaySec, maxDelaySec);
+            this.logger.info(`Added ${addedThisPage} new items → total ${collected.length}/${totalLimit}. ` +
+                `Waiting ~${Math.round(delayMs / 1000)} seconds...`);
+            await new Promise(resolve => setTimeout(resolve, delayMs));
+            return fetchLinkedIn.call(this, accountId, url, totalLimit, apiKey, minDelaySec, maxDelaySec, nextCursor, collected, seenIds, consecutiveEmptyPages, pageCount + 1);
         }
     }
-    catch (error) {
-        console.warn('Error fetching page, returning collected data so far.');
-        console.warn('Failed cursor/page:', cursor);
+    catch (err) {
+        const error = err;
+        const status = error?.response?.status ?? null;
+        const message = error.message || 'Unknown fetch error';
+        this.logger.warn(`Page fetch failed (cursor: ${startCursor ?? 'initial'}): ${message} (status: ${status})`);
         return {
-            items: items.slice(0, totalLimit),
-            lastFailedCursor: cursor,
+            items: collected,
+            nextCursor: null,
+            lastFailedCursor: startCursor,
+            estimatedTotal: null,
             error: {
-                code: error.code || null,
-                status: error.response?.status || null,
-                message: error.message || 'Unknown error'
-            }
+                code: error?.code ?? null,
+                status,
+                message,
+            },
+            totalFetched: collected.length,
         };
     }
-    return { items: items.slice(0, totalLimit), lastFailedCursor: last_page_coursor, error: null };
+    // Fallback (should rarely reach here)
+    return {
+        items: collected,
+        nextCursor: apiNextCursor,
+        lastFailedCursor: null,
+        estimatedTotal: apiEstimatedTotal,
+        error: null,
+        totalFetched: collected.length,
+    };
 }
 class LinkedInFetcher {
     constructor() {
         this.description = {
-            displayName: 'LinkedIn Fetcher',
+            displayName: 'LinkedIn Fetcher (Unipile)',
             name: 'linkedInFetcher',
             icon: 'file:linkedin.svg',
             group: ['transform'],
             version: 1,
-            description: 'Fetch LinkedIn data with pagination and error handling',
+            description: 'Fetch people / company results from LinkedIn via Unipile API (max 2500 default)',
             defaults: {
                 name: 'LinkedIn Fetcher',
             },
@@ -82,60 +167,55 @@ class LinkedInFetcher {
                     name: 'accountId',
                     type: 'string',
                     default: '',
-                    placeholder: 'Enter account ID',
-                    description: 'The LinkedIn account ID to use for the API',
+                    placeholder: 'e.g. x4SmGz5ASq6pD66Gn0rPxA',
                     required: true,
                 },
                 {
-                    displayName: 'URL',
+                    displayName: 'LinkedIn URL',
                     name: 'url',
                     type: 'string',
                     default: '',
-                    placeholder: 'Enter LinkedIn URL',
-                    description: 'The LinkedIn URL to fetch data from',
+                    placeholder: 'https://www.linkedin.com/sales/search/people?...',
+                    description: 'Full LinkedIn search, profile, or company page URL',
                     required: true,
                 },
                 {
-                    displayName: 'Total Items to Scrape',
+                    displayName: 'Max Items to Fetch',
                     name: 'total',
                     type: 'number',
-                    default: 100,
-                    description: 'Maximum number of items to fetch',
+                    default: 2500,
+                    typeOptions: { minValue: 1, maxValue: 10000 },
+                    description: 'Maximum number of records to collect (after deduplication)',
                     required: true,
                 },
                 {
                     displayName: 'API Key',
                     name: 'apiKey',
                     type: 'string',
-                    typeOptions: {
-                        password: true,
-                    },
+                    typeOptions: { password: true },
                     default: '',
-                    placeholder: 'Enter API key',
-                    description: 'The Unipile API key for authentication',
                     required: true,
                 },
                 {
-                    displayName: 'Start Cursor',
+                    displayName: 'Resume From Cursor',
                     name: 'startCursor',
                     type: 'string',
                     default: '',
-                    placeholder: 'Optional start cursor',
-                    description: 'Optional cursor to resume from a previous fetch',
+                    placeholder: '(optional) Paste cursor from previous run',
                 },
                 {
-                    displayName: 'Min Delay (seconds)',
+                    displayName: 'Min Delay Between Requests (seconds)',
                     name: 'minDelay',
                     type: 'number',
                     default: 45,
-                    description: 'Minimum delay between requests in seconds',
+                    typeOptions: { minValue: 10 },
                 },
                 {
-                    displayName: 'Max Delay (seconds)',
+                    displayName: 'Max Delay Between Requests (seconds)',
                     name: 'maxDelay',
                     type: 'number',
                     default: 90,
-                    description: 'Maximum delay between requests in seconds',
+                    typeOptions: { minValue: 30 },
                 },
             ],
         };
@@ -146,42 +226,43 @@ class LinkedInFetcher {
         for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
             try {
                 const accountId = this.getNodeParameter('accountId', itemIndex);
-                const url = this.getNodeParameter('url', itemIndex);
+                const url = this.getNodeParameter('url', itemIndex)?.trim();
                 const total = this.getNodeParameter('total', itemIndex);
                 const apiKey = this.getNodeParameter('apiKey', itemIndex);
-                const startCursor = this.getNodeParameter('startCursor', itemIndex, '');
+                const startCursorRaw = this.getNodeParameter('startCursor', itemIndex, '');
+                let startCursor = startCursorRaw ? startCursorRaw.trim() : null;
                 const minDelay = this.getNodeParameter('minDelay', itemIndex);
                 const maxDelay = this.getNodeParameter('maxDelay', itemIndex);
-                // Fetch data - treat empty string cursor as null
-                const { items: allItems, lastFailedCursor, error } = await fetchLinkedIn(accountId, url, total, apiKey, minDelay, maxDelay, startCursor && startCursor.trim() !== '' ? startCursor : null);
-                // Remove duplicates by `id`
-                const uniqueItems = [
-                    ...new Map(allItems.map(i => [i.id, i])).values()
-                ];
-                // Final output includes last failed cursor and error info
+                if (!url) {
+                    throw new Error('LinkedIn URL is required');
+                }
+                if (startCursor === '' || startCursor === 'null')
+                    startCursor = null;
+                const result = await fetchLinkedIn.call(this, accountId, url, total, apiKey, minDelay, maxDelay, startCursor);
                 returnData.push({
                     json: {
-                        result: uniqueItems,
-                        lastFailedCursor,
-                        error,
-                        totalFetched: uniqueItems.length,
+                        result: result.items,
+                        nextCursor: result.nextCursor,
+                        lastFailedCursor: result.lastFailedCursor,
+                        estimatedTotal: result.estimatedTotal,
+                        error: result.error,
+                        totalFetched: result.totalFetched,
+                        requestedLimit: total,
+                        pagesUsed: Math.ceil(result.totalFetched / 100),
                     },
-                    pairedItem: itemIndex,
+                    pairedItem: { item: itemIndex },
                 });
             }
-            catch (error) {
+            catch (err) {
+                const error = err;
                 if (this.continueOnFail()) {
                     returnData.push({
-                        json: {
-                            error: error.message,
-                        },
-                        pairedItem: itemIndex,
+                        json: { error: error.message || 'Execution failed' },
+                        pairedItem: { item: itemIndex },
                     });
                     continue;
                 }
-                throw new n8n_workflow_1.NodeOperationError(this.getNode(), error, {
-                    itemIndex,
-                });
+                throw new n8n_workflow_1.NodeOperationError(this.getNode(), error, { itemIndex });
             }
         }
         return [returnData];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "n8n-nodes-script-runner",
-  "version": "1.7.0",
+  "version": "1.9.0",
   "description": "Custom n8n nodes for script execution (jsdom, cheerio) and HTTP requests (axios, fetch)",
   "main": "index.js",
   "scripts": {