n8n-nodes-script-runner 1.4.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { IExecuteFunctions, INodeExecutionData, INodeType, INodeTypeDescription } from 'n8n-workflow';
|
|
2
|
+
export declare class LinkedInFetcher implements INodeType {
|
|
3
|
+
description: INodeTypeDescription;
|
|
4
|
+
execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]>;
|
|
5
|
+
}
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.LinkedInFetcher = void 0;
|
|
7
|
+
const n8n_workflow_1 = require("n8n-workflow");
|
|
8
|
+
const axios_1 = __importDefault(require("axios"));
|
|
9
|
+
function randomDelay(min = 45, max = 90) {
|
|
10
|
+
return Math.floor(Math.random() * (max - min + 1)) + min;
|
|
11
|
+
}
|
|
12
|
+
// Fetch function with cursor loop + total limit + error tracking + optional start cursor
|
|
13
|
+
async function fetchLinkedIn(accountId, url, totalLimit, apiKey, minDelay, maxDelay, cursor = null, items = []) {
|
|
14
|
+
const limit = 100;
|
|
15
|
+
let last_page_coursor = cursor;
|
|
16
|
+
// STOP if we already collected enough
|
|
17
|
+
if (items.length >= totalLimit) {
|
|
18
|
+
return { items: items.slice(0, totalLimit), lastFailedCursor: null, error: null };
|
|
19
|
+
}
|
|
20
|
+
try {
|
|
21
|
+
const config = {
|
|
22
|
+
method: 'POST',
|
|
23
|
+
url: `https://api15.unipile.com:14554/api/v1/linkedin/search?limit=${limit}&account_id=${accountId}${cursor ? `&cursor=${cursor}` : ''}`,
|
|
24
|
+
headers: {
|
|
25
|
+
'X-API-KEY': apiKey,
|
|
26
|
+
'accept': 'application/json',
|
|
27
|
+
'Content-Type': 'application/json'
|
|
28
|
+
},
|
|
29
|
+
data: { url }
|
|
30
|
+
};
|
|
31
|
+
const response = await (0, axios_1.default)(config);
|
|
32
|
+
const data = response.data;
|
|
33
|
+
// Add items (don't exceed totalLimit)
|
|
34
|
+
if (data.items?.length) {
|
|
35
|
+
items.push(...data.items);
|
|
36
|
+
}
|
|
37
|
+
if (!data.cursor) {
|
|
38
|
+
return { items: items.slice(0, totalLimit), lastFailedCursor: last_page_coursor, error: null };
|
|
39
|
+
}
|
|
40
|
+
// Continue ONLY if cursor exists and we still need more items
|
|
41
|
+
if (data.cursor && items.length < totalLimit) {
|
|
42
|
+
const delay = randomDelay(minDelay, maxDelay);
|
|
43
|
+
console.log(`Fetched ${items.length} items, waiting ${delay}s before next request...`);
|
|
44
|
+
last_page_coursor = data.cursor;
|
|
45
|
+
console.log('Next cursor:', data.cursor);
|
|
46
|
+
await new Promise(res => setTimeout(res, delay * 1000));
|
|
47
|
+
return fetchLinkedIn(accountId, url, totalLimit, apiKey, minDelay, maxDelay, data.cursor, items);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
catch (error) {
|
|
51
|
+
console.warn('Error fetching page, returning collected data so far.');
|
|
52
|
+
console.warn('Failed cursor/page:', cursor);
|
|
53
|
+
return {
|
|
54
|
+
items: items.slice(0, totalLimit),
|
|
55
|
+
lastFailedCursor: cursor,
|
|
56
|
+
error: {
|
|
57
|
+
code: error.code || null,
|
|
58
|
+
status: error.response?.status || null,
|
|
59
|
+
message: error.message || 'Unknown error'
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
return { items: items.slice(0, totalLimit), lastFailedCursor: last_page_coursor, error: null };
|
|
64
|
+
}
|
|
65
|
+
class LinkedInFetcher {
|
|
66
|
+
constructor() {
|
|
67
|
+
this.description = {
|
|
68
|
+
displayName: 'LinkedIn Fetcher',
|
|
69
|
+
name: 'linkedInFetcher',
|
|
70
|
+
icon: 'file:linkedin.svg',
|
|
71
|
+
group: ['transform'],
|
|
72
|
+
version: 1,
|
|
73
|
+
description: 'Fetch LinkedIn data with pagination and error handling',
|
|
74
|
+
defaults: {
|
|
75
|
+
name: 'LinkedIn Fetcher',
|
|
76
|
+
},
|
|
77
|
+
inputs: ['main'],
|
|
78
|
+
outputs: ['main'],
|
|
79
|
+
properties: [
|
|
80
|
+
{
|
|
81
|
+
displayName: 'Account ID',
|
|
82
|
+
name: 'accountId',
|
|
83
|
+
type: 'string',
|
|
84
|
+
default: '',
|
|
85
|
+
placeholder: 'Enter account ID',
|
|
86
|
+
description: 'The LinkedIn account ID to use for the API',
|
|
87
|
+
required: true,
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
displayName: 'URL',
|
|
91
|
+
name: 'url',
|
|
92
|
+
type: 'string',
|
|
93
|
+
default: '',
|
|
94
|
+
placeholder: 'Enter LinkedIn URL',
|
|
95
|
+
description: 'The LinkedIn URL to fetch data from',
|
|
96
|
+
required: true,
|
|
97
|
+
},
|
|
98
|
+
{
|
|
99
|
+
displayName: 'Total Items to Scrape',
|
|
100
|
+
name: 'total',
|
|
101
|
+
type: 'number',
|
|
102
|
+
default: 100,
|
|
103
|
+
description: 'Maximum number of items to fetch',
|
|
104
|
+
required: true,
|
|
105
|
+
},
|
|
106
|
+
{
|
|
107
|
+
displayName: 'API Key',
|
|
108
|
+
name: 'apiKey',
|
|
109
|
+
type: 'string',
|
|
110
|
+
typeOptions: {
|
|
111
|
+
password: true,
|
|
112
|
+
},
|
|
113
|
+
default: '',
|
|
114
|
+
placeholder: 'Enter API key',
|
|
115
|
+
description: 'The Unipile API key for authentication',
|
|
116
|
+
required: true,
|
|
117
|
+
},
|
|
118
|
+
{
|
|
119
|
+
displayName: 'Start Cursor',
|
|
120
|
+
name: 'startCursor',
|
|
121
|
+
type: 'string',
|
|
122
|
+
default: '',
|
|
123
|
+
placeholder: 'Optional start cursor',
|
|
124
|
+
description: 'Optional cursor to resume from a previous fetch',
|
|
125
|
+
},
|
|
126
|
+
{
|
|
127
|
+
displayName: 'Min Delay (seconds)',
|
|
128
|
+
name: 'minDelay',
|
|
129
|
+
type: 'number',
|
|
130
|
+
default: 45,
|
|
131
|
+
description: 'Minimum delay between requests in seconds',
|
|
132
|
+
},
|
|
133
|
+
{
|
|
134
|
+
displayName: 'Max Delay (seconds)',
|
|
135
|
+
name: 'maxDelay',
|
|
136
|
+
type: 'number',
|
|
137
|
+
default: 90,
|
|
138
|
+
description: 'Maximum delay between requests in seconds',
|
|
139
|
+
},
|
|
140
|
+
],
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
async execute() {
|
|
144
|
+
const items = this.getInputData();
|
|
145
|
+
const returnData = [];
|
|
146
|
+
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
|
|
147
|
+
try {
|
|
148
|
+
const accountId = this.getNodeParameter('accountId', itemIndex);
|
|
149
|
+
const url = this.getNodeParameter('url', itemIndex);
|
|
150
|
+
const total = this.getNodeParameter('total', itemIndex);
|
|
151
|
+
const apiKey = this.getNodeParameter('apiKey', itemIndex);
|
|
152
|
+
const startCursor = this.getNodeParameter('startCursor', itemIndex, '');
|
|
153
|
+
const minDelay = this.getNodeParameter('minDelay', itemIndex);
|
|
154
|
+
const maxDelay = this.getNodeParameter('maxDelay', itemIndex);
|
|
155
|
+
// Fetch data - treat empty string cursor as null
|
|
156
|
+
const { items: allItems, lastFailedCursor, error } = await fetchLinkedIn(accountId, url, total, apiKey, minDelay, maxDelay, startCursor && startCursor.trim() !== '' ? startCursor : null);
|
|
157
|
+
// Remove duplicates by `id`
|
|
158
|
+
const uniqueItems = [
|
|
159
|
+
...new Map(allItems.map(i => [i.id, i])).values()
|
|
160
|
+
];
|
|
161
|
+
// Final output includes last failed cursor and error info
|
|
162
|
+
returnData.push({
|
|
163
|
+
json: {
|
|
164
|
+
result: uniqueItems,
|
|
165
|
+
lastFailedCursor,
|
|
166
|
+
error,
|
|
167
|
+
totalFetched: uniqueItems.length,
|
|
168
|
+
},
|
|
169
|
+
pairedItem: itemIndex,
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
catch (error) {
|
|
173
|
+
if (this.continueOnFail()) {
|
|
174
|
+
returnData.push({
|
|
175
|
+
json: {
|
|
176
|
+
error: error.message,
|
|
177
|
+
},
|
|
178
|
+
pairedItem: itemIndex,
|
|
179
|
+
});
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), error, {
|
|
183
|
+
itemIndex,
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
return [returnData];
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
exports.LinkedInFetcher = LinkedInFetcher;
|