n8n-nodes-alterlab 0.2.2 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/credentials/AlterLabOAuth2Api.credentials.d.ts +2 -1
- package/dist/credentials/AlterLabOAuth2Api.credentials.d.ts.map +1 -1
- package/dist/credentials/AlterLabOAuth2Api.credentials.js +7 -0
- package/dist/credentials/AlterLabOAuth2Api.credentials.js.map +1 -1
- package/dist/nodes/AlterLab/AlterLab.node.d.ts +1 -1
- package/dist/nodes/AlterLab/AlterLab.node.d.ts.map +1 -1
- package/dist/nodes/AlterLab/AlterLab.node.js +379 -257
- package/dist/nodes/AlterLab/AlterLab.node.js.map +1 -1
- package/dist/nodes/AlterLab/AlterLab.node.json +33 -30
- package/dist/nodes/AlterLab/alterlab.svg +10 -3
- package/package.json +60 -61
|
@@ -2,137 +2,179 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.AlterLab = void 0;
|
|
4
4
|
const n8n_workflow_1 = require("n8n-workflow");
|
|
5
|
-
const UTM =
|
|
5
|
+
const UTM = "utm_source=n8n&utm_medium=integration&utm_campaign=community_node";
|
|
6
|
+
const BASE_URL = "https://api.alterlab.io";
|
|
7
|
+
function sleep(ms) {
|
|
8
|
+
return new Promise((resolve) => {
|
|
9
|
+
globalThis.setTimeout(resolve, ms);
|
|
10
|
+
});
|
|
11
|
+
}
|
|
6
12
|
class AlterLab {
|
|
7
13
|
constructor() {
|
|
8
14
|
this.description = {
|
|
9
|
-
displayName:
|
|
10
|
-
name:
|
|
11
|
-
icon:
|
|
12
|
-
group: [
|
|
15
|
+
displayName: "AlterLab",
|
|
16
|
+
name: "alterLab",
|
|
17
|
+
icon: "file:alterlab.svg",
|
|
18
|
+
group: ["transform"],
|
|
13
19
|
version: 1,
|
|
14
|
-
subtitle: '={{$parameter["mode"] + " scrape"}}',
|
|
15
|
-
description:
|
|
20
|
+
subtitle: '={{$parameter["operation"] === "estimateCost" ? "cost estimate" : $parameter["mode"] + " scrape"}}',
|
|
21
|
+
description: "Scrape any website with anti-bot bypass, JS rendering, structured extraction, OCR, and more",
|
|
16
22
|
defaults: {
|
|
17
|
-
name:
|
|
23
|
+
name: "AlterLab",
|
|
18
24
|
},
|
|
19
|
-
inputs: [
|
|
20
|
-
outputs: [
|
|
25
|
+
inputs: ["main"],
|
|
26
|
+
outputs: ["main"],
|
|
21
27
|
credentials: [
|
|
22
28
|
{
|
|
23
|
-
name:
|
|
24
|
-
displayName:
|
|
29
|
+
name: "alterLabApi",
|
|
30
|
+
displayName: "API Key",
|
|
25
31
|
},
|
|
26
32
|
{
|
|
27
|
-
name:
|
|
28
|
-
displayName:
|
|
33
|
+
name: "alterLabOAuth2Api",
|
|
34
|
+
displayName: "OAuth2 (Recommended)",
|
|
29
35
|
},
|
|
30
36
|
],
|
|
37
|
+
requestDefaults: {
|
|
38
|
+
baseURL: BASE_URL,
|
|
39
|
+
},
|
|
31
40
|
properties: [
|
|
41
|
+
// ── Operation ────────────────────────────────────────
|
|
42
|
+
{
|
|
43
|
+
displayName: "Operation",
|
|
44
|
+
name: "operation",
|
|
45
|
+
type: "options",
|
|
46
|
+
noDataExpression: true,
|
|
47
|
+
default: "scrape",
|
|
48
|
+
options: [
|
|
49
|
+
{
|
|
50
|
+
name: "Scrape",
|
|
51
|
+
value: "scrape",
|
|
52
|
+
description: "Scrape a URL and return its content",
|
|
53
|
+
action: "Scrape a URL",
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
name: "Estimate Cost",
|
|
57
|
+
value: "estimateCost",
|
|
58
|
+
description: "Estimate the cost of scraping a URL without actually scraping it",
|
|
59
|
+
action: "Estimate scraping cost",
|
|
60
|
+
},
|
|
61
|
+
],
|
|
62
|
+
description: "The operation to perform",
|
|
63
|
+
},
|
|
32
64
|
// ── Primary ──────────────────────────────────────────
|
|
33
65
|
{
|
|
34
|
-
displayName:
|
|
35
|
-
name:
|
|
36
|
-
type:
|
|
37
|
-
default:
|
|
66
|
+
displayName: "URL",
|
|
67
|
+
name: "url",
|
|
68
|
+
type: "string",
|
|
69
|
+
default: "",
|
|
38
70
|
required: true,
|
|
39
|
-
placeholder:
|
|
40
|
-
description:
|
|
71
|
+
placeholder: "https://www.example.com/page",
|
|
72
|
+
description: "The URL to scrape",
|
|
41
73
|
},
|
|
42
74
|
{
|
|
43
|
-
displayName:
|
|
44
|
-
name:
|
|
45
|
-
type:
|
|
46
|
-
default:
|
|
75
|
+
displayName: "Mode",
|
|
76
|
+
name: "mode",
|
|
77
|
+
type: "options",
|
|
78
|
+
default: "auto",
|
|
47
79
|
options: [
|
|
48
80
|
{
|
|
49
|
-
name:
|
|
50
|
-
value:
|
|
51
|
-
description:
|
|
81
|
+
name: "Auto",
|
|
82
|
+
value: "auto",
|
|
83
|
+
description: "Automatically choose the best scraping method",
|
|
52
84
|
},
|
|
53
85
|
{
|
|
54
|
-
name:
|
|
55
|
-
value:
|
|
56
|
-
description:
|
|
86
|
+
name: "HTML",
|
|
87
|
+
value: "html",
|
|
88
|
+
description: "Fast HTTP-only scraping for static pages",
|
|
57
89
|
},
|
|
58
90
|
{
|
|
59
|
-
name:
|
|
60
|
-
value:
|
|
61
|
-
description:
|
|
91
|
+
name: "JavaScript",
|
|
92
|
+
value: "js",
|
|
93
|
+
description: "Render JavaScript with headless browser",
|
|
62
94
|
},
|
|
63
95
|
{
|
|
64
|
-
name:
|
|
65
|
-
value:
|
|
66
|
-
description:
|
|
96
|
+
name: "PDF",
|
|
97
|
+
value: "pdf",
|
|
98
|
+
description: "Extract text from PDF documents",
|
|
67
99
|
},
|
|
68
100
|
{
|
|
69
|
-
name:
|
|
70
|
-
value:
|
|
71
|
-
description:
|
|
101
|
+
name: "OCR",
|
|
102
|
+
value: "ocr",
|
|
103
|
+
description: "Extract text from images",
|
|
72
104
|
},
|
|
73
105
|
],
|
|
74
|
-
description:
|
|
106
|
+
description: "Scraping mode to use",
|
|
75
107
|
},
|
|
76
108
|
// ── Output Options ───────────────────────────────────
|
|
77
109
|
{
|
|
78
|
-
displayName:
|
|
79
|
-
name:
|
|
80
|
-
type:
|
|
81
|
-
placeholder:
|
|
110
|
+
displayName: "Output Options",
|
|
111
|
+
name: "outputOptions",
|
|
112
|
+
type: "collection",
|
|
113
|
+
placeholder: "Add Option",
|
|
82
114
|
default: {},
|
|
115
|
+
displayOptions: {
|
|
116
|
+
show: {
|
|
117
|
+
operation: ["scrape"],
|
|
118
|
+
},
|
|
119
|
+
},
|
|
83
120
|
options: [
|
|
84
121
|
{
|
|
85
|
-
displayName:
|
|
86
|
-
name:
|
|
87
|
-
type:
|
|
88
|
-
default: [
|
|
122
|
+
displayName: "Formats",
|
|
123
|
+
name: "formats",
|
|
124
|
+
type: "multiOptions",
|
|
125
|
+
default: ["markdown", "json"],
|
|
89
126
|
options: [
|
|
90
|
-
{ name:
|
|
91
|
-
{ name:
|
|
92
|
-
{ name:
|
|
93
|
-
{ name:
|
|
127
|
+
{ name: "Markdown", value: "markdown" },
|
|
128
|
+
{ name: "JSON", value: "json" },
|
|
129
|
+
{ name: "HTML", value: "html" },
|
|
130
|
+
{ name: "Text", value: "text" },
|
|
94
131
|
],
|
|
95
|
-
description:
|
|
132
|
+
description: "Output formats for content transformation",
|
|
96
133
|
},
|
|
97
134
|
{
|
|
98
|
-
displayName:
|
|
99
|
-
name:
|
|
100
|
-
type:
|
|
135
|
+
displayName: "Include Raw HTML",
|
|
136
|
+
name: "includeRawHtml",
|
|
137
|
+
type: "boolean",
|
|
101
138
|
default: false,
|
|
102
|
-
description:
|
|
139
|
+
description: "Whether to include the raw HTML in the response",
|
|
103
140
|
},
|
|
104
141
|
{
|
|
105
|
-
displayName:
|
|
106
|
-
name:
|
|
107
|
-
type:
|
|
142
|
+
displayName: "Timeout (Seconds)",
|
|
143
|
+
name: "timeout",
|
|
144
|
+
type: "number",
|
|
108
145
|
default: 90,
|
|
109
146
|
typeOptions: { minValue: 1, maxValue: 300 },
|
|
110
|
-
description:
|
|
147
|
+
description: "Request timeout in seconds (1-300)",
|
|
111
148
|
},
|
|
112
149
|
],
|
|
113
150
|
},
|
|
114
151
|
// ── Execution Mode ───────────────────────────────────
|
|
115
152
|
{
|
|
116
|
-
displayName:
|
|
117
|
-
name:
|
|
118
|
-
type:
|
|
119
|
-
placeholder:
|
|
153
|
+
displayName: "Execution Mode",
|
|
154
|
+
name: "executionMode",
|
|
155
|
+
type: "collection",
|
|
156
|
+
placeholder: "Add Option",
|
|
120
157
|
default: {},
|
|
158
|
+
displayOptions: {
|
|
159
|
+
show: {
|
|
160
|
+
operation: ["scrape"],
|
|
161
|
+
},
|
|
162
|
+
},
|
|
121
163
|
options: [
|
|
122
164
|
{
|
|
123
|
-
displayName:
|
|
124
|
-
name:
|
|
125
|
-
type:
|
|
165
|
+
displayName: "Cache",
|
|
166
|
+
name: "cache",
|
|
167
|
+
type: "boolean",
|
|
126
168
|
default: false,
|
|
127
|
-
description:
|
|
169
|
+
description: "Whether to enable response caching",
|
|
128
170
|
},
|
|
129
171
|
{
|
|
130
|
-
displayName:
|
|
131
|
-
name:
|
|
132
|
-
type:
|
|
172
|
+
displayName: "Cache TTL (Seconds)",
|
|
173
|
+
name: "cacheTtl",
|
|
174
|
+
type: "number",
|
|
133
175
|
default: 900,
|
|
134
176
|
typeOptions: { minValue: 60, maxValue: 86400 },
|
|
135
|
-
description:
|
|
177
|
+
description: "Cache time-to-live in seconds (60-86400)",
|
|
136
178
|
displayOptions: {
|
|
137
179
|
show: {
|
|
138
180
|
cache: [true],
|
|
@@ -143,25 +185,25 @@ class AlterLab {
|
|
|
143
185
|
},
|
|
144
186
|
// ── Advanced Options ─────────────────────────────────
|
|
145
187
|
{
|
|
146
|
-
displayName:
|
|
147
|
-
name:
|
|
148
|
-
type:
|
|
149
|
-
placeholder:
|
|
188
|
+
displayName: "Advanced Options",
|
|
189
|
+
name: "advancedOptions",
|
|
190
|
+
type: "collection",
|
|
191
|
+
placeholder: "Add Option",
|
|
150
192
|
default: {},
|
|
151
193
|
options: [
|
|
152
194
|
{
|
|
153
|
-
displayName:
|
|
154
|
-
name:
|
|
155
|
-
type:
|
|
195
|
+
displayName: "Render JavaScript",
|
|
196
|
+
name: "renderJs",
|
|
197
|
+
type: "boolean",
|
|
156
198
|
default: false,
|
|
157
|
-
description:
|
|
199
|
+
description: "Whether to render JavaScript with a headless browser (+$0.0006)",
|
|
158
200
|
},
|
|
159
201
|
{
|
|
160
|
-
displayName:
|
|
161
|
-
name:
|
|
162
|
-
type:
|
|
202
|
+
displayName: "Screenshot",
|
|
203
|
+
name: "screenshot",
|
|
204
|
+
type: "boolean",
|
|
163
205
|
default: false,
|
|
164
|
-
description:
|
|
206
|
+
description: "Whether to capture a full-page screenshot (+$0.0002, requires Render JavaScript)",
|
|
165
207
|
displayOptions: {
|
|
166
208
|
show: {
|
|
167
209
|
renderJs: [true],
|
|
@@ -169,11 +211,11 @@ class AlterLab {
|
|
|
169
211
|
},
|
|
170
212
|
},
|
|
171
213
|
{
|
|
172
|
-
displayName:
|
|
173
|
-
name:
|
|
174
|
-
type:
|
|
214
|
+
displayName: "Generate PDF",
|
|
215
|
+
name: "generatePdf",
|
|
216
|
+
type: "boolean",
|
|
175
217
|
default: false,
|
|
176
|
-
description:
|
|
218
|
+
description: "Whether to generate a PDF of the rendered page (+$0.0004, requires Render JavaScript)",
|
|
177
219
|
displayOptions: {
|
|
178
220
|
show: {
|
|
179
221
|
renderJs: [true],
|
|
@@ -181,26 +223,26 @@ class AlterLab {
|
|
|
181
223
|
},
|
|
182
224
|
},
|
|
183
225
|
{
|
|
184
|
-
displayName:
|
|
185
|
-
name:
|
|
186
|
-
type:
|
|
226
|
+
displayName: "OCR",
|
|
227
|
+
name: "ocr",
|
|
228
|
+
type: "boolean",
|
|
187
229
|
default: false,
|
|
188
|
-
description:
|
|
230
|
+
description: "Whether to extract text from images using OCR (+$0.001, refunded if no images found)",
|
|
189
231
|
},
|
|
190
232
|
{
|
|
191
|
-
displayName:
|
|
192
|
-
name:
|
|
193
|
-
type:
|
|
233
|
+
displayName: "Use Proxy",
|
|
234
|
+
name: "useProxy",
|
|
235
|
+
type: "boolean",
|
|
194
236
|
default: false,
|
|
195
|
-
description:
|
|
237
|
+
description: "Whether to route through a premium proxy (+$0.0002)",
|
|
196
238
|
},
|
|
197
239
|
{
|
|
198
|
-
displayName:
|
|
199
|
-
name:
|
|
200
|
-
type:
|
|
201
|
-
default:
|
|
202
|
-
placeholder:
|
|
203
|
-
description:
|
|
240
|
+
displayName: "Proxy Country",
|
|
241
|
+
name: "proxyCountry",
|
|
242
|
+
type: "string",
|
|
243
|
+
default: "",
|
|
244
|
+
placeholder: "US",
|
|
245
|
+
description: "Preferred proxy country code for geo-targeting (e.g. US, DE, GB)",
|
|
204
246
|
displayOptions: {
|
|
205
247
|
show: {
|
|
206
248
|
useProxy: [true],
|
|
@@ -208,28 +250,28 @@ class AlterLab {
|
|
|
208
250
|
},
|
|
209
251
|
},
|
|
210
252
|
{
|
|
211
|
-
displayName:
|
|
212
|
-
name:
|
|
213
|
-
type:
|
|
214
|
-
default:
|
|
253
|
+
displayName: "Wait Condition",
|
|
254
|
+
name: "waitCondition",
|
|
255
|
+
type: "options",
|
|
256
|
+
default: "networkidle",
|
|
215
257
|
options: [
|
|
216
258
|
{
|
|
217
|
-
name:
|
|
218
|
-
value:
|
|
219
|
-
description:
|
|
259
|
+
name: "Network Idle",
|
|
260
|
+
value: "networkidle",
|
|
261
|
+
description: "Wait until network is idle",
|
|
220
262
|
},
|
|
221
263
|
{
|
|
222
|
-
name:
|
|
223
|
-
value:
|
|
224
|
-
description:
|
|
264
|
+
name: "DOM Content Loaded",
|
|
265
|
+
value: "domcontentloaded",
|
|
266
|
+
description: "Wait until DOM content is loaded",
|
|
225
267
|
},
|
|
226
268
|
{
|
|
227
|
-
name:
|
|
228
|
-
value:
|
|
229
|
-
description:
|
|
269
|
+
name: "Load",
|
|
270
|
+
value: "load",
|
|
271
|
+
description: "Wait until page load event",
|
|
230
272
|
},
|
|
231
273
|
],
|
|
232
|
-
description:
|
|
274
|
+
description: "When to consider the page ready (JS rendering only)",
|
|
233
275
|
displayOptions: {
|
|
234
276
|
show: {
|
|
235
277
|
renderJs: [true],
|
|
@@ -237,137 +279,142 @@ class AlterLab {
|
|
|
237
279
|
},
|
|
238
280
|
},
|
|
239
281
|
{
|
|
240
|
-
displayName:
|
|
241
|
-
name:
|
|
242
|
-
type:
|
|
282
|
+
displayName: "Remove Cookie Banners",
|
|
283
|
+
name: "removeCookieBanners",
|
|
284
|
+
type: "boolean",
|
|
243
285
|
default: true,
|
|
244
|
-
description:
|
|
286
|
+
description: "Whether to remove cookie consent banners before content extraction",
|
|
245
287
|
},
|
|
246
288
|
],
|
|
247
289
|
},
|
|
248
290
|
// ── Extraction ───────────────────────────────────────
|
|
249
291
|
{
|
|
250
|
-
displayName:
|
|
251
|
-
name:
|
|
252
|
-
type:
|
|
253
|
-
placeholder:
|
|
292
|
+
displayName: "Extraction",
|
|
293
|
+
name: "extraction",
|
|
294
|
+
type: "collection",
|
|
295
|
+
placeholder: "Add Option",
|
|
254
296
|
default: {},
|
|
297
|
+
displayOptions: {
|
|
298
|
+
show: {
|
|
299
|
+
operation: ["scrape"],
|
|
300
|
+
},
|
|
301
|
+
},
|
|
255
302
|
options: [
|
|
256
303
|
{
|
|
257
|
-
displayName:
|
|
258
|
-
name:
|
|
259
|
-
type:
|
|
260
|
-
default:
|
|
304
|
+
displayName: "Extraction Profile",
|
|
305
|
+
name: "extractionProfile",
|
|
306
|
+
type: "options",
|
|
307
|
+
default: "auto",
|
|
261
308
|
options: [
|
|
262
|
-
{ name:
|
|
263
|
-
{ name:
|
|
264
|
-
{ name:
|
|
265
|
-
{ name:
|
|
266
|
-
{ name:
|
|
267
|
-
{ name:
|
|
268
|
-
{ name:
|
|
309
|
+
{ name: "Auto", value: "auto" },
|
|
310
|
+
{ name: "Product", value: "product" },
|
|
311
|
+
{ name: "Article", value: "article" },
|
|
312
|
+
{ name: "Job Posting", value: "job_posting" },
|
|
313
|
+
{ name: "FAQ", value: "faq" },
|
|
314
|
+
{ name: "Recipe", value: "recipe" },
|
|
315
|
+
{ name: "Event", value: "event" },
|
|
269
316
|
],
|
|
270
|
-
description:
|
|
317
|
+
description: "Pre-defined extraction profile for structured data",
|
|
271
318
|
},
|
|
272
319
|
{
|
|
273
|
-
displayName:
|
|
274
|
-
name:
|
|
275
|
-
type:
|
|
320
|
+
displayName: "Extraction Prompt",
|
|
321
|
+
name: "extractionPrompt",
|
|
322
|
+
type: "string",
|
|
276
323
|
typeOptions: { rows: 4 },
|
|
277
|
-
default:
|
|
278
|
-
placeholder:
|
|
279
|
-
description:
|
|
324
|
+
default: "",
|
|
325
|
+
placeholder: "Extract the product name, price, and rating...",
|
|
326
|
+
description: "Natural language instructions for what data to extract",
|
|
280
327
|
},
|
|
281
328
|
{
|
|
282
|
-
displayName:
|
|
283
|
-
name:
|
|
284
|
-
type:
|
|
285
|
-
default:
|
|
329
|
+
displayName: "Extraction Schema (JSON)",
|
|
330
|
+
name: "extractionSchema",
|
|
331
|
+
type: "json",
|
|
332
|
+
default: "",
|
|
286
333
|
placeholder: '{"name": "string", "price": "number"}',
|
|
287
|
-
description:
|
|
334
|
+
description: "JSON Schema to filter and structure extracted data",
|
|
288
335
|
},
|
|
289
336
|
{
|
|
290
|
-
displayName:
|
|
291
|
-
name:
|
|
292
|
-
type:
|
|
337
|
+
displayName: "Promote Schema.org",
|
|
338
|
+
name: "promoteSchemaOrg",
|
|
339
|
+
type: "boolean",
|
|
293
340
|
default: true,
|
|
294
|
-
description:
|
|
341
|
+
description: "Whether to use Schema.org structured data as primary output when available",
|
|
295
342
|
},
|
|
296
343
|
{
|
|
297
|
-
displayName:
|
|
298
|
-
name:
|
|
299
|
-
type:
|
|
344
|
+
displayName: "Evidence",
|
|
345
|
+
name: "evidence",
|
|
346
|
+
type: "boolean",
|
|
300
347
|
default: false,
|
|
301
|
-
description:
|
|
348
|
+
description: "Whether to include provenance/evidence for extracted fields",
|
|
302
349
|
},
|
|
303
350
|
],
|
|
304
351
|
},
|
|
305
352
|
// ── Cost Controls ────────────────────────────────────
|
|
306
353
|
{
|
|
307
|
-
displayName:
|
|
308
|
-
name:
|
|
309
|
-
type:
|
|
310
|
-
placeholder:
|
|
354
|
+
displayName: "Cost Controls",
|
|
355
|
+
name: "costControls",
|
|
356
|
+
type: "collection",
|
|
357
|
+
placeholder: "Add Option",
|
|
311
358
|
default: {},
|
|
312
359
|
options: [
|
|
313
360
|
{
|
|
314
|
-
displayName:
|
|
315
|
-
name:
|
|
316
|
-
type:
|
|
361
|
+
displayName: "Max Spend",
|
|
362
|
+
name: "maxCredits",
|
|
363
|
+
type: "number",
|
|
317
364
|
default: 0,
|
|
318
365
|
typeOptions: { minValue: 0 },
|
|
319
|
-
description:
|
|
366
|
+
description: "Maximum to spend per request in microcents (0 = no limit)",
|
|
320
367
|
},
|
|
321
368
|
{
|
|
322
|
-
displayName:
|
|
323
|
-
name:
|
|
324
|
-
type:
|
|
325
|
-
default:
|
|
369
|
+
displayName: "Force Tier",
|
|
370
|
+
name: "forceTier",
|
|
371
|
+
type: "options",
|
|
372
|
+
default: "",
|
|
326
373
|
options: [
|
|
327
|
-
{ name:
|
|
328
|
-
{ name:
|
|
329
|
-
{ name:
|
|
330
|
-
{ name:
|
|
331
|
-
{ name:
|
|
332
|
-
{ name:
|
|
374
|
+
{ name: "None", value: "" },
|
|
375
|
+
{ name: "T1 Curl — $0.0002", value: "1" },
|
|
376
|
+
{ name: "T2 HTTP — $0.0003", value: "2" },
|
|
377
|
+
{ name: "T3 Stealth — $0.0005", value: "3" },
|
|
378
|
+
{ name: "T3.5 Light JS — $0.0007", value: "3.5" },
|
|
379
|
+
{ name: "T4 Browser — $0.001", value: "4" },
|
|
333
380
|
],
|
|
334
|
-
description:
|
|
381
|
+
description: "Force a specific scraping tier (skip escalation)",
|
|
335
382
|
},
|
|
336
383
|
{
|
|
337
|
-
displayName:
|
|
338
|
-
name:
|
|
339
|
-
type:
|
|
340
|
-
default:
|
|
384
|
+
displayName: "Max Tier",
|
|
385
|
+
name: "maxTier",
|
|
386
|
+
type: "options",
|
|
387
|
+
default: "",
|
|
341
388
|
options: [
|
|
342
|
-
{ name:
|
|
343
|
-
{ name:
|
|
344
|
-
{ name:
|
|
345
|
-
{ name:
|
|
346
|
-
{ name:
|
|
347
|
-
{ name:
|
|
389
|
+
{ name: "None", value: "" },
|
|
390
|
+
{ name: "T1 Curl — $0.0002", value: "1" },
|
|
391
|
+
{ name: "T2 HTTP — $0.0003", value: "2" },
|
|
392
|
+
{ name: "T3 Stealth — $0.0005", value: "3" },
|
|
393
|
+
{ name: "T3.5 Light JS — $0.0007", value: "3.5" },
|
|
394
|
+
{ name: "T4 Browser — $0.001", value: "4" },
|
|
348
395
|
],
|
|
349
|
-
description:
|
|
396
|
+
description: "Maximum tier to escalate to",
|
|
350
397
|
},
|
|
351
398
|
{
|
|
352
|
-
displayName:
|
|
353
|
-
name:
|
|
354
|
-
type:
|
|
399
|
+
displayName: "Prefer Cost",
|
|
400
|
+
name: "preferCost",
|
|
401
|
+
type: "boolean",
|
|
355
402
|
default: false,
|
|
356
|
-
description:
|
|
403
|
+
description: "Whether to optimize for lower cost (try cheaper tiers first)",
|
|
357
404
|
},
|
|
358
405
|
{
|
|
359
|
-
displayName:
|
|
360
|
-
name:
|
|
361
|
-
type:
|
|
406
|
+
displayName: "Prefer Speed",
|
|
407
|
+
name: "preferSpeed",
|
|
408
|
+
type: "boolean",
|
|
362
409
|
default: false,
|
|
363
|
-
description:
|
|
410
|
+
description: "Whether to optimize for speed (skip to reliable tier)",
|
|
364
411
|
},
|
|
365
412
|
{
|
|
366
|
-
displayName:
|
|
367
|
-
name:
|
|
368
|
-
type:
|
|
413
|
+
displayName: "Fail Fast",
|
|
414
|
+
name: "failFast",
|
|
415
|
+
type: "boolean",
|
|
369
416
|
default: false,
|
|
370
|
-
description:
|
|
417
|
+
description: "Whether to return an error instead of escalating to expensive tiers",
|
|
371
418
|
},
|
|
372
419
|
],
|
|
373
420
|
},
|
|
@@ -375,18 +422,88 @@ class AlterLab {
|
|
|
375
422
|
};
|
|
376
423
|
}
|
|
377
424
|
async execute() {
|
|
378
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _0, _1;
|
|
425
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _0, _1, _2, _3, _4, _5, _6, _7;
|
|
379
426
|
const items = this.getInputData();
|
|
380
427
|
const results = [];
|
|
381
428
|
for (let i = 0; i < items.length; i++) {
|
|
382
429
|
try {
|
|
383
|
-
const
|
|
384
|
-
const
|
|
385
|
-
const
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
430
|
+
const operation = this.getNodeParameter("operation", i);
|
|
431
|
+
const url = this.getNodeParameter("url", i);
|
|
432
|
+
const mode = this.getNodeParameter("mode", i);
|
|
433
|
+
// ── Estimate Cost operation ───────────────────────
|
|
434
|
+
if (operation === "estimateCost") {
|
|
435
|
+
const advancedOptions = this.getNodeParameter("advancedOptions", i, {});
|
|
436
|
+
const costControls = this.getNodeParameter("costControls", i, {});
|
|
437
|
+
const body = { url, mode };
|
|
438
|
+
const advanced = {};
|
|
439
|
+
if (advancedOptions.renderJs)
|
|
440
|
+
advanced.render_js = true;
|
|
441
|
+
if (advancedOptions.useProxy)
|
|
442
|
+
advanced.use_proxy = true;
|
|
443
|
+
if (advancedOptions.proxyCountry) {
|
|
444
|
+
advanced.proxy_country = advancedOptions.proxyCountry;
|
|
445
|
+
}
|
|
446
|
+
if (Object.keys(advanced).length > 0) {
|
|
447
|
+
body.advanced = advanced;
|
|
448
|
+
}
|
|
449
|
+
const costCtrl = {};
|
|
450
|
+
if (costControls.maxCredits && costControls.maxCredits > 0) {
|
|
451
|
+
costCtrl.max_credits = costControls.maxCredits;
|
|
452
|
+
}
|
|
453
|
+
if (costControls.forceTier)
|
|
454
|
+
costCtrl.force_tier = costControls.forceTier;
|
|
455
|
+
if (costControls.maxTier)
|
|
456
|
+
costCtrl.max_tier = costControls.maxTier;
|
|
457
|
+
if (costControls.preferCost)
|
|
458
|
+
costCtrl.prefer_cost = true;
|
|
459
|
+
if (costControls.preferSpeed)
|
|
460
|
+
costCtrl.prefer_speed = true;
|
|
461
|
+
if (costControls.failFast)
|
|
462
|
+
costCtrl.fail_fast = true;
|
|
463
|
+
if (Object.keys(costCtrl).length > 0) {
|
|
464
|
+
body.cost_controls = costCtrl;
|
|
465
|
+
}
|
|
466
|
+
let authName = "alterLabApi";
|
|
467
|
+
try {
|
|
468
|
+
await this.getCredentials("alterLabOAuth2Api");
|
|
469
|
+
authName = "alterLabOAuth2Api";
|
|
470
|
+
}
|
|
471
|
+
catch {
|
|
472
|
+
// OAuth2 not configured, fall back to API key
|
|
473
|
+
}
|
|
474
|
+
const response = await this.helpers.httpRequestWithAuthentication.call(this, authName, {
|
|
475
|
+
method: "POST",
|
|
476
|
+
url: `${BASE_URL}/api/v1/scrape/estimate`,
|
|
477
|
+
body,
|
|
478
|
+
json: true,
|
|
479
|
+
returnFullResponse: true,
|
|
480
|
+
ignoreHttpStatusErrors: true,
|
|
481
|
+
});
|
|
482
|
+
const statusCode = response.statusCode;
|
|
483
|
+
const responseBody = response
|
|
484
|
+
.body;
|
|
485
|
+
if (statusCode >= 400) {
|
|
486
|
+
handleApiError(this, statusCode, responseBody, i);
|
|
487
|
+
}
|
|
488
|
+
const data = responseBody;
|
|
489
|
+
results.push({
|
|
490
|
+
json: {
|
|
491
|
+
url: (_a = data.url) !== null && _a !== void 0 ? _a : "",
|
|
492
|
+
estimatedTier: (_b = data.estimated_tier) !== null && _b !== void 0 ? _b : "unknown",
|
|
493
|
+
estimatedCredits: (_c = data.estimated_credits) !== null && _c !== void 0 ? _c : 0,
|
|
494
|
+
confidence: (_d = data.confidence) !== null && _d !== void 0 ? _d : "low",
|
|
495
|
+
maxPossibleCredits: (_e = data.max_possible_credits) !== null && _e !== void 0 ? _e : 0,
|
|
496
|
+
reasoning: (_f = data.reasoning) !== null && _f !== void 0 ? _f : "",
|
|
497
|
+
},
|
|
498
|
+
});
|
|
499
|
+
continue;
|
|
500
|
+
}
|
|
501
|
+
// ── Scrape operation ──────────────────────────────
|
|
502
|
+
const outputOptions = this.getNodeParameter("outputOptions", i, {});
|
|
503
|
+
const executionMode = this.getNodeParameter("executionMode", i, {});
|
|
504
|
+
const advancedOptions = this.getNodeParameter("advancedOptions", i, {});
|
|
505
|
+
const extraction = this.getNodeParameter("extraction", i, {});
|
|
506
|
+
const costControls = this.getNodeParameter("costControls", i, {});
|
|
390
507
|
// Build request body
|
|
391
508
|
const body = {
|
|
392
509
|
url,
|
|
@@ -394,7 +511,7 @@ class AlterLab {
|
|
|
394
511
|
sync: true,
|
|
395
512
|
};
|
|
396
513
|
// Output options
|
|
397
|
-
if ((
|
|
514
|
+
if ((_g = outputOptions.formats) === null || _g === void 0 ? void 0 : _g.length) {
|
|
398
515
|
body.formats = outputOptions.formats;
|
|
399
516
|
}
|
|
400
517
|
if (outputOptions.includeRawHtml) {
|
|
@@ -425,7 +542,8 @@ class AlterLab {
|
|
|
425
542
|
if (advancedOptions.proxyCountry) {
|
|
426
543
|
advanced.proxy_country = advancedOptions.proxyCountry;
|
|
427
544
|
}
|
|
428
|
-
if (advancedOptions.waitCondition &&
|
|
545
|
+
if (advancedOptions.waitCondition &&
|
|
546
|
+
advancedOptions.waitCondition !== "networkidle") {
|
|
429
547
|
advanced.wait_condition = advancedOptions.waitCondition;
|
|
430
548
|
}
|
|
431
549
|
if (advancedOptions.removeCookieBanners === false) {
|
|
@@ -435,7 +553,8 @@ class AlterLab {
|
|
|
435
553
|
body.advanced = advanced;
|
|
436
554
|
}
|
|
437
555
|
// Extraction
|
|
438
|
-
if (extraction.extractionProfile &&
|
|
556
|
+
if (extraction.extractionProfile &&
|
|
557
|
+
extraction.extractionProfile !== "auto") {
|
|
439
558
|
body.extraction_profile = extraction.extractionProfile;
|
|
440
559
|
}
|
|
441
560
|
if (extraction.extractionPrompt) {
|
|
@@ -444,12 +563,12 @@ class AlterLab {
|
|
|
444
563
|
if (extraction.extractionSchema) {
|
|
445
564
|
try {
|
|
446
565
|
body.extraction_schema =
|
|
447
|
-
typeof extraction.extractionSchema ===
|
|
566
|
+
typeof extraction.extractionSchema === "string"
|
|
448
567
|
? JSON.parse(extraction.extractionSchema)
|
|
449
568
|
: extraction.extractionSchema;
|
|
450
569
|
}
|
|
451
570
|
catch {
|
|
452
|
-
throw new n8n_workflow_1.NodeOperationError(this.getNode(),
|
|
571
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), "Invalid JSON in Extraction Schema", { itemIndex: i });
|
|
453
572
|
}
|
|
454
573
|
}
|
|
455
574
|
if (extraction.promoteSchemaOrg === false) {
|
|
@@ -479,18 +598,18 @@ class AlterLab {
|
|
|
479
598
|
body.cost_controls = costCtrl;
|
|
480
599
|
}
|
|
481
600
|
// ── Detect credential type ────────────────────────
|
|
482
|
-
let authName =
|
|
601
|
+
let authName = "alterLabApi";
|
|
483
602
|
try {
|
|
484
|
-
await this.getCredentials(
|
|
485
|
-
authName =
|
|
603
|
+
await this.getCredentials("alterLabOAuth2Api");
|
|
604
|
+
authName = "alterLabOAuth2Api";
|
|
486
605
|
}
|
|
487
606
|
catch {
|
|
488
607
|
// OAuth2 not configured, fall back to API key
|
|
489
608
|
}
|
|
490
609
|
// ── Make the API call ─────────────────────────────
|
|
491
610
|
let response = await this.helpers.httpRequestWithAuthentication.call(this, authName, {
|
|
492
|
-
method:
|
|
493
|
-
url:
|
|
611
|
+
method: "POST",
|
|
612
|
+
url: `${BASE_URL}/api/v1/scrape`,
|
|
494
613
|
body,
|
|
495
614
|
json: true,
|
|
496
615
|
returnFullResponse: true,
|
|
@@ -503,20 +622,22 @@ class AlterLab {
|
|
|
503
622
|
const jobId = responseBody.job_id;
|
|
504
623
|
let delay = 500;
|
|
505
624
|
const maxDelay = 5000;
|
|
506
|
-
const maxPollTime = (((
|
|
625
|
+
const maxPollTime = (((_h = outputOptions.timeout) !== null && _h !== void 0 ? _h : 90) + 30) * 1000; // timeout + 30s buffer
|
|
507
626
|
const pollStart = Date.now();
|
|
508
627
|
while (Date.now() - pollStart < maxPollTime) {
|
|
509
|
-
await
|
|
628
|
+
await sleep(delay);
|
|
510
629
|
delay = Math.min(delay * 2, maxDelay);
|
|
511
630
|
const pollResponse = await this.helpers.httpRequestWithAuthentication.call(this, authName, {
|
|
512
|
-
method:
|
|
513
|
-
url:
|
|
631
|
+
method: "GET",
|
|
632
|
+
url: `${BASE_URL}/api/v1/jobs/${jobId}`,
|
|
514
633
|
json: true,
|
|
515
634
|
returnFullResponse: true,
|
|
516
635
|
ignoreHttpStatusErrors: true,
|
|
517
636
|
});
|
|
518
|
-
const pollStatus = pollResponse
|
|
519
|
-
|
|
637
|
+
const pollStatus = pollResponse
|
|
638
|
+
.statusCode;
|
|
639
|
+
const pollBody = pollResponse
|
|
640
|
+
.body;
|
|
520
641
|
if (pollStatus === 200 && (pollBody === null || pollBody === void 0 ? void 0 : pollBody.status_code)) {
|
|
521
642
|
statusCode = 200;
|
|
522
643
|
responseBody = pollBody;
|
|
@@ -530,7 +651,7 @@ class AlterLab {
|
|
|
530
651
|
// Still processing (200 with status: "processing") — continue polling
|
|
531
652
|
}
|
|
532
653
|
if (statusCode === 202) {
|
|
533
|
-
throw new n8n_workflow_1.NodeOperationError(this.getNode(),
|
|
654
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), "Scrape job timed out while waiting for results. Try increasing the timeout or using a simpler scraping mode.", { itemIndex: i });
|
|
534
655
|
}
|
|
535
656
|
}
|
|
536
657
|
// ── Handle errors ─────────────────────────────────
|
|
@@ -541,40 +662,41 @@ class AlterLab {
|
|
|
541
662
|
const data = responseBody;
|
|
542
663
|
const content = data.content;
|
|
543
664
|
const output = {
|
|
544
|
-
url: (
|
|
545
|
-
statusCode: (
|
|
546
|
-
title: (
|
|
547
|
-
author: (
|
|
548
|
-
publishedAt: (
|
|
549
|
-
cached: (
|
|
550
|
-
responseTimeMs: (
|
|
551
|
-
sizeBytes: (
|
|
665
|
+
url: (_j = data.url) !== null && _j !== void 0 ? _j : "",
|
|
666
|
+
statusCode: (_k = data.status_code) !== null && _k !== void 0 ? _k : 0,
|
|
667
|
+
title: (_l = data.title) !== null && _l !== void 0 ? _l : null,
|
|
668
|
+
author: (_m = data.author) !== null && _m !== void 0 ? _m : null,
|
|
669
|
+
publishedAt: (_o = data.published_at) !== null && _o !== void 0 ? _o : null,
|
|
670
|
+
cached: (_p = data.cached) !== null && _p !== void 0 ? _p : false,
|
|
671
|
+
responseTimeMs: (_q = data.response_time_ms) !== null && _q !== void 0 ? _q : 0,
|
|
672
|
+
sizeBytes: (_r = data.size_bytes) !== null && _r !== void 0 ? _r : 0,
|
|
552
673
|
};
|
|
553
674
|
// Flatten multi-format content
|
|
554
|
-
if (content && typeof content ===
|
|
555
|
-
output.markdown =
|
|
556
|
-
|
|
557
|
-
output.
|
|
558
|
-
output.
|
|
675
|
+
if (content && typeof content === "object") {
|
|
676
|
+
output.markdown =
|
|
677
|
+
(_s = content.markdown) !== null && _s !== void 0 ? _s : null;
|
|
678
|
+
output.text = (_t = content.text) !== null && _t !== void 0 ? _t : null;
|
|
679
|
+
output.json = (_u = content.json) !== null && _u !== void 0 ? _u : null;
|
|
680
|
+
output.html = (_v = content.html) !== null && _v !== void 0 ? _v : null;
|
|
559
681
|
}
|
|
560
682
|
else {
|
|
561
683
|
output.markdown = content !== null && content !== void 0 ? content : null;
|
|
562
684
|
}
|
|
563
685
|
// Extraction results
|
|
564
|
-
output.filteredContent = (
|
|
565
|
-
output.extractionMethod = (
|
|
686
|
+
output.filteredContent = (_w = data.filtered_content) !== null && _w !== void 0 ? _w : null;
|
|
687
|
+
output.extractionMethod = (_x = data.extraction_method) !== null && _x !== void 0 ? _x : null;
|
|
566
688
|
// Advanced outputs
|
|
567
|
-
output.screenshotUrl = (
|
|
568
|
-
output.pdfUrl = (
|
|
569
|
-
output.ocrResults = (
|
|
570
|
-
output.rawHtml = (
|
|
689
|
+
output.screenshotUrl = (_y = data.screenshot_url) !== null && _y !== void 0 ? _y : null;
|
|
690
|
+
output.pdfUrl = (_z = data.pdf_url) !== null && _z !== void 0 ? _z : null;
|
|
691
|
+
output.ocrResults = (_0 = data.ocr_results) !== null && _0 !== void 0 ? _0 : null;
|
|
692
|
+
output.rawHtml = (_1 = data.raw_html) !== null && _1 !== void 0 ? _1 : null;
|
|
571
693
|
// Billing breakdown (flattened)
|
|
572
694
|
const billing = data.billing;
|
|
573
695
|
output.billing = {
|
|
574
|
-
cost: (
|
|
575
|
-
tier: (
|
|
576
|
-
savings: (
|
|
577
|
-
suggestion: (
|
|
696
|
+
cost: (_3 = (_2 = billing === null || billing === void 0 ? void 0 : billing.total_credits) !== null && _2 !== void 0 ? _2 : data.credits_used) !== null && _3 !== void 0 ? _3 : 0,
|
|
697
|
+
tier: (_5 = (_4 = billing === null || billing === void 0 ? void 0 : billing.tier_used) !== null && _4 !== void 0 ? _4 : data.tier_used) !== null && _5 !== void 0 ? _5 : "unknown",
|
|
698
|
+
savings: (_6 = billing === null || billing === void 0 ? void 0 : billing.savings) !== null && _6 !== void 0 ? _6 : 0,
|
|
699
|
+
suggestion: (_7 = billing === null || billing === void 0 ? void 0 : billing.optimization_suggestion) !== null && _7 !== void 0 ? _7 : null,
|
|
578
700
|
};
|
|
579
701
|
results.push({ json: output });
|
|
580
702
|
}
|
|
@@ -595,41 +717,41 @@ class AlterLab {
|
|
|
595
717
|
exports.AlterLab = AlterLab;
|
|
596
718
|
function handleApiError(ctx, statusCode, body, itemIndex) {
|
|
597
719
|
var _a, _b;
|
|
598
|
-
const detail = (_b = (_a = body === null || body === void 0 ? void 0 : body.detail) !== null && _a !== void 0 ? _a : body === null || body === void 0 ? void 0 : body.message) !== null && _b !== void 0 ? _b :
|
|
720
|
+
const detail = (_b = (_a = body === null || body === void 0 ? void 0 : body.detail) !== null && _a !== void 0 ? _a : body === null || body === void 0 ? void 0 : body.message) !== null && _b !== void 0 ? _b : "Unknown error";
|
|
599
721
|
switch (statusCode) {
|
|
600
722
|
case 401:
|
|
601
723
|
throw new n8n_workflow_1.NodeApiError(ctx.getNode(), body, {
|
|
602
|
-
message:
|
|
724
|
+
message: "Invalid API key",
|
|
603
725
|
description: `${detail}. Check your API key or get a new one at https://app.alterlab.io/dashboard/keys?${UTM}`,
|
|
604
|
-
httpCode:
|
|
726
|
+
httpCode: "401",
|
|
605
727
|
itemIndex,
|
|
606
728
|
});
|
|
607
729
|
case 402:
|
|
608
730
|
throw new n8n_workflow_1.NodeApiError(ctx.getNode(), body, {
|
|
609
|
-
message:
|
|
731
|
+
message: "Insufficient balance",
|
|
610
732
|
description: `${detail}. Top up your balance at https://app.alterlab.io/dashboard/billing?${UTM}`,
|
|
611
|
-
httpCode:
|
|
733
|
+
httpCode: "402",
|
|
612
734
|
itemIndex,
|
|
613
735
|
});
|
|
614
736
|
case 429:
|
|
615
737
|
throw new n8n_workflow_1.NodeApiError(ctx.getNode(), body, {
|
|
616
|
-
message:
|
|
738
|
+
message: "Rate limit exceeded",
|
|
617
739
|
description: `${detail}. Upgrade your plan for higher rate limits at https://alterlab.io/pricing?${UTM}`,
|
|
618
|
-
httpCode:
|
|
740
|
+
httpCode: "429",
|
|
619
741
|
itemIndex,
|
|
620
742
|
});
|
|
621
743
|
case 403:
|
|
622
744
|
throw new n8n_workflow_1.NodeApiError(ctx.getNode(), body, {
|
|
623
|
-
message:
|
|
745
|
+
message: "Blocked by anti-bot protection",
|
|
624
746
|
description: `${detail}. Try enabling "Use Proxy" in Advanced Options, or use a higher tier via Cost Controls.`,
|
|
625
|
-
httpCode:
|
|
747
|
+
httpCode: "403",
|
|
626
748
|
itemIndex,
|
|
627
749
|
});
|
|
628
750
|
case 504:
|
|
629
751
|
throw new n8n_workflow_1.NodeApiError(ctx.getNode(), body, {
|
|
630
|
-
message:
|
|
752
|
+
message: "Request timed out",
|
|
631
753
|
description: `${detail}. Try increasing the timeout, using async mode, or a simpler scraping mode.`,
|
|
632
|
-
httpCode:
|
|
754
|
+
httpCode: "504",
|
|
633
755
|
itemIndex,
|
|
634
756
|
});
|
|
635
757
|
default:
|