@mendable/n8n-nodes-firecrawl 2.1.3 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/credentials/FirecrawlApi.credentials.d.ts +1 -0
- package/dist/credentials/FirecrawlApi.credentials.js +1 -0
- package/dist/credentials/FirecrawlApi.credentials.js.map +1 -1
- package/dist/credentials/firecrawl-logo.svg +33 -0
- package/dist/nodes/Firecrawl/Firecrawl.node.js +2 -1
- package/dist/nodes/Firecrawl/Firecrawl.node.js.map +1 -1
- package/dist/nodes/Firecrawl/api/agent/index.js +7 -6
- package/dist/nodes/Firecrawl/api/agent/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/agentAsync/index.js +8 -6
- package/dist/nodes/Firecrawl/api/agentAsync/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/batchScrape/index.d.ts +3 -1
- package/dist/nodes/Firecrawl/api/batchScrape/index.js +5 -4
- package/dist/nodes/Firecrawl/api/batchScrape/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/batchScrapeErrors/index.d.ts +3 -1
- package/dist/nodes/Firecrawl/api/batchScrapeErrors/index.js +7 -4
- package/dist/nodes/Firecrawl/api/batchScrapeErrors/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/batchScrapeStatus/index.d.ts +3 -1
- package/dist/nodes/Firecrawl/api/batchScrapeStatus/index.js +7 -4
- package/dist/nodes/Firecrawl/api/batchScrapeStatus/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/browserContext/index.d.ts +3 -1
- package/dist/nodes/Firecrawl/api/browserContext/index.js +5 -3
- package/dist/nodes/Firecrawl/api/browserContext/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/browserCreate/index.d.ts +3 -1
- package/dist/nodes/Firecrawl/api/browserCreate/index.js +7 -9
- package/dist/nodes/Firecrawl/api/browserCreate/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/browserDelete/index.d.ts +3 -1
- package/dist/nodes/Firecrawl/api/browserDelete/index.js +8 -9
- package/dist/nodes/Firecrawl/api/browserDelete/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/browserExecute/index.d.ts +3 -1
- package/dist/nodes/Firecrawl/api/browserExecute/index.js +7 -6
- package/dist/nodes/Firecrawl/api/browserExecute/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/browserList/index.d.ts +3 -1
- package/dist/nodes/Firecrawl/api/browserList/index.js +6 -7
- package/dist/nodes/Firecrawl/api/browserList/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/cancelBatchScrape/index.d.ts +3 -1
- package/dist/nodes/Firecrawl/api/cancelBatchScrape/index.js +7 -4
- package/dist/nodes/Firecrawl/api/cancelBatchScrape/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/cancelCrawl/index.d.ts +3 -1
- package/dist/nodes/Firecrawl/api/cancelCrawl/index.js +7 -4
- package/dist/nodes/Firecrawl/api/cancelCrawl/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/common.d.ts +5 -3
- package/dist/nodes/Firecrawl/api/common.js +225 -211
- package/dist/nodes/Firecrawl/api/common.js.map +1 -1
- package/dist/nodes/Firecrawl/api/crawl/index.d.ts +3 -1
- package/dist/nodes/Firecrawl/api/crawl/index.js +84 -21
- package/dist/nodes/Firecrawl/api/crawl/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/crawlActive/index.d.ts +4 -3
- package/dist/nodes/Firecrawl/api/crawlActive/index.js +5 -6
- package/dist/nodes/Firecrawl/api/crawlActive/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/crawlParamsPreview/index.d.ts +3 -1
- package/dist/nodes/Firecrawl/api/crawlParamsPreview/index.js +7 -4
- package/dist/nodes/Firecrawl/api/crawlParamsPreview/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/creditUsageHistorical/index.d.ts +4 -3
- package/dist/nodes/Firecrawl/api/creditUsageHistorical/index.js +5 -6
- package/dist/nodes/Firecrawl/api/creditUsageHistorical/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/extract/index.js +10 -4
- package/dist/nodes/Firecrawl/api/extract/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/getAgentStatus/index.js +5 -3
- package/dist/nodes/Firecrawl/api/getAgentStatus/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/getCrawlErrors/index.d.ts +3 -1
- package/dist/nodes/Firecrawl/api/getCrawlErrors/index.js +7 -4
- package/dist/nodes/Firecrawl/api/getCrawlErrors/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/getCrawlStatus/index.d.ts +3 -1
- package/dist/nodes/Firecrawl/api/getCrawlStatus/index.js +6 -5
- package/dist/nodes/Firecrawl/api/getCrawlStatus/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/getExtractStatus/index.js +5 -3
- package/dist/nodes/Firecrawl/api/getExtractStatus/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/interact/index.d.ts +3 -1
- package/dist/nodes/Firecrawl/api/interact/index.js +7 -6
- package/dist/nodes/Firecrawl/api/interact/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/interactContext/index.d.ts +3 -1
- package/dist/nodes/Firecrawl/api/interactContext/index.js +5 -3
- package/dist/nodes/Firecrawl/api/interactContext/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/interactStop/index.d.ts +3 -1
- package/dist/nodes/Firecrawl/api/interactStop/index.js +7 -8
- package/dist/nodes/Firecrawl/api/interactStop/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/map/index.d.ts +3 -1
- package/dist/nodes/Firecrawl/api/map/index.js +6 -5
- package/dist/nodes/Firecrawl/api/map/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/scrape/index.d.ts +3 -1
- package/dist/nodes/Firecrawl/api/scrape/index.js +6 -4
- package/dist/nodes/Firecrawl/api/scrape/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/search/index.d.ts +3 -1
- package/dist/nodes/Firecrawl/api/search/index.js +8 -6
- package/dist/nodes/Firecrawl/api/search/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/teamCreditUsage/index.d.ts +4 -3
- package/dist/nodes/Firecrawl/api/teamCreditUsage/index.js +5 -6
- package/dist/nodes/Firecrawl/api/teamCreditUsage/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/teamQueueStatus/index.d.ts +4 -3
- package/dist/nodes/Firecrawl/api/teamQueueStatus/index.js +5 -6
- package/dist/nodes/Firecrawl/api/teamQueueStatus/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/teamTokenUsage/index.d.ts +4 -3
- package/dist/nodes/Firecrawl/api/teamTokenUsage/index.js +5 -6
- package/dist/nodes/Firecrawl/api/teamTokenUsage/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/teamTokenUsageHistorical/index.d.ts +4 -3
- package/dist/nodes/Firecrawl/api/teamTokenUsageHistorical/index.js +5 -6
- package/dist/nodes/Firecrawl/api/teamTokenUsageHistorical/index.js.map +1 -1
- package/dist/package.json +2 -2
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
|
@@ -113,13 +113,14 @@ function formatOperationName(name) {
|
|
|
113
113
|
.join(' ');
|
|
114
114
|
}
|
|
115
115
|
exports.formatOperationName = formatOperationName;
|
|
116
|
-
function createOperationNotice(resourceName, operationName,
|
|
116
|
+
function createOperationNotice(resourceName, operationName, text, options = {}) {
|
|
117
|
+
var _a;
|
|
117
118
|
return {
|
|
118
|
-
displayName:
|
|
119
|
-
name:
|
|
119
|
+
displayName: text,
|
|
120
|
+
name: `${operationName}Notice`,
|
|
120
121
|
type: 'notice',
|
|
121
122
|
typeOptions: {
|
|
122
|
-
theme: 'info',
|
|
123
|
+
theme: (_a = options.theme) !== null && _a !== void 0 ? _a : 'info',
|
|
123
124
|
},
|
|
124
125
|
default: '',
|
|
125
126
|
displayOptions: {
|
|
@@ -173,46 +174,49 @@ function createActionsProperty(operationName, omitDisplayOptions = false, useNes
|
|
|
173
174
|
name: 'items',
|
|
174
175
|
values: [
|
|
175
176
|
{
|
|
176
|
-
displayName: '
|
|
177
|
+
displayName: 'Direction',
|
|
177
178
|
type: 'options',
|
|
178
|
-
default: '
|
|
179
|
+
default: 'down',
|
|
180
|
+
description: 'Direction for `scroll` action',
|
|
181
|
+
name: 'direction',
|
|
179
182
|
options: [
|
|
180
183
|
{
|
|
181
|
-
name: '
|
|
182
|
-
value: '
|
|
183
|
-
},
|
|
184
|
-
{
|
|
185
|
-
name: 'Press',
|
|
186
|
-
value: 'press',
|
|
187
|
-
},
|
|
188
|
-
{
|
|
189
|
-
name: 'Screenshot',
|
|
190
|
-
value: 'screenshot',
|
|
184
|
+
name: 'Down',
|
|
185
|
+
value: 'down',
|
|
191
186
|
},
|
|
192
187
|
{
|
|
193
|
-
name: '
|
|
194
|
-
value: '
|
|
188
|
+
name: 'Up',
|
|
189
|
+
value: 'up',
|
|
195
190
|
},
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
191
|
+
],
|
|
192
|
+
displayOptions: {
|
|
193
|
+
show: {
|
|
194
|
+
type: ['scroll'],
|
|
199
195
|
},
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
196
|
+
},
|
|
197
|
+
},
|
|
198
|
+
{
|
|
199
|
+
displayName: 'Full Page',
|
|
200
|
+
type: 'boolean',
|
|
201
|
+
default: false,
|
|
202
|
+
description: 'Whether the screenshot should be full-page or viewport sized',
|
|
203
|
+
name: 'fullPage',
|
|
204
|
+
displayOptions: {
|
|
205
|
+
show: {
|
|
206
|
+
type: ['screenshot'],
|
|
203
207
|
},
|
|
204
|
-
|
|
205
|
-
name: 'type',
|
|
208
|
+
},
|
|
206
209
|
},
|
|
207
210
|
{
|
|
208
|
-
displayName: '
|
|
211
|
+
displayName: 'Key',
|
|
209
212
|
type: 'string',
|
|
210
213
|
default: '',
|
|
211
|
-
|
|
212
|
-
|
|
214
|
+
placeholder: 'e.g. Enter',
|
|
215
|
+
description: 'Keyboard key to press (e.g., "Enter", "Tab", "Escape", "ArrowDown"). Use for form submission, navigation, or triggering keyboard shortcuts.',
|
|
216
|
+
name: 'key',
|
|
213
217
|
displayOptions: {
|
|
214
218
|
show: {
|
|
215
|
-
type: ['
|
|
219
|
+
type: ['press'],
|
|
216
220
|
},
|
|
217
221
|
},
|
|
218
222
|
},
|
|
@@ -229,14 +233,15 @@ function createActionsProperty(operationName, omitDisplayOptions = false, useNes
|
|
|
229
233
|
},
|
|
230
234
|
},
|
|
231
235
|
{
|
|
232
|
-
displayName: '
|
|
233
|
-
type: '
|
|
234
|
-
default:
|
|
235
|
-
|
|
236
|
-
|
|
236
|
+
displayName: 'Selector',
|
|
237
|
+
type: 'string',
|
|
238
|
+
default: '',
|
|
239
|
+
placeholder: 'e.g. #submit-btn',
|
|
240
|
+
description: 'CSS selector to target an element (e.g., "#submit-btn", ".load-more", "[data-testid=login]"). Used for click, write, and scroll actions.',
|
|
241
|
+
name: 'selector',
|
|
237
242
|
displayOptions: {
|
|
238
243
|
show: {
|
|
239
|
-
type: ['
|
|
244
|
+
type: ['click', 'scroll'],
|
|
240
245
|
},
|
|
241
246
|
},
|
|
242
247
|
},
|
|
@@ -244,6 +249,7 @@ function createActionsProperty(operationName, omitDisplayOptions = false, useNes
|
|
|
244
249
|
displayName: 'Text',
|
|
245
250
|
type: 'string',
|
|
246
251
|
default: '',
|
|
252
|
+
placeholder: 'e.g. n8n automation',
|
|
247
253
|
description: 'Text to type into an input field. Use with selector to target the input element. Useful for search boxes, login forms, or any text input.',
|
|
248
254
|
name: 'text',
|
|
249
255
|
displayOptions: {
|
|
@@ -253,38 +259,36 @@ function createActionsProperty(operationName, omitDisplayOptions = false, useNes
|
|
|
253
259
|
},
|
|
254
260
|
},
|
|
255
261
|
{
|
|
256
|
-
displayName: '
|
|
257
|
-
type: 'string',
|
|
258
|
-
default: '',
|
|
259
|
-
description: 'Keyboard key to press (e.g., "Enter", "Tab", "Escape", "ArrowDown"). Use for form submission, navigation, or triggering keyboard shortcuts.',
|
|
260
|
-
name: 'key',
|
|
261
|
-
displayOptions: {
|
|
262
|
-
show: {
|
|
263
|
-
type: ['press'],
|
|
264
|
-
},
|
|
265
|
-
},
|
|
266
|
-
},
|
|
267
|
-
{
|
|
268
|
-
displayName: 'Direction',
|
|
262
|
+
displayName: 'Type',
|
|
269
263
|
type: 'options',
|
|
270
|
-
default: '
|
|
271
|
-
description: 'Direction for `scroll` action',
|
|
272
|
-
name: 'direction',
|
|
264
|
+
default: 'wait',
|
|
273
265
|
options: [
|
|
274
266
|
{
|
|
275
|
-
name: '
|
|
276
|
-
value: '
|
|
267
|
+
name: 'Click',
|
|
268
|
+
value: 'click',
|
|
277
269
|
},
|
|
278
270
|
{
|
|
279
|
-
name: '
|
|
280
|
-
value: '
|
|
271
|
+
name: 'Press',
|
|
272
|
+
value: 'press',
|
|
281
273
|
},
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
type: ['scroll'],
|
|
274
|
+
{
|
|
275
|
+
name: 'Screenshot',
|
|
276
|
+
value: 'screenshot',
|
|
286
277
|
},
|
|
287
|
-
|
|
278
|
+
{
|
|
279
|
+
name: 'Scroll',
|
|
280
|
+
value: 'scroll',
|
|
281
|
+
},
|
|
282
|
+
{
|
|
283
|
+
name: 'Wait',
|
|
284
|
+
value: 'wait',
|
|
285
|
+
},
|
|
286
|
+
{
|
|
287
|
+
name: 'Write',
|
|
288
|
+
value: 'write',
|
|
289
|
+
},
|
|
290
|
+
],
|
|
291
|
+
name: 'type',
|
|
288
292
|
},
|
|
289
293
|
],
|
|
290
294
|
},
|
|
@@ -372,6 +376,7 @@ function createLocationProperty(operationName, omitDisplayOptions = false, useNe
|
|
|
372
376
|
name: 'code',
|
|
373
377
|
type: 'string',
|
|
374
378
|
default: '',
|
|
379
|
+
placeholder: 'e.g. en',
|
|
375
380
|
description: "Language code (e.g., 'en', 'fr', 'de', 'ja')",
|
|
376
381
|
},
|
|
377
382
|
],
|
|
@@ -429,7 +434,7 @@ function createIncludeTagsProperty(operationName, omitDisplayOptions = false, us
|
|
|
429
434
|
name: 'tag',
|
|
430
435
|
type: 'string',
|
|
431
436
|
default: '',
|
|
432
|
-
placeholder: 'header',
|
|
437
|
+
placeholder: 'e.g. header',
|
|
433
438
|
description: 'Tag to include in the output',
|
|
434
439
|
},
|
|
435
440
|
],
|
|
@@ -473,7 +478,7 @@ function createBatchUrlsProperty(operationName, defaultUrl = 'https://firecrawl.
|
|
|
473
478
|
default: defaultUrl,
|
|
474
479
|
required: true,
|
|
475
480
|
description: 'URLs to scrape in batch. Accepts multiple formats: a single URL, multiple URLs separated by commas or newlines, a JSON array like ["url1", "url2"], or an array expression from a previous node. Each URL is scraped independently with the same settings. Example: "https://example.com/page1, https://example.com/page2" or paste one URL per line.',
|
|
476
|
-
placeholder: 'https://example.com/page1\nhttps://example.com/page2',
|
|
481
|
+
placeholder: 'e.g. https://example.com/page1\nhttps://example.com/page2',
|
|
477
482
|
routing: {
|
|
478
483
|
request: {
|
|
479
484
|
body: {
|
|
@@ -531,7 +536,7 @@ function createExcludeTagsProperty(operationName, omitDisplayOptions = false, us
|
|
|
531
536
|
name: 'tag',
|
|
532
537
|
type: 'string',
|
|
533
538
|
default: '',
|
|
534
|
-
placeholder: 'footer',
|
|
539
|
+
placeholder: 'e.g. footer',
|
|
535
540
|
description: 'Tag to exclude from the output',
|
|
536
541
|
},
|
|
537
542
|
],
|
|
@@ -583,6 +588,7 @@ function createBatchSpecificProperties() {
|
|
|
583
588
|
type: 'string',
|
|
584
589
|
required: true,
|
|
585
590
|
default: '',
|
|
591
|
+
placeholder: 'e.g. https://example.com/webhook',
|
|
586
592
|
description: 'The URL to send the webhook to. Triggers for batchScrape.started, batchScrape.page, batchScrape.completed, or batchScrape.failed events.',
|
|
587
593
|
},
|
|
588
594
|
{
|
|
@@ -604,6 +610,7 @@ function createBatchSpecificProperties() {
|
|
|
604
610
|
name: 'key',
|
|
605
611
|
type: 'string',
|
|
606
612
|
default: '',
|
|
613
|
+
placeholder: 'e.g. X-Api-Key',
|
|
607
614
|
description: 'Header key',
|
|
608
615
|
},
|
|
609
616
|
{
|
|
@@ -611,6 +618,7 @@ function createBatchSpecificProperties() {
|
|
|
611
618
|
name: 'value',
|
|
612
619
|
type: 'string',
|
|
613
620
|
default: '',
|
|
621
|
+
placeholder: 'e.g. abc123',
|
|
614
622
|
description: 'Header value',
|
|
615
623
|
},
|
|
616
624
|
],
|
|
@@ -715,6 +723,13 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
715
723
|
displayName: 'Options',
|
|
716
724
|
name: 'options',
|
|
717
725
|
values: [
|
|
726
|
+
{
|
|
727
|
+
displayName: 'Block Ads',
|
|
728
|
+
name: 'blockAds',
|
|
729
|
+
type: 'boolean',
|
|
730
|
+
default: true,
|
|
731
|
+
description: 'Whether to enable ad-blocking and cookie consent popup blocking for cleaner content extraction. Recommended for most scraping tasks.',
|
|
732
|
+
},
|
|
718
733
|
{
|
|
719
734
|
displayName: 'Formats',
|
|
720
735
|
name: 'formats',
|
|
@@ -731,51 +746,36 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
731
746
|
name: 'format',
|
|
732
747
|
values: [
|
|
733
748
|
{
|
|
734
|
-
displayName: '
|
|
735
|
-
name: '
|
|
736
|
-
type: '
|
|
737
|
-
default:
|
|
738
|
-
description: '
|
|
739
|
-
|
|
740
|
-
{
|
|
741
|
-
|
|
742
|
-
value: 'changeTracking',
|
|
743
|
-
description: 'Track changes between page versions over time',
|
|
749
|
+
displayName: 'Full Page',
|
|
750
|
+
name: 'fullPage',
|
|
751
|
+
type: 'boolean',
|
|
752
|
+
default: false,
|
|
753
|
+
description: 'Whether to capture the full page screenshot',
|
|
754
|
+
displayOptions: {
|
|
755
|
+
show: {
|
|
756
|
+
type: ['screenshot'],
|
|
744
757
|
},
|
|
758
|
+
},
|
|
759
|
+
},
|
|
760
|
+
{
|
|
761
|
+
displayName: 'Modes',
|
|
762
|
+
name: 'modes',
|
|
763
|
+
type: 'multiOptions',
|
|
764
|
+
default: ['git-diff'],
|
|
765
|
+
description: 'Modes for change tracking. At least one mode must be selected.',
|
|
766
|
+
displayOptions: {
|
|
767
|
+
show: {
|
|
768
|
+
type: ['changeTracking'],
|
|
769
|
+
},
|
|
770
|
+
},
|
|
771
|
+
options: [
|
|
745
772
|
{
|
|
746
|
-
name: '
|
|
747
|
-
value: '
|
|
748
|
-
description: 'Clean HTML with scripts and styles removed',
|
|
773
|
+
name: 'Git Diff',
|
|
774
|
+
value: 'git-diff',
|
|
749
775
|
},
|
|
750
776
|
{
|
|
751
777
|
name: 'JSON',
|
|
752
778
|
value: 'json',
|
|
753
|
-
description: 'Extract structured data using AI with a schema',
|
|
754
|
-
},
|
|
755
|
-
{
|
|
756
|
-
name: 'Links',
|
|
757
|
-
value: 'links',
|
|
758
|
-
description: 'Extract all links from the page',
|
|
759
|
-
},
|
|
760
|
-
{
|
|
761
|
-
name: 'Markdown',
|
|
762
|
-
value: 'markdown',
|
|
763
|
-
description: 'Full page content as markdown - best for AI/LLM processing and reading full content',
|
|
764
|
-
},
|
|
765
|
-
{
|
|
766
|
-
name: 'Raw HTML',
|
|
767
|
-
value: 'rawHtml',
|
|
768
|
-
description: 'Original HTML including scripts and styles',
|
|
769
|
-
},
|
|
770
|
-
{
|
|
771
|
-
name: 'Screenshot',
|
|
772
|
-
value: 'screenshot',
|
|
773
|
-
description: 'Visual screenshot of the page as base64 image',
|
|
774
|
-
},
|
|
775
|
-
{
|
|
776
|
-
name: 'Summary',
|
|
777
|
-
value: 'summary',
|
|
778
|
-
description: 'AI-generated concise summary of page content - USE THIS when user asks for a summary, overview, brief description, or wants to quickly understand what a page is about',
|
|
779
779
|
},
|
|
780
780
|
],
|
|
781
781
|
},
|
|
@@ -784,6 +784,7 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
784
784
|
name: 'prompt',
|
|
785
785
|
type: 'string',
|
|
786
786
|
default: '',
|
|
787
|
+
placeholder: 'e.g. Extract the product name and price',
|
|
787
788
|
description: 'Prompt for JSON format extraction',
|
|
788
789
|
displayOptions: {
|
|
789
790
|
show: {
|
|
@@ -792,57 +793,52 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
792
793
|
},
|
|
793
794
|
},
|
|
794
795
|
{
|
|
795
|
-
displayName: '
|
|
796
|
-
name: '
|
|
797
|
-
type: '
|
|
798
|
-
default: '
|
|
799
|
-
|
|
796
|
+
displayName: 'Prompt',
|
|
797
|
+
name: 'prompt',
|
|
798
|
+
type: 'string',
|
|
799
|
+
default: '',
|
|
800
|
+
placeholder: 'e.g. Track changes to pricing only',
|
|
801
|
+
description: 'Prompt for change tracking',
|
|
800
802
|
displayOptions: {
|
|
801
803
|
show: {
|
|
802
|
-
type: ['
|
|
804
|
+
type: ['changeTracking'],
|
|
803
805
|
},
|
|
804
806
|
},
|
|
805
807
|
},
|
|
806
808
|
{
|
|
807
|
-
displayName: '
|
|
808
|
-
name: '
|
|
809
|
-
type: '
|
|
810
|
-
|
|
811
|
-
|
|
809
|
+
displayName: 'Quality',
|
|
810
|
+
name: 'quality',
|
|
811
|
+
type: 'number',
|
|
812
|
+
typeOptions: {
|
|
813
|
+
minValue: 1,
|
|
814
|
+
maxValue: 100,
|
|
815
|
+
},
|
|
816
|
+
default: 100,
|
|
817
|
+
description: 'Screenshot quality (1-100)',
|
|
812
818
|
displayOptions: {
|
|
813
819
|
show: {
|
|
814
|
-
type: ['
|
|
820
|
+
type: ['screenshot'],
|
|
815
821
|
},
|
|
816
822
|
},
|
|
817
|
-
options: [
|
|
818
|
-
{
|
|
819
|
-
name: 'Git Diff',
|
|
820
|
-
value: 'git-diff',
|
|
821
|
-
},
|
|
822
|
-
{
|
|
823
|
-
name: 'JSON',
|
|
824
|
-
value: 'json',
|
|
825
|
-
},
|
|
826
|
-
],
|
|
827
823
|
},
|
|
828
824
|
{
|
|
829
825
|
displayName: 'Schema',
|
|
830
826
|
name: 'schema',
|
|
831
827
|
type: 'json',
|
|
832
828
|
default: '{}',
|
|
833
|
-
description: 'JSON schema for
|
|
829
|
+
description: 'JSON schema for JSON format extraction',
|
|
834
830
|
displayOptions: {
|
|
835
831
|
show: {
|
|
836
|
-
type: ['
|
|
832
|
+
type: ['json'],
|
|
837
833
|
},
|
|
838
834
|
},
|
|
839
835
|
},
|
|
840
836
|
{
|
|
841
|
-
displayName: '
|
|
842
|
-
name: '
|
|
843
|
-
type: '
|
|
844
|
-
default: '',
|
|
845
|
-
description: '
|
|
837
|
+
displayName: 'Schema',
|
|
838
|
+
name: 'schema',
|
|
839
|
+
type: 'json',
|
|
840
|
+
default: '{}',
|
|
841
|
+
description: 'JSON schema for change tracking',
|
|
846
842
|
displayOptions: {
|
|
847
843
|
show: {
|
|
848
844
|
type: ['changeTracking'],
|
|
@@ -854,6 +850,7 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
854
850
|
name: 'tag',
|
|
855
851
|
type: 'string',
|
|
856
852
|
default: '',
|
|
853
|
+
placeholder: 'e.g. pricing-page',
|
|
857
854
|
description: 'Tag for change tracking',
|
|
858
855
|
displayOptions: {
|
|
859
856
|
show: {
|
|
@@ -862,27 +859,63 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
862
859
|
},
|
|
863
860
|
},
|
|
864
861
|
{
|
|
865
|
-
displayName: '
|
|
866
|
-
name: '
|
|
867
|
-
type: '
|
|
868
|
-
default:
|
|
869
|
-
description: '
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
862
|
+
displayName: 'Type',
|
|
863
|
+
name: 'type',
|
|
864
|
+
type: 'options',
|
|
865
|
+
default: 'markdown',
|
|
866
|
+
description: 'Output format for scraped content. Use "Summary" when user asks for a summary, overview, or condensed version of page content. Use "Markdown" for full content in AI-readable format.',
|
|
867
|
+
options: [
|
|
868
|
+
{
|
|
869
|
+
name: 'Change Tracking',
|
|
870
|
+
value: 'changeTracking',
|
|
871
|
+
description: 'Track changes between page versions over time',
|
|
873
872
|
},
|
|
874
|
-
|
|
873
|
+
{
|
|
874
|
+
name: 'HTML',
|
|
875
|
+
value: 'html',
|
|
876
|
+
description: 'Clean HTML with scripts and styles removed',
|
|
877
|
+
},
|
|
878
|
+
{
|
|
879
|
+
name: 'JSON',
|
|
880
|
+
value: 'json',
|
|
881
|
+
description: 'Extract structured data using AI with a schema',
|
|
882
|
+
},
|
|
883
|
+
{
|
|
884
|
+
name: 'Links',
|
|
885
|
+
value: 'links',
|
|
886
|
+
description: 'Extract all links from the page',
|
|
887
|
+
},
|
|
888
|
+
{
|
|
889
|
+
name: 'Markdown',
|
|
890
|
+
value: 'markdown',
|
|
891
|
+
description: 'Full page content as markdown - best for AI/LLM processing and reading full content',
|
|
892
|
+
},
|
|
893
|
+
{
|
|
894
|
+
name: 'Raw HTML',
|
|
895
|
+
value: 'rawHtml',
|
|
896
|
+
description: 'Original HTML including scripts and styles',
|
|
897
|
+
},
|
|
898
|
+
{
|
|
899
|
+
name: 'Screenshot',
|
|
900
|
+
value: 'screenshot',
|
|
901
|
+
description: 'Visual screenshot of the page as base64 image',
|
|
902
|
+
},
|
|
903
|
+
{
|
|
904
|
+
name: 'Summary',
|
|
905
|
+
value: 'summary',
|
|
906
|
+
description: 'AI-generated concise summary of page content - USE THIS when user asks for a summary, overview, brief description, or wants to quickly understand what a page is about',
|
|
907
|
+
},
|
|
908
|
+
],
|
|
875
909
|
},
|
|
876
910
|
{
|
|
877
|
-
displayName: '
|
|
878
|
-
name: '
|
|
911
|
+
displayName: 'Viewport Height',
|
|
912
|
+
name: 'viewportHeight',
|
|
879
913
|
type: 'number',
|
|
880
914
|
typeOptions: {
|
|
881
915
|
minValue: 1,
|
|
882
|
-
maxValue: 100,
|
|
883
916
|
},
|
|
884
|
-
default:
|
|
885
|
-
description: '
|
|
917
|
+
default: 768,
|
|
918
|
+
description: 'Viewport height for screenshot',
|
|
886
919
|
displayOptions: {
|
|
887
920
|
show: {
|
|
888
921
|
type: ['screenshot'],
|
|
@@ -904,21 +937,6 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
904
937
|
},
|
|
905
938
|
},
|
|
906
939
|
},
|
|
907
|
-
{
|
|
908
|
-
displayName: 'Viewport Height',
|
|
909
|
-
name: 'viewportHeight',
|
|
910
|
-
type: 'number',
|
|
911
|
-
typeOptions: {
|
|
912
|
-
minValue: 1,
|
|
913
|
-
},
|
|
914
|
-
default: 768,
|
|
915
|
-
description: 'Viewport height for screenshot',
|
|
916
|
-
displayOptions: {
|
|
917
|
-
show: {
|
|
918
|
-
type: ['screenshot'],
|
|
919
|
-
},
|
|
920
|
-
},
|
|
921
|
-
},
|
|
922
940
|
],
|
|
923
941
|
},
|
|
924
942
|
],
|
|
@@ -930,13 +948,6 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
930
948
|
},
|
|
931
949
|
},
|
|
932
950
|
},
|
|
933
|
-
{
|
|
934
|
-
displayName: 'Only Main Content',
|
|
935
|
-
name: 'onlyMainContent',
|
|
936
|
-
type: 'boolean',
|
|
937
|
-
default: true,
|
|
938
|
-
description: 'Whether to automatically remove navigation, headers, footers, sidebars, and other boilerplate content. Best for extracting article text or primary page content. Disable to capture the full page.',
|
|
939
|
-
},
|
|
940
951
|
createIncludeTagsProperty(operationName, true, useNestedScrapeOptions),
|
|
941
952
|
createExcludeTagsProperty(operationName, true, useNestedScrapeOptions),
|
|
942
953
|
{
|
|
@@ -951,6 +962,7 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
951
962
|
name: 'key',
|
|
952
963
|
type: 'string',
|
|
953
964
|
default: '',
|
|
965
|
+
placeholder: 'e.g. Authorization',
|
|
954
966
|
description: 'Key of the header',
|
|
955
967
|
},
|
|
956
968
|
{
|
|
@@ -958,17 +970,11 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
958
970
|
name: 'value',
|
|
959
971
|
type: 'string',
|
|
960
972
|
default: '',
|
|
973
|
+
placeholder: 'e.g. Bearer abc123',
|
|
961
974
|
description: 'Value of the header',
|
|
962
975
|
},
|
|
963
976
|
],
|
|
964
977
|
},
|
|
965
|
-
{
|
|
966
|
-
displayName: 'Wait For (Ms)',
|
|
967
|
-
name: 'waitFor',
|
|
968
|
-
type: 'number',
|
|
969
|
-
default: 0,
|
|
970
|
-
description: 'Additional wait time in milliseconds after page load before extracting content. Use for JavaScript-heavy sites where content renders dynamically. 0 uses intelligent auto-detection.',
|
|
971
|
-
},
|
|
972
978
|
{
|
|
973
979
|
displayName: 'Mobile',
|
|
974
980
|
name: 'mobile',
|
|
@@ -977,21 +983,29 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
977
983
|
description: 'Whether to emulate a mobile device when scraping. Useful for sites with mobile-specific content, responsive layouts, or mobile-only features. Changes viewport and user-agent.',
|
|
978
984
|
},
|
|
979
985
|
{
|
|
980
|
-
displayName: '
|
|
981
|
-
name: '
|
|
986
|
+
displayName: 'Only Main Content',
|
|
987
|
+
name: 'onlyMainContent',
|
|
982
988
|
type: 'boolean',
|
|
983
|
-
default:
|
|
984
|
-
description: 'Whether to
|
|
989
|
+
default: true,
|
|
990
|
+
description: 'Whether to automatically remove navigation, headers, footers, sidebars, and other boilerplate content. Best for extracting article text or primary page content. Disable to capture the full page.',
|
|
985
991
|
},
|
|
986
992
|
{
|
|
987
|
-
displayName: '
|
|
988
|
-
name: '
|
|
989
|
-
type: '
|
|
990
|
-
default:
|
|
991
|
-
description: '
|
|
993
|
+
displayName: 'Proxy',
|
|
994
|
+
name: 'proxy',
|
|
995
|
+
type: 'options',
|
|
996
|
+
default: 'basic',
|
|
997
|
+
description: 'Proxy strategy for bypassing anti-bot measures. Basic is faster and cheaper. Stealth uses residential proxies for better success on protected sites.',
|
|
998
|
+
options: [
|
|
999
|
+
{
|
|
1000
|
+
name: 'Basic',
|
|
1001
|
+
value: 'basic',
|
|
1002
|
+
},
|
|
1003
|
+
{
|
|
1004
|
+
name: 'Stealth',
|
|
1005
|
+
value: 'stealth',
|
|
1006
|
+
},
|
|
1007
|
+
],
|
|
992
1008
|
},
|
|
993
|
-
createActionsProperty(operationName, true, useNestedScrapeOptions),
|
|
994
|
-
createLocationProperty(operationName, true, useNestedScrapeOptions),
|
|
995
1009
|
{
|
|
996
1010
|
displayName: 'Remove Base64 Images',
|
|
997
1011
|
name: 'removeBase64Images',
|
|
@@ -999,12 +1013,14 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
999
1013
|
default: true,
|
|
1000
1014
|
description: 'Whether to strip embedded base64 images from output while preserving alt text. Reduces output size significantly. Disable if you need inline image data.',
|
|
1001
1015
|
},
|
|
1016
|
+
createActionsProperty(operationName, true, useNestedScrapeOptions),
|
|
1017
|
+
createLocationProperty(operationName, true, useNestedScrapeOptions),
|
|
1002
1018
|
{
|
|
1003
|
-
displayName: '
|
|
1004
|
-
name: '
|
|
1019
|
+
displayName: 'Skip TLS Verification',
|
|
1020
|
+
name: 'skipTlsVerification',
|
|
1005
1021
|
type: 'boolean',
|
|
1006
|
-
default:
|
|
1007
|
-
description: 'Whether to
|
|
1022
|
+
default: false,
|
|
1023
|
+
description: 'Whether to bypass SSL/TLS certificate validation. Enable for sites with self-signed or expired certificates. Use with caution as it reduces security.',
|
|
1008
1024
|
},
|
|
1009
1025
|
{
|
|
1010
1026
|
displayName: 'Store In Cache',
|
|
@@ -1014,21 +1030,18 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
1014
1030
|
description: 'Whether to cache the scraped page for faster subsequent requests. Disable for real-time data needs, sensitive content, or when privacy is a concern.',
|
|
1015
1031
|
},
|
|
1016
1032
|
{
|
|
1017
|
-
displayName: '
|
|
1018
|
-
name: '
|
|
1019
|
-
type: '
|
|
1020
|
-
default:
|
|
1021
|
-
description: '
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
value: 'stealth',
|
|
1030
|
-
},
|
|
1031
|
-
],
|
|
1033
|
+
displayName: 'Timeout (Ms)',
|
|
1034
|
+
name: 'timeout',
|
|
1035
|
+
type: 'number',
|
|
1036
|
+
default: 30000,
|
|
1037
|
+
description: 'Maximum time in milliseconds to wait for the page to load. Increase for slow sites or complex pages. Default 30000ms (30 seconds).',
|
|
1038
|
+
},
|
|
1039
|
+
{
|
|
1040
|
+
displayName: 'Wait For (Ms)',
|
|
1041
|
+
name: 'waitFor',
|
|
1042
|
+
type: 'number',
|
|
1043
|
+
default: 0,
|
|
1044
|
+
description: 'Additional wait time in milliseconds after page load before extracting content. Use for JavaScript-heavy sites where content renders dynamically. 0 uses intelligent auto-detection.',
|
|
1032
1045
|
},
|
|
1033
1046
|
...(batchMode ? createBatchSpecificProperties() : []),
|
|
1034
1047
|
],
|
|
@@ -1051,22 +1064,23 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
1051
1064
|
};
|
|
1052
1065
|
}
|
|
1053
1066
|
exports.createScrapeOptionsProperty = createScrapeOptionsProperty;
|
|
1054
|
-
function createOperationOption(
|
|
1067
|
+
function createOperationOption(operationName, displayName, action, description) {
|
|
1055
1068
|
return {
|
|
1056
|
-
name:
|
|
1057
|
-
value:
|
|
1069
|
+
name: displayName,
|
|
1070
|
+
value: operationName,
|
|
1058
1071
|
action,
|
|
1072
|
+
description,
|
|
1059
1073
|
routing: {
|
|
1060
1074
|
request: {
|
|
1061
1075
|
method: 'POST',
|
|
1062
|
-
url: `=/${
|
|
1076
|
+
url: `=/${operationName}`,
|
|
1063
1077
|
},
|
|
1064
1078
|
},
|
|
1065
1079
|
};
|
|
1066
1080
|
}
|
|
1067
1081
|
exports.createOperationOption = createOperationOption;
|
|
1068
|
-
function buildApiProperties(
|
|
1069
|
-
const option = createOperationOption(
|
|
1082
|
+
function buildApiProperties(operationName, displayName, action, description, properties = []) {
|
|
1083
|
+
const option = createOperationOption(operationName, displayName, action, description);
|
|
1070
1084
|
return (0, helpers_1.buildPropertiesWithOptions)(option, properties);
|
|
1071
1085
|
}
|
|
1072
1086
|
exports.buildApiProperties = buildApiProperties;
|