@mendable/n8n-nodes-firecrawl 2.1.2 → 2.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/credentials/FirecrawlApi.credentials.d.ts +1 -0
- package/dist/credentials/FirecrawlApi.credentials.js +1 -0
- package/dist/credentials/FirecrawlApi.credentials.js.map +1 -1
- package/dist/credentials/firecrawl-logo.svg +33 -0
- package/dist/nodes/Firecrawl/api/agent/index.js +1 -1
- package/dist/nodes/Firecrawl/api/agent/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/agentAsync/index.js +2 -1
- package/dist/nodes/Firecrawl/api/agentAsync/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/common.js +220 -209
- package/dist/nodes/Firecrawl/api/common.js.map +1 -1
- package/dist/nodes/Firecrawl/api/crawl/index.js +1 -1
- package/dist/nodes/Firecrawl/api/crawl/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/map/index.js +1 -1
- package/dist/nodes/Firecrawl/api/map/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/search/index.js +1 -1
- package/dist/nodes/Firecrawl/api/search/index.js.map +1 -1
- package/dist/package.json +2 -2
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
|
@@ -155,12 +155,13 @@ function createUrlProperty(operationName, defaultUrl = 'https://firecrawl.dev',
|
|
|
155
155
|
};
|
|
156
156
|
}
|
|
157
157
|
exports.createUrlProperty = createUrlProperty;
|
|
158
|
+
const TAGS_EXPRESSION = '={{Array.isArray($value.items) && $value.items.some(item => item.tag) ? $value.items.map(item => item.tag).filter(tag => tag) : undefined}}';
|
|
158
159
|
function createActionsProperty(operationName, omitDisplayOptions = false, useNestedScrapeOptions = true, resourceName = 'Scraping') {
|
|
159
160
|
return {
|
|
160
161
|
displayName: 'Actions',
|
|
161
162
|
name: 'actions',
|
|
162
163
|
type: 'fixedCollection',
|
|
163
|
-
default:
|
|
164
|
+
default: {},
|
|
164
165
|
typeOptions: {
|
|
165
166
|
multipleValues: true,
|
|
166
167
|
},
|
|
@@ -172,46 +173,48 @@ function createActionsProperty(operationName, omitDisplayOptions = false, useNes
|
|
|
172
173
|
name: 'items',
|
|
173
174
|
values: [
|
|
174
175
|
{
|
|
175
|
-
displayName: '
|
|
176
|
+
displayName: 'Direction',
|
|
176
177
|
type: 'options',
|
|
177
|
-
default: '
|
|
178
|
+
default: 'down',
|
|
179
|
+
description: 'Direction for `scroll` action',
|
|
180
|
+
name: 'direction',
|
|
178
181
|
options: [
|
|
179
182
|
{
|
|
180
|
-
name: '
|
|
181
|
-
value: '
|
|
182
|
-
},
|
|
183
|
-
{
|
|
184
|
-
name: 'Press',
|
|
185
|
-
value: 'press',
|
|
186
|
-
},
|
|
187
|
-
{
|
|
188
|
-
name: 'Screenshot',
|
|
189
|
-
value: 'screenshot',
|
|
183
|
+
name: 'Down',
|
|
184
|
+
value: 'down',
|
|
190
185
|
},
|
|
191
186
|
{
|
|
192
|
-
name: '
|
|
193
|
-
value: '
|
|
187
|
+
name: 'Up',
|
|
188
|
+
value: 'up',
|
|
194
189
|
},
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
190
|
+
],
|
|
191
|
+
displayOptions: {
|
|
192
|
+
show: {
|
|
193
|
+
type: ['scroll'],
|
|
198
194
|
},
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
195
|
+
},
|
|
196
|
+
},
|
|
197
|
+
{
|
|
198
|
+
displayName: 'Full Page',
|
|
199
|
+
type: 'boolean',
|
|
200
|
+
default: false,
|
|
201
|
+
description: 'Whether the screenshot should be full-page or viewport sized',
|
|
202
|
+
name: 'fullPage',
|
|
203
|
+
displayOptions: {
|
|
204
|
+
show: {
|
|
205
|
+
type: ['screenshot'],
|
|
202
206
|
},
|
|
203
|
-
|
|
204
|
-
name: 'type',
|
|
207
|
+
},
|
|
205
208
|
},
|
|
206
209
|
{
|
|
207
|
-
displayName: '
|
|
210
|
+
displayName: 'Key',
|
|
208
211
|
type: 'string',
|
|
209
212
|
default: '',
|
|
210
|
-
description: '
|
|
211
|
-
name: '
|
|
213
|
+
description: 'Keyboard key to press (e.g., "Enter", "Tab", "Escape", "ArrowDown"). Use for form submission, navigation, or triggering keyboard shortcuts.',
|
|
214
|
+
name: 'key',
|
|
212
215
|
displayOptions: {
|
|
213
216
|
show: {
|
|
214
|
-
type: ['
|
|
217
|
+
type: ['press'],
|
|
215
218
|
},
|
|
216
219
|
},
|
|
217
220
|
},
|
|
@@ -228,14 +231,14 @@ function createActionsProperty(operationName, omitDisplayOptions = false, useNes
|
|
|
228
231
|
},
|
|
229
232
|
},
|
|
230
233
|
{
|
|
231
|
-
displayName: '
|
|
232
|
-
type: '
|
|
233
|
-
default:
|
|
234
|
-
description: '
|
|
235
|
-
name: '
|
|
234
|
+
displayName: 'Selector',
|
|
235
|
+
type: 'string',
|
|
236
|
+
default: '',
|
|
237
|
+
description: 'CSS selector to target an element (e.g., "#submit-btn", ".load-more", "[data-testid=login]"). Used for click, write, and scroll actions.',
|
|
238
|
+
name: 'selector',
|
|
236
239
|
displayOptions: {
|
|
237
240
|
show: {
|
|
238
|
-
type: ['
|
|
241
|
+
type: ['click', 'scroll'],
|
|
239
242
|
},
|
|
240
243
|
},
|
|
241
244
|
},
|
|
@@ -252,38 +255,36 @@ function createActionsProperty(operationName, omitDisplayOptions = false, useNes
|
|
|
252
255
|
},
|
|
253
256
|
},
|
|
254
257
|
{
|
|
255
|
-
displayName: '
|
|
256
|
-
type: 'string',
|
|
257
|
-
default: '',
|
|
258
|
-
description: 'Keyboard key to press (e.g., "Enter", "Tab", "Escape", "ArrowDown"). Use for form submission, navigation, or triggering keyboard shortcuts.',
|
|
259
|
-
name: 'key',
|
|
260
|
-
displayOptions: {
|
|
261
|
-
show: {
|
|
262
|
-
type: ['press'],
|
|
263
|
-
},
|
|
264
|
-
},
|
|
265
|
-
},
|
|
266
|
-
{
|
|
267
|
-
displayName: 'Direction',
|
|
258
|
+
displayName: 'Type',
|
|
268
259
|
type: 'options',
|
|
269
|
-
default: '
|
|
270
|
-
description: 'Direction for `scroll` action',
|
|
271
|
-
name: 'direction',
|
|
260
|
+
default: 'wait',
|
|
272
261
|
options: [
|
|
273
262
|
{
|
|
274
|
-
name: '
|
|
275
|
-
value: '
|
|
263
|
+
name: 'Click',
|
|
264
|
+
value: 'click',
|
|
276
265
|
},
|
|
277
266
|
{
|
|
278
|
-
name: '
|
|
279
|
-
value: '
|
|
267
|
+
name: 'Press',
|
|
268
|
+
value: 'press',
|
|
280
269
|
},
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
type: ['scroll'],
|
|
270
|
+
{
|
|
271
|
+
name: 'Screenshot',
|
|
272
|
+
value: 'screenshot',
|
|
285
273
|
},
|
|
286
|
-
|
|
274
|
+
{
|
|
275
|
+
name: 'Scroll',
|
|
276
|
+
value: 'scroll',
|
|
277
|
+
},
|
|
278
|
+
{
|
|
279
|
+
name: 'Wait',
|
|
280
|
+
value: 'wait',
|
|
281
|
+
},
|
|
282
|
+
{
|
|
283
|
+
name: 'Write',
|
|
284
|
+
value: 'write',
|
|
285
|
+
},
|
|
286
|
+
],
|
|
287
|
+
name: 'type',
|
|
287
288
|
},
|
|
288
289
|
],
|
|
289
290
|
},
|
|
@@ -439,11 +440,11 @@ function createIncludeTagsProperty(operationName, omitDisplayOptions = false, us
|
|
|
439
440
|
body: useNestedScrapeOptions
|
|
440
441
|
? {
|
|
441
442
|
scrapeOptions: {
|
|
442
|
-
includeTags:
|
|
443
|
+
includeTags: TAGS_EXPRESSION,
|
|
443
444
|
},
|
|
444
445
|
}
|
|
445
446
|
: {
|
|
446
|
-
includeTags:
|
|
447
|
+
includeTags: TAGS_EXPRESSION,
|
|
447
448
|
},
|
|
448
449
|
},
|
|
449
450
|
},
|
|
@@ -541,11 +542,11 @@ function createExcludeTagsProperty(operationName, omitDisplayOptions = false, us
|
|
|
541
542
|
body: useNestedScrapeOptions
|
|
542
543
|
? {
|
|
543
544
|
scrapeOptions: {
|
|
544
|
-
excludeTags:
|
|
545
|
+
excludeTags: TAGS_EXPRESSION,
|
|
545
546
|
},
|
|
546
547
|
}
|
|
547
548
|
: {
|
|
548
|
-
excludeTags:
|
|
549
|
+
excludeTags: TAGS_EXPRESSION,
|
|
549
550
|
},
|
|
550
551
|
},
|
|
551
552
|
},
|
|
@@ -689,10 +690,20 @@ function createBatchSpecificProperties() {
|
|
|
689
690
|
},
|
|
690
691
|
];
|
|
691
692
|
}
|
|
693
|
+
const SELF_ROUTED_SCRAPE_OPTIONS = [
|
|
694
|
+
'formats',
|
|
695
|
+
'includeTags',
|
|
696
|
+
'excludeTags',
|
|
697
|
+
'actions',
|
|
698
|
+
'location',
|
|
699
|
+
'webhook',
|
|
700
|
+
];
|
|
692
701
|
function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = true, batchMode = false, resourceName = 'Scraping') {
|
|
702
|
+
const rawScrapeOptions = `={{Object.fromEntries(Object.entries($value.options || {}).filter(([k]) => !${JSON.stringify(SELF_ROUTED_SCRAPE_OPTIONS)}.includes(k)))}}`;
|
|
693
703
|
const scrapeOptionsBody = useNestedScrapeOptions
|
|
694
|
-
? { scrapeOptions:
|
|
695
|
-
:
|
|
704
|
+
? { scrapeOptions: rawScrapeOptions }
|
|
705
|
+
: rawScrapeOptions;
|
|
706
|
+
const formatsExpression = '={{Array.isArray($value.format) && $value.format.length > 0 ? $value.format.map(f => { if (f.type === "json" || f.type === "changeTracking") { const format = { type: f.type }; if (f.prompt) format.prompt = f.prompt; if (f.schema) { const schema = JSON.parse(f.schema); if (Object.keys(schema).length > 0) format.schema = schema; } if (f.modes) format.modes = f.modes; if (f.tag) format.tag = f.tag; return format; } else if (f.type === "screenshot") { const format = { type: f.type }; if (f.fullPage !== undefined) format.fullPage = f.fullPage; if (f.quality !== undefined && f.quality !== "" && f.quality !== null) format.quality = f.quality; if (f.viewportWidth !== undefined && f.viewportWidth !== "" && f.viewportWidth !== null && f.viewportHeight !== undefined && f.viewportHeight !== "" && f.viewportHeight !== null) { format.viewport = { width: f.viewportWidth, height: f.viewportHeight }; } return format; } else { return f.type; } }) : undefined}}';
|
|
696
707
|
return {
|
|
697
708
|
displayName: 'Scrape Options',
|
|
698
709
|
name: 'scrapeOptions',
|
|
@@ -704,11 +715,18 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
704
715
|
displayName: 'Options',
|
|
705
716
|
name: 'options',
|
|
706
717
|
values: [
|
|
718
|
+
{
|
|
719
|
+
displayName: 'Block Ads',
|
|
720
|
+
name: 'blockAds',
|
|
721
|
+
type: 'boolean',
|
|
722
|
+
default: true,
|
|
723
|
+
description: 'Whether to enable ad-blocking and cookie consent popup blocking for cleaner content extraction. Recommended for most scraping tasks.',
|
|
724
|
+
},
|
|
707
725
|
{
|
|
708
726
|
displayName: 'Formats',
|
|
709
727
|
name: 'formats',
|
|
710
728
|
type: 'fixedCollection',
|
|
711
|
-
default: [{ type: 'markdown' }],
|
|
729
|
+
default: { format: [{ type: 'markdown' }] },
|
|
712
730
|
typeOptions: {
|
|
713
731
|
multipleValues: true,
|
|
714
732
|
},
|
|
@@ -720,51 +738,36 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
720
738
|
name: 'format',
|
|
721
739
|
values: [
|
|
722
740
|
{
|
|
723
|
-
displayName: '
|
|
724
|
-
name: '
|
|
725
|
-
type: '
|
|
726
|
-
default:
|
|
727
|
-
description: '
|
|
728
|
-
|
|
729
|
-
{
|
|
730
|
-
|
|
731
|
-
value: 'changeTracking',
|
|
732
|
-
description: 'Track changes between page versions over time',
|
|
741
|
+
displayName: 'Full Page',
|
|
742
|
+
name: 'fullPage',
|
|
743
|
+
type: 'boolean',
|
|
744
|
+
default: false,
|
|
745
|
+
description: 'Whether to capture the full page screenshot',
|
|
746
|
+
displayOptions: {
|
|
747
|
+
show: {
|
|
748
|
+
type: ['screenshot'],
|
|
733
749
|
},
|
|
750
|
+
},
|
|
751
|
+
},
|
|
752
|
+
{
|
|
753
|
+
displayName: 'Modes',
|
|
754
|
+
name: 'modes',
|
|
755
|
+
type: 'multiOptions',
|
|
756
|
+
default: ['git-diff'],
|
|
757
|
+
description: 'Modes for change tracking. At least one mode must be selected.',
|
|
758
|
+
displayOptions: {
|
|
759
|
+
show: {
|
|
760
|
+
type: ['changeTracking'],
|
|
761
|
+
},
|
|
762
|
+
},
|
|
763
|
+
options: [
|
|
734
764
|
{
|
|
735
|
-
name: '
|
|
736
|
-
value: '
|
|
737
|
-
description: 'Clean HTML with scripts and styles removed',
|
|
765
|
+
name: 'Git Diff',
|
|
766
|
+
value: 'git-diff',
|
|
738
767
|
},
|
|
739
768
|
{
|
|
740
769
|
name: 'JSON',
|
|
741
770
|
value: 'json',
|
|
742
|
-
description: 'Extract structured data using AI with a schema',
|
|
743
|
-
},
|
|
744
|
-
{
|
|
745
|
-
name: 'Links',
|
|
746
|
-
value: 'links',
|
|
747
|
-
description: 'Extract all links from the page',
|
|
748
|
-
},
|
|
749
|
-
{
|
|
750
|
-
name: 'Markdown',
|
|
751
|
-
value: 'markdown',
|
|
752
|
-
description: 'Full page content as markdown - best for AI/LLM processing and reading full content',
|
|
753
|
-
},
|
|
754
|
-
{
|
|
755
|
-
name: 'Raw HTML',
|
|
756
|
-
value: 'rawHtml',
|
|
757
|
-
description: 'Original HTML including scripts and styles',
|
|
758
|
-
},
|
|
759
|
-
{
|
|
760
|
-
name: 'Screenshot',
|
|
761
|
-
value: 'screenshot',
|
|
762
|
-
description: 'Visual screenshot of the page as base64 image',
|
|
763
|
-
},
|
|
764
|
-
{
|
|
765
|
-
name: 'Summary',
|
|
766
|
-
value: 'summary',
|
|
767
|
-
description: 'AI-generated concise summary of page content - USE THIS when user asks for a summary, overview, brief description, or wants to quickly understand what a page is about',
|
|
768
771
|
},
|
|
769
772
|
],
|
|
770
773
|
},
|
|
@@ -781,57 +784,51 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
781
784
|
},
|
|
782
785
|
},
|
|
783
786
|
{
|
|
784
|
-
displayName: '
|
|
785
|
-
name: '
|
|
786
|
-
type: '
|
|
787
|
-
default: '
|
|
788
|
-
description: '
|
|
787
|
+
displayName: 'Prompt',
|
|
788
|
+
name: 'prompt',
|
|
789
|
+
type: 'string',
|
|
790
|
+
default: '',
|
|
791
|
+
description: 'Prompt for change tracking',
|
|
789
792
|
displayOptions: {
|
|
790
793
|
show: {
|
|
791
|
-
type: ['
|
|
794
|
+
type: ['changeTracking'],
|
|
792
795
|
},
|
|
793
796
|
},
|
|
794
797
|
},
|
|
795
798
|
{
|
|
796
|
-
displayName: '
|
|
797
|
-
name: '
|
|
798
|
-
type: '
|
|
799
|
-
|
|
800
|
-
|
|
799
|
+
displayName: 'Quality',
|
|
800
|
+
name: 'quality',
|
|
801
|
+
type: 'number',
|
|
802
|
+
typeOptions: {
|
|
803
|
+
minValue: 1,
|
|
804
|
+
maxValue: 100,
|
|
805
|
+
},
|
|
806
|
+
default: 100,
|
|
807
|
+
description: 'Screenshot quality (1-100)',
|
|
801
808
|
displayOptions: {
|
|
802
809
|
show: {
|
|
803
|
-
type: ['
|
|
810
|
+
type: ['screenshot'],
|
|
804
811
|
},
|
|
805
812
|
},
|
|
806
|
-
options: [
|
|
807
|
-
{
|
|
808
|
-
name: 'Git Diff',
|
|
809
|
-
value: 'git-diff',
|
|
810
|
-
},
|
|
811
|
-
{
|
|
812
|
-
name: 'JSON',
|
|
813
|
-
value: 'json',
|
|
814
|
-
},
|
|
815
|
-
],
|
|
816
813
|
},
|
|
817
814
|
{
|
|
818
815
|
displayName: 'Schema',
|
|
819
816
|
name: 'schema',
|
|
820
817
|
type: 'json',
|
|
821
818
|
default: '{}',
|
|
822
|
-
description: 'JSON schema for
|
|
819
|
+
description: 'JSON schema for JSON format extraction',
|
|
823
820
|
displayOptions: {
|
|
824
821
|
show: {
|
|
825
|
-
type: ['
|
|
822
|
+
type: ['json'],
|
|
826
823
|
},
|
|
827
824
|
},
|
|
828
825
|
},
|
|
829
826
|
{
|
|
830
|
-
displayName: '
|
|
831
|
-
name: '
|
|
832
|
-
type: '
|
|
833
|
-
default: '',
|
|
834
|
-
description: '
|
|
827
|
+
displayName: 'Schema',
|
|
828
|
+
name: 'schema',
|
|
829
|
+
type: 'json',
|
|
830
|
+
default: '{}',
|
|
831
|
+
description: 'JSON schema for change tracking',
|
|
835
832
|
displayOptions: {
|
|
836
833
|
show: {
|
|
837
834
|
type: ['changeTracking'],
|
|
@@ -851,27 +848,63 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
851
848
|
},
|
|
852
849
|
},
|
|
853
850
|
{
|
|
854
|
-
displayName: '
|
|
855
|
-
name: '
|
|
856
|
-
type: '
|
|
857
|
-
default:
|
|
858
|
-
description: '
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
851
|
+
displayName: 'Type',
|
|
852
|
+
name: 'type',
|
|
853
|
+
type: 'options',
|
|
854
|
+
default: 'markdown',
|
|
855
|
+
description: 'Output format for scraped content. Use "Summary" when user asks for a summary, overview, or condensed version of page content. Use "Markdown" for full content in AI-readable format.',
|
|
856
|
+
options: [
|
|
857
|
+
{
|
|
858
|
+
name: 'Change Tracking',
|
|
859
|
+
value: 'changeTracking',
|
|
860
|
+
description: 'Track changes between page versions over time',
|
|
862
861
|
},
|
|
863
|
-
|
|
862
|
+
{
|
|
863
|
+
name: 'HTML',
|
|
864
|
+
value: 'html',
|
|
865
|
+
description: 'Clean HTML with scripts and styles removed',
|
|
866
|
+
},
|
|
867
|
+
{
|
|
868
|
+
name: 'JSON',
|
|
869
|
+
value: 'json',
|
|
870
|
+
description: 'Extract structured data using AI with a schema',
|
|
871
|
+
},
|
|
872
|
+
{
|
|
873
|
+
name: 'Links',
|
|
874
|
+
value: 'links',
|
|
875
|
+
description: 'Extract all links from the page',
|
|
876
|
+
},
|
|
877
|
+
{
|
|
878
|
+
name: 'Markdown',
|
|
879
|
+
value: 'markdown',
|
|
880
|
+
description: 'Full page content as markdown - best for AI/LLM processing and reading full content',
|
|
881
|
+
},
|
|
882
|
+
{
|
|
883
|
+
name: 'Raw HTML',
|
|
884
|
+
value: 'rawHtml',
|
|
885
|
+
description: 'Original HTML including scripts and styles',
|
|
886
|
+
},
|
|
887
|
+
{
|
|
888
|
+
name: 'Screenshot',
|
|
889
|
+
value: 'screenshot',
|
|
890
|
+
description: 'Visual screenshot of the page as base64 image',
|
|
891
|
+
},
|
|
892
|
+
{
|
|
893
|
+
name: 'Summary',
|
|
894
|
+
value: 'summary',
|
|
895
|
+
description: 'AI-generated concise summary of page content - USE THIS when user asks for a summary, overview, brief description, or wants to quickly understand what a page is about',
|
|
896
|
+
},
|
|
897
|
+
],
|
|
864
898
|
},
|
|
865
899
|
{
|
|
866
|
-
displayName: '
|
|
867
|
-
name: '
|
|
900
|
+
displayName: 'Viewport Height',
|
|
901
|
+
name: 'viewportHeight',
|
|
868
902
|
type: 'number',
|
|
869
903
|
typeOptions: {
|
|
870
904
|
minValue: 1,
|
|
871
|
-
maxValue: 100,
|
|
872
905
|
},
|
|
873
|
-
default:
|
|
874
|
-
description: '
|
|
906
|
+
default: 768,
|
|
907
|
+
description: 'Viewport height for screenshot',
|
|
875
908
|
displayOptions: {
|
|
876
909
|
show: {
|
|
877
910
|
type: ['screenshot'],
|
|
@@ -893,39 +926,17 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
893
926
|
},
|
|
894
927
|
},
|
|
895
928
|
},
|
|
896
|
-
{
|
|
897
|
-
displayName: 'Viewport Height',
|
|
898
|
-
name: 'viewportHeight',
|
|
899
|
-
type: 'number',
|
|
900
|
-
typeOptions: {
|
|
901
|
-
minValue: 1,
|
|
902
|
-
},
|
|
903
|
-
default: 768,
|
|
904
|
-
description: 'Viewport height for screenshot',
|
|
905
|
-
displayOptions: {
|
|
906
|
-
show: {
|
|
907
|
-
type: ['screenshot'],
|
|
908
|
-
},
|
|
909
|
-
},
|
|
910
|
-
},
|
|
911
929
|
],
|
|
912
930
|
},
|
|
913
931
|
],
|
|
914
932
|
routing: {
|
|
915
933
|
request: {
|
|
916
|
-
body:
|
|
917
|
-
|
|
918
|
-
|
|
934
|
+
body: useNestedScrapeOptions
|
|
935
|
+
? { scrapeOptions: { formats: formatsExpression } }
|
|
936
|
+
: { formats: formatsExpression },
|
|
919
937
|
},
|
|
920
938
|
},
|
|
921
939
|
},
|
|
922
|
-
{
|
|
923
|
-
displayName: 'Only Main Content',
|
|
924
|
-
name: 'onlyMainContent',
|
|
925
|
-
type: 'boolean',
|
|
926
|
-
default: true,
|
|
927
|
-
description: 'Whether to automatically remove navigation, headers, footers, sidebars, and other boilerplate content. Best for extracting article text or primary page content. Disable to capture the full page.',
|
|
928
|
-
},
|
|
929
940
|
createIncludeTagsProperty(operationName, true, useNestedScrapeOptions),
|
|
930
941
|
createExcludeTagsProperty(operationName, true, useNestedScrapeOptions),
|
|
931
942
|
{
|
|
@@ -951,13 +962,6 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
951
962
|
},
|
|
952
963
|
],
|
|
953
964
|
},
|
|
954
|
-
{
|
|
955
|
-
displayName: 'Wait For (Ms)',
|
|
956
|
-
name: 'waitFor',
|
|
957
|
-
type: 'number',
|
|
958
|
-
default: 0,
|
|
959
|
-
description: 'Additional wait time in milliseconds after page load before extracting content. Use for JavaScript-heavy sites where content renders dynamically. 0 uses intelligent auto-detection.',
|
|
960
|
-
},
|
|
961
965
|
{
|
|
962
966
|
displayName: 'Mobile',
|
|
963
967
|
name: 'mobile',
|
|
@@ -966,21 +970,29 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
966
970
|
description: 'Whether to emulate a mobile device when scraping. Useful for sites with mobile-specific content, responsive layouts, or mobile-only features. Changes viewport and user-agent.',
|
|
967
971
|
},
|
|
968
972
|
{
|
|
969
|
-
displayName: '
|
|
970
|
-
name: '
|
|
973
|
+
displayName: 'Only Main Content',
|
|
974
|
+
name: 'onlyMainContent',
|
|
971
975
|
type: 'boolean',
|
|
972
|
-
default:
|
|
973
|
-
description: 'Whether to
|
|
976
|
+
default: true,
|
|
977
|
+
description: 'Whether to automatically remove navigation, headers, footers, sidebars, and other boilerplate content. Best for extracting article text or primary page content. Disable to capture the full page.',
|
|
974
978
|
},
|
|
975
979
|
{
|
|
976
|
-
displayName: '
|
|
977
|
-
name: '
|
|
978
|
-
type: '
|
|
979
|
-
default:
|
|
980
|
-
description: '
|
|
980
|
+
displayName: 'Proxy',
|
|
981
|
+
name: 'proxy',
|
|
982
|
+
type: 'options',
|
|
983
|
+
default: 'basic',
|
|
984
|
+
description: 'Proxy strategy for bypassing anti-bot measures. Basic is faster and cheaper. Stealth uses residential proxies for better success on protected sites.',
|
|
985
|
+
options: [
|
|
986
|
+
{
|
|
987
|
+
name: 'Basic',
|
|
988
|
+
value: 'basic',
|
|
989
|
+
},
|
|
990
|
+
{
|
|
991
|
+
name: 'Stealth',
|
|
992
|
+
value: 'stealth',
|
|
993
|
+
},
|
|
994
|
+
],
|
|
981
995
|
},
|
|
982
|
-
createActionsProperty(operationName, true, useNestedScrapeOptions),
|
|
983
|
-
createLocationProperty(operationName, true, useNestedScrapeOptions),
|
|
984
996
|
{
|
|
985
997
|
displayName: 'Remove Base64 Images',
|
|
986
998
|
name: 'removeBase64Images',
|
|
@@ -988,12 +1000,14 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
988
1000
|
default: true,
|
|
989
1001
|
description: 'Whether to strip embedded base64 images from output while preserving alt text. Reduces output size significantly. Disable if you need inline image data.',
|
|
990
1002
|
},
|
|
1003
|
+
createActionsProperty(operationName, true, useNestedScrapeOptions),
|
|
1004
|
+
createLocationProperty(operationName, true, useNestedScrapeOptions),
|
|
991
1005
|
{
|
|
992
|
-
displayName: '
|
|
993
|
-
name: '
|
|
1006
|
+
displayName: 'Skip TLS Verification',
|
|
1007
|
+
name: 'skipTlsVerification',
|
|
994
1008
|
type: 'boolean',
|
|
995
|
-
default:
|
|
996
|
-
description: 'Whether to
|
|
1009
|
+
default: false,
|
|
1010
|
+
description: 'Whether to bypass SSL/TLS certificate validation. Enable for sites with self-signed or expired certificates. Use with caution as it reduces security.',
|
|
997
1011
|
},
|
|
998
1012
|
{
|
|
999
1013
|
displayName: 'Store In Cache',
|
|
@@ -1003,21 +1017,18 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
1003
1017
|
description: 'Whether to cache the scraped page for faster subsequent requests. Disable for real-time data needs, sensitive content, or when privacy is a concern.',
|
|
1004
1018
|
},
|
|
1005
1019
|
{
|
|
1006
|
-
displayName: '
|
|
1007
|
-
name: '
|
|
1008
|
-
type: '
|
|
1009
|
-
default:
|
|
1010
|
-
description: '
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
value: 'stealth',
|
|
1019
|
-
},
|
|
1020
|
-
],
|
|
1020
|
+
displayName: 'Timeout (Ms)',
|
|
1021
|
+
name: 'timeout',
|
|
1022
|
+
type: 'number',
|
|
1023
|
+
default: 30000,
|
|
1024
|
+
description: 'Maximum time in milliseconds to wait for the page to load. Increase for slow sites or complex pages. Default 30000ms (30 seconds).',
|
|
1025
|
+
},
|
|
1026
|
+
{
|
|
1027
|
+
displayName: 'Wait For (Ms)',
|
|
1028
|
+
name: 'waitFor',
|
|
1029
|
+
type: 'number',
|
|
1030
|
+
default: 0,
|
|
1031
|
+
description: 'Additional wait time in milliseconds after page load before extracting content. Use for JavaScript-heavy sites where content renders dynamically. 0 uses intelligent auto-detection.',
|
|
1021
1032
|
},
|
|
1022
1033
|
...(batchMode ? createBatchSpecificProperties() : []),
|
|
1023
1034
|
],
|