@mendable/n8n-nodes-firecrawl 1.0.5 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -0
- package/dist/nodes/Firecrawl/Firecrawl.node.d.ts +3 -0
- package/dist/nodes/Firecrawl/api/batchScrape/index.d.ts +6 -0
- package/dist/nodes/Firecrawl/api/batchScrape/index.js +124 -0
- package/dist/nodes/Firecrawl/api/batchScrape/index.js.map +1 -0
- package/dist/nodes/Firecrawl/api/{form → batchScrapeErrors}/index.d.ts +2 -3
- package/dist/nodes/Firecrawl/api/batchScrapeErrors/index.js +39 -0
- package/dist/nodes/Firecrawl/api/batchScrapeErrors/index.js.map +1 -0
- package/dist/nodes/Firecrawl/api/batchScrapeStatus/index.d.ts +5 -0
- package/dist/nodes/Firecrawl/api/batchScrapeStatus/index.js +39 -0
- package/dist/nodes/Firecrawl/api/batchScrapeStatus/index.js.map +1 -0
- package/dist/nodes/Firecrawl/api/cancelBatchScrape/index.d.ts +5 -0
- package/dist/nodes/Firecrawl/api/cancelBatchScrape/index.js +39 -0
- package/dist/nodes/Firecrawl/api/cancelBatchScrape/index.js.map +1 -0
- package/dist/nodes/Firecrawl/api/cancelCrawl/index.d.ts +5 -0
- package/dist/nodes/Firecrawl/api/cancelCrawl/index.js +39 -0
- package/dist/nodes/Firecrawl/api/cancelCrawl/index.js.map +1 -0
- package/dist/nodes/Firecrawl/api/common.d.ts +2 -1
- package/dist/nodes/Firecrawl/api/common.js +172 -3
- package/dist/nodes/Firecrawl/api/common.js.map +1 -1
- package/dist/nodes/Firecrawl/api/crawlActive/index.d.ts +5 -0
- package/dist/nodes/Firecrawl/api/crawlActive/index.js +19 -0
- package/dist/nodes/Firecrawl/api/crawlActive/index.js.map +1 -0
- package/dist/nodes/Firecrawl/api/crawlParamsPreview/index.d.ts +5 -0
- package/dist/nodes/Firecrawl/api/crawlParamsPreview/index.js +41 -0
- package/dist/nodes/Firecrawl/api/crawlParamsPreview/index.js.map +1 -0
- package/dist/nodes/Firecrawl/api/creditUsageHistorical/index.d.ts +5 -0
- package/dist/nodes/Firecrawl/api/creditUsageHistorical/index.js +19 -0
- package/dist/nodes/Firecrawl/api/creditUsageHistorical/index.js.map +1 -0
- package/dist/nodes/Firecrawl/api/getCrawlErrors/index.d.ts +5 -0
- package/dist/nodes/Firecrawl/api/getCrawlErrors/index.js +39 -0
- package/dist/nodes/Firecrawl/api/getCrawlErrors/index.js.map +1 -0
- package/dist/nodes/Firecrawl/api/index.d.ts +39 -0
- package/dist/nodes/Firecrawl/api/index.js +104 -0
- package/dist/nodes/Firecrawl/api/index.js.map +1 -1
- package/dist/nodes/Firecrawl/api/teamCreditUsage/index.d.ts +5 -0
- package/dist/nodes/Firecrawl/api/teamCreditUsage/index.js +19 -0
- package/dist/nodes/Firecrawl/api/teamCreditUsage/index.js.map +1 -0
- package/dist/nodes/Firecrawl/api/teamQueueStatus/index.d.ts +5 -0
- package/dist/nodes/Firecrawl/api/teamQueueStatus/index.js +19 -0
- package/dist/nodes/Firecrawl/api/teamQueueStatus/index.js.map +1 -0
- package/dist/nodes/Firecrawl/api/teamTokenUsage/index.d.ts +5 -0
- package/dist/nodes/Firecrawl/api/teamTokenUsage/index.js +19 -0
- package/dist/nodes/Firecrawl/api/teamTokenUsage/index.js.map +1 -0
- package/dist/nodes/Firecrawl/api/teamTokenUsageHistorical/index.d.ts +5 -0
- package/dist/nodes/Firecrawl/api/teamTokenUsageHistorical/index.js +19 -0
- package/dist/nodes/Firecrawl/api/teamTokenUsageHistorical/index.js.map +1 -0
- package/dist/nodes/Firecrawl/methods.d.ts +3 -0
- package/dist/package.json +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/dist/nodes/Firecrawl/api/form/index.js +0 -163
- package/dist/nodes/Firecrawl/api/form/index.js.map +0 -1
- package/dist/nodes/Firecrawl/helpers/http/debug.d.ts +0 -2
- package/dist/nodes/Firecrawl/helpers/http/debug.js +0 -18
- package/dist/nodes/Firecrawl/helpers/http/debug.js.map +0 -1
- package/dist/nodes/Firecrawl/openapi.json +0 -2744
- package/dist/nodes/Firecrawl/v1-openapi.json +0 -1915
package/README.md
CHANGED
|
@@ -33,6 +33,27 @@ The **Firecrawl** node supports the following operations:
|
|
|
33
33
|
### Crawl
|
|
34
34
|
- Scrapes all the URLs of a web page and return content in LLM-ready format
|
|
35
35
|
|
|
36
|
+
### Batch Scrape
|
|
37
|
+
- Start a batch job to scrape multiple URLs at once
|
|
38
|
+
|
|
39
|
+
### Batch Scrape Status
|
|
40
|
+
- Get the status/result of a batch scrape job by ID
|
|
41
|
+
|
|
42
|
+
### Batch Scrape Errors
|
|
43
|
+
- Retrieve errors for a batch scrape job by ID
|
|
44
|
+
|
|
45
|
+
### Crawl Active
|
|
46
|
+
- List all currently active crawl jobs for your team
|
|
47
|
+
|
|
48
|
+
### Crawl Params Preview
|
|
49
|
+
- Preview crawl parameters generated from a natural-language prompt
|
|
50
|
+
|
|
51
|
+
### Cancel Crawl
|
|
52
|
+
- Cancel a running crawl job by ID
|
|
53
|
+
|
|
54
|
+
### Get Crawl Errors
|
|
55
|
+
- Retrieve errors for a crawl job by ID
|
|
56
|
+
|
|
36
57
|
### Get Crawl Status
|
|
37
58
|
- Check the current status of a crawl job
|
|
38
59
|
|
|
@@ -42,6 +63,21 @@ The **Firecrawl** node supports the following operations:
|
|
|
42
63
|
### Get Extract Status
|
|
43
64
|
- Get the current status of an extraction job
|
|
44
65
|
|
|
66
|
+
### Team Token Usage
|
|
67
|
+
- Get remaining and plan tokens for the authenticated team
|
|
68
|
+
|
|
69
|
+
### Team Credit Usage
|
|
70
|
+
- Get remaining and plan credits for the authenticated team
|
|
71
|
+
|
|
72
|
+
### Historical Credit Usage
|
|
73
|
+
- Get historical credit usage for your team
|
|
74
|
+
|
|
75
|
+
### Historical Token Usage
|
|
76
|
+
- Get historical token usage for your team
|
|
77
|
+
|
|
78
|
+
### Team Queue Status
|
|
79
|
+
- Get your team’s current queue load (waiting, active, max concurrency)
|
|
80
|
+
|
|
45
81
|
## Credentials
|
|
46
82
|
|
|
47
83
|
To use the Firecrawl node, you need to:
|
|
@@ -67,7 +103,22 @@ To use the Firecrawl node, you need to:
|
|
|
67
103
|
|
|
68
104
|
## Version history
|
|
69
105
|
|
|
106
|
+
### 1.0.6
|
|
107
|
+
- Add support for additional Firecrawl endpoints:
|
|
108
|
+
- Batch Scrape (start/status/errors)
|
|
109
|
+
- Crawl Active
|
|
110
|
+
- Crawl Params Preview
|
|
111
|
+
- Cancel Crawl
|
|
112
|
+
- Get Crawl Errors
|
|
113
|
+
- Team Token Usage
|
|
114
|
+
- Team Credit Usage
|
|
115
|
+
- Historical Credit Usage
|
|
116
|
+
- Historical Token Usage
|
|
117
|
+
- Team Queue Status
|
|
118
|
+
- Wire new operations into the node and align with Firecrawl API v2
|
|
119
|
+
|
|
70
120
|
### 1.0.5
|
|
121
|
+
- API version updated to [/v2](https://docs.firecrawl.dev/migrate-to-v2)
|
|
71
122
|
- Unified sitemap configuration parameters in Map operation
|
|
72
123
|
- Replaced `ignoreSitemap` and `sitemapOnly` with unified `sitemap` parameter
|
|
73
124
|
- `sitemap` parameter now accepts: "include" (default), "only", or "skip"
|
|
@@ -14,6 +14,9 @@ export declare class Firecrawl implements INodeType {
|
|
|
14
14
|
resourceMapping?: {
|
|
15
15
|
[functionName: string]: (this: import("n8n-workflow").ILoadOptionsFunctions) => Promise<import("n8n-workflow").ResourceMapperFields>;
|
|
16
16
|
} | undefined;
|
|
17
|
+
localResourceMapping?: {
|
|
18
|
+
[functionName: string]: (this: import("n8n-workflow").ILocalLoadOptionsFunctions) => Promise<import("n8n-workflow").ResourceMapperFields>;
|
|
19
|
+
} | undefined;
|
|
17
20
|
actionHandler?: {
|
|
18
21
|
[functionName: string]: (this: import("n8n-workflow").ILoadOptionsFunctions, payload: string | import("n8n-workflow").IDataObject | undefined) => Promise<import("n8n-workflow").NodeParameterValueType>;
|
|
19
22
|
} | undefined;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { INodeProperties } from 'n8n-workflow';
|
|
2
|
+
export declare const name = "batchScrape";
|
|
3
|
+
export declare const displayName = "Batch scrape multiple URLs";
|
|
4
|
+
export declare const operationName = "batchScrape";
|
|
5
|
+
declare const options: import("n8n-workflow").INodePropertyOptions, properties: INodeProperties[];
|
|
6
|
+
export { options, properties };
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var _a;
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.properties = exports.options = exports.operationName = exports.displayName = exports.name = void 0;
|
|
5
|
+
const common_1 = require("../common");
|
|
6
|
+
exports.name = 'batchScrape';
|
|
7
|
+
exports.displayName = 'Batch scrape multiple URLs';
|
|
8
|
+
exports.operationName = 'batchScrape';
|
|
9
|
+
function createParsersProperty(operationName) {
|
|
10
|
+
return {
|
|
11
|
+
displayName: 'Parsers',
|
|
12
|
+
name: 'parsers',
|
|
13
|
+
type: 'multiOptions',
|
|
14
|
+
options: [
|
|
15
|
+
{
|
|
16
|
+
name: 'PDF',
|
|
17
|
+
value: 'pdf',
|
|
18
|
+
},
|
|
19
|
+
],
|
|
20
|
+
default: [],
|
|
21
|
+
description: 'Controls how PDF files are processed during scraping. When PDF parser is enabled, the PDF content is extracted and converted to markdown format, with billing based on the number of pages (1 credit per page). When disabled, the PDF file is returned in base64 encoding with a flat rate of 1 credit total.',
|
|
22
|
+
routing: {
|
|
23
|
+
request: {
|
|
24
|
+
body: {
|
|
25
|
+
parsers: '={{ $value }}',
|
|
26
|
+
},
|
|
27
|
+
},
|
|
28
|
+
},
|
|
29
|
+
displayOptions: {
|
|
30
|
+
hide: {
|
|
31
|
+
useCustomBody: [true],
|
|
32
|
+
},
|
|
33
|
+
show: {
|
|
34
|
+
resource: ['Default'],
|
|
35
|
+
operation: [operationName],
|
|
36
|
+
},
|
|
37
|
+
},
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
function createAdditionalFieldsProperty(operation) {
|
|
41
|
+
return {
|
|
42
|
+
displayName: 'Additional Fields',
|
|
43
|
+
name: 'additionalFields',
|
|
44
|
+
type: 'collection',
|
|
45
|
+
placeholder: 'Add Field',
|
|
46
|
+
default: {},
|
|
47
|
+
description: 'Additional fields to send in the request body',
|
|
48
|
+
options: [
|
|
49
|
+
{
|
|
50
|
+
displayName: 'Custom Properties (JSON)',
|
|
51
|
+
name: 'customProperties',
|
|
52
|
+
type: 'json',
|
|
53
|
+
default: '{}',
|
|
54
|
+
description: 'Custom JSON properties to add to the request body',
|
|
55
|
+
},
|
|
56
|
+
],
|
|
57
|
+
routing: {
|
|
58
|
+
request: {
|
|
59
|
+
body: {
|
|
60
|
+
additionalFields: '={{ $value }}',
|
|
61
|
+
},
|
|
62
|
+
},
|
|
63
|
+
send: {
|
|
64
|
+
preSend: [
|
|
65
|
+
async function (requestOptions) {
|
|
66
|
+
if (typeof requestOptions.body !== 'object' || !requestOptions.body) {
|
|
67
|
+
return requestOptions;
|
|
68
|
+
}
|
|
69
|
+
const body = requestOptions.body;
|
|
70
|
+
const additionalFields = body.additionalFields;
|
|
71
|
+
if (additionalFields) {
|
|
72
|
+
if (additionalFields.customProperties) {
|
|
73
|
+
try {
|
|
74
|
+
const customProps = JSON.parse(additionalFields.customProperties);
|
|
75
|
+
Object.assign(requestOptions.body, customProps);
|
|
76
|
+
}
|
|
77
|
+
catch (error) {
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
delete body.additionalFields;
|
|
81
|
+
}
|
|
82
|
+
return requestOptions;
|
|
83
|
+
},
|
|
84
|
+
],
|
|
85
|
+
},
|
|
86
|
+
},
|
|
87
|
+
displayOptions: {
|
|
88
|
+
show: {
|
|
89
|
+
operation: [operation],
|
|
90
|
+
useCustomBody: [true],
|
|
91
|
+
},
|
|
92
|
+
},
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
function createScrapeProperties() {
|
|
96
|
+
return [
|
|
97
|
+
(0, common_1.createOperationNotice)('Default', exports.name, 'POST'),
|
|
98
|
+
(0, common_1.createBatchUrlsProperty)(exports.name, 'https://firecrawl.dev'),
|
|
99
|
+
createParsersProperty(exports.operationName),
|
|
100
|
+
(0, common_1.createScrapeOptionsProperty)(exports.operationName, false, true),
|
|
101
|
+
];
|
|
102
|
+
}
|
|
103
|
+
const { options, properties } = (0, common_1.buildApiProperties)(exports.name, exports.displayName, createScrapeProperties());
|
|
104
|
+
exports.options = options;
|
|
105
|
+
exports.properties = properties;
|
|
106
|
+
options.routing = {
|
|
107
|
+
...options.routing,
|
|
108
|
+
request: {
|
|
109
|
+
...(((_a = options.routing) === null || _a === void 0 ? void 0 : _a.request) || {}),
|
|
110
|
+
url: '/batch/scrape',
|
|
111
|
+
},
|
|
112
|
+
output: {
|
|
113
|
+
postReceive: [
|
|
114
|
+
{
|
|
115
|
+
type: 'setKeyValue',
|
|
116
|
+
properties: {
|
|
117
|
+
data: '={{$response.body}}',
|
|
118
|
+
},
|
|
119
|
+
},
|
|
120
|
+
],
|
|
121
|
+
},
|
|
122
|
+
};
|
|
123
|
+
properties.push(createAdditionalFieldsProperty(exports.name));
|
|
124
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../nodes/Firecrawl/api/batchScrape/index.ts"],"names":[],"mappings":";;;;AAMA,sCAKmB;AAGN,QAAA,IAAI,GAAG,aAAa,CAAC;AACrB,QAAA,WAAW,GAAG,4BAA4B,CAAC;AAC3C,QAAA,aAAa,GAAG,aAAa,CAAC;AAO3C,SAAS,qBAAqB,CAAC,aAAqB;IACnD,OAAO;QACN,WAAW,EAAE,SAAS;QACtB,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,cAAc;QACpB,OAAO,EAAE;YACR;gBACC,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,KAAK;aACZ;SACD;QACD,OAAO,EAAE,EAAE;QACX,WAAW,EACV,gTAAgT;QACjT,OAAO,EAAE;YACR,OAAO,EAAE;gBACR,IAAI,EAAE;oBACL,OAAO,EAAE,eAAe;iBACxB;aACD;SACD;QACD,cAAc,EAAE;YACf,IAAI,EAAE;gBACL,aAAa,EAAE,CAAC,IAAI,CAAC;aACrB;YACD,IAAI,EAAE;gBACL,QAAQ,EAAE,CAAC,SAAS,CAAC;gBACrB,SAAS,EAAE,CAAC,aAAa,CAAC;aAC1B;SACD;KACD,CAAC;AACH,CAAC;AAKD,SAAS,8BAA8B,CAAC,SAAiB;IACxD,OAAO;QACN,WAAW,EAAE,mBAAmB;QAChC,IAAI,EAAE,kBAAkB;QACxB,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,WAAW;QACxB,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,+CAA+C;QAC5D,OAAO,EAAE;YACR;gBACC,WAAW,EAAE,0BAA0B;gBACvC,IAAI,EAAE,kBAAkB;gBACxB,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,IAAI;gBACb,WAAW,EAAE,mDAAmD;aAChE;SACD;QACD,OAAO,EAAE;YACR,OAAO,EAAE;gBACR,IAAI,EAAE;oBACL,gBAAgB,EAAE,eAAe;iBACjC;aACD;YACD,IAAI,EAAE;gBACL,OAAO,EAAE;oBACR,KAAK,WAEJ,cAAmC;wBAEnC,IAAI,OAAO,cAAc,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE;4BACpE,OAAO,cAAc,CAAC;yBACtB;wBAED,MAAM,IAAI,GAAG,cAAc,CAAC,IAAmB,CAAC;wBAChD,MAAM,gBAAgB,GAAG,IAAI,CAAC,gBAA+B,CAAC;wBAE9D,IAAI,gBAAgB,EAAE;4BAErB,IAAI,gBAAgB,CAAC,gBAAgB,EAAE;gCACtC,IAAI;oCACH,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,gBAA0B,CAAC,CAAC;oCAC5E,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,IAAmB,EAAE,WAAW,CAAC,CAAC;iCAC/D;gCAAC,OAAO,KAAK,EAAE;iCAEf;6BACD;4BAGD,OAAO,IAAI,CAAC,gBAAgB,CAAC;yBAC7B;wBAED,OAAO,cAAc,CAAC;oBACvB,CAAC;iBACD;aACD;SACD;QACD,cAAc,EAAE;YACf,IAAI,EAAE;gBACL,SAAS,EAAE,CAAC,SAAS,CAAC;gBACtB,aAAa,EAAE,CAAC,IAAI,CAAC;aACrB;SACD;KACD,CAAC;AACH,CAAC;AAKD,SAAS,sBAAsB;IAC9B,OAAO;QAEN,IAAA,8BAAqB,EAAC,SAAS,EAAE,YAAI,EAAE,MAAM,CAAC;QAG9C,IAAA,gCAAuB,EAAC,YAAI,EAAE,uBAAuB,CAAC;QAGtD,qBAAqB,CAAC,qBAAa,CAAC;QAGpC,IAAA,oCAA2B,EAAC,qBAAa,EAAE,KAAK,EAAE,IAAI,CAAC;KACvD,CAAC;AACH,CAAC;AAGD,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,IAAA,2BAAkB,EAAC,YAAI,EAAE,mBAAW,EAAE,sBAAsB,EAAE,CAAC,CAAC;AAwBvF,0BAAO;AAAE,gCAAU;AArB5B,OAAO,CAAC,OAAO,GAAG;IACjB,GAAG,OAAO,CAAC,OAAO;IAClB,OAAO,EAAE;QACR,GAAG,CAAC,CAAA,MAAA,OAAO,CAAC,OAAO,0CAAE,OAAO,KAAI,EAAE,CAAC;QACnC,GAAG,EAAE,eAAe;KACpB;IACD,MAAM,EAAE;QACP,WAAW,EAAE;YACZ;gBACC,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE;oBACX,IAAI,EAAE,qBAAqB;iBAC3B;aACD;SACD;KACD;CACD,CAAC;AAGF,UAAU,CAAC,IAAI,CAAC,8BAA8B,CAAC,YAAI,CAAC,CAAC,CAAC"}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { INodeProperties } from 'n8n-workflow';
|
|
2
|
-
export declare const name = "
|
|
3
|
-
export declare const displayName = "
|
|
4
|
-
export declare const operationName = "form";
|
|
2
|
+
export declare const name = "batchScrapeErrors";
|
|
3
|
+
export declare const displayName = "Get batch scrape errors";
|
|
5
4
|
declare const options: import("n8n-workflow").INodePropertyOptions, properties: INodeProperties[];
|
|
6
5
|
export { options, properties };
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.properties = exports.options = exports.displayName = exports.name = void 0;
|
|
4
|
+
const common_1 = require("../common");
|
|
5
|
+
exports.name = 'batchScrapeErrors';
|
|
6
|
+
exports.displayName = 'Get batch scrape errors';
|
|
7
|
+
function createBatchIdProperty() {
|
|
8
|
+
return {
|
|
9
|
+
displayName: 'Batch ID',
|
|
10
|
+
name: 'batchId',
|
|
11
|
+
type: 'string',
|
|
12
|
+
required: true,
|
|
13
|
+
default: '',
|
|
14
|
+
description: 'ID of the batch scrape job',
|
|
15
|
+
routing: {
|
|
16
|
+
request: {
|
|
17
|
+
url: '=/batch/scrape/{{$value}}/errors',
|
|
18
|
+
},
|
|
19
|
+
},
|
|
20
|
+
displayOptions: {
|
|
21
|
+
show: {
|
|
22
|
+
resource: ['Default'],
|
|
23
|
+
operation: [exports.name],
|
|
24
|
+
},
|
|
25
|
+
},
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
function createProperties() {
|
|
29
|
+
return [(0, common_1.createOperationNotice)('Default', exports.name, 'GET'), createBatchIdProperty()];
|
|
30
|
+
}
|
|
31
|
+
const { options, properties } = (0, common_1.buildApiProperties)(exports.name, exports.displayName, createProperties());
|
|
32
|
+
exports.options = options;
|
|
33
|
+
exports.properties = properties;
|
|
34
|
+
options.routing = {
|
|
35
|
+
request: {
|
|
36
|
+
method: 'GET',
|
|
37
|
+
},
|
|
38
|
+
};
|
|
39
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../nodes/Firecrawl/api/batchScrapeErrors/index.ts"],"names":[],"mappings":";;;AACA,sCAAsE;AAEzD,QAAA,IAAI,GAAG,mBAAmB,CAAC;AAC3B,QAAA,WAAW,GAAG,yBAAyB,CAAC;AAErD,SAAS,qBAAqB;IAC7B,OAAO;QACN,WAAW,EAAE,UAAU;QACvB,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,IAAI;QACd,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,4BAA4B;QACzC,OAAO,EAAE;YACR,OAAO,EAAE;gBACR,GAAG,EAAE,kCAAkC;aACvC;SACD;QACD,cAAc,EAAE;YACf,IAAI,EAAE;gBACL,QAAQ,EAAE,CAAC,SAAS,CAAC;gBACrB,SAAS,EAAE,CAAC,YAAI,CAAC;aACjB;SACD;KACD,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB;IACxB,OAAO,CAAC,IAAA,8BAAqB,EAAC,SAAS,EAAE,YAAI,EAAE,KAAK,CAAC,EAAE,qBAAqB,EAAE,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,IAAA,2BAAkB,EAAC,YAAI,EAAE,mBAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC;AAQjF,0BAAO;AAAE,gCAAU;AAN5B,OAAO,CAAC,OAAO,GAAG;IACjB,OAAO,EAAE;QACR,MAAM,EAAE,KAAK;KACb;CACD,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { INodeProperties } from 'n8n-workflow';
|
|
2
|
+
export declare const name = "batchScrapeStatus";
|
|
3
|
+
export declare const displayName = "Get batch scrape status";
|
|
4
|
+
declare const options: import("n8n-workflow").INodePropertyOptions, properties: INodeProperties[];
|
|
5
|
+
export { options, properties };
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.properties = exports.options = exports.displayName = exports.name = void 0;
|
|
4
|
+
const common_1 = require("../common");
|
|
5
|
+
exports.name = 'batchScrapeStatus';
|
|
6
|
+
exports.displayName = 'Get batch scrape status';
|
|
7
|
+
function createBatchIdProperty() {
|
|
8
|
+
return {
|
|
9
|
+
displayName: 'Batch ID',
|
|
10
|
+
name: 'batchId',
|
|
11
|
+
type: 'string',
|
|
12
|
+
required: true,
|
|
13
|
+
default: '',
|
|
14
|
+
description: 'ID of the batch scrape job',
|
|
15
|
+
routing: {
|
|
16
|
+
request: {
|
|
17
|
+
url: '=/batch/scrape/{{$value}}',
|
|
18
|
+
},
|
|
19
|
+
},
|
|
20
|
+
displayOptions: {
|
|
21
|
+
show: {
|
|
22
|
+
resource: ['Default'],
|
|
23
|
+
operation: [exports.name],
|
|
24
|
+
},
|
|
25
|
+
},
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
function createProperties() {
|
|
29
|
+
return [(0, common_1.createOperationNotice)('Default', exports.name, 'GET'), createBatchIdProperty()];
|
|
30
|
+
}
|
|
31
|
+
const { options, properties } = (0, common_1.buildApiProperties)(exports.name, exports.displayName, createProperties());
|
|
32
|
+
exports.options = options;
|
|
33
|
+
exports.properties = properties;
|
|
34
|
+
options.routing = {
|
|
35
|
+
request: {
|
|
36
|
+
method: 'GET',
|
|
37
|
+
},
|
|
38
|
+
};
|
|
39
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../nodes/Firecrawl/api/batchScrapeStatus/index.ts"],"names":[],"mappings":";;;AACA,sCAAsE;AAEzD,QAAA,IAAI,GAAG,mBAAmB,CAAC;AAC3B,QAAA,WAAW,GAAG,yBAAyB,CAAC;AAErD,SAAS,qBAAqB;IAC7B,OAAO;QACN,WAAW,EAAE,UAAU;QACvB,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,IAAI;QACd,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,4BAA4B;QACzC,OAAO,EAAE;YACR,OAAO,EAAE;gBACR,GAAG,EAAE,2BAA2B;aAChC;SACD;QACD,cAAc,EAAE;YACf,IAAI,EAAE;gBACL,QAAQ,EAAE,CAAC,SAAS,CAAC;gBACrB,SAAS,EAAE,CAAC,YAAI,CAAC;aACjB;SACD;KACD,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB;IACxB,OAAO,CAAC,IAAA,8BAAqB,EAAC,SAAS,EAAE,YAAI,EAAE,KAAK,CAAC,EAAE,qBAAqB,EAAE,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,IAAA,2BAAkB,EAAC,YAAI,EAAE,mBAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC;AAQjF,0BAAO;AAAE,gCAAU;AAN5B,OAAO,CAAC,OAAO,GAAG;IACjB,OAAO,EAAE;QACR,MAAM,EAAE,KAAK;KACb;CACD,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { INodeProperties } from 'n8n-workflow';
|
|
2
|
+
export declare const name = "cancelBatchScrape";
|
|
3
|
+
export declare const displayName = "Cancel batch scrape job";
|
|
4
|
+
declare const options: import("n8n-workflow").INodePropertyOptions, properties: INodeProperties[];
|
|
5
|
+
export { options, properties };
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.properties = exports.options = exports.displayName = exports.name = void 0;
|
|
4
|
+
const common_1 = require("../common");
|
|
5
|
+
exports.name = 'cancelBatchScrape';
|
|
6
|
+
exports.displayName = 'Cancel batch scrape job';
|
|
7
|
+
function createBatchIdProperty() {
|
|
8
|
+
return {
|
|
9
|
+
displayName: 'Batch ID',
|
|
10
|
+
name: 'batchId',
|
|
11
|
+
type: 'string',
|
|
12
|
+
required: true,
|
|
13
|
+
default: '',
|
|
14
|
+
description: 'ID of the batch scrape job to cancel',
|
|
15
|
+
routing: {
|
|
16
|
+
request: {
|
|
17
|
+
url: '=/batch/scrape/{{$value}}',
|
|
18
|
+
},
|
|
19
|
+
},
|
|
20
|
+
displayOptions: {
|
|
21
|
+
show: {
|
|
22
|
+
resource: ['Default'],
|
|
23
|
+
operation: [exports.name],
|
|
24
|
+
},
|
|
25
|
+
},
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
function createProperties() {
|
|
29
|
+
return [(0, common_1.createOperationNotice)('Default', exports.name, 'DELETE'), createBatchIdProperty()];
|
|
30
|
+
}
|
|
31
|
+
const { options, properties } = (0, common_1.buildApiProperties)(exports.name, exports.displayName, createProperties());
|
|
32
|
+
exports.options = options;
|
|
33
|
+
exports.properties = properties;
|
|
34
|
+
options.routing = {
|
|
35
|
+
request: {
|
|
36
|
+
method: 'DELETE',
|
|
37
|
+
},
|
|
38
|
+
};
|
|
39
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../nodes/Firecrawl/api/cancelBatchScrape/index.ts"],"names":[],"mappings":";;;AACA,sCAAsE;AAEzD,QAAA,IAAI,GAAG,mBAAmB,CAAC;AAC3B,QAAA,WAAW,GAAG,yBAAyB,CAAC;AAErD,SAAS,qBAAqB;IAC7B,OAAO;QACN,WAAW,EAAE,UAAU;QACvB,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,IAAI;QACd,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,sCAAsC;QACnD,OAAO,EAAE;YACR,OAAO,EAAE;gBACR,GAAG,EAAE,2BAA2B;aAChC;SACD;QACD,cAAc,EAAE;YACf,IAAI,EAAE;gBACL,QAAQ,EAAE,CAAC,SAAS,CAAC;gBACrB,SAAS,EAAE,CAAC,YAAI,CAAC;aACjB;SACD;KACD,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB;IACxB,OAAO,CAAC,IAAA,8BAAqB,EAAC,SAAS,EAAE,YAAI,EAAE,QAAQ,CAAC,EAAE,qBAAqB,EAAE,CAAC,CAAC;AACpF,CAAC;AAED,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,IAAA,2BAAkB,EAAC,YAAI,EAAE,mBAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC;AAQjF,0BAAO;AAAE,gCAAU;AAN5B,OAAO,CAAC,OAAO,GAAG;IACjB,OAAO,EAAE;QACR,MAAM,EAAE,QAAQ;KAChB;CACD,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { INodeProperties } from 'n8n-workflow';
|
|
2
|
+
export declare const name = "cancelCrawl";
|
|
3
|
+
export declare const displayName = "Cancel a crawl job";
|
|
4
|
+
declare const options: import("n8n-workflow").INodePropertyOptions, properties: INodeProperties[];
|
|
5
|
+
export { options, properties };
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.properties = exports.options = exports.displayName = exports.name = void 0;
|
|
4
|
+
const common_1 = require("../common");
|
|
5
|
+
exports.name = 'cancelCrawl';
|
|
6
|
+
exports.displayName = 'Cancel a crawl job';
|
|
7
|
+
function createCrawlIdProperty() {
|
|
8
|
+
return {
|
|
9
|
+
displayName: 'Crawl ID',
|
|
10
|
+
name: 'crawlId',
|
|
11
|
+
type: 'string',
|
|
12
|
+
required: true,
|
|
13
|
+
default: '',
|
|
14
|
+
description: 'ID of the crawl job to cancel',
|
|
15
|
+
routing: {
|
|
16
|
+
request: {
|
|
17
|
+
url: '=/crawl/{{$value}}',
|
|
18
|
+
},
|
|
19
|
+
},
|
|
20
|
+
displayOptions: {
|
|
21
|
+
show: {
|
|
22
|
+
resource: ['Default'],
|
|
23
|
+
operation: [exports.name],
|
|
24
|
+
},
|
|
25
|
+
},
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
function createProperties() {
|
|
29
|
+
return [(0, common_1.createOperationNotice)('Default', exports.name, 'DELETE'), createCrawlIdProperty()];
|
|
30
|
+
}
|
|
31
|
+
const { options, properties } = (0, common_1.buildApiProperties)(exports.name, exports.displayName, createProperties());
|
|
32
|
+
exports.options = options;
|
|
33
|
+
exports.properties = properties;
|
|
34
|
+
options.routing = {
|
|
35
|
+
request: {
|
|
36
|
+
method: 'DELETE',
|
|
37
|
+
},
|
|
38
|
+
};
|
|
39
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../nodes/Firecrawl/api/cancelCrawl/index.ts"],"names":[],"mappings":";;;AACA,sCAAsE;AAEzD,QAAA,IAAI,GAAG,aAAa,CAAC;AACrB,QAAA,WAAW,GAAG,oBAAoB,CAAC;AAEhD,SAAS,qBAAqB;IAC7B,OAAO;QACN,WAAW,EAAE,UAAU;QACvB,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,IAAI;QACd,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,+BAA+B;QAC5C,OAAO,EAAE;YACR,OAAO,EAAE;gBACR,GAAG,EAAE,oBAAoB;aACzB;SACD;QACD,cAAc,EAAE;YACf,IAAI,EAAE;gBACL,QAAQ,EAAE,CAAC,SAAS,CAAC;gBACrB,SAAS,EAAE,CAAC,YAAI,CAAC;aACjB;SACD;KACD,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB;IACxB,OAAO,CAAC,IAAA,8BAAqB,EAAC,SAAS,EAAE,YAAI,EAAE,QAAQ,CAAC,EAAE,qBAAqB,EAAE,CAAC,CAAC;AACpF,CAAC;AAED,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,IAAA,2BAAkB,EAAC,YAAI,EAAE,mBAAW,EAAE,gBAAgB,EAAE,CAAC,CAAC;AAQjF,0BAAO;AAAE,gCAAU;AAN5B,OAAO,CAAC,OAAO,GAAG;IACjB,OAAO,EAAE;QACR,MAAM,EAAE,QAAQ;KAChB;CACD,CAAC"}
|
|
@@ -5,7 +5,8 @@ export declare function createUrlProperty(operationName: string, defaultUrl?: st
|
|
|
5
5
|
export declare function createActionsProperty(operationName: string, omitDisplayOptions?: boolean, useNestedScrapeOptions?: boolean): INodeProperties;
|
|
6
6
|
export declare function createLocationProperty(operationName: string, omitDisplayOptions?: boolean, useNestedScrapeOptions?: boolean): INodeProperties;
|
|
7
7
|
export declare function createIncludeTagsProperty(operationName: string, omitDisplayOptions?: boolean, useNestedScrapeOptions?: boolean): INodeProperties;
|
|
8
|
+
export declare function createBatchUrlsProperty(operationName: string, defaultUrl?: string, resourceName?: string): INodeProperties;
|
|
8
9
|
export declare function createExcludeTagsProperty(operationName: string, omitDisplayOptions?: boolean, useNestedScrapeOptions?: boolean): INodeProperties;
|
|
9
|
-
export declare function createScrapeOptionsProperty(operationName: string, useNestedScrapeOptions?: boolean): INodeProperties;
|
|
10
|
+
export declare function createScrapeOptionsProperty(operationName: string, useNestedScrapeOptions?: boolean, batchMode?: boolean): INodeProperties;
|
|
10
11
|
export declare function createOperationOption(name: string, action: string): INodePropertyOptions;
|
|
11
12
|
export declare function buildApiProperties(name: string, action: string, properties: INodeProperties[]): import("../helpers").IPropertiesWithOptions;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.buildApiProperties = exports.createOperationOption = exports.createScrapeOptionsProperty = exports.createExcludeTagsProperty = exports.createIncludeTagsProperty = exports.createLocationProperty = exports.createActionsProperty = exports.createUrlProperty = exports.createOperationNotice = exports.formatOperationName = void 0;
|
|
3
|
+
exports.buildApiProperties = exports.createOperationOption = exports.createScrapeOptionsProperty = exports.createExcludeTagsProperty = exports.createBatchUrlsProperty = exports.createIncludeTagsProperty = exports.createLocationProperty = exports.createActionsProperty = exports.createUrlProperty = exports.createOperationNotice = exports.formatOperationName = void 0;
|
|
4
4
|
const helpers_1 = require("../helpers");
|
|
5
5
|
function formatOperationName(name) {
|
|
6
6
|
return name
|
|
@@ -355,6 +355,48 @@ function createIncludeTagsProperty(operationName, omitDisplayOptions = false, us
|
|
|
355
355
|
};
|
|
356
356
|
}
|
|
357
357
|
exports.createIncludeTagsProperty = createIncludeTagsProperty;
|
|
358
|
+
function createBatchUrlsProperty(operationName, defaultUrl = 'https://firecrawl.dev', resourceName = 'Default') {
|
|
359
|
+
return {
|
|
360
|
+
displayName: 'URLs',
|
|
361
|
+
name: 'urls',
|
|
362
|
+
type: 'fixedCollection',
|
|
363
|
+
typeOptions: {
|
|
364
|
+
multipleValues: true,
|
|
365
|
+
},
|
|
366
|
+
default: {},
|
|
367
|
+
description: 'List of URLs to process in batch',
|
|
368
|
+
placeholder: 'Add URL',
|
|
369
|
+
options: [
|
|
370
|
+
{
|
|
371
|
+
name: 'urlValues',
|
|
372
|
+
displayName: 'URL',
|
|
373
|
+
values: [
|
|
374
|
+
{
|
|
375
|
+
displayName: 'URL',
|
|
376
|
+
name: 'url',
|
|
377
|
+
type: 'string',
|
|
378
|
+
default: defaultUrl,
|
|
379
|
+
description: 'URL to process',
|
|
380
|
+
},
|
|
381
|
+
],
|
|
382
|
+
},
|
|
383
|
+
],
|
|
384
|
+
routing: {
|
|
385
|
+
request: {
|
|
386
|
+
body: {
|
|
387
|
+
urls: '={{ $value.urlValues ? $value.urlValues.map(u => u.url) : [] }}',
|
|
388
|
+
},
|
|
389
|
+
},
|
|
390
|
+
},
|
|
391
|
+
displayOptions: {
|
|
392
|
+
show: {
|
|
393
|
+
resource: [resourceName],
|
|
394
|
+
operation: [operationName],
|
|
395
|
+
},
|
|
396
|
+
},
|
|
397
|
+
};
|
|
398
|
+
}
|
|
399
|
+
exports.createBatchUrlsProperty = createBatchUrlsProperty;
|
|
358
400
|
function createExcludeTagsProperty(operationName, omitDisplayOptions = false, useNestedScrapeOptions = true) {
|
|
359
401
|
return {
|
|
360
402
|
displayName: 'Exclude Tags',
|
|
@@ -409,10 +451,136 @@ function createExcludeTagsProperty(operationName, omitDisplayOptions = false, us
|
|
|
409
451
|
};
|
|
410
452
|
}
|
|
411
453
|
exports.createExcludeTagsProperty = createExcludeTagsProperty;
|
|
412
|
-
function
|
|
454
|
+
function createBatchSpecificProperties() {
|
|
455
|
+
return [
|
|
456
|
+
{
|
|
457
|
+
displayName: 'Webhook',
|
|
458
|
+
name: 'webhook',
|
|
459
|
+
type: 'fixedCollection',
|
|
460
|
+
default: {},
|
|
461
|
+
description: 'Webhook settings to receive notifications for batch scrape events',
|
|
462
|
+
options: [
|
|
463
|
+
{
|
|
464
|
+
displayName: 'Webhook Settings',
|
|
465
|
+
name: 'settings',
|
|
466
|
+
values: [
|
|
467
|
+
{
|
|
468
|
+
displayName: 'URL',
|
|
469
|
+
name: 'url',
|
|
470
|
+
type: 'string',
|
|
471
|
+
required: true,
|
|
472
|
+
default: '',
|
|
473
|
+
description: 'The URL to send the webhook to. Triggers for batchScrape.started, batchScrape.page, batchScrape.completed, or batchScrape.failed events.',
|
|
474
|
+
},
|
|
475
|
+
{
|
|
476
|
+
displayName: 'Headers',
|
|
477
|
+
name: 'headers',
|
|
478
|
+
type: 'fixedCollection',
|
|
479
|
+
default: {},
|
|
480
|
+
typeOptions: {
|
|
481
|
+
multipleValues: true,
|
|
482
|
+
},
|
|
483
|
+
description: 'Headers to send to the webhook URL',
|
|
484
|
+
options: [
|
|
485
|
+
{
|
|
486
|
+
displayName: 'Header',
|
|
487
|
+
name: 'header',
|
|
488
|
+
values: [
|
|
489
|
+
{
|
|
490
|
+
displayName: 'Key',
|
|
491
|
+
name: 'key',
|
|
492
|
+
type: 'string',
|
|
493
|
+
default: '',
|
|
494
|
+
description: 'Header key',
|
|
495
|
+
},
|
|
496
|
+
{
|
|
497
|
+
displayName: 'Value',
|
|
498
|
+
name: 'value',
|
|
499
|
+
type: 'string',
|
|
500
|
+
default: '',
|
|
501
|
+
description: 'Header value',
|
|
502
|
+
},
|
|
503
|
+
],
|
|
504
|
+
},
|
|
505
|
+
],
|
|
506
|
+
},
|
|
507
|
+
{
|
|
508
|
+
displayName: 'Metadata',
|
|
509
|
+
name: 'metadata',
|
|
510
|
+
type: 'json',
|
|
511
|
+
default: '{}',
|
|
512
|
+
description: 'Custom metadata that will be included in all webhook payloads for this crawl',
|
|
513
|
+
},
|
|
514
|
+
{
|
|
515
|
+
displayName: 'Events',
|
|
516
|
+
name: 'events',
|
|
517
|
+
type: 'multiOptions',
|
|
518
|
+
options: [
|
|
519
|
+
{
|
|
520
|
+
name: 'Started',
|
|
521
|
+
value: 'started',
|
|
522
|
+
},
|
|
523
|
+
{
|
|
524
|
+
name: 'Page',
|
|
525
|
+
value: 'page',
|
|
526
|
+
},
|
|
527
|
+
{
|
|
528
|
+
name: 'Completed',
|
|
529
|
+
value: 'completed',
|
|
530
|
+
},
|
|
531
|
+
{
|
|
532
|
+
name: 'Failed',
|
|
533
|
+
value: 'failed',
|
|
534
|
+
},
|
|
535
|
+
],
|
|
536
|
+
default: ['completed', 'page', 'failed', 'started'],
|
|
537
|
+
description: 'Type of events that should be sent to the webhook URL',
|
|
538
|
+
},
|
|
539
|
+
],
|
|
540
|
+
},
|
|
541
|
+
],
|
|
542
|
+
routing: {
|
|
543
|
+
request: {
|
|
544
|
+
body: {
|
|
545
|
+
webhook: '={{$value.settings ? { url: $value.settings.url, headers: $value.settings.headers?.header?.reduce((acc, h) => ({ ...acc, [h.key]: h.value }), {}) || {}, metadata: $value.settings.metadata ? JSON.parse($value.settings.metadata) : {}, events: $value.settings.events || [] } : undefined}}',
|
|
546
|
+
},
|
|
547
|
+
},
|
|
548
|
+
},
|
|
549
|
+
},
|
|
550
|
+
{
|
|
551
|
+
displayName: 'Max Concurrency',
|
|
552
|
+
name: 'maxConcurrency',
|
|
553
|
+
type: 'number',
|
|
554
|
+
default: 100,
|
|
555
|
+
description: "Maximum number of concurrent scrapes. If not specified, adheres to your team's concurrency limit.",
|
|
556
|
+
},
|
|
557
|
+
{
|
|
558
|
+
displayName: 'Ignore Invalid URLs',
|
|
559
|
+
name: 'ignoreInvalidURLs',
|
|
560
|
+
type: 'boolean',
|
|
561
|
+
default: true,
|
|
562
|
+
description: 'Whether invalid URLs should be ignored. Instead of failing the entire request, a batch scrape using the remaining valid URLs will be created.',
|
|
563
|
+
},
|
|
564
|
+
{
|
|
565
|
+
displayName: 'Store In Cache',
|
|
566
|
+
name: 'storeInCache',
|
|
567
|
+
type: 'boolean',
|
|
568
|
+
default: true,
|
|
569
|
+
description: 'Whether the page will be stored in the Firecrawl index and cache. Setting this to false is useful if your scraping activity may have data protection concerns.',
|
|
570
|
+
},
|
|
571
|
+
{
|
|
572
|
+
displayName: 'Zero Data Retention',
|
|
573
|
+
name: 'zeroDataRetention',
|
|
574
|
+
type: 'boolean',
|
|
575
|
+
default: false,
|
|
576
|
+
description: 'Whether to ensure no data is retained on the server after the scrape is complete',
|
|
577
|
+
},
|
|
578
|
+
];
|
|
579
|
+
}
|
|
580
|
+
function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = true, batchMode = false) {
|
|
413
581
|
const scrapeOptionsBody = useNestedScrapeOptions
|
|
414
582
|
? { scrapeOptions: '={{$value.options}}' }
|
|
415
|
-
: '={{$value.options}}';
|
|
583
|
+
: '={{Object.fromEntries(Object.entries($value.options || {}).filter(([k]) => k !== "webhook"))}}';
|
|
416
584
|
return {
|
|
417
585
|
displayName: 'Scrape Options',
|
|
418
586
|
name: 'scrapeOptions',
|
|
@@ -731,6 +899,7 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
|
|
|
731
899
|
},
|
|
732
900
|
],
|
|
733
901
|
},
|
|
902
|
+
...(batchMode ? createBatchSpecificProperties() : []),
|
|
734
903
|
],
|
|
735
904
|
},
|
|
736
905
|
],
|