n8n-nodes-crawlbrulee 0.3.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -47,7 +47,7 @@ pick a resource, then an operation:
|
|
|
47
47
|
| Scrape | Scrape URL | fetch one url, get back the content you asked for |
|
|
48
48
|
| Scrape | Scrape URL (Async) | start a background job, get back its job id |
|
|
49
49
|
| Scrape | Get Scrape Status | pending, running, done or failed |
|
|
50
|
-
| Scrape | Get Scrape Result | the content of a finished job |
|
|
50
|
+
| Scrape | Get Scrape Result | the content of a finished job, for 24 hours after it was submitted |
|
|
51
51
|
| Map | Map Website | a site's urls, from its sitemap and its homepage links |
|
|
52
52
|
| Account | Get Credit Usage | credits used and left this billing cycle, plus your concurrency limit |
|
|
53
53
|
| Account | Get Account Info | the organization and token behind the api key |
|
|
@@ -60,9 +60,44 @@ every field has a hint in n8n. three things the hints don't tell you:
|
|
|
60
60
|
- a **Screenshot** renders the page in a real browser and is billed at the screenshot rate, the priciest one — leave it on None when you don't need the image.
|
|
61
61
|
- **Scrape URL (Async)** takes the same fields as **Scrape URL**, minus **Download Screenshot** (a submit gives you a job id, not an image), plus **Webhook URL** and **Webhook Metadata**.
|
|
62
62
|
|
|
63
|
+
### elements
|
|
64
|
+
|
|
65
|
+
**Elements** reads named values from the page by CSS selector. add one row per value: a **Name**, a **CSS Selector**, the **Output** (Text, HTML or an Attribute such as `href`), and **All Matches** to get every match as a list. they come back in `elements`, under the names you gave. elements cost no extra credits. **Extract** starts on Cleaned HTML and Metadata, so clear it to get only the elements.
|
|
66
|
+
|
|
67
|
+
for a value per match, like a title, price and link per product card, put the request in **Elements (JSON)** under **Options**. it is merged with the list, and each name may be used once:
|
|
68
|
+
|
|
69
|
+
```json
|
|
70
|
+
{
|
|
71
|
+
"books": {
|
|
72
|
+
"selector": "article.product_pod",
|
|
73
|
+
"all": true,
|
|
74
|
+
"fields": {
|
|
75
|
+
"title": { "selector": "h3 a", "output": "attribute", "attribute": "title" },
|
|
76
|
+
"price": ".price_color"
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
with a **heading** row (`h1`, Text), the item's json gets this (the page has 20 books; two are shown here):
|
|
83
|
+
|
|
84
|
+
```json
|
|
85
|
+
{
|
|
86
|
+
"elements": {
|
|
87
|
+
"heading": "All products",
|
|
88
|
+
"books": [
|
|
89
|
+
{ "title": "A Light in the Attic", "price": "£51.77" },
|
|
90
|
+
{ "title": "Tipping the Velvet", "price": "£53.74" }
|
|
91
|
+
]
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
a name with no match is `null` (`[]` with All Matches). selector rules and limits: see [elements](https://crawlbrulee.com/docs/scrape/elements).
|
|
97
|
+
|
|
63
98
|
### screenshots as files
|
|
64
99
|
|
|
65
|
-
turn on **Download Screenshot** — it's on **Scrape URL** and on **Get Scrape Result** — and the node attaches the image as binary data under `screenshot`, plus one key per tile (`screenshot_slice_0`, `screenshot_slice_1`, …) when you set Slice Height. the urls stay in the json too. a download can fail, since a signed url can expire; the item still comes through with its page content and the reason lands on the json as `screenshot_download_error`.
|
|
100
|
+
turn on **Download Screenshot** — it's on **Scrape URL** and on **Get Scrape Result** — and the node attaches the image as binary data under `screenshot`, plus one key per tile (`screenshot_slice_0`, `screenshot_slice_1`, …) when you set Slice Height. the urls stay in the json too, but they are signed links that expire 24 hours after the scrape (for **Scrape URL (Async)**, 24 hours after the job was submitted), so turn this on when you need the image later in the workflow. a download can fail, since a signed url can expire; the item still comes through with its page content and the reason lands on the json as `screenshot_download_error`.
|
|
66
101
|
|
|
67
102
|
### what comes back
|
|
68
103
|
|
|
@@ -57,6 +57,7 @@ function scrapeParams(i) {
|
|
|
57
57
|
return {
|
|
58
58
|
url: this.getNodeParameter('url', i),
|
|
59
59
|
extract: this.getNodeParameter('extract', i, []),
|
|
60
|
+
elements: this.getNodeParameter('elements', i, {}),
|
|
60
61
|
screenshotType: this.getNodeParameter('screenshotType', i, 'none'),
|
|
61
62
|
screenshotOptions: this.getNodeParameter('screenshotOptions', i, {}),
|
|
62
63
|
options: this.getNodeParameter('options', i, {}),
|
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.EXTRACT_KEYS = void 0;
|
|
4
4
|
exports.parseCacheMaxAge = parseCacheMaxAge;
|
|
5
5
|
exports.isSet = isSet;
|
|
6
|
+
exports.buildElements = buildElements;
|
|
6
7
|
exports.buildScrapeBody = buildScrapeBody;
|
|
7
8
|
exports.buildAsyncScrapeBody = buildAsyncScrapeBody;
|
|
8
9
|
const n8n_workflow_1 = require("n8n-workflow");
|
|
@@ -67,11 +68,98 @@ function buildScreenshot(node, itemIndex, p) {
|
|
|
67
68
|
}
|
|
68
69
|
return shot;
|
|
69
70
|
}
|
|
71
|
+
function parseJsonText(node, itemIndex, text, field) {
|
|
72
|
+
try {
|
|
73
|
+
return JSON.parse(text);
|
|
74
|
+
}
|
|
75
|
+
catch (error) {
|
|
76
|
+
const reason = error instanceof Error ? error.message : '';
|
|
77
|
+
const safe = reason !== '' && reason.length <= 120 && !reason.includes('"');
|
|
78
|
+
throw new n8n_workflow_1.NodeOperationError(node, `${field} is not valid JSON`, {
|
|
79
|
+
itemIndex,
|
|
80
|
+
description: safe ? `${reason}.` : 'Check for a missing quote, comma or bracket.',
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
function parseElementsJson(node, itemIndex, raw) {
|
|
85
|
+
if (raw === undefined || raw === null)
|
|
86
|
+
return {};
|
|
87
|
+
if (typeof raw === 'string' && raw.trim() === '')
|
|
88
|
+
return {};
|
|
89
|
+
const parsed = typeof raw === 'string' ? parseJsonText(node, itemIndex, raw, 'Elements (JSON)') : raw;
|
|
90
|
+
if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
|
|
91
|
+
throw new n8n_workflow_1.NodeOperationError(node, 'Elements (JSON) must be a JSON object', {
|
|
92
|
+
itemIndex,
|
|
93
|
+
description: 'Each key is a name, each value a CSS selector or an object.',
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
for (const [name, value] of Object.entries(parsed)) {
|
|
97
|
+
const ok = typeof value === 'string' || (typeof value === 'object' && value !== null);
|
|
98
|
+
if (!ok || Array.isArray(value)) {
|
|
99
|
+
throw new n8n_workflow_1.NodeOperationError(node, `Element "${name}" in Elements (JSON) must be a CSS selector or an object`, { itemIndex });
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
return parsed;
|
|
103
|
+
}
|
|
104
|
+
function elementFromRow(node, itemIndex, row, name) {
|
|
105
|
+
var _a, _b, _c;
|
|
106
|
+
const selector = String((_a = row.selector) !== null && _a !== void 0 ? _a : '').trim();
|
|
107
|
+
if (selector === '') {
|
|
108
|
+
throw new n8n_workflow_1.NodeOperationError(node, `Element "${name}" needs a CSS Selector`, { itemIndex });
|
|
109
|
+
}
|
|
110
|
+
const output = (_b = row.output) !== null && _b !== void 0 ? _b : 'text';
|
|
111
|
+
const all = row.all === true;
|
|
112
|
+
if (output === 'text' && !all)
|
|
113
|
+
return selector;
|
|
114
|
+
const spec = { selector };
|
|
115
|
+
if (output !== 'text')
|
|
116
|
+
spec.output = output;
|
|
117
|
+
if (output === 'attribute') {
|
|
118
|
+
const attribute = String((_c = row.attribute) !== null && _c !== void 0 ? _c : '').trim();
|
|
119
|
+
if (attribute === '') {
|
|
120
|
+
throw new n8n_workflow_1.NodeOperationError(node, `Element "${name}" needs an Attribute`, {
|
|
121
|
+
itemIndex,
|
|
122
|
+
description: 'Set the attribute to read, for example href.',
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
spec.attribute = attribute;
|
|
126
|
+
}
|
|
127
|
+
if (all)
|
|
128
|
+
spec.all = true;
|
|
129
|
+
return spec;
|
|
130
|
+
}
|
|
131
|
+
function buildElements(node, itemIndex, p) {
|
|
132
|
+
var _a, _b, _c, _d, _e;
|
|
133
|
+
const elements = {};
|
|
134
|
+
const twice = (name) => new n8n_workflow_1.NodeOperationError(node, `Element name "${name}" is used twice`, {
|
|
135
|
+
itemIndex,
|
|
136
|
+
description: 'Each name in Elements and Elements (JSON) must be different.',
|
|
137
|
+
});
|
|
138
|
+
for (const row of (_b = (_a = p.elements) === null || _a === void 0 ? void 0 : _a.element) !== null && _b !== void 0 ? _b : []) {
|
|
139
|
+
const name = String((_c = row.name) !== null && _c !== void 0 ? _c : '').trim();
|
|
140
|
+
if (name === '' && String((_d = row.selector) !== null && _d !== void 0 ? _d : '').trim() === '')
|
|
141
|
+
continue;
|
|
142
|
+
if (name === '') {
|
|
143
|
+
throw new n8n_workflow_1.NodeOperationError(node, 'Each Element needs a Name', { itemIndex });
|
|
144
|
+
}
|
|
145
|
+
if (Object.prototype.hasOwnProperty.call(elements, name))
|
|
146
|
+
throw twice(name);
|
|
147
|
+
elements[name] = elementFromRow(node, itemIndex, row, name);
|
|
148
|
+
}
|
|
149
|
+
const fromJson = parseElementsJson(node, itemIndex, (_e = p.options) === null || _e === void 0 ? void 0 : _e.elementsJson);
|
|
150
|
+
for (const [name, value] of Object.entries(fromJson)) {
|
|
151
|
+
if (Object.prototype.hasOwnProperty.call(elements, name))
|
|
152
|
+
throw twice(name);
|
|
153
|
+
elements[name] = value;
|
|
154
|
+
}
|
|
155
|
+
return Object.keys(elements).length > 0 ? elements : undefined;
|
|
156
|
+
}
|
|
70
157
|
function buildScrapeBody(node, itemIndex, p) {
|
|
71
158
|
var _a, _b, _c, _d, _e;
|
|
72
159
|
const chosen = new Set((_a = p.extract) !== null && _a !== void 0 ? _a : []);
|
|
73
|
-
|
|
74
|
-
|
|
160
|
+
const elements = buildElements(node, itemIndex, p);
|
|
161
|
+
if (chosen.size === 0 && !elements && p.screenshotType === 'none') {
|
|
162
|
+
throw new n8n_workflow_1.NodeOperationError(node, 'Pick at least one Extract output, an Element or a Screenshot', {
|
|
75
163
|
itemIndex,
|
|
76
164
|
description: 'The request would ask for nothing.',
|
|
77
165
|
});
|
|
@@ -79,6 +167,8 @@ function buildScrapeBody(node, itemIndex, p) {
|
|
|
79
167
|
const extract = {};
|
|
80
168
|
for (const key of exports.EXTRACT_KEYS)
|
|
81
169
|
extract[key] = chosen.has(key);
|
|
170
|
+
if (elements)
|
|
171
|
+
extract.elements = elements;
|
|
82
172
|
const screenshot = buildScreenshot(node, itemIndex, p);
|
|
83
173
|
if (screenshot)
|
|
84
174
|
extract.screenshot = screenshot;
|
|
@@ -117,17 +207,8 @@ function parseMetadata(node, itemIndex, raw) {
|
|
|
117
207
|
return undefined;
|
|
118
208
|
if (typeof raw === 'object')
|
|
119
209
|
return raw;
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
try {
|
|
123
|
-
parsed = JSON.parse(raw);
|
|
124
|
-
if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed))
|
|
125
|
-
ok = false;
|
|
126
|
-
}
|
|
127
|
-
catch {
|
|
128
|
-
ok = false;
|
|
129
|
-
}
|
|
130
|
-
if (!ok)
|
|
210
|
+
const parsed = parseJsonText(node, itemIndex, raw, 'Webhook Metadata');
|
|
211
|
+
if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed))
|
|
131
212
|
throw new n8n_workflow_1.NodeOperationError(node, 'Webhook Metadata must be a JSON object', { itemIndex });
|
|
132
213
|
return parsed;
|
|
133
214
|
}
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.scrapeFields = exports.scrapeOperations = void 0;
|
|
4
4
|
const shared_1 = require("./shared");
|
|
5
|
+
const ELEMENTS_DOCS = 'Selector rules and limits: <a href="https://crawlbrulee.com/docs/scrape/elements">elements docs</a>';
|
|
5
6
|
const resourceShow = { resource: ['scrape'] };
|
|
6
7
|
const urlShow = { resource: ['scrape'], operation: ['scrape', 'scrapeAsync'] };
|
|
7
8
|
const screenshotShow = { ...urlShow, screenshotType: ['full_page', 'viewport'] };
|
|
@@ -17,13 +18,13 @@ exports.scrapeOperations = {
|
|
|
17
18
|
name: 'Get Scrape Result',
|
|
18
19
|
value: 'getScrapeResult',
|
|
19
20
|
action: 'Get scrape result',
|
|
20
|
-
description: "Fetch the content of a finished async scrape. The site's own HTTP status is in page_status_code.",
|
|
21
|
+
description: "Fetch the content of a finished async scrape, up to 24 hours after it was submitted. The site's own HTTP status is in page_status_code.",
|
|
21
22
|
},
|
|
22
23
|
{
|
|
23
24
|
name: 'Get Scrape Status',
|
|
24
25
|
value: 'getScrapeStatus',
|
|
25
26
|
action: 'Get scrape status',
|
|
26
|
-
description: 'Check whether an async scrape is pending, running, done or failed',
|
|
27
|
+
description: 'Check whether an async scrape is pending, running, done or failed. Available for 24 hours after it was submitted.',
|
|
27
28
|
},
|
|
28
29
|
{
|
|
29
30
|
name: 'Scrape URL',
|
|
@@ -87,6 +88,72 @@ exports.scrapeFields = [
|
|
|
87
88
|
description: 'Which outputs to return',
|
|
88
89
|
displayOptions: { show: urlShow },
|
|
89
90
|
},
|
|
91
|
+
{
|
|
92
|
+
displayName: 'Elements',
|
|
93
|
+
name: 'elements',
|
|
94
|
+
type: 'fixedCollection',
|
|
95
|
+
typeOptions: { multipleValues: true },
|
|
96
|
+
placeholder: 'Add Element',
|
|
97
|
+
default: {},
|
|
98
|
+
description: `Values to read from the page by CSS selector. They come back in elements, under the names you give. No extra credits. ${ELEMENTS_DOCS}.`,
|
|
99
|
+
displayOptions: { show: urlShow },
|
|
100
|
+
options: [
|
|
101
|
+
{
|
|
102
|
+
name: 'element',
|
|
103
|
+
displayName: 'Element',
|
|
104
|
+
values: [
|
|
105
|
+
{
|
|
106
|
+
displayName: 'All Matches',
|
|
107
|
+
name: 'all',
|
|
108
|
+
type: 'boolean',
|
|
109
|
+
default: false,
|
|
110
|
+
description: 'Whether to return every match as a list. Off returns the first match, or null when nothing matches.',
|
|
111
|
+
},
|
|
112
|
+
{
|
|
113
|
+
displayName: 'Attribute',
|
|
114
|
+
name: 'attribute',
|
|
115
|
+
type: 'string',
|
|
116
|
+
default: '',
|
|
117
|
+
placeholder: 'e.g. href',
|
|
118
|
+
description: 'The attribute to read',
|
|
119
|
+
displayOptions: { show: { output: ['attribute'] } },
|
|
120
|
+
},
|
|
121
|
+
{
|
|
122
|
+
displayName: 'CSS Selector',
|
|
123
|
+
name: 'selector',
|
|
124
|
+
type: 'string',
|
|
125
|
+
default: '',
|
|
126
|
+
placeholder: 'e.g. .price',
|
|
127
|
+
description: 'Which element on the page to read',
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
displayName: 'Name',
|
|
131
|
+
name: 'name',
|
|
132
|
+
type: 'string',
|
|
133
|
+
default: '',
|
|
134
|
+
placeholder: 'e.g. price',
|
|
135
|
+
description: 'The key this value gets in elements',
|
|
136
|
+
},
|
|
137
|
+
{
|
|
138
|
+
displayName: 'Output',
|
|
139
|
+
name: 'output',
|
|
140
|
+
type: 'options',
|
|
141
|
+
options: [
|
|
142
|
+
{
|
|
143
|
+
name: 'Attribute',
|
|
144
|
+
value: 'attribute',
|
|
145
|
+
description: 'One attribute, like href. href and src come back as full URLs.',
|
|
146
|
+
},
|
|
147
|
+
{ name: 'HTML', value: 'html', description: 'The HTML of the element itself' },
|
|
148
|
+
{ name: 'Text', value: 'text', description: 'The text inside the element' },
|
|
149
|
+
],
|
|
150
|
+
default: 'text',
|
|
151
|
+
description: 'What to read from each match',
|
|
152
|
+
},
|
|
153
|
+
],
|
|
154
|
+
},
|
|
155
|
+
],
|
|
156
|
+
},
|
|
90
157
|
{
|
|
91
158
|
displayName: 'Screenshot',
|
|
92
159
|
name: 'screenshotType',
|
|
@@ -200,7 +267,7 @@ exports.scrapeFields = [
|
|
|
200
267
|
name: 'downloadScreenshot',
|
|
201
268
|
type: 'boolean',
|
|
202
269
|
default: false,
|
|
203
|
-
description: 'Whether to download the screenshot (and its slices) into binary data on the item',
|
|
270
|
+
description: 'Whether to download the screenshot (and its slices) into binary data on the item. Screenshot links are signed and expire 24 hours after the scrape.',
|
|
204
271
|
displayOptions: {
|
|
205
272
|
show: {
|
|
206
273
|
resource: ['scrape'],
|
|
@@ -219,6 +286,14 @@ exports.scrapeFields = [
|
|
|
219
286
|
options: [
|
|
220
287
|
(0, shared_1.cacheMaxAgeOption)('2 days'),
|
|
221
288
|
shared_1.countryOption,
|
|
289
|
+
{
|
|
290
|
+
displayName: 'Elements (JSON)',
|
|
291
|
+
name: 'elementsJson',
|
|
292
|
+
type: 'json',
|
|
293
|
+
default: '',
|
|
294
|
+
placeholder: 'e.g. {"books": {"selector": "article.product_pod", "all": true, "fields": {"title": {"selector": "h3 a", "output": "attribute", "attribute": "title"}, "price": ".price_color"}}}',
|
|
295
|
+
description: `Elements as a JSON object, merged with the Elements list. Use it for fields, which read values inside each match, like a title and price per product card. ${ELEMENTS_DOCS}.`,
|
|
296
|
+
},
|
|
222
297
|
{
|
|
223
298
|
displayName: 'Exclude Selectors',
|
|
224
299
|
name: 'excludeSelectors',
|
|
@@ -226,7 +301,7 @@ exports.scrapeFields = [
|
|
|
226
301
|
typeOptions: { multipleValues: true, multipleValueButtonText: 'Add Selector' },
|
|
227
302
|
default: [],
|
|
228
303
|
placeholder: 'e.g. .cookie-banner',
|
|
229
|
-
description: 'CSS selectors removed before anything is extracted.
|
|
304
|
+
description: 'CSS selectors removed before anything is extracted, Elements included, except Raw HTML. Requests with different selectors do not share cached results.',
|
|
230
305
|
},
|
|
231
306
|
shared_1.localeOption,
|
|
232
307
|
shared_1.proxyOption,
|
|
@@ -269,7 +344,7 @@ exports.scrapeFields = [
|
|
|
269
344
|
name: 'downloadScreenshot',
|
|
270
345
|
type: 'boolean',
|
|
271
346
|
default: false,
|
|
272
|
-
description: 'Whether to download the screenshot (and its slices) into binary data when the result has one',
|
|
347
|
+
description: 'Whether to download the screenshot (and its slices) into binary data when the result has one. Screenshot links expire 24 hours after the job was submitted.',
|
|
273
348
|
displayOptions: { show: { resource: ['scrape'], operation: ['getScrapeResult'] } },
|
|
274
349
|
},
|
|
275
350
|
];
|
|
@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.USER_AGENT = void 0;
|
|
4
4
|
exports.crawlbruleeRequest = crawlbruleeRequest;
|
|
5
5
|
const errors_1 = require("./errors");
|
|
6
|
-
exports.USER_AGENT = 'n8n-nodes-crawlbrulee/0.
|
|
6
|
+
exports.USER_AGENT = 'n8n-nodes-crawlbrulee/1.0.0';
|
|
7
7
|
function stripTrailingSlash(url) {
|
|
8
8
|
return url.replace(/\/+$/, '');
|
|
9
9
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "n8n-nodes-crawlbrulee",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "1.0.0",
|
|
4
4
|
"description": "crawlbrulee for n8n — EU-native web scraping for AI agents & developers. scrape any url to markdown, html, links, images, metadata or a screenshot, map a site's urls, and react to finished background scrapes.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"homepage": "https://crawlbrulee.com",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
]
|
|
58
58
|
},
|
|
59
59
|
"devDependencies": {
|
|
60
|
-
"@crawlbrulee/sdk": "^1.
|
|
60
|
+
"@crawlbrulee/sdk": "^1.3.0",
|
|
61
61
|
"@n8n/node-cli": "0.48.6",
|
|
62
62
|
"@types/node": "^22.0.0",
|
|
63
63
|
"eslint": "9.32.0",
|