n8n-nodes-crawlbrulee 0.3.1 → 1.0.1

package/README.md

@@ -8,7 +8,7 @@
 put crawlbrulee in your n8n workflows. this community node package turns any url into clean markdown, html, links, images, metadata or a screenshot, maps a site's urls, and starts a workflow when a background scrape finishes. it also works as an [AI Agent](https://docs.n8n.io/advanced-ai/) tool, so an agent in n8n can scrape a page on its own.
 
 - **everything runs in the EU.** the fetch, the render, the cache and your result never leave EU servers. the proxy exit is the one hop you choose: pick an EU exit and nothing leaves at all. gdpr-aligned, with a data processing agreement.
-- **output made for models.** markdown with the page chrome stripped and the links kept, ready for the prompt. full-page screenshots can come back as tiles sized for an image model.
+- **output made for models.** markdown with ads, popups and cookie banners removed and the links kept, ready for the prompt. full-page screenshots can come back as tiles sized for an image model.
 - **the hard parts, handled.** headless Chrome when a page needs it, rotating proxies with country selection, automatic retries, ad and cookie-banner removal, caching, background jobs and signed webhooks.
 - **start free.** 750 credits, no credit card.
 
@@ -60,6 +60,41 @@ every field has a hint in n8n. three things the hints don't tell you:
 - a **Screenshot** renders the page in a real browser and is billed at the screenshot rate, the priciest one — leave it on None when you don't need the image.
 - **Scrape URL (Async)** takes the same fields as **Scrape URL**, minus **Download Screenshot** (a submit gives you a job id, not an image), plus **Webhook URL** and **Webhook Metadata**.
 
+### elements
+
+**Elements** reads named values from the page by CSS selector. add one row per value: a **Name**, a **CSS Selector**, the **Output** (Text, HTML or an Attribute such as `href`), and **All Matches** to get every match as a list. they come back in `elements`, under the names you gave. elements cost no extra credits. **Extract** starts on Cleaned HTML and Metadata, so clear it to get only the elements.
+
+for a value per match, like a title, price and link per product card, put the request in **Elements (JSON)** under **Options**. it is merged with the list, and each name may be used once:
+
+```json
+{
+  "books": {
+    "selector": "article.product_pod",
+    "all": true,
+    "fields": {
+      "title": { "selector": "h3 a", "output": "attribute", "attribute": "title" },
+      "price": ".price_color"
+    }
+  }
+}
+```
+
+with a **heading** row (`h1`, Text), the item's json gets this (the page has 20 books; two are shown here):
+
+```json
+{
+  "elements": {
+    "heading": "All products",
+    "books": [
+      { "title": "A Light in the Attic", "price": "£51.77" },
+      { "title": "Tipping the Velvet", "price": "£53.74" }
+    ]
+  }
+}
+```
+
+a name with no match is `null` (`[]` with All Matches). selector rules and limits: see [elements](https://crawlbrulee.com/docs/scrape/elements).
+
 ### screenshots as files
 
 turn on **Download Screenshot** — it's on **Scrape URL** and on **Get Scrape Result** — and the node attaches the image as binary data under `screenshot`, plus one key per tile (`screenshot_slice_0`, `screenshot_slice_1`, …) when you set Slice Height. the urls stay in the json too, but they are signed links that expire 24 hours after the scrape (for **Scrape URL (Async)**, 24 hours after the job was submitted), so turn this on when you need the image later in the workflow. a download can fail, since a signed url can expire; the item still comes through with its page content and the reason lands on the json as `screenshot_download_error`.

package/dist/nodes/Crawlbrulee/Crawlbrulee.node.js

@@ -57,6 +57,7 @@ function scrapeParams(i) {
     return {
         url: this.getNodeParameter('url', i),
         extract: this.getNodeParameter('extract', i, []),
+        elements: this.getNodeParameter('elements', i, {}),
         screenshotType: this.getNodeParameter('screenshotType', i, 'none'),
         screenshotOptions: this.getNodeParameter('screenshotOptions', i, {}),
         options: this.getNodeParameter('options', i, {}),

package/dist/nodes/Crawlbrulee/builders/scrape.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.EXTRACT_KEYS = void 0;
 exports.parseCacheMaxAge = parseCacheMaxAge;
 exports.isSet = isSet;
+exports.buildElements = buildElements;
 exports.buildScrapeBody = buildScrapeBody;
 exports.buildAsyncScrapeBody = buildAsyncScrapeBody;
 const n8n_workflow_1 = require("n8n-workflow");
@@ -67,11 +68,98 @@ function buildScreenshot(node, itemIndex, p) {
     }
     return shot;
 }
+function parseJsonText(node, itemIndex, text, field) {
+    try {
+        return JSON.parse(text);
+    }
+    catch (error) {
+        const reason = error instanceof Error ? error.message : '';
+        const safe = reason !== '' && reason.length <= 120 && !reason.includes('"');
+        throw new n8n_workflow_1.NodeOperationError(node, `${field} is not valid JSON`, {
+            itemIndex,
+            description: safe ? `${reason}.` : 'Check for a missing quote, comma or bracket.',
+        });
+    }
+}
+function parseElementsJson(node, itemIndex, raw) {
+    if (raw === undefined || raw === null)
+        return {};
+    if (typeof raw === 'string' && raw.trim() === '')
+        return {};
+    const parsed = typeof raw === 'string' ? parseJsonText(node, itemIndex, raw, 'Elements (JSON)') : raw;
+    if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+        throw new n8n_workflow_1.NodeOperationError(node, 'Elements (JSON) must be a JSON object', {
+            itemIndex,
+            description: 'Each key is a name, each value a CSS selector or an object.',
+        });
+    }
+    for (const [name, value] of Object.entries(parsed)) {
+        const ok = typeof value === 'string' || (typeof value === 'object' && value !== null);
+        if (!ok || Array.isArray(value)) {
+            throw new n8n_workflow_1.NodeOperationError(node, `Element "${name}" in Elements (JSON) must be a CSS selector or an object`, { itemIndex });
+        }
+    }
+    return parsed;
+}
+function elementFromRow(node, itemIndex, row, name) {
+    var _a, _b, _c;
+    const selector = String((_a = row.selector) !== null && _a !== void 0 ? _a : '').trim();
+    if (selector === '') {
+        throw new n8n_workflow_1.NodeOperationError(node, `Element "${name}" needs a CSS Selector`, { itemIndex });
+    }
+    const output = (_b = row.output) !== null && _b !== void 0 ? _b : 'text';
+    const all = row.all === true;
+    if (output === 'text' && !all)
+        return selector;
+    const spec = { selector };
+    if (output !== 'text')
+        spec.output = output;
+    if (output === 'attribute') {
+        const attribute = String((_c = row.attribute) !== null && _c !== void 0 ? _c : '').trim();
+        if (attribute === '') {
+            throw new n8n_workflow_1.NodeOperationError(node, `Element "${name}" needs an Attribute`, {
+                itemIndex,
+                description: 'Set the attribute to read, for example href.',
+            });
+        }
+        spec.attribute = attribute;
+    }
+    if (all)
+        spec.all = true;
+    return spec;
+}
+function buildElements(node, itemIndex, p) {
+    var _a, _b, _c, _d, _e;
+    const elements = {};
+    const twice = (name) => new n8n_workflow_1.NodeOperationError(node, `Element name "${name}" is used twice`, {
+        itemIndex,
+        description: 'Each name in Elements and Elements (JSON) must be different.',
+    });
+    for (const row of (_b = (_a = p.elements) === null || _a === void 0 ? void 0 : _a.element) !== null && _b !== void 0 ? _b : []) {
+        const name = String((_c = row.name) !== null && _c !== void 0 ? _c : '').trim();
+        if (name === '' && String((_d = row.selector) !== null && _d !== void 0 ? _d : '').trim() === '')
+            continue;
+        if (name === '') {
+            throw new n8n_workflow_1.NodeOperationError(node, 'Each Element needs a Name', { itemIndex });
+        }
+        if (Object.prototype.hasOwnProperty.call(elements, name))
+            throw twice(name);
+        elements[name] = elementFromRow(node, itemIndex, row, name);
+    }
+    const fromJson = parseElementsJson(node, itemIndex, (_e = p.options) === null || _e === void 0 ? void 0 : _e.elementsJson);
+    for (const [name, value] of Object.entries(fromJson)) {
+        if (Object.prototype.hasOwnProperty.call(elements, name))
+            throw twice(name);
+        elements[name] = value;
+    }
+    return Object.keys(elements).length > 0 ? elements : undefined;
+}
 function buildScrapeBody(node, itemIndex, p) {
     var _a, _b, _c, _d, _e;
     const chosen = new Set((_a = p.extract) !== null && _a !== void 0 ? _a : []);
-    if (chosen.size === 0 && p.screenshotType === 'none') {
-        throw new n8n_workflow_1.NodeOperationError(node, 'Pick at least one Extract output or a Screenshot', {
+    const elements = buildElements(node, itemIndex, p);
+    if (chosen.size === 0 && !elements && p.screenshotType === 'none') {
+        throw new n8n_workflow_1.NodeOperationError(node, 'Pick at least one Extract output, an Element or a Screenshot', {
             itemIndex,
             description: 'The request would ask for nothing.',
         });
@@ -79,6 +167,8 @@ function buildScrapeBody(node, itemIndex, p) {
     const extract = {};
     for (const key of exports.EXTRACT_KEYS)
         extract[key] = chosen.has(key);
+    if (elements)
+        extract.elements = elements;
     const screenshot = buildScreenshot(node, itemIndex, p);
     if (screenshot)
         extract.screenshot = screenshot;
@@ -117,17 +207,8 @@ function parseMetadata(node, itemIndex, raw) {
         return undefined;
     if (typeof raw === 'object')
         return raw;
-    let parsed;
-    let ok = true;
-    try {
-        parsed = JSON.parse(raw);
-        if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed))
-            ok = false;
-    }
-    catch {
-        ok = false;
-    }
-    if (!ok)
+    const parsed = parseJsonText(node, itemIndex, raw, 'Webhook Metadata');
+    if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed))
         throw new n8n_workflow_1.NodeOperationError(node, 'Webhook Metadata must be a JSON object', { itemIndex });
     return parsed;
 }

package/dist/nodes/Crawlbrulee/descriptions/scrape.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.scrapeFields = exports.scrapeOperations = void 0;
 const shared_1 = require("./shared");
+const ELEMENTS_DOCS = 'Selector rules and limits: <a href="https://crawlbrulee.com/docs/scrape/elements">elements docs</a>';
 const resourceShow = { resource: ['scrape'] };
 const urlShow = { resource: ['scrape'], operation: ['scrape', 'scrapeAsync'] };
 const screenshotShow = { ...urlShow, screenshotType: ['full_page', 'viewport'] };
@@ -87,6 +88,72 @@ exports.scrapeFields = [
         description: 'Which outputs to return',
         displayOptions: { show: urlShow },
     },
+    {
+        displayName: 'Elements',
+        name: 'elements',
+        type: 'fixedCollection',
+        typeOptions: { multipleValues: true },
+        placeholder: 'Add Element',
+        default: {},
+        description: `Values to read from the page by CSS selector. They come back in elements, under the names you give. No extra credits. ${ELEMENTS_DOCS}.`,
+        displayOptions: { show: urlShow },
+        options: [
+            {
+                name: 'element',
+                displayName: 'Element',
+                values: [
+                    {
+                        displayName: 'All Matches',
+                        name: 'all',
+                        type: 'boolean',
+                        default: false,
+                        description: 'Whether to return every match as a list. Off returns the first match, or null when nothing matches.',
+                    },
+                    {
+                        displayName: 'Attribute',
+                        name: 'attribute',
+                        type: 'string',
+                        default: '',
+                        placeholder: 'e.g. href',
+                        description: 'The attribute to read',
+                        displayOptions: { show: { output: ['attribute'] } },
+                    },
+                    {
+                        displayName: 'CSS Selector',
+                        name: 'selector',
+                        type: 'string',
+                        default: '',
+                        placeholder: 'e.g. .price',
+                        description: 'Which element on the page to read',
+                    },
+                    {
+                        displayName: 'Name',
+                        name: 'name',
+                        type: 'string',
+                        default: '',
+                        placeholder: 'e.g. price',
+                        description: 'The key this value gets in elements',
+                    },
+                    {
+                        displayName: 'Output',
+                        name: 'output',
+                        type: 'options',
+                        options: [
+                            {
+                                name: 'Attribute',
+                                value: 'attribute',
+                                description: 'One attribute, like href. href and src come back as full URLs.',
+                            },
+                            { name: 'HTML', value: 'html', description: 'The HTML of the element itself' },
+                            { name: 'Text', value: 'text', description: 'The text inside the element' },
+                        ],
+                        default: 'text',
+                        description: 'What to read from each match',
+                    },
+                ],
+            },
+        ],
+    },
     {
         displayName: 'Screenshot',
         name: 'screenshotType',
@@ -219,6 +286,14 @@ exports.scrapeFields = [
         options: [
             (0, shared_1.cacheMaxAgeOption)('2 days'),
             shared_1.countryOption,
+            {
+                displayName: 'Elements (JSON)',
+                name: 'elementsJson',
+                type: 'json',
+                default: '',
+                placeholder: 'e.g. {"books": {"selector": "article.product_pod", "all": true, "fields": {"title": {"selector": "h3 a", "output": "attribute", "attribute": "title"}, "price": ".price_color"}}}',
+                description: `Elements as a JSON object, merged with the Elements list. Use it for fields, which read values inside each match, like a title and price per product card. ${ELEMENTS_DOCS}.`,
+            },
             {
                 displayName: 'Exclude Selectors',
                 name: 'excludeSelectors',
@@ -226,7 +301,7 @@ exports.scrapeFields = [
                 typeOptions: { multipleValues: true, multipleValueButtonText: 'Add Selector' },
                 default: [],
                 placeholder: 'e.g. .cookie-banner',
-                description: 'CSS selectors removed before anything is extracted. Sending any skips the cache.',
+                description: 'CSS selectors removed before anything is extracted, Elements included, except Raw HTML. Requests with different selectors do not share cached results.',
             },
             shared_1.localeOption,
             shared_1.proxyOption,

package/dist/nodes/Crawlbrulee/transport/request.js

@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.USER_AGENT = void 0;
 exports.crawlbruleeRequest = crawlbruleeRequest;
 const errors_1 = require("./errors");
-exports.USER_AGENT = 'n8n-nodes-crawlbrulee/0.3.1';
+exports.USER_AGENT = 'n8n-nodes-crawlbrulee/1.0.1';
 function stripTrailingSlash(url) {
     return url.replace(/\/+$/, '');
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "n8n-nodes-crawlbrulee",
-	"version": "0.3.1",
+	"version": "1.0.1",
 	"description": "crawlbrulee for n8n — EU-native web scraping for AI agents & developers. scrape any url to markdown, html, links, images, metadata or a screenshot, map a site's urls, and react to finished background scrapes.",
 	"license": "MIT",
 	"homepage": "https://crawlbrulee.com",
@@ -57,7 +57,7 @@
 		]
 	},
 	"devDependencies": {
-		"@crawlbrulee/sdk": "^1.2.1",
+		"@crawlbrulee/sdk": "^1.3.0",
 		"@n8n/node-cli": "0.48.6",
 		"@types/node": "^22.0.0",
 		"eslint": "9.32.0",