n8n-nodes-octopus-scraping 0.1.0

package/dist/nodes/BrowserRun/BrowserRun.node.js

@@ -0,0 +1,706 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BrowserRun = void 0;
+const n8n_workflow_1 = require("n8n-workflow");
+class BrowserRun {
+    constructor() {
+        this.description = {
+            displayName: 'Octopus Scraping',
+            name: 'browserRun',
+            icon: 'file:octopus.svg',
+            group: ['transform'],
+            version: 1,
+            subtitle: '={{$parameter.action}}',
+            description: 'Interact with Octopus Scraping Worker for web scraping, screenshots, PDFs, and browser automation',
+            defaults: {
+                name: 'Octopus Scraping',
+            },
+            inputs: [n8n_workflow_1.NodeConnectionTypes.Main],
+            outputs: [n8n_workflow_1.NodeConnectionTypes.Main],
+            usableAsTool: true,
+            credentials: [
+                {
+                    name: 'browserRunApi',
+                    required: true,
+                },
+            ],
+            properties: [
+                {
+                    displayName: 'Action',
+                    name: 'action',
+                    type: 'options',
+                    noDataExpression: true,
+                    options: [
+                        { name: 'Batch', value: 'batch' },
+                        { name: 'Click', value: 'click' },
+                        { name: 'Cookies', value: 'cookies' },
+                        { name: 'Crawl', value: 'crawl' },
+                        { name: 'Evaluate JavaScript', value: 'evaluate' },
+                        { name: 'Extract Links', value: 'links' },
+                        { name: 'Health Check', value: 'health' },
+                        { name: 'JSON Extraction (AI)', value: 'json' },
+                        { name: 'Markdown', value: 'markdown' },
+                        { name: 'PDF', value: 'pdf' },
+                        { name: 'Run Custom Steps', value: 'run' },
+                        { name: 'Scrape (HTML)', value: 'scrape' },
+                        { name: 'Screenshot', value: 'screenshot' },
+                        { name: 'SEO Analysis', value: 'seo' },
+                        { name: 'Snapshot (Accessibility Tree)', value: 'snapshot' },
+                    ],
+                    default: 'screenshot',
+                },
+                {
+                    displayName: 'URL',
+                    name: 'url',
+                    type: 'string',
+                    default: '',
+                    required: true,
+                    placeholder: 'https://example.com',
+                    displayOptions: {
+                        show: {
+                            action: ['screenshot', 'pdf', 'scrape', 'markdown', 'snapshot', 'links', 'json', 'crawl', 'seo', 'click', 'cookies', 'evaluate'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Full Page',
+                    name: 'fullPage',
+                    type: 'boolean',
+                    default: true,
+                    description: 'Whether to capture the full page or just the viewport',
+                    displayOptions: {
+                        show: {
+                            action: ['screenshot'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Format',
+                    name: 'format',
+                    type: 'options',
+                    options: [
+                        { name: 'JPEG', value: 'jpeg' },
+                        { name: 'PNG', value: 'png' },
+                    ],
+                    default: 'png',
+                    displayOptions: {
+                        show: {
+                            action: ['screenshot'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Viewport Width',
+                    name: 'viewportWidth',
+                    type: 'number',
+                    default: 1920,
+                    description: 'Viewport width in pixels',
+                    displayOptions: {
+                        show: {
+                            action: ['screenshot', 'pdf'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Viewport Height',
+                    name: 'viewportHeight',
+                    type: 'number',
+                    default: 1080,
+                    description: 'Viewport height in pixels',
+                    displayOptions: {
+                        show: {
+                            action: ['screenshot', 'pdf'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Clip Region',
+                    name: 'clipRegion',
+                    type: 'collection',
+                    placeholder: 'Add Clip Region',
+                    default: {},
+                    displayOptions: {
+                        show: {
+                            action: ['screenshot'],
+                        },
+                    },
+                    options: [
+                        { displayName: 'X', name: 'x', type: 'number', default: 0 },
+                        { displayName: 'Y', name: 'y', type: 'number', default: 0 },
+                        { displayName: 'Width', name: 'width', type: 'number', default: 800 },
+                        { displayName: 'Height', name: 'height', type: 'number', default: 600 },
+                    ],
+                },
+                {
+                    displayName: 'Page Format',
+                    name: 'pageFormat',
+                    type: 'options',
+                    options: [
+                        { name: 'A4', value: 'A4' },
+                        { name: 'Legal', value: 'Legal' },
+                        { name: 'Letter', value: 'Letter' },
+                        { name: 'Tabloid', value: 'Tabloid' },
+                    ],
+                    default: 'A4',
+                    displayOptions: {
+                        show: {
+                            action: ['pdf'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Landscape',
+                    name: 'landscape',
+                    type: 'boolean',
+                    default: false,
+                    description: 'Whether to use landscape orientation',
+                    displayOptions: {
+                        show: {
+                            action: ['pdf'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Margin',
+                    name: 'margin',
+                    type: 'collection',
+                    placeholder: 'Add Margin',
+                    default: {},
+                    displayOptions: {
+                        show: {
+                            action: ['pdf'],
+                        },
+                    },
+                    options: [
+                        { displayName: 'Top', name: 'top', type: 'string', default: '1cm' },
+                        { displayName: 'Bottom', name: 'bottom', type: 'string', default: '1cm' },
+                        { displayName: 'Left', name: 'left', type: 'string', default: '1cm' },
+                        { displayName: 'Right', name: 'right', type: 'string', default: '1cm' },
+                    ],
+                },
+                {
+                    displayName: 'Selectors',
+                    name: 'selectors',
+                    type: 'json',
+                    default: '{}',
+                    description: 'CSS selectors to extract elements (JSON object)',
+                    displayOptions: {
+                        show: {
+                            action: ['scrape'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Wait Until',
+                    name: 'waitUntil',
+                    type: 'options',
+                    options: [
+                        { name: 'DOM Content Loaded', value: 'domcontentloaded' },
+                        { name: 'Load', value: 'load' },
+                        { name: 'Network Idle 0', value: 'networkidle0' },
+                        { name: 'Network Idle 2', value: 'networkidle2' },
+                    ],
+                    default: 'load',
+                    description: 'When to consider the page loaded',
+                    displayOptions: {
+                        show: {
+                            action: ['scrape', 'markdown', 'snapshot', 'links', 'json'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Wait For Selector',
+                    name: 'waitForSelector',
+                    type: 'string',
+                    default: '',
+                    description: 'CSS selector to wait for before scraping',
+                    displayOptions: {
+                        show: {
+                            action: ['scrape', 'markdown'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Prompt',
+                    name: 'prompt',
+                    type: 'string',
+                    default: '',
+                    description: 'Natural language prompt for AI extraction',
+                    displayOptions: {
+                        show: {
+                            action: ['json'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Schema',
+                    name: 'schema',
+                    type: 'json',
+                    default: '{}',
+                    description: 'JSON schema for structured extraction',
+                    displayOptions: {
+                        show: {
+                            action: ['json'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Max Depth',
+                    name: 'maxDepth',
+                    type: 'number',
+                    default: 2,
+                    description: 'Maximum crawl depth',
+                    displayOptions: {
+                        show: {
+                            action: ['crawl'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Max Pages',
+                    name: 'maxPages',
+                    type: 'number',
+                    default: 10,
+                    description: 'Maximum number of pages to crawl',
+                    displayOptions: {
+                        show: {
+                            action: ['crawl'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Include Pattern',
+                    name: 'includePattern',
+                    type: 'string',
+                    default: '',
+                    description: 'URL pattern to include (e.g., /blog/)',
+                    displayOptions: {
+                        show: {
+                            action: ['crawl'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Exclude Pattern',
+                    name: 'excludePattern',
+                    type: 'string',
+                    default: '',
+                    description: 'URL pattern to exclude (e.g., /tag/)',
+                    displayOptions: {
+                        show: {
+                            action: ['crawl'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Selector',
+                    name: 'selector',
+                    type: 'string',
+                    default: '',
+                    description: 'CSS selector for the element to click',
+                    displayOptions: {
+                        show: {
+                            action: ['click'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Wait After Click (MS)',
+                    name: 'waitFor',
+                    type: 'number',
+                    default: 2000,
+                    description: 'Time to wait after clicking in milliseconds',
+                    displayOptions: {
+                        show: {
+                            action: ['click'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Take Screenshot',
+                    name: 'takeScreenshot',
+                    type: 'boolean',
+                    default: false,
+                    description: 'Whether to take a screenshot after clicking',
+                    displayOptions: {
+                        show: {
+                            action: ['click'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Operation',
+                    name: 'cookieOperation',
+                    type: 'options',
+                    options: [
+                        { name: 'Get', value: 'get' },
+                    ],
+                    default: 'get',
+                    displayOptions: {
+                        show: {
+                            action: ['cookies'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'JavaScript',
+                    name: 'script',
+                    type: 'string',
+                    default: '() => document.title',
+                    description: 'JavaScript function to execute in page context',
+                    displayOptions: {
+                        show: {
+                            action: ['evaluate'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Wait Before (MS)',
+                    name: 'waitBefore',
+                    type: 'number',
+                    default: 0,
+                    description: 'Time to wait before executing script',
+                    displayOptions: {
+                        show: {
+                            action: ['evaluate'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'URLs',
+                    name: 'urls',
+                    type: 'string',
+                    default: '',
+                    description: 'Comma-separated list of URLs to process',
+                    displayOptions: {
+                        show: {
+                            action: ['batch'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Batch Action',
+                    name: 'batchAction',
+                    type: 'options',
+                    options: [
+                        { name: 'Markdown', value: 'markdown' },
+                        { name: 'Scrape', value: 'scrape' },
+                        { name: 'Screenshot', value: 'screenshot' },
+                    ],
+                    default: 'scrape',
+                    displayOptions: {
+                        show: {
+                            action: ['batch'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Parallel',
+                    name: 'parallel',
+                    type: 'boolean',
+                    default: false,
+                    description: 'Whether to process URLs in parallel',
+                    displayOptions: {
+                        show: {
+                            action: ['batch'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Steps',
+                    name: 'steps',
+                    type: 'json',
+                    default: '[]',
+                    description: 'Array of browser automation steps (JSON)',
+                    displayOptions: {
+                        show: {
+                            action: ['run'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Timeout (Ms)',
+                    name: 'timeout',
+                    type: 'number',
+                    default: 30000,
+                    description: 'Maximum execution time in milliseconds',
+                    displayOptions: {
+                        show: {
+                            action: ['run', 'crawl', 'batch'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Return Binary Data',
+                    name: 'returnBinary',
+                    type: 'boolean',
+                    default: false,
+                    description: 'Whether to return binary data (for screenshot/PDF) instead of base64 string',
+                    displayOptions: {
+                        show: {
+                            action: ['screenshot', 'pdf'],
+                        },
+                    },
+                },
+                {
+                    displayName: 'Binary Property',
+                    name: 'binaryProperty',
+                    type: 'string',
+                    default: 'data',
+                    description: 'Name of the binary property to save the file',
+                    displayOptions: {
+                        show: {
+                            action: ['screenshot', 'pdf'],
+                            returnBinary: [true],
+                        },
+                    },
+                },
+            ],
+        };
+    }
+    async execute() {
+        const items = this.getInputData();
+        const returnData = [];
+        const credentials = await this.getCredentials('browserRunApi');
+        const workerUrl = credentials.workerUrl;
+        const apiKey = credentials.apiKey;
+        for (let i = 0; i < items.length; i++) {
+            try {
+                const action = this.getNodeParameter('action', i);
+                const body = { action };
+                switch (action) {
+                    case 'health':
+                        break;
+                    case 'screenshot': {
+                        const url = this.getNodeParameter('url', i);
+                        body.url = url;
+                        body.fullPage = this.getNodeParameter('fullPage', i);
+                        body.format = this.getNodeParameter('format', i);
+                        const viewportWidth = this.getNodeParameter('viewportWidth', i);
+                        const viewportHeight = this.getNodeParameter('viewportHeight', i);
+                        if (viewportWidth || viewportHeight) {
+                            body.viewport = { width: viewportWidth, height: viewportHeight };
+                        }
+                        const clipRegion = this.getNodeParameter('clipRegion', i);
+                        if (Object.keys(clipRegion).length > 0) {
+                            body.clip = clipRegion;
+                        }
+                        break;
+                    }
+                    case 'pdf': {
+                        const url = this.getNodeParameter('url', i);
+                        body.url = url;
+                        body.format = this.getNodeParameter('pageFormat', i);
+                        body.landscape = this.getNodeParameter('landscape', i);
+                        const margin = this.getNodeParameter('margin', i);
+                        if (Object.keys(margin).length > 0) {
+                            body.margin = margin;
+                        }
+                        break;
+                    }
+                    case 'scrape': {
+                        const url = this.getNodeParameter('url', i);
+                        body.url = url;
+                        const selectors = this.getNodeParameter('selectors', i);
+                        if (selectors && selectors !== '{}') {
+                            body.selectors = JSON.parse(selectors);
+                        }
+                        const waitUntil = this.getNodeParameter('waitUntil', i);
+                        if (waitUntil !== 'load') {
+                            body.waitUntil = waitUntil;
+                        }
+                        const waitForSelector = this.getNodeParameter('waitForSelector', i);
+                        if (waitForSelector) {
+                            body.waitForSelector = waitForSelector;
+                        }
+                        break;
+                    }
+                    case 'markdown': {
+                        const url = this.getNodeParameter('url', i);
+                        body.url = url;
+                        const waitUntil = this.getNodeParameter('waitUntil', i);
+                        if (waitUntil !== 'load') {
+                            body.waitUntil = waitUntil;
+                        }
+                        const waitForSelector = this.getNodeParameter('waitForSelector', i);
+                        if (waitForSelector) {
+                            body.waitForSelector = waitForSelector;
+                        }
+                        break;
+                    }
+                    case 'snapshot': {
+                        const url = this.getNodeParameter('url', i);
+                        body.url = url;
+                        const waitUntil = this.getNodeParameter('waitUntil', i);
+                        if (waitUntil !== 'load') {
+                            body.waitUntil = waitUntil;
+                        }
+                        break;
+                    }
+                    case 'links': {
+                        const url = this.getNodeParameter('url', i);
+                        body.url = url;
+                        const waitUntil = this.getNodeParameter('waitUntil', i);
+                        if (waitUntil !== 'load') {
+                            body.waitUntil = waitUntil;
+                        }
+                        break;
+                    }
+                    case 'json': {
+                        const url = this.getNodeParameter('url', i);
+                        body.url = url;
+                        const prompt = this.getNodeParameter('prompt', i);
+                        if (prompt) {
+                            body.prompt = prompt;
+                        }
+                        const schema = this.getNodeParameter('schema', i);
+                        if (schema && schema !== '{}') {
+                            body.schema = JSON.parse(schema);
+                        }
+                        const waitUntil = this.getNodeParameter('waitUntil', i);
+                        if (waitUntil !== 'load') {
+                            body.waitUntil = waitUntil;
+                        }
+                        break;
+                    }
+                    case 'crawl': {
+                        const url = this.getNodeParameter('url', i);
+                        body.url = url;
+                        body.maxDepth = this.getNodeParameter('maxDepth', i);
+                        body.maxPages = this.getNodeParameter('maxPages', i);
+                        body.timeout = this.getNodeParameter('timeout', i);
+                        const includePattern = this.getNodeParameter('includePattern', i);
+                        if (includePattern) {
+                            body.includePattern = includePattern;
+                        }
+                        const excludePattern = this.getNodeParameter('excludePattern', i);
+                        if (excludePattern) {
+                            body.excludePattern = excludePattern;
+                        }
+                        break;
+                    }
+                    case 'run': {
+                        const steps = this.getNodeParameter('steps', i);
+                        body.steps = JSON.parse(steps);
+                        body.timeout = this.getNodeParameter('timeout', i);
+                        break;
+                    }
+                    case 'seo': {
+                        const url = this.getNodeParameter('url', i);
+                        body.url = url;
+                        break;
+                    }
+                    case 'click': {
+                        const url = this.getNodeParameter('url', i);
+                        body.url = url;
+                        body.selector = this.getNodeParameter('selector', i);
+                        body.waitFor = this.getNodeParameter('waitFor', i);
+                        body.screenshot = this.getNodeParameter('takeScreenshot', i);
+                        break;
+                    }
+                    case 'cookies': {
+                        const url = this.getNodeParameter('url', i);
+                        body.url = url;
+                        body.operation = this.getNodeParameter('cookieOperation', i);
+                        break;
+                    }
+                    case 'evaluate': {
+                        const url = this.getNodeParameter('url', i);
+                        body.url = url;
+                        body.script = this.getNodeParameter('script', i);
+                        const waitBefore = this.getNodeParameter('waitBefore', i);
+                        if (waitBefore > 0) {
+                            body.waitFor = waitBefore;
+                        }
+                        break;
+                    }
+                    case 'batch': {
+                        const urls = this.getNodeParameter('urls', i);
+                        body.urls = urls.split(',').map((u) => u.trim());
+                        body.batchAction = this.getNodeParameter('batchAction', i);
+                        body.parallel = this.getNodeParameter('parallel', i);
+                        body.timeout = this.getNodeParameter('timeout', i);
+                        break;
+                    }
+                    default:
+                        throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Unknown action: ${action}`, { itemIndex: i });
+                }
+                const headers = {
+                    'Content-Type': 'application/json',
+                };
+                if (apiKey) {
+                    headers['x-api-key'] = apiKey;
+                }
+                const response = await this.helpers.httpRequestWithAuthentication.call(this, 'generic', {
+                    method: 'POST',
+                    url: workerUrl,
+                    headers,
+                    body,
+                    json: true,
+                    timeout: body.timeout || 60000,
+                });
+                if ((action === 'screenshot' || action === 'pdf') && response.success && response.result) {
+                    const returnBinary = this.getNodeParameter('returnBinary', i);
+                    if (returnBinary) {
+                        const binaryProperty = this.getNodeParameter('binaryProperty', i);
+                        const buffer = Buffer.from(response.result, 'base64');
+                        const mimeType = action === 'screenshot'
+                            ? (body.format === 'jpeg' ? 'image/jpeg' : 'image/png')
+                            : 'application/pdf';
+                        const fileName = action === 'screenshot'
+                            ? `screenshot.${body.format || 'png'}`
+                            : 'page.pdf';
+                        const binaryData = await this.helpers.prepareBinaryData(buffer, fileName, mimeType);
+                        returnData.push({
+                            json: { success: true, fileName, mimeType },
+                            binary: { [binaryProperty]: binaryData },
+                            pairedItem: { item: i },
+                        });
+                    }
+                    else {
+                        returnData.push({
+                            json: response,
+                            pairedItem: { item: i },
+                        });
+                    }
+                }
+                else {
+                    returnData.push({
+                        json: response,
+                        pairedItem: { item: i },
+                    });
+                }
+            }
+            catch (error) {
+                const errorObj = error;
+                const errorDetails = error;
+                if (this.continueOnFail()) {
+                    returnData.push({
+                        json: {
+                            success: false,
+                            error: errorObj.message || 'Unknown error',
+                        },
+                        pairedItem: { item: i },
+                    });
+                    continue;
+                }
+                if (errorDetails.statusCode === 401) {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Invalid API Key - Please check your API key in the credentials section', { itemIndex: i });
+                }
+                if (errorDetails.statusCode === 429) {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Rate limit exceeded - Please wait before making more requests', { itemIndex: i });
+                }
+                if (errorDetails.code === 'MISSING_ACTION') {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Missing required field: action', { itemIndex: i });
+                }
+                if (errorDetails.code === 'INVALID_URL') {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Invalid URL: ${errorObj.message}`, { itemIndex: i });
+                }
+                throw new n8n_workflow_1.NodeOperationError(this.getNode(), errorObj.message || 'Request failed', { itemIndex: i });
+            }
+        }
+        return [returnData];
+    }
+}
+exports.BrowserRun = BrowserRun;
+//# sourceMappingURL=BrowserRun.node.js.map