@nodetool-ai/base-nodes 0.7.0-rc.11

package/dist/nodes/lib-browser.js

@@ -0,0 +1,692 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+import { BaseNode, prop } from "@nodetool-ai/node-sdk";
+const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
+export class WebFetchLibNode extends BaseNode {
+    static nodeType = "lib.browser.WebFetch";
+    static title = "Web Fetch";
+    static description = "Fetches HTML content from a URL and converts it to text.\n    web, fetch, html, markdown, http\n\n    Use cases:\n    - Extract text content from web pages\n    - Process web content for analysis\n    - Save web content to files";
+    static metadataOutputTypes = {
+        output: "str"
+    };
+    static exposeAsTool = true;
+    async process() {
+        const url = String(this.url ?? "");
+        const selector = String(this.selector ?? "body");
+        if (!url)
+            throw new Error("URL is required");
+        const cheerio = await import("cheerio");
+        const TurndownService = (await import("turndown")).default;
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), 30000);
+        let response;
+        try {
+            response = await fetch(url, {
+                headers: { "User-Agent": USER_AGENT, Accept: "*/*" },
+                signal: controller.signal
+            });
+        }
+        finally {
+            clearTimeout(timeoutId);
+        }
+        const responseText = await response.text();
+        const contentType = String(response.headers.get("content-type") ?? "").toLowerCase();
+        if (!contentType.includes("text/html") &&
+            !contentType.includes("application/xhtml+xml")) {
+            return { output: responseText };
+        }
+        const $ = cheerio.load(responseText);
+        const elements = $(selector);
+        if (elements.length === 0) {
+            throw new Error(`No elements found matching selector: ${selector}`);
+        }
+        let html = "";
+        elements.each((_i, el) => {
+            html += $.html(el) ?? "";
+        });
+        const turndown = new TurndownService();
+        const markdown = turndown.turndown(html);
+        return { output: markdown };
+    }
+}
+__decorate([
+    prop({
+        type: "str",
+        default: "",
+        title: "Url",
+        description: "URL to fetch content from"
+    })
+], WebFetchLibNode.prototype, "url", void 0);
+__decorate([
+    prop({
+        type: "str",
+        default: "body",
+        title: "Selector",
+        description: "CSS selector to extract specific elements"
+    })
+], WebFetchLibNode.prototype, "selector", void 0);
+export class DownloadFileLibNode extends BaseNode {
+    static nodeType = "lib.browser.DownloadFile";
+    static title = "Download File";
+    static description = "Downloads a file from a URL and saves it to disk.\n    download, file, web, save\n\n    Use cases:\n    - Download documents, images, or other files from the web\n    - Save data for further processing\n    - Retrieve file assets for analysis";
+    static metadataOutputTypes = {
+        output: "bytes"
+    };
+    static exposeAsTool = true;
+    async process() {
+        const url = String(this.url ?? "");
+        if (!url)
+            throw new Error("URL is required");
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), 60000);
+        let response;
+        try {
+            response = await fetch(url, {
+                headers: { "User-Agent": USER_AGENT, Accept: "*/*" },
+                signal: controller.signal
+            });
+        }
+        finally {
+            clearTimeout(timeoutId);
+        }
+        const data = Buffer.from(await response.arrayBuffer()).toString("base64");
+        return { output: { __bytes__: data } };
+    }
+}
+__decorate([
+    prop({
+        type: "str",
+        default: "",
+        title: "Url",
+        description: "URL of the file to download"
+    })
+], DownloadFileLibNode.prototype, "url", void 0);
+export class BrowserLibNode extends BaseNode {
+    static nodeType = "lib.browser.Browser";
+    static title = "Browser";
+    static description = "Fetches content from a web page using a headless browser.\n    browser, web, scraping, content, fetch\n\n    Use cases:\n    - Extract content from JavaScript-heavy websites\n    - Retrieve text content from web pages\n    - Get metadata from web pages\n    - Save extracted content to files";
+    static metadataOutputTypes = {
+        success: "bool",
+        content: "str",
+        metadata: "dict[str, any]"
+    };
+    static exposeAsTool = true;
+    async process() {
+        const url = String(this.url ?? "");
+        const timeout = Number(this.timeout ?? 20000);
+        if (!url)
+            throw new Error("URL is required");
+        const { chromium } = await import("playwright");
+        const TurndownService = (await import("turndown")).default;
+        const browser = await chromium.launch({ headless: true });
+        try {
+            const ctx = await browser.newContext({ bypassCSP: true });
+            const page = await ctx.newPage();
+            await page.goto(url, { waitUntil: "networkidle", timeout });
+            const html = await page.content();
+            const turndown = new TurndownService();
+            const content = turndown.turndown(html);
+            const title = await page.title();
+            return {
+                success: true,
+                content,
+                metadata: { title }
+            };
+        }
+        finally {
+            await browser.close();
+        }
+    }
+}
+__decorate([
+    prop({
+        type: "str",
+        default: "",
+        title: "Url",
+        description: "URL to navigate to"
+    })
+], BrowserLibNode.prototype, "url", void 0);
+__decorate([
+    prop({
+        type: "int",
+        default: 20000,
+        title: "Timeout",
+        description: "Timeout in milliseconds for page navigation"
+    })
+], BrowserLibNode.prototype, "timeout", void 0);
+export class ScreenshotLibNode extends BaseNode {
+    static nodeType = "lib.browser.Screenshot";
+    static title = "Screenshot";
+    static description = "Takes a screenshot of a web page or specific element.\n    browser, screenshot, capture, image\n\n    Use cases:\n    - Capture visual representation of web pages\n    - Document specific UI elements\n    - Create visual records of web content";
+    static metadataOutputTypes = {
+        output: "dict[str, any]"
+    };
+    static exposeAsTool = true;
+    async process() {
+        const url = String(this.url ?? "");
+        const selector = String(this.selector ?? "");
+        const timeout = Number(this.timeout ?? 30000);
+        if (!url)
+            throw new Error("URL is required");
+        const { chromium } = await import("playwright");
+        const browser = await chromium.launch({ headless: true });
+        try {
+            const ctx = await browser.newContext({ bypassCSP: true });
+            const page = await ctx.newPage();
+            await page.goto(url, { waitUntil: "domcontentloaded", timeout });
+            let buffer;
+            if (selector) {
+                const el = await page.waitForSelector(selector, { timeout });
+                if (!el)
+                    throw new Error(`No element found matching selector: ${selector}`);
+                buffer = await el.screenshot();
+            }
+            else {
+                buffer = await page.screenshot();
+            }
+            const data = buffer.toString("base64");
+            return {
+                success: true,
+                output: { type: "image", data },
+                url
+            };
+        }
+        finally {
+            await browser.close();
+        }
+    }
+}
+__decorate([
+    prop({
+        type: "str",
+        default: "",
+        title: "Url",
+        description: "URL to navigate to before taking screenshot"
+    })
+], ScreenshotLibNode.prototype, "url", void 0);
+__decorate([
+    prop({
+        type: "str",
+        default: "",
+        title: "Selector",
+        description: "Optional CSS selector for capturing a specific element"
+    })
+], ScreenshotLibNode.prototype, "selector", void 0);
+__decorate([
+    prop({
+        type: "str",
+        default: "screenshot.png",
+        title: "Output File",
+        description: "Path to save the screenshot (relative to workspace)"
+    })
+], ScreenshotLibNode.prototype, "output_file", void 0);
+__decorate([
+    prop({
+        type: "int",
+        default: 30000,
+        title: "Timeout",
+        description: "Timeout in milliseconds for page navigation"
+    })
+], ScreenshotLibNode.prototype, "timeout", void 0);
+export class BrowserNavigationLibNode extends BaseNode {
+    static nodeType = "lib.browser.BrowserNavigation";
+    static title = "Browser Navigation";
+    static description = "Navigates and interacts with web pages in a browser session.\n    browser, navigation, interaction, click, extract\n\n    Use cases:\n    - Perform complex web interactions\n    - Navigate through multi-step web processes\n    - Extract content after interaction";
+    static metadataOutputTypes = {
+        output: "dict[str, any]"
+    };
+    async process() {
+        const url = String(this.url ?? "");
+        const action = String(this.action ?? "goto");
+        const selector = String(this.selector ?? "");
+        const timeout = Number(this.timeout ?? 30000);
+        const waitFor = String(this.wait_for ?? "");
+        const extractType = String(this.extract_type ?? "text");
+        const attribute = String(this.attribute ?? "");
+        if (action === "goto" && !url)
+            throw new Error("URL is required for goto action");
+        const { chromium } = await import("playwright");
+        const browser = await chromium.launch({ headless: true });
+        try {
+            const ctx = await browser.newContext({ bypassCSP: true });
+            const page = await ctx.newPage();
+            if (action === "goto") {
+                await page.goto(url, { waitUntil: "domcontentloaded", timeout });
+            }
+            else if (action === "reload") {
+                await page.goto(url, { waitUntil: "domcontentloaded", timeout });
+                await page.reload({ waitUntil: "domcontentloaded", timeout });
+            }
+            else if (action === "click") {
+                await page.goto(url, { waitUntil: "domcontentloaded", timeout });
+                const el = await page.waitForSelector(selector, { timeout });
+                if (!el)
+                    throw new Error(`Element not found: ${selector}`);
+                await el.click();
+            }
+            else if (action === "extract") {
+                await page.goto(url, { waitUntil: "domcontentloaded", timeout });
+            }
+            else if (action === "back") {
+                await page.goBack({ timeout, waitUntil: "domcontentloaded" });
+            }
+            else if (action === "forward") {
+                await page.goForward({ timeout, waitUntil: "domcontentloaded" });
+            }
+            if (waitFor) {
+                await page.waitForSelector(waitFor, { timeout });
+            }
+            let extracted = null;
+            if (action === "extract") {
+                if (selector) {
+                    const el = await page.waitForSelector(selector, { timeout });
+                    if (!el)
+                        throw new Error(`Element not found: ${selector}`);
+                    if (extractType === "html") {
+                        extracted = await el.evaluate((e) => e.outerHTML);
+                    }
+                    else if (extractType === "value") {
+                        extracted = await el.evaluate((e) => e.value ?? "");
+                    }
+                    else if (extractType === "attribute") {
+                        extracted = await el.evaluate((e, a) => e.getAttribute(a), attribute);
+                    }
+                    else {
+                        extracted = await el.evaluate((e) => e.innerText ?? e.textContent ?? "");
+                    }
+                }
+                else {
+                    if (extractType === "html") {
+                        extracted = await page.content();
+                    }
+                    else {
+                        extracted = await page.evaluate(() => document.body ? document.body.innerText : "");
+                    }
+                }
+            }
+            return { success: true, action, extracted };
+        }
+        finally {
+            await browser.close();
+        }
+    }
+}
+__decorate([
+    prop({
+        type: "str",
+        default: "",
+        title: "Url",
+        description: "URL to navigate to (required for 'goto' action)"
+    })
+], BrowserNavigationLibNode.prototype, "url", void 0);
+__decorate([
+    prop({
+        type: "enum",
+        default: "goto",
+        title: "Action",
+        description: "Navigation or extraction action to perform",
+        values: ["click", "goto", "back", "forward", "reload", "extract"]
+    })
+], BrowserNavigationLibNode.prototype, "action", void 0);
+__decorate([
+    prop({
+        type: "str",
+        default: "",
+        title: "Selector",
+        description: "CSS selector for the element to interact with or extract from"
+    })
+], BrowserNavigationLibNode.prototype, "selector", void 0);
+__decorate([
+    prop({
+        type: "int",
+        default: 30000,
+        title: "Timeout",
+        description: "Timeout in milliseconds for the action"
+    })
+], BrowserNavigationLibNode.prototype, "timeout", void 0);
+__decorate([
+    prop({
+        type: "str",
+        default: "",
+        title: "Wait For",
+        description: "Optional selector to wait for after performing the action"
+    })
+], BrowserNavigationLibNode.prototype, "wait_for", void 0);
+__decorate([
+    prop({
+        type: "enum",
+        default: "text",
+        title: "Extract Type",
+        description: "Type of content to extract (for 'extract' action)",
+        values: ["text", "html", "value", "attribute"]
+    })
+], BrowserNavigationLibNode.prototype, "extract_type", void 0);
+__decorate([
+    prop({
+        type: "str",
+        default: "",
+        title: "Attribute",
+        description: "Attribute name to extract (when extract_type is 'attribute')"
+    })
+], BrowserNavigationLibNode.prototype, "attribute", void 0);
+export class SpiderCrawlLibNode extends BaseNode {
+    static nodeType = "lib.browser.SpiderCrawl";
+    static title = "Spider Crawl";
+    static description = "Crawls websites following links and emitting URLs with optional HTML content.\n    spider, crawler, web scraping, links, sitemap\n\n    Use cases:\n    - Build sitemaps and discover website structure\n    - Collect URLs for bulk processing\n    - Find all pages on a website\n    - Extract content from multiple pages\n    - Feed agentic workflows with discovered pages\n    - Analyze website content and structure";
+    static metadataOutputTypes = {
+        url: "str",
+        depth: "int",
+        html: "str",
+        title: "str",
+        status_code: "int",
+        pages: "list"
+    };
+    static exposeAsTool = true;
+    static isStreamingOutput = true;
+    async process() {
+        const allPages = [];
+        for await (const page of this._crawlPages()) {
+            allPages.push(page);
+        }
+        const first = allPages[0] ?? {
+            url: "",
+            depth: 0,
+            html: null,
+            title: null,
+            status_code: 0
+        };
+        return {
+            url: first.url,
+            depth: first.depth,
+            html: first.html,
+            title: first.title,
+            status_code: first.status_code,
+            pages: allPages
+        };
+    }
+    async *_crawlPages() {
+        const startUrl = String(this.start_url ?? "");
+        const maxDepth = Number(this.max_depth ?? 2);
+        const maxPages = Number(this.max_pages ?? 50);
+        const sameDomainOnly = Boolean(this.same_domain_only ?? true);
+        const includeHtml = Boolean(this.include_html ?? false);
+        const respectRobotsTxt = Boolean(this.respect_robots_txt ?? true);
+        const delayMs = Number(this.delay_ms ?? 1000);
+        const timeout = Number(this.timeout ?? 30000);
+        const urlPattern = String(this.url_pattern ?? "");
+        const excludePattern = String(this.exclude_pattern ?? "");
+        if (!startUrl)
+            throw new Error("start_url is required");
+        const cheerio = await import("cheerio");
+        const startParsed = new URL(startUrl);
+        const startDomain = `${startParsed.protocol}//${startParsed.host}`;
+        const urlPatternRe = urlPattern ? new RegExp(urlPattern) : null;
+        const excludePatternRe = excludePattern ? new RegExp(excludePattern) : null;
+        // robots.txt cache and parser
+        const robotsCache = new Map(); // origin -> disallowed paths
+        const fetchRobotsTxt = async (origin) => {
+            if (robotsCache.has(origin))
+                return robotsCache.get(origin);
+            const disallowed = [];
+            try {
+                const robotsController = new AbortController();
+                const robotsTimeoutId = setTimeout(() => robotsController.abort(), 10000);
+                let res;
+                try {
+                    res = await fetch(`${origin}/robots.txt`, {
+                        headers: { "User-Agent": USER_AGENT },
+                        signal: robotsController.signal
+                    });
+                }
+                finally {
+                    clearTimeout(robotsTimeoutId);
+                }
+                const resText = await res.text();
+                if (res.ok && typeof resText === "string") {
+                    // Parse robots.txt — look for User-agent: * sections
+                    const lines = resText.split("\n");
+                    let inWildcard = false;
+                    for (const rawLine of lines) {
+                        const line = rawLine.trim();
+                        if (/^user-agent\s*:/i.test(line)) {
+                            const agent = line.replace(/^user-agent\s*:\s*/i, "").trim();
+                            inWildcard = agent === "*";
+                        }
+                        else if (inWildcard && /^disallow\s*:/i.test(line)) {
+                            const path = line.replace(/^disallow\s*:\s*/i, "").trim();
+                            if (path)
+                                disallowed.push(path);
+                        }
+                    }
+                }
+            }
+            catch {
+                // robots.txt not available — allow all
+            }
+            robotsCache.set(origin, disallowed);
+            return disallowed;
+        };
+        const isAllowedByRobots = async (urlStr) => {
+            if (!respectRobotsTxt)
+                return true;
+            try {
+                const parsed = new URL(urlStr);
+                const origin = `${parsed.protocol}//${parsed.host}`;
+                const disallowed = await fetchRobotsTxt(origin);
+                const path = parsed.pathname;
+                for (const rule of disallowed) {
+                    // Handle wildcard rules with trailing *
+                    if (rule.endsWith("*")) {
+                        if (path.startsWith(rule.slice(0, -1)))
+                            return false;
+                    }
+                    else if (path.startsWith(rule)) {
+                        return false;
+                    }
+                }
+                return true;
+            }
+            catch {
+                return true;
+            }
+        };
+        const visited = new Set();
+        const toVisit = [
+            { url: startUrl, depth: 0 }
+        ];
+        let pagesCrawled = 0;
+        while (toVisit.length > 0 && pagesCrawled < maxPages) {
+            const item = toVisit.shift();
+            const { url: currentUrl, depth } = item;
+            if (visited.has(currentUrl))
+                continue;
+            if (depth > maxDepth)
+                continue;
+            if (urlPatternRe && !urlPatternRe.test(currentUrl))
+                continue;
+            if (excludePatternRe && excludePatternRe.test(currentUrl))
+                continue;
+            // Check robots.txt before crawling
+            if (!(await isAllowedByRobots(currentUrl))) {
+                visited.add(currentUrl);
+                continue;
+            }
+            visited.add(currentUrl);
+            try {
+                const crawlController = new AbortController();
+                const crawlTimeoutId = setTimeout(() => crawlController.abort(), timeout);
+                let response;
+                try {
+                    response = await fetch(currentUrl, {
+                        headers: { "User-Agent": USER_AGENT },
+                        signal: crawlController.signal
+                    });
+                }
+                finally {
+                    clearTimeout(crawlTimeoutId);
+                }
+                const statusCode = response.status;
+                const htmlContent = await response.text();
+                const $ = cheerio.load(htmlContent);
+                const title = $("title").text() || null;
+                yield {
+                    url: currentUrl,
+                    depth,
+                    html: includeHtml ? htmlContent : null,
+                    title,
+                    status_code: statusCode
+                };
+                pagesCrawled++;
+                if (depth < maxDepth) {
+                    $("a[href]").each((_i, el) => {
+                        try {
+                            const href = $(el).attr("href") ?? "";
+                            if (!href ||
+                                href.startsWith("javascript:") ||
+                                href.startsWith("mailto:") ||
+                                href.startsWith("tel:"))
+                                return;
+                            const resolved = new URL(href, currentUrl).href.split("#")[0];
+                            if (visited.has(resolved))
+                                return;
+                            if (sameDomainOnly) {
+                                const linkParsed = new URL(resolved);
+                                const linkDomain = `${linkParsed.protocol}//${linkParsed.host}`;
+                                if (linkDomain !== startDomain)
+                                    return;
+                            }
+                            if (!toVisit.some((v) => v.url === resolved)) {
+                                toVisit.push({ url: resolved, depth: depth + 1 });
+                            }
+                        }
+                        catch {
+                            // invalid URL, skip
+                        }
+                    });
+                }
+                if (delayMs > 0 && toVisit.length > 0) {
+                    await new Promise((r) => setTimeout(r, delayMs));
+                }
+            }
+            catch {
+                yield {
+                    url: currentUrl,
+                    depth,
+                    html: null,
+                    title: null,
+                    status_code: 0
+                };
+            }
+        }
+    }
+    async *genProcess() {
+        const allPages = [];
+        for await (const page of this._crawlPages()) {
+            allPages.push(page);
+            yield page;
+        }
+        // Emit collected list as final output
+        yield { pages: allPages };
+    }
+}
+__decorate([
+    prop({
+        type: "str",
+        default: "",
+        title: "Start Url",
+        description: "The starting URL to begin crawling from"
+    })
+], SpiderCrawlLibNode.prototype, "start_url", void 0);
+__decorate([
+    prop({
+        type: "int",
+        default: 2,
+        title: "Max Depth",
+        description: "Maximum depth to crawl (0 = start page only, 1 = start + linked pages, etc.)",
+        min: 0,
+        max: 10
+    })
+], SpiderCrawlLibNode.prototype, "max_depth", void 0);
+__decorate([
+    prop({
+        type: "int",
+        default: 50,
+        title: "Max Pages",
+        description: "Maximum number of pages to crawl (safety limit)",
+        min: 1,
+        max: 1000
+    })
+], SpiderCrawlLibNode.prototype, "max_pages", void 0);
+__decorate([
+    prop({
+        type: "bool",
+        default: true,
+        title: "Same Domain Only",
+        description: "Only follow links within the same domain as the start URL"
+    })
+], SpiderCrawlLibNode.prototype, "same_domain_only", void 0);
+__decorate([
+    prop({
+        type: "bool",
+        default: false,
+        title: "Include Html",
+        description: "Include the HTML content of each page in the output (increases bandwidth)"
+    })
+], SpiderCrawlLibNode.prototype, "include_html", void 0);
+__decorate([
+    prop({
+        type: "bool",
+        default: true,
+        title: "Respect Robots Txt",
+        description: "Respect robots.txt rules (follows web crawler best practices)"
+    })
+], SpiderCrawlLibNode.prototype, "respect_robots_txt", void 0);
+__decorate([
+    prop({
+        type: "int",
+        default: 1000,
+        title: "Delay Ms",
+        description: "Delay in milliseconds between requests (politeness policy)",
+        min: 0,
+        max: 10000
+    })
+], SpiderCrawlLibNode.prototype, "delay_ms", void 0);
+__decorate([
+    prop({
+        type: "int",
+        default: 30000,
+        title: "Timeout",
+        description: "Timeout in milliseconds for each page load",
+        min: 1000,
+        max: 120000
+    })
+], SpiderCrawlLibNode.prototype, "timeout", void 0);
+__decorate([
+    prop({
+        type: "str",
+        default: "",
+        title: "Url Pattern",
+        description: "Optional regex pattern to filter URLs (only crawl matching URLs)"
+    })
+], SpiderCrawlLibNode.prototype, "url_pattern", void 0);
+__decorate([
+    prop({
+        type: "str",
+        default: "",
+        title: "Exclude Pattern",
+        description: "Optional regex pattern to exclude URLs (skip matching URLs)"
+    })
+], SpiderCrawlLibNode.prototype, "exclude_pattern", void 0);
+export const LIB_BROWSER_NODES = [
+    WebFetchLibNode,
+    DownloadFileLibNode,
+    BrowserLibNode,
+    ScreenshotLibNode,
+    BrowserNavigationLibNode,
+    SpiderCrawlLibNode
+];
+//# sourceMappingURL=lib-browser.js.map