@nodetool-ai/base-nodes 0.7.0-rc.23 → 0.8.1

package/dist/nodes/lib-browser.js

@@ -1,692 +0,0 @@
-var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
-    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
-    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
-    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
-    return c > 3 && r && Object.defineProperty(target, key, r), r;
-};
-import { BaseNode, prop } from "@nodetool-ai/node-sdk";
-const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
-export class WebFetchLibNode extends BaseNode {
-    static nodeType = "lib.browser.WebFetch";
-    static title = "Web Fetch";
-    static description = "Fetches HTML content from a URL and converts it to text.\n    web, fetch, html, markdown, http\n\n    Use cases:\n    - Extract text content from web pages\n    - Process web content for analysis\n    - Save web content to files";
-    static metadataOutputTypes = {
-        output: "str"
-    };
-    static exposeAsTool = true;
-    async process() {
-        const url = String(this.url ?? "");
-        const selector = String(this.selector ?? "body");
-        if (!url)
-            throw new Error("URL is required");
-        const cheerio = await import("cheerio");
-        const TurndownService = (await import("turndown")).default;
-        const controller = new AbortController();
-        const timeoutId = setTimeout(() => controller.abort(), 30000);
-        let response;
-        try {
-            response = await fetch(url, {
-                headers: { "User-Agent": USER_AGENT, Accept: "*/*" },
-                signal: controller.signal
-            });
-        }
-        finally {
-            clearTimeout(timeoutId);
-        }
-        const responseText = await response.text();
-        const contentType = String(response.headers.get("content-type") ?? "").toLowerCase();
-        if (!contentType.includes("text/html") &&
-            !contentType.includes("application/xhtml+xml")) {
-            return { output: responseText };
-        }
-        const $ = cheerio.load(responseText);
-        const elements = $(selector);
-        if (elements.length === 0) {
-            throw new Error(`No elements found matching selector: ${selector}`);
-        }
-        let html = "";
-        elements.each((_i, el) => {
-            html += $.html(el) ?? "";
-        });
-        const turndown = new TurndownService();
-        const markdown = turndown.turndown(html);
-        return { output: markdown };
-    }
-}
-__decorate([
-    prop({
-        type: "str",
-        default: "",
-        title: "Url",
-        description: "URL to fetch content from"
-    })
-], WebFetchLibNode.prototype, "url", void 0);
-__decorate([
-    prop({
-        type: "str",
-        default: "body",
-        title: "Selector",
-        description: "CSS selector to extract specific elements"
-    })
-], WebFetchLibNode.prototype, "selector", void 0);
-export class DownloadFileLibNode extends BaseNode {
-    static nodeType = "lib.browser.DownloadFile";
-    static title = "Download File";
-    static description = "Downloads a file from a URL and saves it to disk.\n    download, file, web, save\n\n    Use cases:\n    - Download documents, images, or other files from the web\n    - Save data for further processing\n    - Retrieve file assets for analysis";
-    static metadataOutputTypes = {
-        output: "bytes"
-    };
-    static exposeAsTool = true;
-    async process() {
-        const url = String(this.url ?? "");
-        if (!url)
-            throw new Error("URL is required");
-        const controller = new AbortController();
-        const timeoutId = setTimeout(() => controller.abort(), 60000);
-        let response;
-        try {
-            response = await fetch(url, {
-                headers: { "User-Agent": USER_AGENT, Accept: "*/*" },
-                signal: controller.signal
-            });
-        }
-        finally {
-            clearTimeout(timeoutId);
-        }
-        const data = Buffer.from(await response.arrayBuffer()).toString("base64");
-        return { output: { __bytes__: data } };
-    }
-}
-__decorate([
-    prop({
-        type: "str",
-        default: "",
-        title: "Url",
-        description: "URL of the file to download"
-    })
-], DownloadFileLibNode.prototype, "url", void 0);
-export class BrowserLibNode extends BaseNode {
-    static nodeType = "lib.browser.Browser";
-    static title = "Browser";
-    static description = "Fetches content from a web page using a headless browser.\n    browser, web, scraping, content, fetch\n\n    Use cases:\n    - Extract content from JavaScript-heavy websites\n    - Retrieve text content from web pages\n    - Get metadata from web pages\n    - Save extracted content to files";
-    static metadataOutputTypes = {
-        success: "bool",
-        content: "str",
-        metadata: "dict[str, any]"
-    };
-    static exposeAsTool = true;
-    async process() {
-        const url = String(this.url ?? "");
-        const timeout = Number(this.timeout ?? 20000);
-        if (!url)
-            throw new Error("URL is required");
-        const { chromium } = await import("playwright");
-        const TurndownService = (await import("turndown")).default;
-        const browser = await chromium.launch({ headless: true });
-        try {
-            const ctx = await browser.newContext({ bypassCSP: true });
-            const page = await ctx.newPage();
-            await page.goto(url, { waitUntil: "networkidle", timeout });
-            const html = await page.content();
-            const turndown = new TurndownService();
-            const content = turndown.turndown(html);
-            const title = await page.title();
-            return {
-                success: true,
-                content,
-                metadata: { title }
-            };
-        }
-        finally {
-            await browser.close();
-        }
-    }
-}
-__decorate([
-    prop({
-        type: "str",
-        default: "",
-        title: "Url",
-        description: "URL to navigate to"
-    })
-], BrowserLibNode.prototype, "url", void 0);
-__decorate([
-    prop({
-        type: "int",
-        default: 20000,
-        title: "Timeout",
-        description: "Timeout in milliseconds for page navigation"
-    })
-], BrowserLibNode.prototype, "timeout", void 0);
-export class ScreenshotLibNode extends BaseNode {
-    static nodeType = "lib.browser.Screenshot";
-    static title = "Screenshot";
-    static description = "Takes a screenshot of a web page or specific element.\n    browser, screenshot, capture, image\n\n    Use cases:\n    - Capture visual representation of web pages\n    - Document specific UI elements\n    - Create visual records of web content";
-    static metadataOutputTypes = {
-        output: "dict[str, any]"
-    };
-    static exposeAsTool = true;
-    async process() {
-        const url = String(this.url ?? "");
-        const selector = String(this.selector ?? "");
-        const timeout = Number(this.timeout ?? 30000);
-        if (!url)
-            throw new Error("URL is required");
-        const { chromium } = await import("playwright");
-        const browser = await chromium.launch({ headless: true });
-        try {
-            const ctx = await browser.newContext({ bypassCSP: true });
-            const page = await ctx.newPage();
-            await page.goto(url, { waitUntil: "domcontentloaded", timeout });
-            let buffer;
-            if (selector) {
-                const el = await page.waitForSelector(selector, { timeout });
-                if (!el)
-                    throw new Error(`No element found matching selector: ${selector}`);
-                buffer = await el.screenshot();
-            }
-            else {
-                buffer = await page.screenshot();
-            }
-            const data = buffer.toString("base64");
-            return {
-                success: true,
-                output: { type: "image", data },
-                url
-            };
-        }
-        finally {
-            await browser.close();
-        }
-    }
-}
-__decorate([
-    prop({
-        type: "str",
-        default: "",
-        title: "Url",
-        description: "URL to navigate to before taking screenshot"
-    })
-], ScreenshotLibNode.prototype, "url", void 0);
-__decorate([
-    prop({
-        type: "str",
-        default: "",
-        title: "Selector",
-        description: "Optional CSS selector for capturing a specific element"
-    })
-], ScreenshotLibNode.prototype, "selector", void 0);
-__decorate([
-    prop({
-        type: "str",
-        default: "screenshot.png",
-        title: "Output File",
-        description: "Path to save the screenshot (relative to workspace)"
-    })
-], ScreenshotLibNode.prototype, "output_file", void 0);
-__decorate([
-    prop({
-        type: "int",
-        default: 30000,
-        title: "Timeout",
-        description: "Timeout in milliseconds for page navigation"
-    })
-], ScreenshotLibNode.prototype, "timeout", void 0);
-export class BrowserNavigationLibNode extends BaseNode {
-    static nodeType = "lib.browser.BrowserNavigation";
-    static title = "Browser Navigation";
-    static description = "Navigates and interacts with web pages in a browser session.\n    browser, navigation, interaction, click, extract\n\n    Use cases:\n    - Perform complex web interactions\n    - Navigate through multi-step web processes\n    - Extract content after interaction";
-    static metadataOutputTypes = {
-        output: "dict[str, any]"
-    };
-    async process() {
-        const url = String(this.url ?? "");
-        const action = String(this.action ?? "goto");
-        const selector = String(this.selector ?? "");
-        const timeout = Number(this.timeout ?? 30000);
-        const waitFor = String(this.wait_for ?? "");
-        const extractType = String(this.extract_type ?? "text");
-        const attribute = String(this.attribute ?? "");
-        if (action === "goto" && !url)
-            throw new Error("URL is required for goto action");
-        const { chromium } = await import("playwright");
-        const browser = await chromium.launch({ headless: true });
-        try {
-            const ctx = await browser.newContext({ bypassCSP: true });
-            const page = await ctx.newPage();
-            if (action === "goto") {
-                await page.goto(url, { waitUntil: "domcontentloaded", timeout });
-            }
-            else if (action === "reload") {
-                await page.goto(url, { waitUntil: "domcontentloaded", timeout });
-                await page.reload({ waitUntil: "domcontentloaded", timeout });
-            }
-            else if (action === "click") {
-                await page.goto(url, { waitUntil: "domcontentloaded", timeout });
-                const el = await page.waitForSelector(selector, { timeout });
-                if (!el)
-                    throw new Error(`Element not found: ${selector}`);
-                await el.click();
-            }
-            else if (action === "extract") {
-                await page.goto(url, { waitUntil: "domcontentloaded", timeout });
-            }
-            else if (action === "back") {
-                await page.goBack({ timeout, waitUntil: "domcontentloaded" });
-            }
-            else if (action === "forward") {
-                await page.goForward({ timeout, waitUntil: "domcontentloaded" });
-            }
-            if (waitFor) {
-                await page.waitForSelector(waitFor, { timeout });
-            }
-            let extracted = null;
-            if (action === "extract") {
-                if (selector) {
-                    const el = await page.waitForSelector(selector, { timeout });
-                    if (!el)
-                        throw new Error(`Element not found: ${selector}`);
-                    if (extractType === "html") {
-                        extracted = await el.evaluate((e) => e.outerHTML);
-                    }
-                    else if (extractType === "value") {
-                        extracted = await el.evaluate((e) => e.value ?? "");
-                    }
-                    else if (extractType === "attribute") {
-                        extracted = await el.evaluate((e, a) => e.getAttribute(a), attribute);
-                    }
-                    else {
-                        extracted = await el.evaluate((e) => e.innerText ?? e.textContent ?? "");
-                    }
-                }
-                else {
-                    if (extractType === "html") {
-                        extracted = await page.content();
-                    }
-                    else {
-                        extracted = await page.evaluate(() => document.body ? document.body.innerText : "");
-                    }
-                }
-            }
-            return { success: true, action, extracted };
-        }
-        finally {
-            await browser.close();
-        }
-    }
-}
-__decorate([
-    prop({
-        type: "str",
-        default: "",
-        title: "Url",
-        description: "URL to navigate to (required for 'goto' action)"
-    })
-], BrowserNavigationLibNode.prototype, "url", void 0);
-__decorate([
-    prop({
-        type: "enum",
-        default: "goto",
-        title: "Action",
-        description: "Navigation or extraction action to perform",
-        values: ["click", "goto", "back", "forward", "reload", "extract"]
-    })
-], BrowserNavigationLibNode.prototype, "action", void 0);
-__decorate([
-    prop({
-        type: "str",
-        default: "",
-        title: "Selector",
-        description: "CSS selector for the element to interact with or extract from"
-    })
-], BrowserNavigationLibNode.prototype, "selector", void 0);
-__decorate([
-    prop({
-        type: "int",
-        default: 30000,
-        title: "Timeout",
-        description: "Timeout in milliseconds for the action"
-    })
-], BrowserNavigationLibNode.prototype, "timeout", void 0);
-__decorate([
-    prop({
-        type: "str",
-        default: "",
-        title: "Wait For",
-        description: "Optional selector to wait for after performing the action"
-    })
-], BrowserNavigationLibNode.prototype, "wait_for", void 0);
-__decorate([
-    prop({
-        type: "enum",
-        default: "text",
-        title: "Extract Type",
-        description: "Type of content to extract (for 'extract' action)",
-        values: ["text", "html", "value", "attribute"]
-    })
-], BrowserNavigationLibNode.prototype, "extract_type", void 0);
-__decorate([
-    prop({
-        type: "str",
-        default: "",
-        title: "Attribute",
-        description: "Attribute name to extract (when extract_type is 'attribute')"
-    })
-], BrowserNavigationLibNode.prototype, "attribute", void 0);
-export class SpiderCrawlLibNode extends BaseNode {
-    static nodeType = "lib.browser.SpiderCrawl";
-    static title = "Spider Crawl";
-    static description = "Crawls websites following links and emitting URLs with optional HTML content.\n    spider, crawler, web scraping, links, sitemap\n\n    Use cases:\n    - Build sitemaps and discover website structure\n    - Collect URLs for bulk processing\n    - Find all pages on a website\n    - Extract content from multiple pages\n    - Feed agentic workflows with discovered pages\n    - Analyze website content and structure";
-    static metadataOutputTypes = {
-        url: "str",
-        depth: "int",
-        html: "str",
-        title: "str",
-        status_code: "int",
-        pages: "list"
-    };
-    static exposeAsTool = true;
-    static isStreamingOutput = true;
-    async process() {
-        const allPages = [];
-        for await (const page of this._crawlPages()) {
-            allPages.push(page);
-        }
-        const first = allPages[0] ?? {
-            url: "",
-            depth: 0,
-            html: null,
-            title: null,
-            status_code: 0
-        };
-        return {
-            url: first.url,
-            depth: first.depth,
-            html: first.html,
-            title: first.title,
-            status_code: first.status_code,
-            pages: allPages
-        };
-    }
-    async *_crawlPages() {
-        const startUrl = String(this.start_url ?? "");
-        const maxDepth = Number(this.max_depth ?? 2);
-        const maxPages = Number(this.max_pages ?? 50);
-        const sameDomainOnly = Boolean(this.same_domain_only ?? true);
-        const includeHtml = Boolean(this.include_html ?? false);
-        const respectRobotsTxt = Boolean(this.respect_robots_txt ?? true);
-        const delayMs = Number(this.delay_ms ?? 1000);
-        const timeout = Number(this.timeout ?? 30000);
-        const urlPattern = String(this.url_pattern ?? "");
-        const excludePattern = String(this.exclude_pattern ?? "");
-        if (!startUrl)
-            throw new Error("start_url is required");
-        const cheerio = await import("cheerio");
-        const startParsed = new URL(startUrl);
-        const startDomain = `${startParsed.protocol}//${startParsed.host}`;
-        const urlPatternRe = urlPattern ? new RegExp(urlPattern) : null;
-        const excludePatternRe = excludePattern ? new RegExp(excludePattern) : null;
-        // robots.txt cache and parser
-        const robotsCache = new Map(); // origin -> disallowed paths
-        const fetchRobotsTxt = async (origin) => {
-            if (robotsCache.has(origin))
-                return robotsCache.get(origin);
-            const disallowed = [];
-            try {
-                const robotsController = new AbortController();
-                const robotsTimeoutId = setTimeout(() => robotsController.abort(), 10000);
-                let res;
-                try {
-                    res = await fetch(`${origin}/robots.txt`, {
-                        headers: { "User-Agent": USER_AGENT },
-                        signal: robotsController.signal
-                    });
-                }
-                finally {
-                    clearTimeout(robotsTimeoutId);
-                }
-                const resText = await res.text();
-                if (res.ok && typeof resText === "string") {
-                    // Parse robots.txt — look for User-agent: * sections
-                    const lines = resText.split("\n");
-                    let inWildcard = false;
-                    for (const rawLine of lines) {
-                        const line = rawLine.trim();
-                        if (/^user-agent\s*:/i.test(line)) {
-                            const agent = line.replace(/^user-agent\s*:\s*/i, "").trim();
-                            inWildcard = agent === "*";
-                        }
-                        else if (inWildcard && /^disallow\s*:/i.test(line)) {
-                            const path = line.replace(/^disallow\s*:\s*/i, "").trim();
-                            if (path)
-                                disallowed.push(path);
-                        }
-                    }
-                }
-            }
-            catch {
-                // robots.txt not available — allow all
-            }
-            robotsCache.set(origin, disallowed);
-            return disallowed;
-        };
-        const isAllowedByRobots = async (urlStr) => {
-            if (!respectRobotsTxt)
-                return true;
-            try {
-                const parsed = new URL(urlStr);
-                const origin = `${parsed.protocol}//${parsed.host}`;
-                const disallowed = await fetchRobotsTxt(origin);
-                const path = parsed.pathname;
-                for (const rule of disallowed) {
-                    // Handle wildcard rules with trailing *
-                    if (rule.endsWith("*")) {
-                        if (path.startsWith(rule.slice(0, -1)))
-                            return false;
-                    }
-                    else if (path.startsWith(rule)) {
-                        return false;
-                    }
-                }
-                return true;
-            }
-            catch {
-                return true;
-            }
-        };
-        const visited = new Set();
-        const toVisit = [
-            { url: startUrl, depth: 0 }
-        ];
-        let pagesCrawled = 0;
-        while (toVisit.length > 0 && pagesCrawled < maxPages) {
-            const item = toVisit.shift();
-            const { url: currentUrl, depth } = item;
-            if (visited.has(currentUrl))
-                continue;
-            if (depth > maxDepth)
-                continue;
-            if (urlPatternRe && !urlPatternRe.test(currentUrl))
-                continue;
-            if (excludePatternRe && excludePatternRe.test(currentUrl))
-                continue;
-            // Check robots.txt before crawling
-            if (!(await isAllowedByRobots(currentUrl))) {
-                visited.add(currentUrl);
-                continue;
-            }
-            visited.add(currentUrl);
-            try {
-                const crawlController = new AbortController();
-                const crawlTimeoutId = setTimeout(() => crawlController.abort(), timeout);
-                let response;
-                try {
-                    response = await fetch(currentUrl, {
-                        headers: { "User-Agent": USER_AGENT },
-                        signal: crawlController.signal
-                    });
-                }
-                finally {
-                    clearTimeout(crawlTimeoutId);
-                }
-                const statusCode = response.status;
-                const htmlContent = await response.text();
-                const $ = cheerio.load(htmlContent);
-                const title = $("title").text() || null;
-                yield {
-                    url: currentUrl,
-                    depth,
-                    html: includeHtml ? htmlContent : null,
-                    title,
-                    status_code: statusCode
-                };
-                pagesCrawled++;
-                if (depth < maxDepth) {
-                    $("a[href]").each((_i, el) => {
-                        try {
-                            const href = $(el).attr("href") ?? "";
-                            if (!href ||
-                                href.startsWith("javascript:") ||
-                                href.startsWith("mailto:") ||
-                                href.startsWith("tel:"))
-                                return;
-                            const resolved = new URL(href, currentUrl).href.split("#")[0];
-                            if (visited.has(resolved))
-                                return;
-                            if (sameDomainOnly) {
-                                const linkParsed = new URL(resolved);
-                                const linkDomain = `${linkParsed.protocol}//${linkParsed.host}`;
-                                if (linkDomain !== startDomain)
-                                    return;
-                            }
-                            if (!toVisit.some((v) => v.url === resolved)) {
-                                toVisit.push({ url: resolved, depth: depth + 1 });
-                            }
-                        }
-                        catch {
-                            // invalid URL, skip
-                        }
-                    });
-                }
-                if (delayMs > 0 && toVisit.length > 0) {
-                    await new Promise((r) => setTimeout(r, delayMs));
-                }
-            }
-            catch {
-                yield {
-                    url: currentUrl,
-                    depth,
-                    html: null,
-                    title: null,
-                    status_code: 0
-                };
-            }
-        }
-    }
-    async *genProcess() {
-        const allPages = [];
-        for await (const page of this._crawlPages()) {
-            allPages.push(page);
-            yield page;
-        }
-        // Emit collected list as final output
-        yield { pages: allPages };
-    }
-}
-__decorate([
-    prop({
-        type: "str",
-        default: "",
-        title: "Start Url",
-        description: "The starting URL to begin crawling from"
-    })
-], SpiderCrawlLibNode.prototype, "start_url", void 0);
-__decorate([
-    prop({
-        type: "int",
-        default: 2,
-        title: "Max Depth",
-        description: "Maximum depth to crawl (0 = start page only, 1 = start + linked pages, etc.)",
-        min: 0,
-        max: 10
-    })
-], SpiderCrawlLibNode.prototype, "max_depth", void 0);
-__decorate([
-    prop({
-        type: "int",
-        default: 50,
-        title: "Max Pages",
-        description: "Maximum number of pages to crawl (safety limit)",
-        min: 1,
-        max: 1000
-    })
-], SpiderCrawlLibNode.prototype, "max_pages", void 0);
-__decorate([
-    prop({
-        type: "bool",
-        default: true,
-        title: "Same Domain Only",
-        description: "Only follow links within the same domain as the start URL"
-    })
-], SpiderCrawlLibNode.prototype, "same_domain_only", void 0);
-__decorate([
-    prop({
-        type: "bool",
-        default: false,
-        title: "Include Html",
-        description: "Include the HTML content of each page in the output (increases bandwidth)"
-    })
-], SpiderCrawlLibNode.prototype, "include_html", void 0);
-__decorate([
-    prop({
-        type: "bool",
-        default: true,
-        title: "Respect Robots Txt",
-        description: "Respect robots.txt rules (follows web crawler best practices)"
-    })
-], SpiderCrawlLibNode.prototype, "respect_robots_txt", void 0);
-__decorate([
-    prop({
-        type: "int",
-        default: 1000,
-        title: "Delay Ms",
-        description: "Delay in milliseconds between requests (politeness policy)",
-        min: 0,
-        max: 10000
-    })
-], SpiderCrawlLibNode.prototype, "delay_ms", void 0);
-__decorate([
-    prop({
-        type: "int",
-        default: 30000,
-        title: "Timeout",
-        description: "Timeout in milliseconds for each page load",
-        min: 1000,
-        max: 120000
-    })
-], SpiderCrawlLibNode.prototype, "timeout", void 0);
-__decorate([
-    prop({
-        type: "str",
-        default: "",
-        title: "Url Pattern",
-        description: "Optional regex pattern to filter URLs (only crawl matching URLs)"
-    })
-], SpiderCrawlLibNode.prototype, "url_pattern", void 0);
-__decorate([
-    prop({
-        type: "str",
-        default: "",
-        title: "Exclude Pattern",
-        description: "Optional regex pattern to exclude URLs (skip matching URLs)"
-    })
-], SpiderCrawlLibNode.prototype, "exclude_pattern", void 0);
-export const LIB_BROWSER_NODES = [
-    WebFetchLibNode,
-    DownloadFileLibNode,
-    BrowserLibNode,
-    ScreenshotLibNode,
-    BrowserNavigationLibNode,
-    SpiderCrawlLibNode
-];
-//# sourceMappingURL=lib-browser.js.map