app-manager-edge-worker 1.0.4 → 1.0.6

package/src/handlers/scrapeHandler.js

@@ -1,433 +1,801 @@
-import axios from "axios";
-import * as cheerio from "cheerio";
-import localBrowserManager from "../browser.js";
-
-/**
- * Extracts a single field value from a Puppeteer page given a strategy.
- */
-async function extractBrowserField(page, strategy) {
-    if (!strategy || !strategy.value) return null;
-
-    try {
-        const type = strategy.type || "css";
-        const attr = strategy.attribute || "text";
-
-        if (type === "css") {
-            const el = await page.$(strategy.value);
-            if (!el) return null;
-
-            if (attr === "text") {
-                const text = await page.evaluate((element) => element.textContent, el);
-                return text ? text.trim() : null;
-            } else if (attr === "html") {
-                return await page.evaluate((element) => element.innerHTML, el);
-            } else {
-                const val = await page.evaluate((element, a) => element.getAttribute(a), el, attr);
-                return val !== null && val !== undefined ? String(val).trim() : null;
-            }
-        }
-
-        if (type === "xpath") {
-            const val = await page.evaluate(
-                (xpathStr, attrName) => {
-                    try {
-                        const result = document.evaluate(
-                            xpathStr,
-                            document,
-                            null,
-                            XPathResult.FIRST_ORDERED_NODE_TYPE,
-                            null
-                        );
-                        const node = result.singleNodeValue;
-                        if (!node) return null;
-                        if (attrName === "text") {
-                            return node.textContent ? node.textContent.trim() : null;
-                        }
-                        return node.getAttribute ? node.getAttribute(attrName) : null;
-                    } catch (e) {
-                        return null;
-                    }
-                },
-                strategy.value,
-                attr
-            );
-            return val;
-        }
-
-        if (type === "text-relative") {
-            return await page.evaluate(
-                (anchorText, relation) => {
-                    const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT, null, false);
-                    let node;
-                    while ((node = walker.nextNode())) {
-                        if (node.textContent.includes(anchorText)) {
-                            const parent = node.parentElement;
-                            if (relation === "next-sibling") {
-                                const sibling = parent.nextElementSibling;
-                                return sibling ? (sibling.innerText || sibling.textContent)?.trim() : null;
-                            }
-                            if (relation === "parent") {
-                                const grandParent = parent.parentElement;
-                                return grandParent ? (grandParent.innerText || grandParent.textContent)?.trim() : null;
-                            }
-                            return (parent.innerText || parent.textContent)?.trim();
-                        }
-                    }
-                    return null;
-                },
-                strategy.anchor || strategy.value,
-                strategy.relation || "self"
-            );
-        }
-
-        return null;
-    } catch (err) {
-        return null;
-    }
-}
-
-/**
- * Extracts a single field from a Cheerio instance given a strategy.
- */
-function extractCheerioField($, strategy) {
-    if (!strategy || !strategy.value) return null;
-
-    try {
-        const type = strategy.type || "css";
-        const attr = strategy.attribute || "text";
-
-        if (type === "css") {
-            const el = $(strategy.value);
-            if (!el || el.length === 0) return null;
-
-            if (attr === "text") return el.first().text().trim();
-            if (attr === "html") return el.first().html();
-            return el.first().attr(attr) || null;
-        }
-
-        if (type === "xpath") {
-            let css = String(strategy.value).trim();
-            css = css.replace(/\/\/\*\[@id=["']([^"']+)["']\]/g, "#$1");
-            css = css.replace(/\/([a-zA-Z0-9_-]+)\[([0-9]+)\]/g, " > $1:nth-of-type($2)");
-            css = css.replace(/\/([a-zA-Z0-9_-]+)/g, " > $1");
-            css = css.replace(/^\/+/g, "").replace(/^>\s*/, "").trim();
-
-            if (css) {
-                try {
-                    const el = $(css);
-                    if (el && el.length > 0) {
-                        if (attr === "text") return el.first().text().trim();
-                        if (attr === "html") return el.first().html();
-                        return el.first().attr(attr) || null;
-                    }
-                } catch {
-                    // Ignore parse error
-                }
-            }
-        }
-
-        if (type === "text-relative") {
-            const anchor = strategy.anchor || strategy.value;
-            const match = $(`:contains("${anchor}")`).last();
-            if (match && match.length > 0) return match.text().trim();
-        }
-
-        return null;
-    } catch {
-        return null;
-    }
-}
-
-/**
- * Handles SCRAPE_CHROMIUM task via local stealth Puppeteer.
- */
-export async function handleChromiumScrape(payload) {
-    const startTime = Date.now();
-    const {
-        url,
-        fieldMappings = [],
-        navigation = {},
-        waitFor = {},
-        timeout = 30000,
-        waitForSelector = null,
-        waitForTimeout = 1000,
-        actions = [],
-        extractSelectors = [],
-        headers = {},
-        cookies = [],
-        viewport = { width: 1366, height: 900 },
-        includeHtml = false,
-    } = payload;
-
-    const navTimeout = navigation.timeoutMs || timeout || 30000;
-    const navWaitUntil = navigation.waitUntil || "domcontentloaded";
-
-    const browser = await localBrowserManager.getBrowser();
-    const page = await browser.newPage();
-
-    try {
-        await page.setViewport({
-            width: viewport.width || 1366,
-            height: viewport.height || 900,
-            deviceScaleFactor: 1,
-        });
-
-        if (Object.keys(headers).length > 0) {
-            await page.setExtraHTTPHeaders(headers);
-        }
-
-        if (Array.isArray(cookies) && cookies.length > 0) {
-            await page.setCookie(...cookies);
-        }
-
-        // Navigate
-        const response = await page.goto(url, {
-            waitUntil: navWaitUntil,
-            timeout: navTimeout,
-        });
-
-        const status = response ? response.status() : 200;
-
-        // Handle wait conditions
-        if (waitFor && waitFor.type) {
-            const waitTimeout = waitFor.timeoutMs || 10000;
-            if (waitFor.type === "SELECTOR_EXISTS" && waitFor.selector) {
-                await page.waitForSelector(waitFor.selector, { timeout: waitTimeout }).catch(() => {});
-            } else if (waitFor.type === "SELECTOR_VISIBLE" && waitFor.selector) {
-                await page.waitForSelector(waitFor.selector, { visible: true, timeout: waitTimeout }).catch(() => {});
-            } else if (waitFor.type === "TEXT_PRESENT" && waitFor.text) {
-                await page
-                    .waitForFunction(
-                        (t) => document.body && document.body.innerText.includes(t),
-                        { timeout: waitTimeout },
-                        waitFor.text
-                    )
-                    .catch(() => {});
-            } else if (waitFor.type === "FIXED_DELAY" && waitFor.delayMs > 0) {
-                await new Promise((r) => setTimeout(r, Math.min(waitFor.delayMs, 10000)));
-            }
-        } else if (waitForSelector) {
-            await page.waitForSelector(waitForSelector, { timeout: 10000 }).catch(() => {});
-        }
-
-        // Optional brief delay for dynamic content
-        if (waitForTimeout > 0) {
-            await new Promise((r) => setTimeout(r, Math.min(waitForTimeout, 10000)));
-        }
-
-        // Execute sequential interactive actions if configured
-        if (Array.isArray(actions) && actions.length > 0) {
-            for (const action of actions) {
-                try {
-                    if (action.type === "click" && action.selector) {
-                        await page.click(action.selector);
-                    } else if (action.type === "type" && action.selector && action.text) {
-                        await page.type(action.selector, action.text);
-                    } else if (action.type === "scroll") {
-                        await page.evaluate((y) => window.scrollBy(0, y || 500), action.distance);
-                    } else if (action.type === "wait") {
-                        await new Promise((r) => setTimeout(r, action.duration || 1000));
-                    }
-                } catch (actionErr) {
-                    console.warn(`[ScrapeHandler] Action ${action.type} failed:`, actionErr.message);
-                }
-            }
-        }
-
-        const pageTitle = (await page.title()) || "";
-        const finalUrl = page.url();
-
-        const raw = {};
-        const strategySummary = {};
-
-        // 1. Extract mapped fields (Field Schema & Strategies)
-        if (Array.isArray(fieldMappings) && fieldMappings.length > 0) {
-            for (const mapping of fieldMappings) {
-                let extractedVal = null;
-                let successfulStrategy = null;
-
-                const strategies = mapping.strategies || [];
-                for (let i = 0; i < strategies.length; i++) {
-                    const strat = strategies[i];
-                    const val = await extractBrowserField(page, strat);
-                    if (val !== null && val !== undefined && val !== "") {
-                        extractedVal = val;
-                        successfulStrategy = strat.type || `strategy_${i + 1}`;
-                        break;
-                    }
-                }
-
-                raw[mapping.field] = extractedVal;
-                strategySummary[mapping.field] = {
-                    extracted: extractedVal !== null,
-                    strategyUsed: successfulStrategy,
-                };
-            }
-        }
-
-        // 2. Extract legacy extractSelectors if present
-        if (Array.isArray(extractSelectors) && extractSelectors.length > 0) {
-            for (const item of extractSelectors) {
-                const { field, selector, type = "text", attribute = null } = item;
-                if (!field || !selector) continue;
-
-                try {
-                    const values = await page.evaluate(
-                        (sel, t, attr) => {
-                            const elements = Array.from(document.querySelectorAll(sel));
-                            return elements.map((el) => {
-                                if (t === "attribute" && attr) {
-                                    return el.getAttribute(attr) || "";
-                                }
-                                if (t === "html") {
-                                    return el.innerHTML;
-                                }
-                                return el.innerText ? el.innerText.trim() : el.textContent?.trim() || "";
-                            });
-                        },
-                        selector,
-                        type,
-                        attribute
-                    );
-
-                    const extractedVal = values.length <= 1 ? (values[0] ?? null) : values;
-                    if (raw[field] === undefined) {
-                        raw[field] = extractedVal;
-                    }
-                } catch {
-                    if (raw[field] === undefined) {
-                        raw[field] = null;
-                    }
-                }
-            }
-        }
-
-        const executionTimeMs = Date.now() - startTime;
-        const hasExtractedValues = Object.values(raw).some(
-            (v) => v !== null && v !== undefined && v !== ""
-        );
-
-        // Avoid transferring huge raw HTML unless requested or nothing was extracted
-        let html = "";
-        if (includeHtml || (!hasExtractedValues && fieldMappings.length > 0)) {
-            html = await page.content().catch(() => "");
-        }
-
-        return {
-            status,
-            title: pageTitle,
-            url: finalUrl,
-            raw,
-            strategySummary,
-            extractedData: raw,
-            html,
-            executionTimeMs,
-        };
-    } finally {
-        await page.close().catch(() => {});
-    }
-}
-
-/**
- * Handles fast SCRAPE_HTTP task via Cheerio.
- */
-export async function handleHttpScrape(payload) {
-    const startTime = Date.now();
-    const {
-        url,
-        fieldMappings = [],
-        timeout = 15000,
-        headers = {},
-        extractSelectors = [],
-        includeHtml = false,
-    } = payload;
-
-    const response = await axios.get(url, {
-        timeout,
-        headers: {
-            "User-Agent":
-                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
-            Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-            "Accept-Language": "en-US,en;q=0.9",
-            ...headers,
-        },
-    });
-
-    const pageHtml = typeof response.data === "string" ? response.data : "";
-    const $ = cheerio.load(pageHtml);
-    const title = $("title").first().text().trim() || "";
-
-    const raw = {};
-    const strategySummary = {};
-
-    // 1. Extract fieldMappings
-    if (Array.isArray(fieldMappings) && fieldMappings.length > 0) {
-        for (const mapping of fieldMappings) {
-            let extractedVal = null;
-            let successfulStrategy = null;
-
-            const strategies = mapping.strategies || [];
-            for (let i = 0; i < strategies.length; i++) {
-                const strat = strategies[i];
-                const val = extractCheerioField($, strat);
-                if (val !== null && val !== undefined && val !== "") {
-                    extractedVal = val;
-                    successfulStrategy = strat.type || `strategy_${i + 1}`;
-                    break;
-                }
-            }
-
-            raw[mapping.field] = extractedVal;
-            strategySummary[mapping.field] = {
-                extracted: extractedVal !== null,
-                strategyUsed: successfulStrategy,
-            };
-        }
-    }
-
-    // 2. Extract legacy extractSelectors
-    if (Array.isArray(extractSelectors) && extractSelectors.length > 0) {
-        for (const item of extractSelectors) {
-            const { field, selector, type = "text", attribute = null } = item;
-            if (!field || !selector) continue;
-
-            try {
-                const values = [];
-                $(selector).each((_, el) => {
-                    if (type === "attribute" && attribute) {
-                        values.push($(el).attr(attribute) || "");
-                    } else if (type === "html") {
-                        values.push($(el).html() || "");
-                    } else {
-                        values.push($(el).text().trim());
-                    }
-                });
-
-                const extractedVal = values.length <= 1 ? (values[0] ?? null) : values;
-                if (raw[field] === undefined) {
-                    raw[field] = extractedVal;
-                }
-            } catch {
-                if (raw[field] === undefined) {
-                    raw[field] = null;
-                }
-            }
-        }
-    }
-
-    const executionTimeMs = Date.now() - startTime;
-    const hasExtractedValues = Object.values(raw).some(
-        (v) => v !== null && v !== undefined && v !== ""
-    );
-
-    return {
-        status: response.status,
-        title,
-        url,
-        raw,
-        strategySummary,
-        extractedData: raw,
-        html: includeHtml || (!hasExtractedValues && fieldMappings.length > 0) ? pageHtml : "",
-        executionTimeMs,
-    };
-}
+import axios from "axios";
+import * as cheerio from "cheerio";
+import localBrowserManager from "../browser.js";
+import { extractSmartCheerioText, runTransformPipeline, castValueToType } from "./transformUtils.js";
+
+/**
+ * Extracts a single field value from a Puppeteer page given a strategy.
+ */
+async function extractBrowserField(page, strategy) {
+    if (!strategy || !strategy.value) return null;
+
+    try {
+        const type = strategy.type || "css";
+        const attr = strategy.attribute || "text";
+
+        if (type === "css") {
+            const el = await page.$(strategy.value);
+            if (!el) return null;
+
+            if (attr === "text") {
+                const text = await page.evaluate((element) => element.textContent, el);
+                return text ? text.trim() : null;
+            } else if (attr === "html") {
+                return await page.evaluate((element) => element.innerHTML, el);
+            } else {
+                const val = await page.evaluate((element, a) => element.getAttribute(a), el, attr);
+                return val !== null && val !== undefined ? String(val).trim() : null;
+            }
+        }
+
+        if (type === "xpath") {
+            const val = await page.evaluate(
+                (xpathStr, attrName) => {
+                    try {
+                        const result = document.evaluate(
+                            xpathStr,
+                            document,
+                            null,
+                            XPathResult.FIRST_ORDERED_NODE_TYPE,
+                            null
+                        );
+                        const node = result.singleNodeValue;
+                        if (!node) return null;
+                        if (attrName === "text") {
+                            return node.textContent ? node.textContent.trim() : null;
+                        }
+                        return node.getAttribute ? node.getAttribute(attrName) : null;
+                    } catch (e) {
+                        return null;
+                    }
+                },
+                strategy.value,
+                attr
+            );
+            return val;
+        }
+
+        if (type === "text-relative") {
+            return await page.evaluate(
+                (anchorText, relation) => {
+                    const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT, null, false);
+                    let node;
+                    while ((node = walker.nextNode())) {
+                        if (node.textContent.includes(anchorText)) {
+                            const parent = node.parentElement;
+                            if (relation === "next-sibling") {
+                                const sibling = parent.nextElementSibling;
+                                return sibling ? (sibling.innerText || sibling.textContent)?.trim() : null;
+                            }
+                            if (relation === "parent") {
+                                const grandParent = parent.parentElement;
+                                return grandParent ? (grandParent.innerText || grandParent.textContent)?.trim() : null;
+                            }
+                            return (parent.innerText || parent.textContent)?.trim();
+                        }
+                    }
+                    return null;
+                },
+                strategy.anchor || strategy.value,
+                strategy.relation || "self"
+            );
+        }
+
+        return null;
+    } catch (err) {
+        return null;
+    }
+}
+
+/**
+ * Extracts a single field from a Cheerio instance given a strategy.
+ */
+function extractCheerioField($, strategy) {
+    if (!strategy || !strategy.value) return null;
+
+    try {
+        const type = strategy.type || "css";
+        const attr = strategy.attribute || "text";
+
+        if (type === "css") {
+            const el = $(strategy.value);
+            if (!el || el.length === 0) return null;
+
+            if (attr === "text") return extractSmartCheerioText($, el.first());
+            if (attr === "html") return el.first().html();
+            return el.first().attr(attr) || null;
+        }
+
+        if (type === "xpath") {
+            let css = String(strategy.value).trim();
+            css = css.replace(/\/\/\*\[@id=["']([^"']+)["']\]/g, "#$1");
+            css = css.replace(/\/([a-zA-Z0-9_-]+)\[([0-9]+)\]/g, " > $1:nth-of-type($2)");
+            css = css.replace(/\/([a-zA-Z0-9_-]+)/g, " > $1");
+            css = css.replace(/^\/+/g, "").replace(/^>\s*/, "").trim();
+
+            if (css) {
+                try {
+                    const el = $(css);
+                    if (el && el.length > 0) {
+                        if (attr === "text") return extractSmartCheerioText($, el.first());
+                        if (attr === "html") return el.first().html();
+                        return el.first().attr(attr) || null;
+                    }
+                } catch {
+                    // Ignore parse error
+                }
+            }
+        }
+
+        if (type === "text-relative") {
+            const anchor = strategy.anchor || strategy.value;
+            const match = $(`:contains("${anchor}")`).last();
+            if (match && match.length > 0) return extractSmartCheerioText($, match);
+        }
+
+        return null;
+    } catch {
+        return null;
+    }
+}
+
+/**
+ * Extracts a table collection using Cheerio.
+ */
+function extractTableCollectionCheerio($, collectionDef) {
+    if (!collectionDef) return [];
+    const selector = collectionDef.containerSelector || collectionDef.tableSelector || "table";
+    const tableEl = $(selector).first();
+    if (!tableEl || tableEl.length === 0) return [];
+
+    const hasHeaders = collectionDef.hasHeaders !== false;
+    const headerRowIndex = typeof collectionDef.headerRowIndex === "number" ? collectionDef.headerRowIndex : 0;
+    const maxItems = collectionDef.maxItems || 100;
+    const columns = collectionDef.columns || [];
+
+    const rows = [];
+    tableEl.find("tr").each((rIdx, rowEl) => {
+        const $row = $(rowEl);
+        const cells = [];
+        $row.find("th, td").each((_, cellEl) => {
+            cells.push(extractSmartCheerioText($, $(cellEl)));
+        });
+
+        if (cells.length > 0) {
+            rows.push({ rowIndex: rIdx, cells, rowEl: $row });
+        }
+    });
+
+    if (rows.length === 0) return [];
+
+    let activeColumns = [...columns];
+    if (activeColumns.length === 0) {
+        if (hasHeaders && rows[headerRowIndex]) {
+            activeColumns = rows[headerRowIndex].cells.map((headerText, colIdx) => {
+                const normalizedKey =
+                    headerText
+                        .toLowerCase()
+                        .replace(/[^a-z0-9]+/g, "_")
+                        .replace(/^_+|_+$/g, "") || `column_${colIdx + 1}`;
+                return {
+                    key: normalizedKey,
+                    label: headerText || `Column ${colIdx + 1}`,
+                    columnIndex: colIdx,
+                    type: "string",
+                    excluded: false,
+                };
+            });
+        } else {
+            activeColumns = rows[0].cells.map((_, colIdx) => ({
+                key: `column_${colIdx + 1}`,
+                label: `Column ${colIdx + 1}`,
+                columnIndex: colIdx,
+                type: "string",
+                excluded: false,
+            }));
+        }
+    }
+
+    const startRowIdx = hasHeaders ? headerRowIndex + 1 : 0;
+    const dataRows = rows.slice(startRowIdx, startRowIdx + maxItems);
+    const results = [];
+
+    for (const { cells, rowEl } of dataRows) {
+        const record = {};
+        for (let colIdx = 0; colIdx < activeColumns.length; colIdx++) {
+            const col = activeColumns[colIdx];
+            if (col.excluded) continue;
+            let rawVal = null;
+            const colSelector = col.relativeSelector || col.selector;
+
+            if (colSelector) {
+                const subEl = rowEl.find(colSelector).first();
+                if (subEl && subEl.length > 0) {
+                    const attr = col.attribute || "text";
+                    rawVal = attr === "text" ? extractSmartCheerioText($, subEl) : subEl.attr(attr) || null;
+                }
+            } else if (typeof col.columnIndex === "number" && col.columnIndex < cells.length) {
+                rawVal = cells[col.columnIndex];
+            } else if (colIdx < cells.length) {
+                rawVal = cells[colIdx];
+            }
+
+            if ((rawVal === null || rawVal === undefined || rawVal === "") && col.defaultValue !== undefined) {
+                rawVal = col.defaultValue;
+            }
+
+            // Apply column transforms
+            if (col.transforms && col.transforms.length > 0) {
+                rawVal = runTransformPipeline(rawVal, col.transforms);
+            }
+
+            // Cast to target type
+            if (col.type) {
+                rawVal = castValueToType(rawVal, col.type);
+            }
+
+            record[col.key] = rawVal;
+        }
+        results.push(record);
+    }
+
+    return results;
+}
+
+/**
+ * Extracts repeating items/cards using Cheerio.
+ */
+function extractRepeatingCollectionCheerio($, collectionDef) {
+    if (!collectionDef) return [];
+
+    let items;
+    if (collectionDef.rowSelector) {
+        if (collectionDef.containerSelector) {
+            items = $(collectionDef.containerSelector).find(collectionDef.rowSelector);
+        }
+        if (!items || items.length === 0) {
+            items = $(collectionDef.rowSelector);
+        }
+    } else if (collectionDef.containerSelector) {
+        items = $(collectionDef.containerSelector);
+    } else {
+        return [];
+    }
+
+    if (!items || items.length === 0) return [];
+
+    const maxItems = collectionDef.maxItems || 100;
+    const columns = collectionDef.columns || [];
+    const results = [];
+
+    items.slice(0, maxItems).each((_, itemEl) => {
+        const $item = $(itemEl);
+        const record = {};
+
+        for (const col of columns) {
+            if (col.excluded) continue;
+            let rawVal = null;
+            const colSelector = col.relativeSelector || col.selector;
+
+            if (colSelector) {
+                const target = $item.find(colSelector).first();
+                if (target && target.length > 0) {
+                    const attr = col.attribute || "text";
+                    rawVal = attr === "text" ? extractSmartCheerioText($, target) : target.attr(attr) || null;
+                }
+            } else {
+                const attr = col.attribute || "text";
+                rawVal = attr === "text" ? extractSmartCheerioText($, $item) : $item.attr(attr) || null;
+            }
+
+            if ((rawVal === null || rawVal === undefined || rawVal === "") && col.defaultValue !== undefined) {
+                rawVal = col.defaultValue;
+            }
+
+            // Apply column transforms
+            if (col.transforms && col.transforms.length > 0) {
+                rawVal = runTransformPipeline(rawVal, col.transforms);
+            }
+
+            // Cast to target type
+            if (col.type) {
+                rawVal = castValueToType(rawVal, col.type);
+            }
+
+            record[col.key] = rawVal;
+        }
+
+        const hasValue = Object.values(record).some((v) => v !== null && v !== undefined && v !== "");
+        if (hasValue) {
+            results.push(record);
+        }
+    });
+
+    return results;
+}
+
+/**
+ * Handles SCRAPE_CHROMIUM task via local stealth Puppeteer.
+ */
+export async function handleChromiumScrape(payload) {
+    const startTime = Date.now();
+    const {
+        url,
+        fieldMappings = [],
+        collections = [],
+        navigation = {},
+        waitFor = {},
+        timeout = 30000,
+        waitForSelector = null,
+        waitForTimeout = 1000,
+        actions = [],
+        extractSelectors = [],
+        headers = {},
+        cookies = [],
+        viewport = { width: 1366, height: 900 },
+        includeHtml = false,
+    } = payload;
+
+    const navTimeout = navigation.timeoutMs || timeout || 30000;
+    const navWaitUntil = navigation.waitUntil || "domcontentloaded";
+
+    const browser = await localBrowserManager.getBrowser();
+    const page = await browser.newPage();
+
+    try {
+        await page.setViewport({
+            width: viewport.width || 1366,
+            height: viewport.height || 900,
+            deviceScaleFactor: 1,
+        });
+
+        if (Object.keys(headers).length > 0) {
+            await page.setExtraHTTPHeaders(headers);
+        }
+
+        if (Array.isArray(cookies) && cookies.length > 0) {
+            await page.setCookie(...cookies);
+        }
+
+        // Navigate
+        const response = await page.goto(url, {
+            waitUntil: navWaitUntil,
+            timeout: navTimeout,
+        });
+
+        const status = response ? response.status() : 200;
+
+        // Handle wait conditions
+        if (waitFor && waitFor.type) {
+            const waitTimeout = waitFor.timeoutMs || 10000;
+            if (waitFor.type === "SELECTOR_EXISTS" && waitFor.selector) {
+                await page.waitForSelector(waitFor.selector, { timeout: waitTimeout }).catch(() => {});
+            } else if (waitFor.type === "SELECTOR_VISIBLE" && waitFor.selector) {
+                await page.waitForSelector(waitFor.selector, { visible: true, timeout: waitTimeout }).catch(() => {});
+            } else if (waitFor.type === "TEXT_PRESENT" && waitFor.text) {
+                await page
+                    .waitForFunction(
+                        (t) => document.body && document.body.innerText.includes(t),
+                        { timeout: waitTimeout },
+                        waitFor.text
+                    )
+                    .catch(() => {});
+            } else if (waitFor.type === "FIXED_DELAY" && waitFor.delayMs > 0) {
+                await new Promise((r) => setTimeout(r, Math.min(waitFor.delayMs, 10000)));
+            }
+        } else if (waitForSelector) {
+            await page.waitForSelector(waitForSelector, { timeout: 10000 }).catch(() => {});
+        }
+
+        // Optional brief delay for dynamic content
+        if (waitForTimeout > 0) {
+            await new Promise((r) => setTimeout(r, Math.min(waitForTimeout, 10000)));
+        }
+
+        // Execute sequential interactive actions if configured
+        if (Array.isArray(actions) && actions.length > 0) {
+            for (const action of actions) {
+                try {
+                    if (action.type === "click" && action.selector) {
+                        await page.click(action.selector);
+                    } else if (action.type === "type" && action.selector && action.text) {
+                        await page.type(action.selector, action.text);
+                    } else if (action.type === "scroll") {
+                        await page.evaluate((y) => window.scrollBy(0, y || 500), action.distance);
+                    } else if (action.type === "wait") {
+                        await new Promise((r) => setTimeout(r, action.duration || 1000));
+                    }
+                } catch (actionErr) {
+                    console.warn(`[ScrapeHandler] Action ${action.type} failed:`, actionErr.message);
+                }
+            }
+        }
+
+        const pageTitle = (await page.title()) || "";
+        const finalUrl = page.url();
+
+        const raw = {};
+        const strategySummary = {};
+
+        // 1. Extract mapped fields (Field Schema & Strategies)
+        if (Array.isArray(fieldMappings) && fieldMappings.length > 0) {
+            for (const mapping of fieldMappings) {
+                let extractedVal = null;
+                let successfulStrategy = null;
+
+                const strategies = mapping.strategies || [];
+                for (let i = 0; i < strategies.length; i++) {
+                    const strat = strategies[i];
+                    const val = await extractBrowserField(page, strat);
+                    if (val !== null && val !== undefined && val !== "") {
+                        extractedVal = val;
+                        successfulStrategy = strat.type || `strategy_${i + 1}`;
+                        break;
+                    }
+                }
+
+                if (mapping.transforms && mapping.transforms.length > 0) {
+                    extractedVal = runTransformPipeline(extractedVal, mapping.transforms);
+                }
+                if (mapping.type) {
+                    extractedVal = castValueToType(extractedVal, mapping.type);
+                }
+
+                raw[mapping.field] = extractedVal;
+                strategySummary[mapping.field] = {
+                    extracted: extractedVal !== null,
+                    strategyUsed: successfulStrategy,
+                };
+            }
+        }
+
+        // 2. Extract legacy extractSelectors if present
+        if (Array.isArray(extractSelectors) && extractSelectors.length > 0) {
+            for (const item of extractSelectors) {
+                const { field, selector, type = "text", attribute = null } = item;
+                if (!field || !selector) continue;
+
+                try {
+                    const values = await page.evaluate(
+                        (sel, t, attr) => {
+                            const elements = Array.from(document.querySelectorAll(sel));
+                            return elements.map((el) => {
+                                if (t === "attribute" && attr) {
+                                    return el.getAttribute(attr) || "";
+                                }
+                                if (t === "html") {
+                                    return el.innerHTML;
+                                }
+                                return el.innerText ? el.innerText.trim() : el.textContent?.trim() || "";
+                            });
+                        },
+                        selector,
+                        type,
+                        attribute
+                    );
+
+                    const extractedVal = values.length <= 1 ? (values[0] ?? null) : values;
+                    if (raw[field] === undefined) {
+                        raw[field] = extractedVal;
+                    }
+                } catch {
+                    if (raw[field] === undefined) {
+                        raw[field] = null;
+                    }
+                }
+            }
+        }
+
+        // 5. Extract structured collections (Tables & Repeating items)
+        const extractedCollections = {};
+        if (Array.isArray(collections) && collections.length > 0) {
+            for (const coll of collections) {
+                if (!coll || !coll.name) continue;
+                try {
+                    if (coll.type === "TABLE") {
+                        extractedCollections[coll.name] = await page.evaluate((cDef) => {
+                            const sel = cDef.containerSelector || "table";
+                            const table = document.querySelector(sel);
+                            if (!table) return [];
+
+                            const hasHeaders = cDef.hasHeaders !== false;
+                            const headerRowIndex = typeof cDef.headerRowIndex === "number" ? cDef.headerRowIndex : 0;
+                            const maxItems = cDef.maxItems || 100;
+                            const columns = cDef.columns || [];
+
+                            const rowEls = Array.from(table.querySelectorAll("tr"));
+                            const rowsData = rowEls
+                                .map((r) => {
+                                    const cells = Array.from(r.querySelectorAll("th, td")).map(
+                                        (c) => c.innerText?.trim() || c.textContent?.trim() || ""
+                                    );
+                                    return { cells, el: r };
+                                })
+                                .filter((r) => r.cells.length > 0);
+
+                            if (rowsData.length === 0) return [];
+
+                            let activeColumns = [...columns];
+                            if (activeColumns.length === 0) {
+                                if (hasHeaders && rowsData[headerRowIndex]) {
+                                    activeColumns = rowsData[headerRowIndex].cells.map((h, idx) => {
+                                        const key =
+                                            h.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "") ||
+                                            `col_${idx + 1}`;
+                                        return { key, label: h || `Column ${idx + 1}`, columnIndex: idx, excluded: false };
+                                    });
+                                } else {
+                                    activeColumns = rowsData[0].cells.map((_, idx) => ({
+                                        key: `col_${idx + 1}`,
+                                        label: `Column ${idx + 1}`,
+                                        columnIndex: idx,
+                                        excluded: false,
+                                    }));
+                                }
+                            }
+
+                            const startIdx = hasHeaders ? headerRowIndex + 1 : 0;
+                            const sliceRows = rowsData.slice(startIdx, startIdx + maxItems);
+                            const results = [];
+
+                            for (const row of sliceRows) {
+                                const rec = {};
+                                for (let colIdx = 0; colIdx < activeColumns.length; colIdx++) {
+                                    const col = activeColumns[colIdx];
+                                    if (col.excluded) continue;
+                                    let val = null;
+                                    const colSel = col.relativeSelector || col.selector;
+                                    if (colSel) {
+                                        const subEl = row.el.querySelector(colSel);
+                                        if (subEl) {
+                                            const attr = col.attribute || "text";
+                                            val = attr === "text" ? (subEl.innerText?.trim() || subEl.textContent?.trim()) : subEl.getAttribute(attr);
+                                        }
+                                    } else if (typeof col.columnIndex === "number" && col.columnIndex < row.cells.length) {
+                                        val = row.cells[col.columnIndex];
+                                    } else if (colIdx < row.cells.length) {
+                                        val = row.cells[colIdx];
+                                    }
+                                    if ((val === null || val === undefined || val === "") && col.defaultValue !== undefined) {
+                                        val = col.defaultValue;
+                                    }
+                                    rec[col.key] = val;
+                                }
+                                results.push(rec);
+                            }
+                            return results;
+                        }, coll);
+                    } else {
+                        extractedCollections[coll.name] = await page.evaluate((cDef) => {
+                            let items = [];
+                            if (cDef.rowSelector) {
+                                if (cDef.containerSelector) {
+                                    const container = document.querySelector(cDef.containerSelector);
+                                    if (container) {
+                                        items = Array.from(container.querySelectorAll(cDef.rowSelector));
+                                    }
+                                }
+                                if (items.length === 0) {
+                                    items = Array.from(document.querySelectorAll(cDef.rowSelector));
+                                }
+                            } else if (cDef.containerSelector) {
+                                items = Array.from(document.querySelectorAll(cDef.containerSelector));
+                            }
+
+                            if (items.length === 0) return [];
+
+                            const maxItems = cDef.maxItems || 100;
+                            const columns = cDef.columns || [];
+                            const results = [];
+
+                            for (const item of items.slice(0, maxItems)) {
+                                const rec = {};
+                                for (const col of columns) {
+                                    if (col.excluded) continue;
+                                    let val = null;
+                                    const colSel = col.relativeSelector || col.selector;
+                                    if (colSel) {
+                                        const target = item.querySelector(colSel);
+                                        if (target) {
+                                            const attr = col.attribute || "text";
+                                            val = attr === "text" ? (target.innerText?.trim() || target.textContent?.trim()) : target.getAttribute(attr);
+                                        }
+                                    } else {
+                                        const attr = col.attribute || "text";
+                                        val = attr === "text" ? (item.innerText?.trim() || item.textContent?.trim()) : item.getAttribute(attr);
+                                    }
+                                    if ((val === null || val === undefined || val === "") && col.defaultValue !== undefined) {
+                                        val = col.defaultValue;
+                                    }
+                                    rec[col.key] = val;
+                                }
+                                const hasVal = Object.values(rec).some((v) => v !== null && v !== undefined && v !== "");
+                                if (hasVal) results.push(rec);
+                            }
+                            return results;
+                        }, coll);
+                    }
+
+                    // Apply column transforms and type casting
+                    if (Array.isArray(extractedCollections[coll.name]) && coll.columns) {
+                        extractedCollections[coll.name] = extractedCollections[coll.name].map((rec) => {
+                            const transformed = { ...rec };
+                            for (const col of coll.columns) {
+                                if (col.excluded) continue;
+                                let val = transformed[col.key];
+                                if ((val === null || val === undefined || val === "") && col.defaultValue !== undefined) {
+                                    val = col.defaultValue;
+                                }
+                                if (col.transforms && col.transforms.length > 0) {
+                                    val = runTransformPipeline(val, col.transforms);
+                                }
+                                if (col.type) {
+                                    val = castValueToType(val, col.type);
+                                }
+                                transformed[col.key] = val;
+                            }
+                            return transformed;
+                        });
+                    }
+                } catch {
+                    extractedCollections[coll.name] = [];
+                }
+            }
+        }
+
+        const executionTimeMs = Date.now() - startTime;
+        const hasExtractedValues = Object.values(raw).some(
+            (v) => v !== null && v !== undefined && v !== ""
+        );
+
+        // Avoid transferring huge raw HTML unless requested or nothing was extracted
+        let html = "";
+        if (includeHtml || (!hasExtractedValues && fieldMappings.length > 0)) {
+            html = await page.content().catch(() => "");
+        }
+
+        return {
+            status,
+            title: pageTitle,
+            url: finalUrl,
+            raw,
+            strategySummary,
+            extractedData: raw,
+            collections: extractedCollections,
+            html,
+            executionTimeMs,
+        };
+    } finally {
+        await page.close().catch(() => {});
+    }
+}
+
+/**
+ * Handles fast SCRAPE_HTTP task via Cheerio.
+ */
+export async function handleHttpScrape(payload) {
+    const startTime = Date.now();
+    const {
+        url,
+        fieldMappings = [],
+        collections = [],
+        timeout = 15000,
+        headers = {},
+        extractSelectors = [],
+        includeHtml = false,
+    } = payload;
+
+    const response = await axios.get(url, {
+        timeout,
+        headers: {
+            "User-Agent":
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+            Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.9",
+            ...headers,
+        },
+    });
+
+    let pageHtml = typeof response.data === "string" ? response.data : Buffer.isBuffer(response.data) ? response.data.toString("utf-8") : "";
+    const $ = cheerio.load(pageHtml);
+    const title = $("title").first().text().trim() || "";
+
+    const raw = {};
+    const strategySummary = {};
+
+    // 1. Extract fieldMappings
+    if (Array.isArray(fieldMappings) && fieldMappings.length > 0) {
+        for (const mapping of fieldMappings) {
+            let extractedVal = null;
+            let successfulStrategy = null;
+
+            const strategies = mapping.strategies || [];
+            for (let i = 0; i < strategies.length; i++) {
+                const strat = strategies[i];
+                const val = extractCheerioField($, strat);
+                if (val !== null && val !== undefined && val !== "") {
+                    extractedVal = val;
+                    successfulStrategy = strat.type || `strategy_${i + 1}`;
+                    break;
+                }
+            }
+
+            if (mapping.transforms && mapping.transforms.length > 0) {
+                extractedVal = runTransformPipeline(extractedVal, mapping.transforms);
+            }
+            if (mapping.type) {
+                extractedVal = castValueToType(extractedVal, mapping.type);
+            }
+
+            raw[mapping.field] = extractedVal;
+            strategySummary[mapping.field] = {
+                extracted: extractedVal !== null,
+                strategyUsed: successfulStrategy,
+            };
+        }
+    }
+
+    // 2. Extract legacy extractSelectors
+    if (Array.isArray(extractSelectors) && extractSelectors.length > 0) {
+        for (const item of extractSelectors) {
+            const { field, selector, type = "text", attribute = null } = item;
+            if (!field || !selector) continue;
+
+            try {
+                const values = [];
+                $(selector).each((_, el) => {
+                    if (type === "attribute" && attribute) {
+                        values.push($(el).attr(attribute) || "");
+                    } else if (type === "html") {
+                        values.push($(el).html() || "");
+                    } else {
+                        values.push($(el).text().trim());
+                    }
+                });
+
+                const extractedVal = values.length <= 1 ? (values[0] ?? null) : values;
+                if (raw[field] === undefined) {
+                    raw[field] = extractedVal;
+                }
+            } catch {
+                if (raw[field] === undefined) {
+                    raw[field] = null;
+                }
+            }
+        }
+    }
+
+    // 3. Extract structured collections (Tables & Repeating items)
+    const extractedCollections = {};
+    if (Array.isArray(collections) && collections.length > 0) {
+        for (const coll of collections) {
+            if (!coll || !coll.name) continue;
+            try {
+                if (coll.type === "TABLE") {
+                    extractedCollections[coll.name] = extractTableCollectionCheerio($, coll);
+                } else {
+                    extractedCollections[coll.name] = extractRepeatingCollectionCheerio($, coll);
+                }
+            } catch {
+                extractedCollections[coll.name] = [];
+            }
+        }
+    }
+
+    const executionTimeMs = Date.now() - startTime;
+    const hasExtractedValues = Object.values(raw).some(
+        (v) => v !== null && v !== undefined && v !== ""
+    );
+
+    return {
+        status: response.status,
+        title,
+        url,
+        raw,
+        strategySummary,
+        extractedData: raw,
+        collections: extractedCollections,
+        html: includeHtml || (!hasExtractedValues && fieldMappings.length > 0) ? pageHtml : "",
+        executionTimeMs,
+    };
+}