app-manager-edge-worker 1.0.1 → 1.0.2

package/README.md

@@ -55,21 +55,48 @@ app-manager-edge-worker
 
 ## 📱 Turn Any Android Phone into a 24/7 Scraping Node (Termux)
 
-You can turn any old Android phone into an ultra-low-power, 24/7 residential scraping node using **Termux** (available free from F-Droid):
+Turn any spare or everyday Android phone into an ultra-low-power, 24/7 residential edge scraping node using **Termux** (download free from [F-Droid](https://f-droid.org/en/packages/com.termux/)):
+
+### Quick 1-Liner Setup (Copy & Paste into Termux)
 
 ```bash
-# 1. Update Termux packages & install Node.js + Chromium
-pkg update -y && pkg install -y nodejs-lts chromium
+pkg update -y && pkg install -y x11-repo && pkg update -y && pkg install -y nodejs-lts chromium termux-tools && termux-wake-lock && npx app-manager-edge-worker --server https://apps-manager.onrender.com --key <YOUR_WORKER_KEY>
+```
+
+---
+
+### Step-by-Step Installation
 
-# 2. Point Puppeteer to Termux's native Chromium ARM64 binary
+```bash
+# 1. Update Termux core packages
+pkg update -y
+
+# 2. Enable x11-repo (Termux hosts the Chromium ARM64 binary inside x11-repo)
+pkg install -y x11-repo
+pkg update -y
+
+# 3. Install Node.js LTS, Chromium browser, and Termux tools
+pkg install -y nodejs-lts chromium termux-tools
+
+# 4. Acquire wake-lock (keeps CPU active when screen turns off)
+termux-wake-lock
+
+# 5. Point Puppeteer to Termux's native Chromium binary (auto-detected, or export explicitly)
 export PUPPETEER_EXECUTABLE_PATH=/data/data/com.termux/files/usr/bin/chromium-browser
+export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
 
-# 3. Launch the edge worker
+# 6. Launch the distributed edge worker
 npx app-manager-edge-worker --server https://apps-manager.onrender.com --key <YOUR_WORKER_KEY>
 ```
 
+> [!IMPORTANT]
+> **Why `x11-repo` is mandatory**: On Android aarch64, Termux maintains the `chromium` package within the `x11-repo` repository. Running `pkg install -y x11-repo` enables this repository so `pkg install chromium` succeeds without "package not found" errors.
+
 > [!TIP]
-> In Termux, swipe down your Android notification shade and tap **"Acquire Wake Lock"**. Disable Android OS battery optimization for Termux so your phone runs uninterrupted in the background.
+> **Android Battery Optimization**:
+> 1. In Termux, run `termux-wake-lock` or swipe down your Android notification panel and tap **"Acquire Wake Lock"** (our worker CLI also automatically acquires the wake-lock on startup).
+> 2. In Android **Settings** → **Apps** → **Termux** → **Battery** → set to **"Unrestricted"** (or **"Don't optimize"**). This prevents Android's Doze mode or manufacturer task killers (Samsung, Xiaomi, OnePlus) from putting Termux to sleep when the screen is locked.
+> 3. For low-RAM phones (2GB–3GB RAM), add `--concurrency 1` or `--concurrency 2` to keep memory consumption ultra-lean.
 
 ---
 
@@ -128,7 +155,7 @@ The worker **automatically downloads and installs it** according to your operati
 | **Windows** (x64/x86) | Downloads official stable Chrome for Testing from Google CDN with a progress bar into `~/.cache/puppeteer`. |
 | **macOS** (Intel / Apple Silicon M1-M4) | Detects system architecture (`mac` or `mac_arm`) and downloads matching stable Chrome from Google CDN. |
 | **Linux Desktop / VPS** (Ubuntu, Debian, Fedora, Arch) | Downloads official Linux x64/arm64 Chrome for Testing via `@puppeteer/browsers`. |
-| **Android (Termux)** | Automatically invokes Termux package manager (`pkg install -y chromium`). |
+| **Android (Termux)** | Automatically enables `x11-repo` and invokes Termux package manager (`pkg update -y && pkg install -y x11-repo && pkg update -y && pkg install -y nodejs-lts chromium termux-tools`). |
 | **Alpine Linux** | Automatically invokes Alpine package manager (`apk add --no-cache chromium`). |
 
 No manual setup required!

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "app-manager-edge-worker",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "description": "Distributed Scraper Edge Worker for Apps Manager. Run on any local machine, VPS, or Termux Android device to serve browser and HTTP scraping jobs.",
   "type": "module",
   "bin": {

package/src/banner.js

@@ -21,7 +21,7 @@ export function printBanner({ workerName, serverUrl, concurrency, platform, isTe
     console.log("  ██║  ██║██║     ██║     ███████║██╗██║ ╚═╝ ██║██║  ██║██║ ╚████║██║  ██║╚██████╔╝███████╗██║  ██║");
     console.log("  ╚═╝  ╚═╝╚═╝     ╚═╝     ╚══════╝╚═╝╚═╝     ╚═╝╚═╝  ╚═╝╚═╝  ╚═══╝╚═╝  ╚═╝ ╚═════╝ ╚══════╝╚═╝  ╚═╝");
     console.log(`${colors.reset}`);
-    console.log(`${colors.bright}  🌐 Distributed Scraper Edge Worker Client v1.0.1${colors.reset}`);
+    console.log(`${colors.bright}  🌐 Distributed Scraper Edge Worker Client v1.0.2${colors.reset}`);
     console.log(`${colors.dim}  Zero-cost, persistent tunneling & residential edge scraping${colors.reset}`);
     console.log("--------------------------------------------------------------------------------");
     console.log(`  ${colors.bright}Device Name  :${colors.reset} ${colors.green}${workerName}${colors.reset}`);

package/src/browser.js

@@ -169,9 +169,11 @@ class LocalBrowserManager {
         // 1. Android Termux
         if (this.isTermux()) {
             logWarn("Termux Android environment detected without Chromium.");
-            logInfo("Attempting auto-install via Termux package manager (pkg install -y chromium)...");
+            logInfo("Attempting auto-install via Termux package manager (enabling x11-repo and installing chromium)...");
             try {
-                execSync("pkg update -y && pkg install -y chromium", { stdio: "inherit" });
+                execSync("pkg update -y && pkg install -y x11-repo && pkg update -y && pkg install -y nodejs-lts chromium termux-tools", {
+                    stdio: "inherit",
+                });
                 const termuxPaths = [
                     "/data/data/com.termux/files/usr/bin/chromium-browser",
                     "/data/data/com.termux/files/usr/bin/chromium",
@@ -184,7 +186,7 @@ class LocalBrowserManager {
                 }
             } catch (err) {
                 logError(`Termux auto-install failed: ${err.message}`);
-                logWarn("Please run manually in Termux: pkg install chromium -y");
+                logWarn("Please run manually in Termux:\n  pkg update -y && pkg install -y x11-repo && pkg update -y && pkg install -y nodejs-lts chromium termux-tools");
             }
         }
 
@@ -319,6 +321,10 @@ class LocalBrowserManager {
                     "--window-size=1366,900",
                 ];
 
+                if (this.isTermux()) {
+                    launchArgs.push("--single-process");
+                }
+
                 const launchOptions = {
                     headless: "new",
                     args: launchArgs,

package/src/client.js

@@ -2,6 +2,7 @@ import { io } from "socket.io-client";
 import os from "os";
 import path from "path";
 import axios from "axios";
+import { execSync } from "child_process";
 import localBrowserManager from "./browser.js";
 import { handleChromiumScrape, handleHttpScrape } from "./handlers/scrapeHandler.js";
 import { handlePreviewPage, handleInspectElement } from "./handlers/previewHandler.js";
@@ -20,6 +21,15 @@ export async function startWorkerClient(config) {
         keepAliveMinutes,
     });
 
+    if (isTermux) {
+        try {
+            execSync("termux-wake-lock", { stdio: "ignore" });
+            logSuccess("[Android Termux] Wake lock acquired to keep CPU awake in background.");
+        } catch {
+            // termux-tools might not be installed yet
+        }
+    }
+
     logInfo("Checking Chromium browser availability...");
     let chromiumExecutable = null;
     try {
@@ -52,11 +62,58 @@ export async function startWorkerClient(config) {
         },
     });
 
+    const CHUNK_SIZE = 256 * 1024; // 256 KB chunks to prevent any WebSocket frame buffer overflow
+
+    const sendJobResult = (payload) => {
+        let payloadStr = "";
+        try {
+            payloadStr = JSON.stringify(payload);
+        } catch {
+            socket.emit("job:result", payload);
+            return;
+        }
+
+        // Small payload: send directly
+        if (payloadStr.length <= CHUNK_SIZE) {
+            socket.emit("job:result", payload);
+            return;
+        }
+
+        // Large payload (e.g. screenshot or large scrape): stream in chunks
+        const totalChunks = Math.ceil(payloadStr.length / CHUNK_SIZE);
+        const jobId = payload.jobId;
+
+        socket.emit("job:result:stream:start", {
+            jobId,
+            totalChunks,
+            totalLength: payloadStr.length,
+        });
+
+        for (let i = 0; i < totalChunks; i++) {
+            const chunk = payloadStr.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE);
+            socket.emit("job:result:stream:chunk", {
+                jobId,
+                index: i,
+                chunk,
+            });
+        }
+
+        socket.emit("job:result:stream:end", { jobId });
+    };
+
     let activeJobs = 0;
+    let activeConcurrency = concurrency;
 
     socket.on("connect", () => {
         logSuccess(`Connected to cluster! (Socket ID: ${socket.id})`);
-        logInfo(`Ready to accept scraping and visual preview jobs (Concurrency: ${concurrency}).\n`);
+        logInfo(`Ready to accept scraping and visual preview jobs (Concurrency: ${activeConcurrency}).\n`);
+    });
+
+    socket.on("worker:registered", (data) => {
+        if (data?.concurrencyLimit && typeof data.concurrencyLimit === "number") {
+            activeConcurrency = data.concurrencyLimit;
+            logInfo(`[Cluster Config] Worker registered as '${data.name || data.workerId}'. Active concurrency: ${activeConcurrency} concurrent tabs.`);
+        }
     });
 
     socket.on("connect_error", (err) => {
@@ -102,7 +159,7 @@ export async function startWorkerClient(config) {
             const durationMs = Date.now() - startTime;
             logSuccess(`Completed ${jobType} for ${targetUrl} in ${durationMs}ms`);
 
-            socket.emit("job:result", {
+            sendJobResult({
                 jobId,
                 success: true,
                 data: resultData,
@@ -112,7 +169,7 @@ export async function startWorkerClient(config) {
             const durationMs = Date.now() - startTime;
             logError(`Failed ${jobType} for ${targetUrl}: ${err.message}`);
 
-            socket.emit("job:result", {
+            sendJobResult({
                 jobId,
                 success: false,
                 error: {
@@ -139,35 +196,49 @@ export async function startWorkerClient(config) {
     let keepAliveTimer = null;
     let initialKeepAliveTimer = null;
 
-    if (keepAliveMinutes > 0) {
-        const pingKeepAlive = async () => {
+    const clearKeepAliveTimers = () => {
+        if (keepAliveTimer) {
+            clearInterval(keepAliveTimer);
+            keepAliveTimer = null;
+        }
+        if (initialKeepAliveTimer) {
+            clearTimeout(initialKeepAliveTimer);
+            initialKeepAliveTimer = null;
+        }
+    };
+
+    const pingKeepAlive = async () => {
+        try {
+            // Try the dedicated keep-alive endpoint first
+            const endpoint = `${serverUrl}/api/workers/keep-alive`;
+            const res = await axios.get(endpoint, {
+                timeout: 15000,
+                headers: {
+                    "x-worker-key": workerKey || "",
+                    "user-agent": "AppsManagerEdgeWorker/1.0.2",
+                },
+            });
+            logInfo(`[Keep-Alive] Server awake ping OK (${res.status}) - Render instance active`);
+        } catch (err) {
+            // Fallback to /health
             try {
-                // Try the dedicated keep-alive endpoint first
-                const endpoint = `${serverUrl}/api/workers/keep-alive`;
-                const res = await axios.get(endpoint, {
-                    timeout: 15000,
+                const fallbackUrl = `${serverUrl}/health`;
+                await axios.get(fallbackUrl, {
+                    timeout: 10000,
                     headers: {
-                        "x-worker-key": workerKey || "",
-                        "user-agent": "AppsManagerEdgeWorker/1.0.1",
+                        "user-agent": "AppsManagerEdgeWorker/1.0.2",
                     },
                 });
-                logInfo(`[Keep-Alive] Server awake ping OK (${res.status}) - Render instance active`);
-            } catch (err) {
-                // Fallback to /health
-                try {
-                    const fallbackUrl = `${serverUrl}/health`;
-                    await axios.get(fallbackUrl, {
-                        timeout: 10000,
-                        headers: {
-                            "user-agent": "AppsManagerEdgeWorker/1.0.1",
-                        },
-                    });
-                    logInfo(`[Keep-Alive] Fallback health ping OK - Render instance active`);
-                } catch (fallbackErr) {
-                    logWarn(`[Keep-Alive] Server awake ping warning (${err.message}) - will retry next cycle`);
-                }
+                logInfo(`[Keep-Alive] Fallback health ping OK - Render instance active`);
+            } catch (fallbackErr) {
+                logWarn(`[Keep-Alive] Server awake ping warning (${err.message}) - will retry next cycle`);
             }
-        };
+        }
+    };
+
+    const startKeepAlive = () => {
+        if (keepAliveMinutes <= 0) return;
+        clearKeepAliveTimers();
 
         // Send initial ping 15s after startup to warm up/verify
         initialKeepAliveTimer = setTimeout(pingKeepAlive, 15000);
@@ -176,15 +247,40 @@ export async function startWorkerClient(config) {
         const intervalMs = Math.max(1, keepAliveMinutes) * 60 * 1000;
         keepAliveTimer = setInterval(pingKeepAlive, intervalMs);
         if (keepAliveTimer.unref) keepAliveTimer.unref();
+    };
+
+    if (keepAliveMinutes > 0) {
+        startKeepAlive();
     }
 
+    // Clean up keep-alive timers on socket disconnect
+    socket.on("disconnect", (reason) => {
+        logWarn(`Disconnected from cluster: ${reason}`);
+        clearKeepAliveTimers();
+    });
+
+    socket.on("connect", () => {
+        if (keepAliveMinutes > 0 && !keepAliveTimer) {
+            startKeepAlive();
+        }
+    });
+
+    // Provide programmatic close for automated test harnesses
+    socket.closeClient = async () => {
+        clearKeepAliveTimers();
+        if (isTermux) {
+            try {
+                execSync("termux-wake-unlock", { stdio: "ignore" });
+            } catch {}
+        }
+        socket.disconnect();
+        await localBrowserManager.close();
+    };
+
     // Graceful shutdown handling
     const shutdown = async () => {
-        if (keepAliveTimer) clearInterval(keepAliveTimer);
-        if (initialKeepAliveTimer) clearTimeout(initialKeepAliveTimer);
         logInfo("\nGracefully shutting down edge worker...");
-        socket.disconnect();
-        await localBrowserManager.close();
+        await socket.closeClient();
         logSuccess("Edge worker stopped cleanly.");
         process.exit(0);
     };

package/src/config.js

@@ -106,7 +106,7 @@ export async function resolveWorkerConfig() {
 
     if (cli.help) {
         console.log(`
-Apps Manager Scraper Edge Worker CLI v1.0.1
+Apps Manager Scraper Edge Worker CLI v1.0.2
 
 Usage:
   npx app-manager-edge-worker [options]

package/src/handlers/previewHandler.js

@@ -12,15 +12,23 @@ export async function handlePreviewPage(payload) {
         await page.setViewport({ width: 1366, height: 900, deviceScaleFactor: 1 });
 
         await page.goto(url, { waitUntil: "domcontentloaded", timeout });
-        await new Promise((r) => setTimeout(r, 1500));
+        try {
+            await page.waitForNetworkIdle({ idleTime: 500, timeout: 3500 }).catch(() => {});
+        } catch (_) {}
+        await page.evaluate(() => document.fonts?.ready).catch(() => {});
+        await new Promise((r) => setTimeout(r, 800));
 
         const title = (await page.title()) || new URL(url).hostname;
         const currentUrl = page.url();
 
-        const dimensions = await page.evaluate(() => ({
+        const rawDimensions = await page.evaluate(() => ({
             width: document.documentElement.scrollWidth || 1366,
             height: document.documentElement.scrollHeight || 900,
         }));
+        const dimensions = {
+            width: Math.min(rawDimensions.width, 3840),
+            height: Math.min(rawDimensions.height, 12000),
+        };
 
         // Hide scrollbars before screenshot
         await page
@@ -35,7 +43,7 @@ export async function handlePreviewPage(payload) {
         const screenshotBuffer = await page.screenshot({
             encoding: "binary",
             type: "jpeg",
-            quality: 85,
+            quality: 75,
             fullPage: true,
         });
 

package/src/handlers/scrapeHandler.js

@@ -2,6 +2,142 @@ import axios from "axios";
 import * as cheerio from "cheerio";
 import localBrowserManager from "../browser.js";
 
+/**
+ * Extracts a single field value from a Puppeteer page given a strategy.
+ */
+async function extractBrowserField(page, strategy) {
+    if (!strategy || !strategy.value) return null;
+
+    try {
+        const type = strategy.type || "css";
+        const attr = strategy.attribute || "text";
+
+        if (type === "css") {
+            const el = await page.$(strategy.value);
+            if (!el) return null;
+
+            if (attr === "text") {
+                const text = await page.evaluate((element) => element.textContent, el);
+                return text ? text.trim() : null;
+            } else if (attr === "html") {
+                return await page.evaluate((element) => element.innerHTML, el);
+            } else {
+                const val = await page.evaluate((element, a) => element.getAttribute(a), el, attr);
+                return val !== null && val !== undefined ? String(val).trim() : null;
+            }
+        }
+
+        if (type === "xpath") {
+            const val = await page.evaluate(
+                (xpathStr, attrName) => {
+                    try {
+                        const result = document.evaluate(
+                            xpathStr,
+                            document,
+                            null,
+                            XPathResult.FIRST_ORDERED_NODE_TYPE,
+                            null
+                        );
+                        const node = result.singleNodeValue;
+                        if (!node) return null;
+                        if (attrName === "text") {
+                            return node.textContent ? node.textContent.trim() : null;
+                        }
+                        return node.getAttribute ? node.getAttribute(attrName) : null;
+                    } catch (e) {
+                        return null;
+                    }
+                },
+                strategy.value,
+                attr
+            );
+            return val;
+        }
+
+        if (type === "text-relative") {
+            return await page.evaluate(
+                (anchorText, relation) => {
+                    const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT, null, false);
+                    let node;
+                    while ((node = walker.nextNode())) {
+                        if (node.textContent.includes(anchorText)) {
+                            const parent = node.parentElement;
+                            if (relation === "next-sibling") {
+                                const sibling = parent.nextElementSibling;
+                                return sibling ? (sibling.innerText || sibling.textContent)?.trim() : null;
+                            }
+                            if (relation === "parent") {
+                                const grandParent = parent.parentElement;
+                                return grandParent ? (grandParent.innerText || grandParent.textContent)?.trim() : null;
+                            }
+                            return (parent.innerText || parent.textContent)?.trim();
+                        }
+                    }
+                    return null;
+                },
+                strategy.anchor || strategy.value,
+                strategy.relation || "self"
+            );
+        }
+
+        return null;
+    } catch (err) {
+        return null;
+    }
+}
+
+/**
+ * Extracts a single field from a Cheerio instance given a strategy.
+ */
+function extractCheerioField($, strategy) {
+    if (!strategy || !strategy.value) return null;
+
+    try {
+        const type = strategy.type || "css";
+        const attr = strategy.attribute || "text";
+
+        if (type === "css") {
+            const el = $(strategy.value);
+            if (!el || el.length === 0) return null;
+
+            if (attr === "text") return el.first().text().trim();
+            if (attr === "html") return el.first().html();
+            return el.first().attr(attr) || null;
+        }
+
+        if (type === "xpath") {
+            let css = String(strategy.value).trim();
+            css = css.replace(/\/\/\*\[@id=["']([^"']+)["']\]/g, "#$1");
+            css = css.replace(/\/([a-zA-Z0-9_-]+)\[([0-9]+)\]/g, " > $1:nth-of-type($2)");
+            css = css.replace(/\/([a-zA-Z0-9_-]+)/g, " > $1");
+            css = css.replace(/^\/+/g, "").replace(/^>\s*/, "").trim();
+
+            if (css) {
+                try {
+                    const el = $(css);
+                    if (el && el.length > 0) {
+                        if (attr === "text") return el.first().text().trim();
+                        if (attr === "html") return el.first().html();
+                        return el.first().attr(attr) || null;
+                    }
+                } catch {
+                    // Ignore parse error
+                }
+            }
+        }
+
+        if (type === "text-relative") {
+            const anchor = strategy.anchor || strategy.value;
+            const match = $(`:contains("${anchor}")`).last();
+            if (match && match.length > 0) return match.text().trim();
+        }
+
+        return null;
+    } catch {
+        return null;
+    }
+}
+
 /**
  * Handles SCRAPE_CHROMIUM task via local stealth Puppeteer.
  */
@@ -9,6 +145,9 @@ export async function handleChromiumScrape(payload) {
     const startTime = Date.now();
     const {
         url,
+        fieldMappings = [],
+        navigation = {},
+        waitFor = {},
         timeout = 30000,
         waitForSelector = null,
         waitForTimeout = 1000,
@@ -17,8 +156,12 @@ export async function handleChromiumScrape(payload) {
         headers = {},
         cookies = [],
         viewport = { width: 1366, height: 900 },
+        includeHtml = false,
     } = payload;
 
+    const navTimeout = navigation.timeoutMs || timeout || 30000;
+    const navWaitUntil = navigation.waitUntil || "domcontentloaded";
+
     const browser = await localBrowserManager.getBrowser();
     const page = await browser.newPage();
 
@@ -39,22 +182,35 @@ export async function handleChromiumScrape(payload) {
 
         // Navigate
         const response = await page.goto(url, {
-            waitUntil: "domcontentloaded",
-            timeout,
+            waitUntil: navWaitUntil,
+            timeout: navTimeout,
         });
 
         const status = response ? response.status() : 200;
 
-        // Custom selector wait
-        if (waitForSelector) {
-            try {
-                await page.waitForSelector(waitForSelector, { timeout: 10000 });
-            } catch {
-                // Ignore timeout on optional selector wait
+        // Handle wait conditions
+        if (waitFor && waitFor.type) {
+            const waitTimeout = waitFor.timeoutMs || 10000;
+            if (waitFor.type === "SELECTOR_EXISTS" && waitFor.selector) {
+                await page.waitForSelector(waitFor.selector, { timeout: waitTimeout }).catch(() => {});
+            } else if (waitFor.type === "SELECTOR_VISIBLE" && waitFor.selector) {
+                await page.waitForSelector(waitFor.selector, { visible: true, timeout: waitTimeout }).catch(() => {});
+            } else if (waitFor.type === "TEXT_PRESENT" && waitFor.text) {
+                await page
+                    .waitForFunction(
+                        (t) => document.body && document.body.innerText.includes(t),
+                        { timeout: waitTimeout },
+                        waitFor.text
+                    )
+                    .catch(() => {});
+            } else if (waitFor.type === "FIXED_DELAY" && waitFor.delayMs > 0) {
+                await new Promise((r) => setTimeout(r, Math.min(waitFor.delayMs, 10000)));
             }
+        } else if (waitForSelector) {
+            await page.waitForSelector(waitForSelector, { timeout: 10000 }).catch(() => {});
         }
 
-        // Brief delay for dynamic content
+        // Optional brief delay for dynamic content
         if (waitForTimeout > 0) {
             await new Promise((r) => setTimeout(r, Math.min(waitForTimeout, 10000)));
         }
@@ -80,10 +236,36 @@ export async function handleChromiumScrape(payload) {
 
         const pageTitle = (await page.title()) || "";
         const finalUrl = page.url();
-        const html = await page.content();
 
-        // Extract structured data if extraction rules are defined
-        const extractedData = {};
+        const raw = {};
+        const strategySummary = {};
+
+        // 1. Extract mapped fields (Field Schema & Strategies)
+        if (Array.isArray(fieldMappings) && fieldMappings.length > 0) {
+            for (const mapping of fieldMappings) {
+                let extractedVal = null;
+                let successfulStrategy = null;
+
+                const strategies = mapping.strategies || [];
+                for (let i = 0; i < strategies.length; i++) {
+                    const strat = strategies[i];
+                    const val = await extractBrowserField(page, strat);
+                    if (val !== null && val !== undefined && val !== "") {
+                        extractedVal = val;
+                        successfulStrategy = strat.type || `strategy_${i + 1}`;
+                        break;
+                    }
+                }
+
+                raw[mapping.field] = extractedVal;
+                strategySummary[mapping.field] = {
+                    extracted: extractedVal !== null,
+                    strategyUsed: successfulStrategy,
+                };
+            }
+        }
+
+        // 2. Extract legacy extractSelectors if present
         if (Array.isArray(extractSelectors) && extractSelectors.length > 0) {
             for (const item of extractSelectors) {
                 const { field, selector, type = "text", attribute = null } = item;
@@ -108,21 +290,37 @@ export async function handleChromiumScrape(payload) {
                         attribute
                     );
 
-                    extractedData[field] = values.length <= 1 ? (values[0] ?? null) : values;
+                    const extractedVal = values.length <= 1 ? (values[0] ?? null) : values;
+                    if (raw[field] === undefined) {
+                        raw[field] = extractedVal;
+                    }
                 } catch {
-                    extractedData[field] = null;
+                    if (raw[field] === undefined) {
+                        raw[field] = null;
+                    }
                 }
             }
         }
 
         const executionTimeMs = Date.now() - startTime;
+        const hasExtractedValues = Object.values(raw).some(
+            (v) => v !== null && v !== undefined && v !== ""
+        );
+
+        // Avoid transferring huge raw HTML unless requested or nothing was extracted
+        let html = "";
+        if (includeHtml || (!hasExtractedValues && fieldMappings.length > 0)) {
+            html = await page.content().catch(() => "");
+        }
 
         return {
             status,
             title: pageTitle,
             url: finalUrl,
+            raw,
+            strategySummary,
+            extractedData: raw,
             html,
-            extractedData,
             executionTimeMs,
         };
     } finally {
@@ -135,7 +333,14 @@ export async function handleChromiumScrape(payload) {
  */
 export async function handleHttpScrape(payload) {
     const startTime = Date.now();
-    const { url, timeout = 15000, headers = {}, extractSelectors = [] } = payload;
+    const {
+        url,
+        fieldMappings = [],
+        timeout = 15000,
+        headers = {},
+        extractSelectors = [],
+        includeHtml = false,
+    } = payload;
 
     const response = await axios.get(url, {
         timeout,
@@ -148,11 +353,39 @@ export async function handleHttpScrape(payload) {
         },
     });
 
-    const html = typeof response.data === "string" ? response.data : "";
-    const $ = cheerio.load(html);
+    const pageHtml = typeof response.data === "string" ? response.data : "";
+    const $ = cheerio.load(pageHtml);
     const title = $("title").first().text().trim() || "";
 
-    const extractedData = {};
+    const raw = {};
+    const strategySummary = {};
+
+    // 1. Extract fieldMappings
+    if (Array.isArray(fieldMappings) && fieldMappings.length > 0) {
+        for (const mapping of fieldMappings) {
+            let extractedVal = null;
+            let successfulStrategy = null;
+
+            const strategies = mapping.strategies || [];
+            for (let i = 0; i < strategies.length; i++) {
+                const strat = strategies[i];
+                const val = extractCheerioField($, strat);
+                if (val !== null && val !== undefined && val !== "") {
+                    extractedVal = val;
+                    successfulStrategy = strat.type || `strategy_${i + 1}`;
+                    break;
+                }
+            }
+
+            raw[mapping.field] = extractedVal;
+            strategySummary[mapping.field] = {
+                extracted: extractedVal !== null,
+                strategyUsed: successfulStrategy,
+            };
+        }
+    }
+
+    // 2. Extract legacy extractSelectors
     if (Array.isArray(extractSelectors) && extractSelectors.length > 0) {
         for (const item of extractSelectors) {
             const { field, selector, type = "text", attribute = null } = item;
@@ -170,21 +403,31 @@ export async function handleHttpScrape(payload) {
                     }
                 });
 
-                extractedData[field] = values.length <= 1 ? (values[0] ?? null) : values;
+                const extractedVal = values.length <= 1 ? (values[0] ?? null) : values;
+                if (raw[field] === undefined) {
+                    raw[field] = extractedVal;
+                }
             } catch {
-                extractedData[field] = null;
+                if (raw[field] === undefined) {
+                    raw[field] = null;
+                }
             }
         }
     }
 
     const executionTimeMs = Date.now() - startTime;
+    const hasExtractedValues = Object.values(raw).some(
+        (v) => v !== null && v !== undefined && v !== ""
+    );
 
     return {
         status: response.status,
         title,
         url,
-        html,
-        extractedData,
+        raw,
+        strategySummary,
+        extractedData: raw,
+        html: includeHtml || (!hasExtractedValues && fieldMappings.length > 0) ? pageHtml : "",
         executionTimeMs,
     };
 }