app-manager-edge-worker 1.0.7 → 1.0.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "app-manager-edge-worker",
-  "version": "1.0.7",
+  "version": "1.0.9",
   "description": "Distributed Scraper Edge Worker for Apps Manager. Run on any local machine, VPS, or Termux Android device to serve browser and HTTP scraping jobs.",
   "type": "module",
   "bin": {

package/src/banner.js

@@ -1,7 +1,7 @@
 import { createRequire } from "module";
 
 const require = createRequire(import.meta.url);
-let workerVersion = "1.0.7";
+let workerVersion = "1.0.9";
 try {
     const pkg = require("../package.json");
     if (pkg?.version) workerVersion = pkg.version;

package/src/client.js

@@ -7,6 +7,7 @@ import localBrowserManager from "./browser.js";
 import { handleChromiumScrape, handleHttpScrape } from "./handlers/scrapeHandler.js";
 import { handlePreviewPage, handleInspectElement } from "./handlers/previewHandler.js";
 import { handleSendEmail } from "./handlers/mailHandler.js";
+import { handleAiSmartRepair } from "./handlers/aiRepairHandler.js";
 import { printBanner, logInfo, logSuccess, logWarn, logError, logJob } from "./banner.js";
 
 export async function startWorkerClient(config) {
@@ -157,6 +158,8 @@ export async function startWorkerClient(config) {
                 resultData = await handleInspectElement(payload);
             } else if (jobType === "SEND_EMAIL") {
                 resultData = await handleSendEmail(payload);
+            } else if (jobType === "AI_SMART_REPAIR") {
+                resultData = await handleAiSmartRepair(payload, config);
             } else {
                 throw new Error(`Unsupported job type: ${jobType}`);
             }

package/src/config.js

@@ -69,6 +69,12 @@ export function parseCliArgs() {
             parsed.keepAlive = parseInt(args[++i], 10);
         } else if (arg === "--no-keep-alive") {
             parsed.keepAlive = false;
+        } else if (arg === "--ai-endpoint") {
+            parsed.aiEndpoint = args[++i];
+        } else if (arg === "--ai-model") {
+            parsed.aiModel = args[++i];
+        } else if (arg === "--ai-key") {
+            parsed.aiApiKey = args[++i];
         } else if (arg === "--reset") {
             parsed.reset = true;
         } else if (arg === "--help" || arg === "-h") {
@@ -105,8 +111,14 @@ export async function resolveWorkerConfig() {
     const cli = parseCliArgs();
 
     if (cli.help) {
+        let pkgVer = "1.0.9";
+        try {
+            const pkg = JSON.parse(fs.readFileSync(new URL("../package.json", import.meta.url), "utf-8"));
+            if (pkg?.version) pkgVer = pkg.version;
+        } catch (_) {}
+
         console.log(`
-Apps Manager Scraper Edge Worker CLI v1.0.4
+Apps Manager Scraper Edge Worker CLI v${pkgVer}
 
 Usage:
   npx app-manager-edge-worker [options]
@@ -118,6 +130,9 @@ Options:
   -n, --name <name>           Custom display name for this worker device
   -p, --keep-alive <minutes>  Interval in minutes to ping server and prevent Render free-tier sleep (default: 7)
       --no-keep-alive         Disable keep-alive ping
+      --ai-endpoint <url>     Local/remote AI endpoint (default: http://127.0.0.1:11434 for Ollama)
+      --ai-model <model>      Model name for Tier 4 AI Smart Repair (e.g. llama3, mistral)
+      --ai-key <apiKey>       Optional API key for AI endpoint
       --reset                 Clear stored credentials and prompt again
   -h, --help                  Display this help message
         `);
@@ -208,11 +223,33 @@ Options:
         keepAliveMinutes = parseInt(process.env.KEEP_ALIVE_MINUTES, 10) || 7;
     }
 
+    const aiEndpoint =
+        cli.aiEndpoint ||
+        process.env.AI_ENDPOINT ||
+        process.env.OLLAMA_ENDPOINT ||
+        stored.aiEndpoint ||
+        "http://localhost:11434/v1";
+
+    const aiModel =
+        cli.aiModel ||
+        process.env.AI_MODEL ||
+        stored.aiModel ||
+        "qwen2.5-coder:7b";
+
+    const aiApiKey =
+        cli.aiApiKey ||
+        process.env.AI_API_KEY ||
+        stored.aiApiKey ||
+        "ollama";
+
     return {
         workerKey: workerKey.trim(),
         serverUrl,
         concurrency,
         workerName,
         keepAliveMinutes,
+        aiEndpoint,
+        aiModel,
+        aiApiKey,
     };
 }

package/src/handlers/aiRepairHandler.js

@@ -0,0 +1,154 @@
+import axios from "axios";
+import * as cheerio from "cheerio";
+
+/**
+ * Handles AI_SMART_REPAIR jobs on the Edge Worker.
+ * Communicates with local Ollama, LocalAI, LM Studio, or OpenAI-compatible endpoints.
+ * This offloads all AI computation from the cloud server (Render free tier) to the local worker.
+ *
+ * @param {Object} payload - { field, expectedType, previousSelector, previousValue, regexPattern, htmlSnippet, aiConfig }
+ * @param {Object} workerConfig - Worker global config with default AI endpoint/model
+ * @returns {Promise<Object>} { success, field, repairedSelector, extractedValue, confidence, explanation, modelUsed }
+ */
+export async function handleAiSmartRepair(payload, workerConfig = {}) {
+    const {
+        field = "field",
+        expectedType = "string",
+        previousSelector = "",
+        previousValue = "",
+        regexPattern = "",
+        htmlSnippet = "",
+        aiConfig = {},
+    } = payload;
+
+    const endpoint =
+        aiConfig.endpoint ||
+        workerConfig.aiEndpoint ||
+        process.env.AI_ENDPOINT ||
+        "http://localhost:11434/v1";
+
+    const model =
+        aiConfig.model ||
+        workerConfig.aiModel ||
+        process.env.AI_MODEL ||
+        "qwen2.5-coder:7b";
+
+    const apiKey =
+        aiConfig.apiKey ||
+        workerConfig.aiApiKey ||
+        process.env.AI_API_KEY ||
+        "ollama";
+
+    if (!htmlSnippet) {
+        throw new Error("Cannot perform AI smart repair: HTML snippet is empty.");
+    }
+
+    const systemPrompt = `You are an automated web scraping repair assistant.
+A website redesign has broken an existing selector. Your job is to analyze the updated HTML snippet and identify the relocated element for the target field.
+You MUST output ONLY a valid JSON object matching the requested schema with no surrounding Markdown or explanation.`;
+
+    const userPrompt = `FIELD REPAIR TARGET:
+- Field Name: "${field}"
+- Expected Data Type: "${expectedType}"
+- Broken Previous Selector: "${previousSelector}"
+- Last Known Sample Value: "${previousValue}"
+${regexPattern ? `- Expected Value Format (Regex): "${regexPattern}"` : ""}
+
+UPDATED HTML SNIPPET:
+\`\`\`html
+${htmlSnippet.slice(0, 8000)}
+\`\`\`
+
+TASK:
+1. Locate the exact HTML element in the snippet that represents "${field}".
+2. Construct a robust, resilient CSS selector (avoid dynamic or hashed classes like '_1a2b3c', prefer semantic tags, ids, data-* attributes, or structural paths).
+3. Return ONLY a JSON object formatted as follows:
+{
+  "repairedSelector": "css_selector_here",
+  "extractedValue": "exact_text_or_attribute_value",
+  "confidence": 0.95,
+  "explanation": "Brief explanation of why this element was chosen"
+}`;
+
+    const cleanEndpoint = endpoint.replace(/\/+$/, "");
+    const chatCompletionsUrl = cleanEndpoint.endsWith("/chat/completions")
+        ? cleanEndpoint
+        : `${cleanEndpoint}/chat/completions`;
+
+    try {
+        const response = await axios.post(
+            chatCompletionsUrl,
+            {
+                model,
+                messages: [
+                    { role: "system", content: systemPrompt },
+                    { role: "user", content: userPrompt },
+                ],
+                temperature: 0.1,
+                response_format: { type: "json_object" },
+            },
+            {
+                headers: {
+                    Authorization: `Bearer ${apiKey}`,
+                    "Content-Type": "application/json",
+                },
+                timeout: 45000,
+            }
+        );
+
+        let content = response.data?.choices?.[0]?.message?.content || "{}";
+
+        // Strip any markdown code fences if model returned ```json ... ```
+        content = content.replace(/^```json\s*/i, "").replace(/^```\s*/i, "").replace(/```$/i, "").trim();
+
+        let parsedResult = {};
+        try {
+            parsedResult = JSON.parse(content);
+        } catch {
+            const jsonMatch = content.match(/\{[\s\S]*\}/);
+            if (jsonMatch) {
+                parsedResult = JSON.parse(jsonMatch[0]);
+            } else {
+                throw new Error("Failed to parse JSON response from local AI model.");
+            }
+        }
+
+        const candidateSelector = parsedResult.repairedSelector?.trim() || "";
+        if (!candidateSelector) {
+            return {
+                success: false,
+                field,
+                error: "AI model could not determine a candidate selector.",
+                confidence: 0,
+            };
+        }
+
+        // Verify the candidate selector locally in the HTML snippet using Cheerio
+        const $ = cheerio.load(htmlSnippet);
+        let matchEl = null;
+        try {
+            matchEl = $(candidateSelector);
+        } catch {
+            // In case model returned an invalid CSS syntax
+            matchEl = null;
+        }
+
+        const isVerified = matchEl && matchEl.length > 0;
+        const verifiedValue = isVerified ? matchEl.first().text().trim() : parsedResult.extractedValue || "";
+
+        return {
+            success: isVerified,
+            field,
+            repairedSelector: candidateSelector,
+            extractedValue: verifiedValue,
+            confidence: isVerified ? Math.min(1.0, Math.max(0.6, parsedResult.confidence || 0.85)) : 0.3,
+            explanation: parsedResult.explanation || "Selector inferred via Edge Worker AI model.",
+            modelUsed: model,
+            endpointUsed: endpoint,
+        };
+    } catch (apiErr) {
+        throw new Error(
+            `Edge Worker AI repair request failed on '${endpoint}' (${model}): ${apiErr.message}`
+        );
+    }
+}

package/src/handlers/previewHandler.js

@@ -4,13 +4,29 @@ import localBrowserManager from "../browser.js";
  * Handles PREVIEW_PAGE task: captures full-page screenshot and dimensions on the edge worker.
  */
 export async function handlePreviewPage(payload) {
-    const { url, timeout = 25000 } = payload;
+    const { url, timeout = 25000, preScrapeActions, authentication } = payload;
     const browser = await localBrowserManager.getBrowser();
     const page = await browser.newPage();
 
     try {
         await page.setViewport({ width: 1366, height: 900, deviceScaleFactor: 1 });
 
+        // Apply authentication cookies & headers if provided
+        if (authentication) {
+            if (authentication.type === "COOKIES" && Array.isArray(authentication.cookies)) {
+                const parsedUrl = new URL(url);
+                await page.setCookie(...authentication.cookies.map(c => ({
+                    name: c.name,
+                    value: c.value,
+                    domain: c.domain || parsedUrl.hostname,
+                    path: c.path || "/",
+                }))).catch(() => {});
+            }
+            if (authentication.type === "HEADERS" && authentication.headers) {
+                await page.setExtraHTTPHeaders(authentication.headers).catch(() => {});
+            }
+        }
+
         await page.goto(url, { waitUntil: "domcontentloaded", timeout });
         try {
             await page.waitForNetworkIdle({ idleTime: 500, timeout: 3500 }).catch(() => {});
@@ -18,6 +34,37 @@ export async function handlePreviewPage(payload) {
         await page.evaluate(() => document.fonts?.ready).catch(() => {});
         await new Promise((r) => setTimeout(r, 800));
 
+        // Execute pre-scrape actions if present
+        if (Array.isArray(preScrapeActions) && preScrapeActions.length > 0) {
+            for (const action of preScrapeActions) {
+                try {
+                    if (action.actionType === "CLICK" && action.selector) {
+                        await page.waitForSelector(action.selector, { timeout: 4000 });
+                        await page.click(action.selector);
+                    } else if (action.actionType === "TYPE" && action.selector && action.value) {
+                        await page.waitForSelector(action.selector, { timeout: 4000 });
+                        await page.type(action.selector, action.value, { delay: 30 });
+                    } else if (action.actionType === "SCROLL") {
+                        const scrollPixels = parseInt(action.value, 10) || 500;
+                        await page.evaluate((px) => window.scrollBy(0, px), scrollPixels);
+                    } else if (action.actionType === "WAIT_FOR" && action.selector) {
+                        await page.waitForSelector(action.selector, { timeout: 8000 });
+                    } else if (action.actionType === "SELECT_OPTION" && action.selector && action.value) {
+                        await page.waitForSelector(action.selector, { timeout: 4000 });
+                        await page.select(action.selector, action.value);
+                    } else if (action.actionType === "PRESS_KEY" && action.value) {
+                        await page.keyboard.press(action.value);
+                    }
+                    if (action.delayMs && action.delayMs > 0) {
+                        await new Promise((r) => setTimeout(r, Math.min(action.delayMs, 5000)));
+                    }
+                } catch (actErr) {
+                    console.warn(`[WorkerPreview] Pre-scrape action ${action.actionType} (${action.selector}) skipped: ${actErr.message}`);
+                }
+            }
+            await new Promise((r) => setTimeout(r, 600));
+        }
+
         const title = (await page.title()) || new URL(url).hostname;
         const currentUrl = page.url();
 
@@ -64,7 +111,7 @@ export async function handlePreviewPage(payload) {
  * Handles INSPECT_ELEMENT task: resolves DOM element coordinates or evaluates candidate selectors on the edge worker.
  */
 export async function handleInspectElement(payload) {
-    const { url, selector, x, y, timeout = 20000 } = payload;
+    const { url, selector, strategyType, strategies, fingerprint, x, y, timeout = 20000 } = payload;
     const browser = await localBrowserManager.getBrowser();
     const page = await browser.newPage();
 
@@ -442,12 +489,98 @@ export async function handleInspectElement(payload) {
             return elementInfo || { found: false, message: "No element found at specified coordinates." };
         }
 
-        if (selector) {
-            const testResult = await page.evaluate((sel) => {
+        if (selector || (Array.isArray(strategies) && strategies.length > 0) || fingerprint) {
+            const testResult = await page.evaluate((target) => {
                 try {
-                    const el = document.querySelector(sel);
-                    if (!el) return { found: false, count: 0 };
+                    const countMatches = (sel) => {
+                        try { return document.querySelectorAll(sel).length; } catch { return 999; }
+                    };
 
+                    const resolveOne = (selStr, typeHint) => {
+                        if (!selStr) return null;
+                        let s = String(selStr).trim();
+                        if (!s) return null;
+
+                        const isXPath = typeHint === "xpath" ||
+                                        s.startsWith("//") ||
+                                        s.startsWith("xpath=") ||
+                                        s.startsWith("xpath:") ||
+                                        s.startsWith("/html") ||
+                                        s.startsWith("(/");
+
+                        if (s.startsWith("xpath=")) s = s.slice(6).trim();
+                        if (s.startsWith("xpath:")) s = s.slice(6).trim();
+
+                        if (isXPath) {
+                            try {
+                                const snap = document.evaluate(s, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+                                if (snap.snapshotLength > 0) {
+                                    const node = snap.snapshotItem(0);
+                                    if (node && node.nodeType === 1) {
+                                        return {
+                                            el: node,
+                                            count: snap.snapshotLength,
+                                            strategyType: "xpath",
+                                            matchedSelector: s,
+                                        };
+                                    }
+                                }
+                            } catch (_) {}
+                        }
+
+                        try {
+                            const matches = document.querySelectorAll(s);
+                            if (matches.length > 0) {
+                                return {
+                                    el: matches[0],
+                                    count: matches.length,
+                                    strategyType: "css",
+                                    matchedSelector: s,
+                                };
+                            }
+                        } catch (_) {
+                            if (!isXPath) {
+                                try {
+                                    const snap = document.evaluate(s, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+                                    if (snap.snapshotLength > 0) {
+                                        const node = snap.snapshotItem(0);
+                                        if (node && node.nodeType === 1) {
+                                            return {
+                                                el: node,
+                                                count: snap.snapshotLength,
+                                                strategyType: "xpath",
+                                                matchedSelector: s,
+                                            };
+                                        }
+                                    }
+                                } catch (_) {}
+                            }
+                        }
+
+                        return null;
+                    };
+
+                    const queue = [];
+                    if (target.selector) queue.push({ value: target.selector, type: target.strategyType });
+                    if (Array.isArray(target.strategies)) {
+                        for (const st of target.strategies) {
+                            if (st && st.value && !queue.some(q => q.value === st.value)) {
+                                queue.push({ value: st.value, type: st.type });
+                            }
+                        }
+                    }
+
+                    let resolved = null;
+                    for (const item of queue) {
+                        resolved = resolveOne(item.value, item.type);
+                        if (resolved) break;
+                    }
+
+                    if (!resolved || !resolved.el) {
+                        return { found: false, count: 0, message: "Element not found on page." };
+                    }
+
+                    const { el, count, strategyType, matchedSelector } = resolved;
                     const rect = el.getBoundingClientRect();
                     const boundingRect = {
                         left: Math.round(rect.left + window.scrollX),
@@ -456,23 +589,53 @@ export async function handleInspectElement(payload) {
                         height: Math.round(rect.height),
                     };
 
-                    const count = document.querySelectorAll(sel).length;
+                    const rawText = el.innerText ? el.innerText.trim() : el.textContent?.trim() || "";
+                    const attrs = Array.from(el.attributes).reduce((acc, a) => {
+                        acc[a.name] = a.value;
+                        return acc;
+                    }, {});
+
+                    const hierarchy = [];
+                    let currH = el;
+                    while (currH && currH !== document.body && hierarchy.length < 6) {
+                        let desc = currH.tagName.toLowerCase();
+                        if (currH.id) desc += `#${currH.id}`;
+                        else if (currH.className && typeof currH.className === "string") {
+                            const cls = currH.className.split(/\s+/).filter(c => /^[a-zA-Z_-][\w-]*$/.test(c)).slice(0, 2);
+                            if (cls.length) desc += `.${cls.join(".")}`;
+                        }
+                        hierarchy.unshift(desc);
+                        currH = currH.parentElement;
+                    }
+
+                    const candidates = [
+                        { type: strategyType, value: matchedSelector, attribute: "text", matchCount: count }
+                    ];
 
                     return {
                         found: true,
                         count,
+                        strategyType,
+                        matchedSelector,
                         tagName: el.tagName.toLowerCase(),
-                        text: el.innerText ? el.innerText.trim() : el.textContent?.trim() || "",
-                        attributes: Array.from(el.attributes).reduce((acc, a) => {
-                            acc[a.name] = a.value;
-                            return acc;
-                        }, {}),
+                        text: rawText,
+                        attributes: attrs,
                         boundingRect,
+                        hierarchy,
+                        candidates,
+                        fingerprint: {
+                            tagName: el.tagName.toLowerCase(),
+                            domHierarchy: hierarchy,
+                            attributes: attrs,
+                            textAnchors: [rawText.slice(0, 80)].filter(Boolean),
+                            dataPattern: "",
+                            sampleValues: [rawText.slice(0, 100)].filter(Boolean),
+                        },
                     };
                 } catch (e) {
                     return { found: false, error: e.message };
                 }
-            }, selector);
+            }, { selector, strategyType, strategies, fingerprint });
 
             return testResult;
         }

package/src/handlers/scrapeHandler.js

@@ -349,15 +349,57 @@ export async function handleChromiumScrape(payload) {
             deviceScaleFactor: 1,
         });
 
-        if (Object.keys(headers).length > 0) {
-            await page.setExtraHTTPHeaders(headers);
+        // Merge auth headers & cookies
+        const effectiveHeaders = {
+            ...(headers || {}),
+            ...(payload.authentication?.headers || {}),
+        };
+        const effectiveCookies = [
+            ...(Array.isArray(cookies) ? cookies : []),
+            ...(Array.isArray(payload.authentication?.cookies) ? payload.authentication.cookies : []),
+        ];
+
+        if (Object.keys(effectiveHeaders).length > 0) {
+            await page.setExtraHTTPHeaders(effectiveHeaders);
         }
 
-        if (Array.isArray(cookies) && cookies.length > 0) {
-            await page.setCookie(...cookies);
+        if (effectiveCookies.length > 0) {
+            await page.setCookie(...effectiveCookies);
+        }
+
+        // Automated Login Flow if configured
+        const auth = payload.authentication || {};
+        if (auth.type === "LOGIN_FLOW" && auth.loginFlow?.loginUrl) {
+            try {
+                const lf = auth.loginFlow;
+                await page.goto(lf.loginUrl, { waitUntil: "domcontentloaded", timeout: 20000 });
+                if (lf.usernameSelector && lf.usernameValue) {
+                    await page.waitForSelector(lf.usernameSelector, { timeout: 6000 });
+                    await page.click(lf.usernameSelector);
+                    await page.type(lf.usernameSelector, lf.usernameValue, { delay: 30 });
+                }
+                if (lf.passwordSelector && lf.passwordValue) {
+                    await page.waitForSelector(lf.passwordSelector, { timeout: 6000 });
+                    await page.click(lf.passwordSelector);
+                    await page.type(lf.passwordSelector, lf.passwordValue, { delay: 30 });
+                }
+                if (lf.submitSelector) {
+                    await page.waitForSelector(lf.submitSelector, { timeout: 6000 });
+                    await Promise.all([
+                        page.click(lf.submitSelector),
+                        page.waitForNavigation({ waitUntil: "domcontentloaded", timeout: 15000 }).catch(() => {}),
+                    ]);
+                }
+                if (lf.successIndicatorSelector) {
+                    await page.waitForSelector(lf.successIndicatorSelector, { timeout: 8000 }).catch(() => {});
+                }
+                await new Promise((r) => setTimeout(r, 1000));
+            } catch (loginErr) {
+                console.warn(`[ScrapeHandler] Automated login flow error: ${loginErr.message}. Continuing with scrape...`);
+            }
         }
 
-        // Navigate
+        // Navigate to target URL
         const response = await page.goto(url, {
             waitUntil: navWaitUntil,
             timeout: navTimeout,
@@ -392,21 +434,53 @@ export async function handleChromiumScrape(payload) {
             await new Promise((r) => setTimeout(r, Math.min(waitForTimeout, 10000)));
         }
 
-        // Execute sequential interactive actions if configured
-        if (Array.isArray(actions) && actions.length > 0) {
-            for (const action of actions) {
+        // Execute sequential interactive actions (preScrapeActions or actions)
+        const combinedActions = [
+            ...(Array.isArray(payload.preScrapeActions) ? payload.preScrapeActions : []),
+            ...(Array.isArray(actions) ? actions : []),
+        ];
+
+        if (combinedActions.length > 0) {
+            for (let aIdx = 0; aIdx < combinedActions.length; aIdx++) {
+                const action = combinedActions[aIdx];
+                const actType = (action.type || "").toUpperCase();
+                const opt = action.optional !== false;
                 try {
-                    if (action.type === "click" && action.selector) {
+                    if ((actType === "CLICK" || action.type === "click") && action.selector) {
+                        await page.waitForSelector(action.selector, { timeout: action.timeoutMs || 4000 });
                         await page.click(action.selector);
-                    } else if (action.type === "type" && action.selector && action.text) {
-                        await page.type(action.selector, action.text);
-                    } else if (action.type === "scroll") {
-                        await page.evaluate((y) => window.scrollBy(0, y || 500), action.distance);
-                    } else if (action.type === "wait") {
-                        await new Promise((r) => setTimeout(r, action.duration || 1000));
+                    } else if ((actType === "TYPE" || action.type === "type") && action.selector) {
+                        const txt = action.value !== undefined ? action.value : action.text || "";
+                        await page.waitForSelector(action.selector, { timeout: action.timeoutMs || 4000 });
+                        await page.click(action.selector);
+                        await page.type(action.selector, txt, { delay: 30 });
+                    } else if (actType === "SELECT_OPTION" && action.selector && action.value) {
+                        await page.waitForSelector(action.selector, { timeout: action.timeoutMs || 4000 });
+                        await page.select(action.selector, action.value);
+                    } else if (actType === "PRESS_KEY") {
+                        await page.keyboard.press(action.key || "Enter");
+                    } else if (actType === "SCROLL" || action.type === "scroll") {
+                        const dist = action.scrollDistance || action.distance || 500;
+                        await page.evaluate((y) => window.scrollBy(0, y), dist);
+                    } else if (actType === "WAIT_FOR" && action.selector) {
+                        await page.waitForSelector(action.selector, {
+                            visible: true,
+                            timeout: action.timeoutMs || 6000,
+                        });
+                    } else if (actType === "DELAY" || action.type === "wait") {
+                        const dur = action.delayMs || action.duration || 1000;
+                        await new Promise((r) => setTimeout(r, dur));
+                    }
+
+                    if (action.delayMs > 0) {
+                        await new Promise((r) => setTimeout(r, action.delayMs));
                     }
                 } catch (actionErr) {
-                    console.warn(`[ScrapeHandler] Action ${action.type} failed:`, actionErr.message);
+                    if (opt) {
+                        console.warn(`[ScrapeHandler] Optional action #${aIdx + 1} (${action.type} '${action.selector || ""}') skipped: ${actionErr.message}`);
+                    } else {
+                        throw actionErr;
+                    }
                 }
             }
         }
@@ -685,15 +759,27 @@ export async function handleHttpScrape(payload) {
         includeHtml = false,
     } = payload;
 
+    const auth = payload.authentication || {};
+    const effectiveHeaders = {
+        "User-Agent":
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+        Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.9",
+        ...(headers || {}),
+        ...(auth.headers || {}),
+    };
+
+    const effectiveCookies = [
+        ...(Array.isArray(payload.cookies) ? payload.cookies : []),
+        ...(Array.isArray(auth.cookies) ? auth.cookies : []),
+    ];
+    if (effectiveCookies.length > 0 && !effectiveHeaders.Cookie) {
+        effectiveHeaders.Cookie = effectiveCookies.map((c) => `${c.name}=${c.value}`).join("; ");
+    }
+
     const response = await axios.get(url, {
         timeout,
-        headers: {
-            "User-Agent":
-                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
-            Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-            "Accept-Language": "en-US,en;q=0.9",
-            ...headers,
-        },
+        headers: effectiveHeaders,
     });
 
     let pageHtml = typeof response.data === "string" ? response.data : Buffer.isBuffer(response.data) ? response.data.toString("utf-8") : "";