app-manager-edge-worker 1.0.6 → 1.0.8

package/src/config.js

@@ -69,6 +69,12 @@ export function parseCliArgs() {
             parsed.keepAlive = parseInt(args[++i], 10);
         } else if (arg === "--no-keep-alive") {
             parsed.keepAlive = false;
+        } else if (arg === "--ai-endpoint") {
+            parsed.aiEndpoint = args[++i];
+        } else if (arg === "--ai-model") {
+            parsed.aiModel = args[++i];
+        } else if (arg === "--ai-key") {
+            parsed.aiApiKey = args[++i];
         } else if (arg === "--reset") {
             parsed.reset = true;
         } else if (arg === "--help" || arg === "-h") {
@@ -105,8 +111,14 @@ export async function resolveWorkerConfig() {
     const cli = parseCliArgs();
 
     if (cli.help) {
+        let pkgVer = "1.0.8";
+        try {
+            const pkg = JSON.parse(fs.readFileSync(new URL("../package.json", import.meta.url), "utf-8"));
+            if (pkg?.version) pkgVer = pkg.version;
+        } catch (_) {}
+
         console.log(`
-Apps Manager Scraper Edge Worker CLI v1.0.4
+Apps Manager Scraper Edge Worker CLI v${pkgVer}
 
 Usage:
   npx app-manager-edge-worker [options]
@@ -118,6 +130,9 @@ Options:
   -n, --name <name>           Custom display name for this worker device
   -p, --keep-alive <minutes>  Interval in minutes to ping server and prevent Render free-tier sleep (default: 7)
       --no-keep-alive         Disable keep-alive ping
+      --ai-endpoint <url>     Local/remote AI endpoint (default: http://127.0.0.1:11434 for Ollama)
+      --ai-model <model>      Model name for Tier 4 AI Smart Repair (e.g. llama3, mistral)
+      --ai-key <apiKey>       Optional API key for AI endpoint
       --reset                 Clear stored credentials and prompt again
   -h, --help                  Display this help message
         `);
@@ -208,11 +223,33 @@ Options:
         keepAliveMinutes = parseInt(process.env.KEEP_ALIVE_MINUTES, 10) || 7;
     }
 
+    const aiEndpoint =
+        cli.aiEndpoint ||
+        process.env.AI_ENDPOINT ||
+        process.env.OLLAMA_ENDPOINT ||
+        stored.aiEndpoint ||
+        "http://localhost:11434/v1";
+
+    const aiModel =
+        cli.aiModel ||
+        process.env.AI_MODEL ||
+        stored.aiModel ||
+        "qwen2.5-coder:7b";
+
+    const aiApiKey =
+        cli.aiApiKey ||
+        process.env.AI_API_KEY ||
+        stored.aiApiKey ||
+        "ollama";
+
     return {
         workerKey: workerKey.trim(),
         serverUrl,
         concurrency,
         workerName,
         keepAliveMinutes,
+        aiEndpoint,
+        aiModel,
+        aiApiKey,
     };
 }

package/src/handlers/aiRepairHandler.js

@@ -0,0 +1,154 @@
+import axios from "axios";
+import * as cheerio from "cheerio";
+
+/**
+ * Handles AI_SMART_REPAIR jobs on the Edge Worker.
+ * Communicates with local Ollama, LocalAI, LM Studio, or OpenAI-compatible endpoints.
+ * This offloads all AI computation from the cloud server (Render free tier) to the local worker.
+ *
+ * @param {Object} payload - { field, expectedType, previousSelector, previousValue, regexPattern, htmlSnippet, aiConfig }
+ * @param {Object} workerConfig - Worker global config with default AI endpoint/model
+ * @returns {Promise<Object>} { success, field, repairedSelector, extractedValue, confidence, explanation, modelUsed }
+ */
+export async function handleAiSmartRepair(payload, workerConfig = {}) {
+    const {
+        field = "field",
+        expectedType = "string",
+        previousSelector = "",
+        previousValue = "",
+        regexPattern = "",
+        htmlSnippet = "",
+        aiConfig = {},
+    } = payload;
+
+    const endpoint =
+        aiConfig.endpoint ||
+        workerConfig.aiEndpoint ||
+        process.env.AI_ENDPOINT ||
+        "http://localhost:11434/v1";
+
+    const model =
+        aiConfig.model ||
+        workerConfig.aiModel ||
+        process.env.AI_MODEL ||
+        "qwen2.5-coder:7b";
+
+    const apiKey =
+        aiConfig.apiKey ||
+        workerConfig.aiApiKey ||
+        process.env.AI_API_KEY ||
+        "ollama";
+
+    if (!htmlSnippet) {
+        throw new Error("Cannot perform AI smart repair: HTML snippet is empty.");
+    }
+
+    const systemPrompt = `You are an automated web scraping repair assistant.
+A website redesign has broken an existing selector. Your job is to analyze the updated HTML snippet and identify the relocated element for the target field.
+You MUST output ONLY a valid JSON object matching the requested schema with no surrounding Markdown or explanation.`;
+
+    const userPrompt = `FIELD REPAIR TARGET:
+- Field Name: "${field}"
+- Expected Data Type: "${expectedType}"
+- Broken Previous Selector: "${previousSelector}"
+- Last Known Sample Value: "${previousValue}"
+${regexPattern ? `- Expected Value Format (Regex): "${regexPattern}"` : ""}
+
+UPDATED HTML SNIPPET:
+\`\`\`html
+${htmlSnippet.slice(0, 8000)}
+\`\`\`
+
+TASK:
+1. Locate the exact HTML element in the snippet that represents "${field}".
+2. Construct a robust, resilient CSS selector (avoid dynamic or hashed classes like '_1a2b3c', prefer semantic tags, ids, data-* attributes, or structural paths).
+3. Return ONLY a JSON object formatted as follows:
+{
+  "repairedSelector": "css_selector_here",
+  "extractedValue": "exact_text_or_attribute_value",
+  "confidence": 0.95,
+  "explanation": "Brief explanation of why this element was chosen"
+}`;
+
+    const cleanEndpoint = endpoint.replace(/\/+$/, "");
+    const chatCompletionsUrl = cleanEndpoint.endsWith("/chat/completions")
+        ? cleanEndpoint
+        : `${cleanEndpoint}/chat/completions`;
+
+    try {
+        const response = await axios.post(
+            chatCompletionsUrl,
+            {
+                model,
+                messages: [
+                    { role: "system", content: systemPrompt },
+                    { role: "user", content: userPrompt },
+                ],
+                temperature: 0.1,
+                response_format: { type: "json_object" },
+            },
+            {
+                headers: {
+                    Authorization: `Bearer ${apiKey}`,
+                    "Content-Type": "application/json",
+                },
+                timeout: 45000,
+            }
+        );
+
+        let content = response.data?.choices?.[0]?.message?.content || "{}";
+
+        // Strip any markdown code fences if model returned ```json ... ```
+        content = content.replace(/^```json\s*/i, "").replace(/^```\s*/i, "").replace(/```$/i, "").trim();
+
+        let parsedResult = {};
+        try {
+            parsedResult = JSON.parse(content);
+        } catch {
+            const jsonMatch = content.match(/\{[\s\S]*\}/);
+            if (jsonMatch) {
+                parsedResult = JSON.parse(jsonMatch[0]);
+            } else {
+                throw new Error("Failed to parse JSON response from local AI model.");
+            }
+        }
+
+        const candidateSelector = parsedResult.repairedSelector?.trim() || "";
+        if (!candidateSelector) {
+            return {
+                success: false,
+                field,
+                error: "AI model could not determine a candidate selector.",
+                confidence: 0,
+            };
+        }
+
+        // Verify the candidate selector locally in the HTML snippet using Cheerio
+        const $ = cheerio.load(htmlSnippet);
+        let matchEl = null;
+        try {
+            matchEl = $(candidateSelector);
+        } catch {
+            // In case model returned an invalid CSS syntax
+            matchEl = null;
+        }
+
+        const isVerified = matchEl && matchEl.length > 0;
+        const verifiedValue = isVerified ? matchEl.first().text().trim() : parsedResult.extractedValue || "";
+
+        return {
+            success: isVerified,
+            field,
+            repairedSelector: candidateSelector,
+            extractedValue: verifiedValue,
+            confidence: isVerified ? Math.min(1.0, Math.max(0.6, parsedResult.confidence || 0.85)) : 0.3,
+            explanation: parsedResult.explanation || "Selector inferred via Edge Worker AI model.",
+            modelUsed: model,
+            endpointUsed: endpoint,
+        };
+    } catch (apiErr) {
+        throw new Error(
+            `Edge Worker AI repair request failed on '${endpoint}' (${model}): ${apiErr.message}`
+        );
+    }
+}

package/src/handlers/mailHandler.js

@@ -0,0 +1,74 @@
+import nodemailer from "nodemailer";
+
+/**
+ * Handles incoming SEND_EMAIL job from the cluster server.
+ *
+ * Security & Reliability Guarantees:
+ * 1. Zero credential retention: Credentials are received in-memory and destroyed after sending.
+ * 2. Fail-fast timeouts: Prevents socket hanging with strict 10s connection & socket timeouts.
+ * 3. Structured result payload: Returns delivery metadata (messageId, accepted, response).
+ *
+ * @param {Object} payload
+ * @param {Object} payload.mailOptions - { to, subject, html, text, fromName, fromEmail }
+ * @param {Object} payload.smtpConfig - { host, port, secure, auth: { user, pass } }
+ * @returns {Promise<Object>}
+ */
+export async function handleSendEmail(payload = {}) {
+    const { mailOptions, smtpConfig } = payload;
+
+    if (!mailOptions || !mailOptions.to || !mailOptions.subject) {
+        throw new Error("Invalid mailOptions: 'to' and 'subject' are required.");
+    }
+    if (!smtpConfig || !smtpConfig.auth?.user || !smtpConfig.auth?.pass) {
+        throw new Error("Invalid smtpConfig: credentials missing.");
+    }
+
+    const host = smtpConfig.host || "smtp.titan.email";
+    const port = Number(smtpConfig.port) || 465;
+    const secure = smtpConfig.secure !== undefined ? Boolean(smtpConfig.secure) : port === 465;
+
+    // Ephemeral Nodemailer transporter created strictly for this in-memory dispatch
+    const transporter = nodemailer.createTransport({
+        host,
+        port,
+        secure,
+        auth: {
+            user: smtpConfig.auth.user,
+            pass: smtpConfig.auth.pass,
+        },
+        connectionTimeout: 10000,
+        greetingTimeout: 10000,
+        socketTimeout: 15000,
+        tls: {
+            rejectUnauthorized: false, // Prevents certificate mismatches on custom domain mail
+        },
+    });
+
+    try {
+        const fromName = mailOptions.fromName || "Apps Manager";
+        const fromAddress = mailOptions.fromEmail || smtpConfig.auth.user;
+
+        const info = await transporter.sendMail({
+            from: `"${fromName}" <${fromAddress}>`,
+            to: mailOptions.to,
+            subject: mailOptions.subject,
+            text: mailOptions.text,
+            html: mailOptions.html,
+        });
+
+        return {
+            success: true,
+            messageId: info.messageId,
+            accepted: info.accepted,
+            rejected: info.rejected,
+            response: info.response,
+        };
+    } finally {
+        // Explicitly close transporter socket to prevent lingering connections
+        try {
+            transporter.close();
+        } catch (_) {
+            // Best effort cleanup
+        }
+    }
+}

package/src/handlers/previewHandler.js

@@ -4,13 +4,29 @@ import localBrowserManager from "../browser.js";
  * Handles PREVIEW_PAGE task: captures full-page screenshot and dimensions on the edge worker.
  */
 export async function handlePreviewPage(payload) {
-    const { url, timeout = 25000 } = payload;
+    const { url, timeout = 25000, preScrapeActions, authentication } = payload;
     const browser = await localBrowserManager.getBrowser();
     const page = await browser.newPage();
 
     try {
         await page.setViewport({ width: 1366, height: 900, deviceScaleFactor: 1 });
 
+        // Apply authentication cookies & headers if provided
+        if (authentication) {
+            if (authentication.type === "COOKIES" && Array.isArray(authentication.cookies)) {
+                const parsedUrl = new URL(url);
+                await page.setCookie(...authentication.cookies.map(c => ({
+                    name: c.name,
+                    value: c.value,
+                    domain: c.domain || parsedUrl.hostname,
+                    path: c.path || "/",
+                }))).catch(() => {});
+            }
+            if (authentication.type === "HEADERS" && authentication.headers) {
+                await page.setExtraHTTPHeaders(authentication.headers).catch(() => {});
+            }
+        }
+
         await page.goto(url, { waitUntil: "domcontentloaded", timeout });
         try {
             await page.waitForNetworkIdle({ idleTime: 500, timeout: 3500 }).catch(() => {});
@@ -18,6 +34,37 @@ export async function handlePreviewPage(payload) {
         await page.evaluate(() => document.fonts?.ready).catch(() => {});
         await new Promise((r) => setTimeout(r, 800));
 
+        // Execute pre-scrape actions if present
+        if (Array.isArray(preScrapeActions) && preScrapeActions.length > 0) {
+            for (const action of preScrapeActions) {
+                try {
+                    if (action.actionType === "CLICK" && action.selector) {
+                        await page.waitForSelector(action.selector, { timeout: 4000 });
+                        await page.click(action.selector);
+                    } else if (action.actionType === "TYPE" && action.selector && action.value) {
+                        await page.waitForSelector(action.selector, { timeout: 4000 });
+                        await page.type(action.selector, action.value, { delay: 30 });
+                    } else if (action.actionType === "SCROLL") {
+                        const scrollPixels = parseInt(action.value, 10) || 500;
+                        await page.evaluate((px) => window.scrollBy(0, px), scrollPixels);
+                    } else if (action.actionType === "WAIT_FOR" && action.selector) {
+                        await page.waitForSelector(action.selector, { timeout: 8000 });
+                    } else if (action.actionType === "SELECT_OPTION" && action.selector && action.value) {
+                        await page.waitForSelector(action.selector, { timeout: 4000 });
+                        await page.select(action.selector, action.value);
+                    } else if (action.actionType === "PRESS_KEY" && action.value) {
+                        await page.keyboard.press(action.value);
+                    }
+                    if (action.delayMs && action.delayMs > 0) {
+                        await new Promise((r) => setTimeout(r, Math.min(action.delayMs, 5000)));
+                    }
+                } catch (actErr) {
+                    console.warn(`[WorkerPreview] Pre-scrape action ${action.actionType} (${action.selector}) skipped: ${actErr.message}`);
+                }
+            }
+            await new Promise((r) => setTimeout(r, 600));
+        }
+
         const title = (await page.title()) || new URL(url).hostname;
         const currentUrl = page.url();
 
@@ -458,16 +505,42 @@ export async function handleInspectElement(payload) {
 
                     const count = document.querySelectorAll(sel).length;
 
+                    const rawText = el.innerText ? el.innerText.trim() : el.textContent?.trim() || "";
+                    const attrs = Array.from(el.attributes).reduce((acc, a) => {
+                        acc[a.name] = a.value;
+                        return acc;
+                    }, {});
+
+                    const hierarchy = [];
+                    let currH = el;
+                    while (currH && currH !== document.body && hierarchy.length < 6) {
+                        let desc = currH.tagName.toLowerCase();
+                        if (currH.id) desc += `#${currH.id}`;
+                        else if (currH.className && typeof currH.className === "string") {
+                            const cls = currH.className.split(/\s+/).filter(c => /^[a-zA-Z_-][\w-]*$/.test(c)).slice(0, 2);
+                            if (cls.length) desc += `.${cls.join(".")}`;
+                        }
+                        hierarchy.unshift(desc);
+                        currH = currH.parentElement;
+                    }
+
+                    const fingerprint = {
+                        tagName: el.tagName.toLowerCase(),
+                        domHierarchy: hierarchy,
+                        attributes: attrs,
+                        textAnchors: [rawText.slice(0, 80)].filter(Boolean),
+                        dataPattern: "",
+                        sampleValues: [rawText.slice(0, 100)].filter(Boolean),
+                    };
+
                     return {
                         found: true,
                         count,
                         tagName: el.tagName.toLowerCase(),
-                        text: el.innerText ? el.innerText.trim() : el.textContent?.trim() || "",
-                        attributes: Array.from(el.attributes).reduce((acc, a) => {
-                            acc[a.name] = a.value;
-                            return acc;
-                        }, {}),
+                        text: rawText,
+                        attributes: attrs,
                         boundingRect,
+                        fingerprint,
                     };
                 } catch (e) {
                     return { found: false, error: e.message };

package/src/handlers/scrapeHandler.js

@@ -349,15 +349,57 @@ export async function handleChromiumScrape(payload) {
             deviceScaleFactor: 1,
         });
 
-        if (Object.keys(headers).length > 0) {
-            await page.setExtraHTTPHeaders(headers);
+        // Merge auth headers & cookies
+        const effectiveHeaders = {
+            ...(headers || {}),
+            ...(payload.authentication?.headers || {}),
+        };
+        const effectiveCookies = [
+            ...(Array.isArray(cookies) ? cookies : []),
+            ...(Array.isArray(payload.authentication?.cookies) ? payload.authentication.cookies : []),
+        ];
+
+        if (Object.keys(effectiveHeaders).length > 0) {
+            await page.setExtraHTTPHeaders(effectiveHeaders);
         }
 
-        if (Array.isArray(cookies) && cookies.length > 0) {
-            await page.setCookie(...cookies);
+        if (effectiveCookies.length > 0) {
+            await page.setCookie(...effectiveCookies);
+        }
+
+        // Automated Login Flow if configured
+        const auth = payload.authentication || {};
+        if (auth.type === "LOGIN_FLOW" && auth.loginFlow?.loginUrl) {
+            try {
+                const lf = auth.loginFlow;
+                await page.goto(lf.loginUrl, { waitUntil: "domcontentloaded", timeout: 20000 });
+                if (lf.usernameSelector && lf.usernameValue) {
+                    await page.waitForSelector(lf.usernameSelector, { timeout: 6000 });
+                    await page.click(lf.usernameSelector);
+                    await page.type(lf.usernameSelector, lf.usernameValue, { delay: 30 });
+                }
+                if (lf.passwordSelector && lf.passwordValue) {
+                    await page.waitForSelector(lf.passwordSelector, { timeout: 6000 });
+                    await page.click(lf.passwordSelector);
+                    await page.type(lf.passwordSelector, lf.passwordValue, { delay: 30 });
+                }
+                if (lf.submitSelector) {
+                    await page.waitForSelector(lf.submitSelector, { timeout: 6000 });
+                    await Promise.all([
+                        page.click(lf.submitSelector),
+                        page.waitForNavigation({ waitUntil: "domcontentloaded", timeout: 15000 }).catch(() => {}),
+                    ]);
+                }
+                if (lf.successIndicatorSelector) {
+                    await page.waitForSelector(lf.successIndicatorSelector, { timeout: 8000 }).catch(() => {});
+                }
+                await new Promise((r) => setTimeout(r, 1000));
+            } catch (loginErr) {
+                console.warn(`[ScrapeHandler] Automated login flow error: ${loginErr.message}. Continuing with scrape...`);
+            }
         }
 
-        // Navigate
+        // Navigate to target URL
         const response = await page.goto(url, {
             waitUntil: navWaitUntil,
             timeout: navTimeout,
@@ -392,21 +434,53 @@ export async function handleChromiumScrape(payload) {
             await new Promise((r) => setTimeout(r, Math.min(waitForTimeout, 10000)));
         }
 
-        // Execute sequential interactive actions if configured
-        if (Array.isArray(actions) && actions.length > 0) {
-            for (const action of actions) {
+        // Execute sequential interactive actions (preScrapeActions or actions)
+        const combinedActions = [
+            ...(Array.isArray(payload.preScrapeActions) ? payload.preScrapeActions : []),
+            ...(Array.isArray(actions) ? actions : []),
+        ];
+
+        if (combinedActions.length > 0) {
+            for (let aIdx = 0; aIdx < combinedActions.length; aIdx++) {
+                const action = combinedActions[aIdx];
+                const actType = (action.type || "").toUpperCase();
+                const opt = action.optional !== false;
                 try {
-                    if (action.type === "click" && action.selector) {
+                    if ((actType === "CLICK" || action.type === "click") && action.selector) {
+                        await page.waitForSelector(action.selector, { timeout: action.timeoutMs || 4000 });
                         await page.click(action.selector);
-                    } else if (action.type === "type" && action.selector && action.text) {
-                        await page.type(action.selector, action.text);
-                    } else if (action.type === "scroll") {
-                        await page.evaluate((y) => window.scrollBy(0, y || 500), action.distance);
-                    } else if (action.type === "wait") {
-                        await new Promise((r) => setTimeout(r, action.duration || 1000));
+                    } else if ((actType === "TYPE" || action.type === "type") && action.selector) {
+                        const txt = action.value !== undefined ? action.value : action.text || "";
+                        await page.waitForSelector(action.selector, { timeout: action.timeoutMs || 4000 });
+                        await page.click(action.selector);
+                        await page.type(action.selector, txt, { delay: 30 });
+                    } else if (actType === "SELECT_OPTION" && action.selector && action.value) {
+                        await page.waitForSelector(action.selector, { timeout: action.timeoutMs || 4000 });
+                        await page.select(action.selector, action.value);
+                    } else if (actType === "PRESS_KEY") {
+                        await page.keyboard.press(action.key || "Enter");
+                    } else if (actType === "SCROLL" || action.type === "scroll") {
+                        const dist = action.scrollDistance || action.distance || 500;
+                        await page.evaluate((y) => window.scrollBy(0, y), dist);
+                    } else if (actType === "WAIT_FOR" && action.selector) {
+                        await page.waitForSelector(action.selector, {
+                            visible: true,
+                            timeout: action.timeoutMs || 6000,
+                        });
+                    } else if (actType === "DELAY" || action.type === "wait") {
+                        const dur = action.delayMs || action.duration || 1000;
+                        await new Promise((r) => setTimeout(r, dur));
+                    }
+
+                    if (action.delayMs > 0) {
+                        await new Promise((r) => setTimeout(r, action.delayMs));
                     }
                 } catch (actionErr) {
-                    console.warn(`[ScrapeHandler] Action ${action.type} failed:`, actionErr.message);
+                    if (opt) {
+                        console.warn(`[ScrapeHandler] Optional action #${aIdx + 1} (${action.type} '${action.selector || ""}') skipped: ${actionErr.message}`);
+                    } else {
+                        throw actionErr;
+                    }
                 }
             }
         }
@@ -685,15 +759,27 @@ export async function handleHttpScrape(payload) {
         includeHtml = false,
     } = payload;
 
+    const auth = payload.authentication || {};
+    const effectiveHeaders = {
+        "User-Agent":
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+        Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.9",
+        ...(headers || {}),
+        ...(auth.headers || {}),
+    };
+
+    const effectiveCookies = [
+        ...(Array.isArray(payload.cookies) ? payload.cookies : []),
+        ...(Array.isArray(auth.cookies) ? auth.cookies : []),
+    ];
+    if (effectiveCookies.length > 0 && !effectiveHeaders.Cookie) {
+        effectiveHeaders.Cookie = effectiveCookies.map((c) => `${c.name}=${c.value}`).join("; ");
+    }
+
     const response = await axios.get(url, {
         timeout,
-        headers: {
-            "User-Agent":
-                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
-            Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-            "Accept-Language": "en-US,en;q=0.9",
-            ...headers,
-        },
+        headers: effectiveHeaders,
     });
 
     let pageHtml = typeof response.data === "string" ? response.data : Buffer.isBuffer(response.data) ? response.data.toString("utf-8") : "";