app-manager-edge-worker 1.0.7 → 1.0.8

package/src/config.js

@@ -69,6 +69,12 @@ export function parseCliArgs() {
             parsed.keepAlive = parseInt(args[++i], 10);
         } else if (arg === "--no-keep-alive") {
             parsed.keepAlive = false;
+        } else if (arg === "--ai-endpoint") {
+            parsed.aiEndpoint = args[++i];
+        } else if (arg === "--ai-model") {
+            parsed.aiModel = args[++i];
+        } else if (arg === "--ai-key") {
+            parsed.aiApiKey = args[++i];
         } else if (arg === "--reset") {
             parsed.reset = true;
         } else if (arg === "--help" || arg === "-h") {
@@ -105,8 +111,14 @@ export async function resolveWorkerConfig() {
     const cli = parseCliArgs();
 
     if (cli.help) {
+        let pkgVer = "1.0.8";
+        try {
+            const pkg = JSON.parse(fs.readFileSync(new URL("../package.json", import.meta.url), "utf-8"));
+            if (pkg?.version) pkgVer = pkg.version;
+        } catch (_) {}
+
         console.log(`
-Apps Manager Scraper Edge Worker CLI v1.0.4
+Apps Manager Scraper Edge Worker CLI v${pkgVer}
 
 Usage:
   npx app-manager-edge-worker [options]
@@ -118,6 +130,9 @@ Options:
   -n, --name <name>           Custom display name for this worker device
   -p, --keep-alive <minutes>  Interval in minutes to ping server and prevent Render free-tier sleep (default: 7)
       --no-keep-alive         Disable keep-alive ping
+      --ai-endpoint <url>     Local/remote AI endpoint (default: http://127.0.0.1:11434 for Ollama)
+      --ai-model <model>      Model name for Tier 4 AI Smart Repair (e.g. llama3, mistral)
+      --ai-key <apiKey>       Optional API key for AI endpoint
       --reset                 Clear stored credentials and prompt again
   -h, --help                  Display this help message
         `);
@@ -208,11 +223,33 @@ Options:
         keepAliveMinutes = parseInt(process.env.KEEP_ALIVE_MINUTES, 10) || 7;
     }
 
+    const aiEndpoint =
+        cli.aiEndpoint ||
+        process.env.AI_ENDPOINT ||
+        process.env.OLLAMA_ENDPOINT ||
+        stored.aiEndpoint ||
+        "http://localhost:11434/v1";
+
+    const aiModel =
+        cli.aiModel ||
+        process.env.AI_MODEL ||
+        stored.aiModel ||
+        "qwen2.5-coder:7b";
+
+    const aiApiKey =
+        cli.aiApiKey ||
+        process.env.AI_API_KEY ||
+        stored.aiApiKey ||
+        "ollama";
+
     return {
         workerKey: workerKey.trim(),
         serverUrl,
         concurrency,
         workerName,
         keepAliveMinutes,
+        aiEndpoint,
+        aiModel,
+        aiApiKey,
     };
 }

package/src/handlers/aiRepairHandler.js

@@ -0,0 +1,154 @@
+import axios from "axios";
+import * as cheerio from "cheerio";
+
+/**
+ * Handles AI_SMART_REPAIR jobs on the Edge Worker.
+ * Communicates with local Ollama, LocalAI, LM Studio, or OpenAI-compatible endpoints.
+ * This offloads all AI computation from the cloud server (Render free tier) to the local worker.
+ *
+ * @param {Object} payload - { field, expectedType, previousSelector, previousValue, regexPattern, htmlSnippet, aiConfig }
+ * @param {Object} workerConfig - Worker global config with default AI endpoint/model
+ * @returns {Promise<Object>} { success, field, repairedSelector, extractedValue, confidence, explanation, modelUsed }
+ */
+export async function handleAiSmartRepair(payload, workerConfig = {}) {
+    const {
+        field = "field",
+        expectedType = "string",
+        previousSelector = "",
+        previousValue = "",
+        regexPattern = "",
+        htmlSnippet = "",
+        aiConfig = {},
+    } = payload;
+
+    const endpoint =
+        aiConfig.endpoint ||
+        workerConfig.aiEndpoint ||
+        process.env.AI_ENDPOINT ||
+        "http://localhost:11434/v1";
+
+    const model =
+        aiConfig.model ||
+        workerConfig.aiModel ||
+        process.env.AI_MODEL ||
+        "qwen2.5-coder:7b";
+
+    const apiKey =
+        aiConfig.apiKey ||
+        workerConfig.aiApiKey ||
+        process.env.AI_API_KEY ||
+        "ollama";
+
+    if (!htmlSnippet) {
+        throw new Error("Cannot perform AI smart repair: HTML snippet is empty.");
+    }
+
+    const systemPrompt = `You are an automated web scraping repair assistant.
+A website redesign has broken an existing selector. Your job is to analyze the updated HTML snippet and identify the relocated element for the target field.
+You MUST output ONLY a valid JSON object matching the requested schema with no surrounding Markdown or explanation.`;
+
+    const userPrompt = `FIELD REPAIR TARGET:
+- Field Name: "${field}"
+- Expected Data Type: "${expectedType}"
+- Broken Previous Selector: "${previousSelector}"
+- Last Known Sample Value: "${previousValue}"
+${regexPattern ? `- Expected Value Format (Regex): "${regexPattern}"` : ""}
+
+UPDATED HTML SNIPPET:
+\`\`\`html
+${htmlSnippet.slice(0, 8000)}
+\`\`\`
+
+TASK:
+1. Locate the exact HTML element in the snippet that represents "${field}".
+2. Construct a robust, resilient CSS selector (avoid dynamic or hashed classes like '_1a2b3c', prefer semantic tags, ids, data-* attributes, or structural paths).
+3. Return ONLY a JSON object formatted as follows:
+{
+  "repairedSelector": "css_selector_here",
+  "extractedValue": "exact_text_or_attribute_value",
+  "confidence": 0.95,
+  "explanation": "Brief explanation of why this element was chosen"
+}`;
+
+    const cleanEndpoint = endpoint.replace(/\/+$/, "");
+    const chatCompletionsUrl = cleanEndpoint.endsWith("/chat/completions")
+        ? cleanEndpoint
+        : `${cleanEndpoint}/chat/completions`;
+
+    try {
+        const response = await axios.post(
+            chatCompletionsUrl,
+            {
+                model,
+                messages: [
+                    { role: "system", content: systemPrompt },
+                    { role: "user", content: userPrompt },
+                ],
+                temperature: 0.1,
+                response_format: { type: "json_object" },
+            },
+            {
+                headers: {
+                    Authorization: `Bearer ${apiKey}`,
+                    "Content-Type": "application/json",
+                },
+                timeout: 45000,
+            }
+        );
+
+        let content = response.data?.choices?.[0]?.message?.content || "{}";
+
+        // Strip any markdown code fences if model returned ```json ... ```
+        content = content.replace(/^```json\s*/i, "").replace(/^```\s*/i, "").replace(/```$/i, "").trim();
+
+        let parsedResult = {};
+        try {
+            parsedResult = JSON.parse(content);
+        } catch {
+            const jsonMatch = content.match(/\{[\s\S]*\}/);
+            if (jsonMatch) {
+                parsedResult = JSON.parse(jsonMatch[0]);
+            } else {
+                throw new Error("Failed to parse JSON response from local AI model.");
+            }
+        }
+
+        const candidateSelector = parsedResult.repairedSelector?.trim() || "";
+        if (!candidateSelector) {
+            return {
+                success: false,
+                field,
+                error: "AI model could not determine a candidate selector.",
+                confidence: 0,
+            };
+        }
+
+        // Verify the candidate selector locally in the HTML snippet using Cheerio
+        const $ = cheerio.load(htmlSnippet);
+        let matchEl = null;
+        try {
+            matchEl = $(candidateSelector);
+        } catch {
+            // In case model returned an invalid CSS syntax
+            matchEl = null;
+        }
+
+        const isVerified = matchEl && matchEl.length > 0;
+        const verifiedValue = isVerified ? matchEl.first().text().trim() : parsedResult.extractedValue || "";
+
+        return {
+            success: isVerified,
+            field,
+            repairedSelector: candidateSelector,
+            extractedValue: verifiedValue,
+            confidence: isVerified ? Math.min(1.0, Math.max(0.6, parsedResult.confidence || 0.85)) : 0.3,
+            explanation: parsedResult.explanation || "Selector inferred via Edge Worker AI model.",
+            modelUsed: model,
+            endpointUsed: endpoint,
+        };
+    } catch (apiErr) {
+        throw new Error(
+            `Edge Worker AI repair request failed on '${endpoint}' (${model}): ${apiErr.message}`
+        );
+    }
+}

package/src/handlers/mailHandler.js

@@ -1,74 +1,74 @@
-import nodemailer from "nodemailer";
-
-/**
- * Handles incoming SEND_EMAIL job from the cluster server.
- *
- * Security & Reliability Guarantees:
- * 1. Zero credential retention: Credentials are received in-memory and destroyed after sending.
- * 2. Fail-fast timeouts: Prevents socket hanging with strict 10s connection & socket timeouts.
- * 3. Structured result payload: Returns delivery metadata (messageId, accepted, response).
- *
- * @param {Object} payload
- * @param {Object} payload.mailOptions - { to, subject, html, text, fromName, fromEmail }
- * @param {Object} payload.smtpConfig - { host, port, secure, auth: { user, pass } }
- * @returns {Promise<Object>}
- */
-export async function handleSendEmail(payload = {}) {
-    const { mailOptions, smtpConfig } = payload;
-
-    if (!mailOptions || !mailOptions.to || !mailOptions.subject) {
-        throw new Error("Invalid mailOptions: 'to' and 'subject' are required.");
-    }
-    if (!smtpConfig || !smtpConfig.auth?.user || !smtpConfig.auth?.pass) {
-        throw new Error("Invalid smtpConfig: credentials missing.");
-    }
-
-    const host = smtpConfig.host || "smtp.titan.email";
-    const port = Number(smtpConfig.port) || 465;
-    const secure = smtpConfig.secure !== undefined ? Boolean(smtpConfig.secure) : port === 465;
-
-    // Ephemeral Nodemailer transporter created strictly for this in-memory dispatch
-    const transporter = nodemailer.createTransport({
-        host,
-        port,
-        secure,
-        auth: {
-            user: smtpConfig.auth.user,
-            pass: smtpConfig.auth.pass,
-        },
-        connectionTimeout: 10000,
-        greetingTimeout: 10000,
-        socketTimeout: 15000,
-        tls: {
-            rejectUnauthorized: false, // Prevents certificate mismatches on custom domain mail
-        },
-    });
-
-    try {
-        const fromName = mailOptions.fromName || "Apps Manager";
-        const fromAddress = mailOptions.fromEmail || smtpConfig.auth.user;
-
-        const info = await transporter.sendMail({
-            from: `"${fromName}" <${fromAddress}>`,
-            to: mailOptions.to,
-            subject: mailOptions.subject,
-            text: mailOptions.text,
-            html: mailOptions.html,
-        });
-
-        return {
-            success: true,
-            messageId: info.messageId,
-            accepted: info.accepted,
-            rejected: info.rejected,
-            response: info.response,
-        };
-    } finally {
-        // Explicitly close transporter socket to prevent lingering connections
-        try {
-            transporter.close();
-        } catch (_) {
-            // Best effort cleanup
-        }
-    }
-}
+import nodemailer from "nodemailer";
+
+/**
+ * Handles incoming SEND_EMAIL job from the cluster server.
+ *
+ * Security & Reliability Guarantees:
+ * 1. Zero credential retention: Credentials are received in-memory and destroyed after sending.
+ * 2. Fail-fast timeouts: Prevents socket hanging with strict 10s connection & socket timeouts.
+ * 3. Structured result payload: Returns delivery metadata (messageId, accepted, response).
+ *
+ * @param {Object} payload
+ * @param {Object} payload.mailOptions - { to, subject, html, text, fromName, fromEmail }
+ * @param {Object} payload.smtpConfig - { host, port, secure, auth: { user, pass } }
+ * @returns {Promise<Object>}
+ */
+export async function handleSendEmail(payload = {}) {
+    const { mailOptions, smtpConfig } = payload;
+
+    if (!mailOptions || !mailOptions.to || !mailOptions.subject) {
+        throw new Error("Invalid mailOptions: 'to' and 'subject' are required.");
+    }
+    if (!smtpConfig || !smtpConfig.auth?.user || !smtpConfig.auth?.pass) {
+        throw new Error("Invalid smtpConfig: credentials missing.");
+    }
+
+    const host = smtpConfig.host || "smtp.titan.email";
+    const port = Number(smtpConfig.port) || 465;
+    const secure = smtpConfig.secure !== undefined ? Boolean(smtpConfig.secure) : port === 465;
+
+    // Ephemeral Nodemailer transporter created strictly for this in-memory dispatch
+    const transporter = nodemailer.createTransport({
+        host,
+        port,
+        secure,
+        auth: {
+            user: smtpConfig.auth.user,
+            pass: smtpConfig.auth.pass,
+        },
+        connectionTimeout: 10000,
+        greetingTimeout: 10000,
+        socketTimeout: 15000,
+        tls: {
+            rejectUnauthorized: false, // Prevents certificate mismatches on custom domain mail
+        },
+    });
+
+    try {
+        const fromName = mailOptions.fromName || "Apps Manager";
+        const fromAddress = mailOptions.fromEmail || smtpConfig.auth.user;
+
+        const info = await transporter.sendMail({
+            from: `"${fromName}" <${fromAddress}>`,
+            to: mailOptions.to,
+            subject: mailOptions.subject,
+            text: mailOptions.text,
+            html: mailOptions.html,
+        });
+
+        return {
+            success: true,
+            messageId: info.messageId,
+            accepted: info.accepted,
+            rejected: info.rejected,
+            response: info.response,
+        };
+    } finally {
+        // Explicitly close transporter socket to prevent lingering connections
+        try {
+            transporter.close();
+        } catch (_) {
+            // Best effort cleanup
+        }
+    }
+}