app-manager-edge-worker 1.0.8 → 1.1.0

package/src/client.js

@@ -1,300 +1,351 @@
-import { io } from "socket.io-client";
-import os from "os";
-import path from "path";
-import axios from "axios";
-import { execSync } from "child_process";
-import localBrowserManager from "./browser.js";
-import { handleChromiumScrape, handleHttpScrape } from "./handlers/scrapeHandler.js";
-import { handlePreviewPage, handleInspectElement } from "./handlers/previewHandler.js";
-import { handleSendEmail } from "./handlers/mailHandler.js";
-import { handleAiSmartRepair } from "./handlers/aiRepairHandler.js";
-import { printBanner, logInfo, logSuccess, logWarn, logError, logJob } from "./banner.js";
-
-export async function startWorkerClient(config) {
-    const { workerKey, serverUrl, concurrency, workerName, keepAliveMinutes = 7 } = config;
-    const isTermux = localBrowserManager.isTermux();
-
-    printBanner({
-        workerName,
-        serverUrl,
-        concurrency,
-        platform: `${os.platform()} ${os.arch()}`,
-        isTermux,
-        keepAliveMinutes,
-    });
-
-    if (isTermux) {
-        try {
-            execSync("termux-wake-lock", { stdio: "ignore" });
-            logSuccess("[Android Termux] Wake lock acquired to keep CPU awake in background.");
-        } catch {
-            // termux-tools might not be installed yet
-        }
-    }
-
-    logInfo("Checking Chromium browser availability...");
-    let chromiumExecutable = null;
-    try {
-        chromiumExecutable = await localBrowserManager.ensureChromiumExecutable();
-        logSuccess(`Chromium ready: ${chromiumExecutable}`);
-    } catch (browserErr) {
-        logWarn(`Chromium initialization warning: ${browserErr.message}`);
-    }
-
-    logInfo("Connecting to Apps Manager cluster...");
-
-    const socket = io(`${serverUrl}/workers`, {
-        transports: ["websocket", "polling"],
-        reconnection: true,
-        reconnectionAttempts: Infinity,
-        reconnectionDelay: 1000,
-        reconnectionDelayMax: 5000,
-        auth: {
-            workerKey,
-            metadata: {
-                hostname: workerName || os.hostname(),
-                platform: os.platform(),
-                arch: os.arch(),
-                osVersion: os.release(),
-                nodeVersion: process.version,
-                chromiumVersion: chromiumExecutable ? path.basename(chromiumExecutable) : "",
-                chromiumPath: chromiumExecutable || "",
-                isTermux,
-            },
-        },
-    });
-
-    const CHUNK_SIZE = 256 * 1024; // 256 KB chunks to prevent any WebSocket frame buffer overflow
-
-    const sendJobResult = (payload) => {
-        let payloadStr = "";
-        try {
-            payloadStr = JSON.stringify(payload);
-        } catch {
-            socket.emit("job:result", payload);
-            return;
-        }
-
-        // Small payload: send directly
-        if (payloadStr.length <= CHUNK_SIZE) {
-            socket.emit("job:result", payload);
-            return;
-        }
-
-        // Large payload (e.g. screenshot or large scrape): stream in chunks
-        const totalChunks = Math.ceil(payloadStr.length / CHUNK_SIZE);
-        const jobId = payload.jobId;
-
-        socket.emit("job:result:stream:start", {
-            jobId,
-            totalChunks,
-            totalLength: payloadStr.length,
-        });
-
-        for (let i = 0; i < totalChunks; i++) {
-            const chunk = payloadStr.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE);
-            socket.emit("job:result:stream:chunk", {
-                jobId,
-                index: i,
-                chunk,
-            });
-        }
-
-        socket.emit("job:result:stream:end", { jobId });
-    };
-
-    let activeJobs = 0;
-    let activeConcurrency = concurrency;
-
-    socket.on("connect", () => {
-        logSuccess(`Connected to cluster! (Socket ID: ${socket.id})`);
-        logInfo(`Ready to accept scraping and visual preview jobs (Concurrency: ${activeConcurrency}).\n`);
-    });
-
-    socket.on("worker:registered", (data) => {
-        if (data?.concurrencyLimit && typeof data.concurrencyLimit === "number") {
-            activeConcurrency = data.concurrencyLimit;
-            logInfo(`[Cluster Config] Worker registered as '${data.name || data.workerId}'. Active concurrency: ${activeConcurrency} concurrent tabs.`);
-        }
-    });
-
-    socket.on("connect_error", (err) => {
-        if (err.message && err.message.includes("Authentication failed")) {
-            logError(`Authentication Failed: ${err.message}`);
-            logWarn("Please verify your worker key or run with '--reset' to enter a new key.");
-        } else {
-            logWarn(`Connection failed (${err.message}). Retrying in a few seconds...`);
-        }
-    });
-
-    socket.on("disconnect", (reason) => {
-        logWarn(`Disconnected from cluster: ${reason}`);
-    });
-
-    // Handle incoming job dispatch from server
-    socket.on("job:dispatch", async (job) => {
-        const jobId = job.jobId;
-        const jobType = job.jobType || job.type;
-        const payload = job.payload || {};
-        const targetUrl = jobType === "SEND_EMAIL"
-            ? (payload.mailOptions?.to ? `to: ${payload.mailOptions.to}` : "email")
-            : (payload.url || "(no url)");
-
-        logJob(jobType, targetUrl);
-        activeJobs++;
-
-        const startTime = Date.now();
-
-        try {
-            let resultData = null;
-
-            if (jobType === "SCRAPE_CHROMIUM") {
-                resultData = await handleChromiumScrape(payload);
-            } else if (jobType === "SCRAPE_HTTP") {
-                resultData = await handleHttpScrape(payload);
-            } else if (jobType === "PREVIEW_PAGE") {
-                resultData = await handlePreviewPage(payload);
-            } else if (jobType === "INSPECT_ELEMENT") {
-                resultData = await handleInspectElement(payload);
-            } else if (jobType === "SEND_EMAIL") {
-                resultData = await handleSendEmail(payload);
-            } else if (jobType === "AI_SMART_REPAIR") {
-                resultData = await handleAiSmartRepair(payload, config);
-            } else {
-                throw new Error(`Unsupported job type: ${jobType}`);
-            }
-
-            const durationMs = Date.now() - startTime;
-            logSuccess(`Completed ${jobType} for ${targetUrl} in ${durationMs}ms`);
-
-            sendJobResult({
-                jobId,
-                success: true,
-                data: resultData,
-                durationMs,
-            });
-        } catch (err) {
-            const durationMs = Date.now() - startTime;
-            logError(`Failed ${jobType} for ${targetUrl}: ${err.message}`);
-
-            sendJobResult({
-                jobId,
-                success: false,
-                error: {
-                    message: err.message,
-                    code: err.code || "JOB_EXECUTION_ERROR",
-                },
-                durationMs,
-            });
-        } finally {
-            activeJobs = Math.max(0, activeJobs - 1);
-        }
-    });
-
-    // Handle cluster heartbeat ping
-    socket.on("ping", () => {
-        socket.emit("pong", {
-            activeJobs,
-            memory: process.memoryUsage(),
-            uptime: process.uptime(),
-        });
-    });
-
-    // Render Anti-Sleep Keep-Alive Ping
-    let keepAliveTimer = null;
-    let initialKeepAliveTimer = null;
-
-    const clearKeepAliveTimers = () => {
-        if (keepAliveTimer) {
-            clearInterval(keepAliveTimer);
-            keepAliveTimer = null;
-        }
-        if (initialKeepAliveTimer) {
-            clearTimeout(initialKeepAliveTimer);
-            initialKeepAliveTimer = null;
-        }
-    };
-
-    const pingKeepAlive = async () => {
-        try {
-            // Try the dedicated keep-alive endpoint first
-            const endpoint = `${serverUrl}/api/workers/keep-alive`;
-            const res = await axios.get(endpoint, {
-                timeout: 15000,
-                headers: {
-                    "x-worker-key": workerKey || "",
-                    "user-agent": "AppsManagerEdgeWorker/1.0.4",
-                },
-            });
-            logInfo(`[Keep-Alive] Server awake ping OK (${res.status}) - Render instance active`);
-        } catch (err) {
-            // Fallback to /health
-            try {
-                const fallbackUrl = `${serverUrl}/health`;
-                await axios.get(fallbackUrl, {
-                    timeout: 10000,
-                    headers: {
-                        "user-agent": "AppsManagerEdgeWorker/1.0.4",
-                    },
-                });
-                logInfo(`[Keep-Alive] Fallback health ping OK - Render instance active`);
-            } catch (fallbackErr) {
-                logWarn(`[Keep-Alive] Server awake ping warning (${err.message}) - will retry next cycle`);
-            }
-        }
-    };
-
-    const startKeepAlive = () => {
-        if (keepAliveMinutes <= 0) return;
-        clearKeepAliveTimers();
-
-        // Send initial ping 15s after startup to warm up/verify
-        initialKeepAliveTimer = setTimeout(pingKeepAlive, 15000);
-        if (initialKeepAliveTimer.unref) initialKeepAliveTimer.unref();
-
-        const intervalMs = Math.max(1, keepAliveMinutes) * 60 * 1000;
-        keepAliveTimer = setInterval(pingKeepAlive, intervalMs);
-        if (keepAliveTimer.unref) keepAliveTimer.unref();
-    };
-
-    if (keepAliveMinutes > 0) {
-        startKeepAlive();
-    }
-
-    // Clean up keep-alive timers on socket disconnect
-    socket.on("disconnect", (reason) => {
-        logWarn(`Disconnected from cluster: ${reason}`);
-        clearKeepAliveTimers();
-    });
-
-    socket.on("connect", () => {
-        if (keepAliveMinutes > 0 && !keepAliveTimer) {
-            startKeepAlive();
-        }
-    });
-
-    // Provide programmatic close for automated test harnesses
-    socket.closeClient = async () => {
-        clearKeepAliveTimers();
-        if (isTermux) {
-            try {
-                execSync("termux-wake-unlock", { stdio: "ignore" });
-            } catch {}
-        }
-        socket.disconnect();
-        await localBrowserManager.close();
-    };
-
-    // Graceful shutdown handling
-    const shutdown = async () => {
-        logInfo("\nGracefully shutting down edge worker...");
-        await socket.closeClient();
-        logSuccess("Edge worker stopped cleanly.");
-        process.exit(0);
-    };
-
-    process.on("SIGINT", shutdown);
-    process.on("SIGTERM", shutdown);
-
-    return socket;
-}
+import { io } from "socket.io-client";
+import os from "os";
+import path from "path";
+import axios from "axios";
+import { execSync } from "child_process";
+import localBrowserManager from "./browser.js";
+import { handleChromiumScrape, handleHttpScrape } from "./handlers/scrapeHandler.js";
+import { handlePreviewPage, handleInspectElement } from "./handlers/previewHandler.js";
+import { handleSendEmail } from "./handlers/mailHandler.js";
+import { handleAiSmartRepair } from "./handlers/aiRepairHandler.js";
+import { getWhatsAppHandler } from "./handlers/whatsappHandler.js";
+import { printBanner, logInfo, logSuccess, logWarn, logError, logJob } from "./banner.js";
+
+export async function startWorkerClient(config) {
+    const { workerKey, serverUrl, concurrency, workerName, keepAliveMinutes = 7 } = config;
+    const isTermux = localBrowserManager.isTermux();
+
+    printBanner({
+        workerName,
+        serverUrl,
+        concurrency,
+        platform: `${os.platform()} ${os.arch()}`,
+        isTermux,
+        keepAliveMinutes,
+    });
+
+    if (isTermux) {
+        try {
+            execSync("termux-wake-lock", { stdio: "ignore" });
+            logSuccess("[Android Termux] Wake lock acquired to keep CPU awake in background.");
+        } catch {
+            // termux-tools might not be installed yet
+        }
+    }
+
+    logInfo("Checking Chromium browser availability...");
+    let chromiumExecutable = null;
+    try {
+        chromiumExecutable = await localBrowserManager.ensureChromiumExecutable();
+        logSuccess(`Chromium ready: ${chromiumExecutable}`);
+    } catch (browserErr) {
+        logWarn(`Chromium initialization warning: ${browserErr.message}`);
+    }
+
+    logInfo("Connecting to Apps Manager cluster...");
+
+    const socket = io(`${serverUrl}/workers`, {
+        transports: ["websocket", "polling"],
+        reconnection: true,
+        reconnectionAttempts: Infinity,
+        reconnectionDelay: 1000,
+        reconnectionDelayMax: 5000,
+        auth: {
+            workerKey,
+            metadata: {
+                hostname: workerName || os.hostname(),
+                platform: os.platform(),
+                arch: os.arch(),
+                osVersion: os.release(),
+                nodeVersion: process.version,
+                chromiumVersion: chromiumExecutable ? path.basename(chromiumExecutable) : "",
+                chromiumPath: chromiumExecutable || "",
+                isTermux,
+            },
+        },
+    });
+
+    const CHUNK_SIZE = 256 * 1024; // 256 KB chunks to prevent any WebSocket frame buffer overflow
+
+    const sendJobResult = (payload) => {
+        let payloadStr = "";
+        try {
+            payloadStr = JSON.stringify(payload);
+        } catch {
+            socket.emit("job:result", payload);
+            return;
+        }
+
+        // Small payload: send directly
+        if (payloadStr.length <= CHUNK_SIZE) {
+            socket.emit("job:result", payload);
+            return;
+        }
+
+        // Large payload (e.g. screenshot or large scrape): stream in chunks
+        const totalChunks = Math.ceil(payloadStr.length / CHUNK_SIZE);
+        const jobId = payload.jobId;
+
+        socket.emit("job:result:stream:start", {
+            jobId,
+            totalChunks,
+            totalLength: payloadStr.length,
+        });
+
+        for (let i = 0; i < totalChunks; i++) {
+            const chunk = payloadStr.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE);
+            socket.emit("job:result:stream:chunk", {
+                jobId,
+                index: i,
+                chunk,
+            });
+        }
+
+        socket.emit("job:result:stream:end", { jobId });
+    };
+
+    let activeJobs = 0;
+    let activeConcurrency = concurrency;
+
+    // Initialize WhatsApp Companion Device Engine
+    const waHandler = getWhatsAppHandler({
+        onStatusChange: (statusData) => {
+            socket.emit("whatsapp:status_update", statusData);
+        },
+        onQrGenerated: ({ qrDataUrl }) => {
+            socket.emit("whatsapp:qr_stream", { qrDataUrl });
+        },
+    });
+
+    socket.on("connect", () => {
+        logSuccess(`Connected to cluster! (Socket ID: ${socket.id})`);
+        logInfo(`Ready to accept scraping and visual preview jobs (Concurrency: ${activeConcurrency}).\n`);
+        
+        // Announce current WhatsApp companion device status
+        const waTelemetry = waHandler.getTelemetry();
+        socket.emit("whatsapp:status_update", waTelemetry);
+    });
+
+    socket.on("worker:registered", (data) => {
+        if (data?.concurrencyLimit && typeof data.concurrencyLimit === "number") {
+            activeConcurrency = data.concurrencyLimit;
+            logInfo(`[Cluster Config] Worker registered as '${data.name || data.workerId}'. Active concurrency: ${activeConcurrency} concurrent tabs.`);
+        }
+    });
+
+    // Handle Admin Panel WhatsApp QR stream request
+    socket.on("whatsapp:request_qr", async () => {
+        logInfo("[WhatsApp] QR pairing requested via Admin Panel...");
+        try {
+            await waHandler.initialize(true);
+        } catch (err) {
+            logError(`[WhatsApp] Failed to initialize QR pairing: ${err.message}`);
+        }
+    });
+
+    // Handle Admin Panel Remote Logout
+    socket.on("whatsapp:remote_logout", async () => {
+        logInfo("[WhatsApp] Remote unpair requested via Admin Panel...");
+        try {
+            await waHandler.logout();
+            logSuccess("[WhatsApp] Companion device unpaired and local session cleared.");
+        } catch (err) {
+            logError(`[WhatsApp] Logout error: ${err.message}`);
+        }
+    });
+
+    socket.on("connect_error", (err) => {
+        if (err.message && err.message.includes("Authentication failed")) {
+            logError(`Authentication Failed: ${err.message}`);
+            logWarn("Please verify your worker key or run with '--reset' to enter a new key.");
+        } else {
+            logWarn(`Connection failed (${err.message}). Retrying in a few seconds...`);
+        }
+    });
+
+    socket.on("disconnect", (reason) => {
+        logWarn(`Disconnected from cluster: ${reason}`);
+    });
+
+    // Handle incoming job dispatch from server
+    socket.on("job:dispatch", async (job) => {
+        const jobId = job.jobId;
+        const jobType = job.jobType || job.type;
+        const payload = job.payload || {};
+        const targetUrl = jobType === "SEND_EMAIL"
+            ? (payload.mailOptions?.to ? `to: ${payload.mailOptions.to}` : "email")
+            : jobType.startsWith("WHATSAPP_")
+            ? (payload.recipientPhone ? `WA: ${payload.recipientPhone}` : "whatsapp")
+            : (payload.url || "(no url)");
+
+        logJob(jobType, targetUrl);
+        activeJobs++;
+
+        const startTime = Date.now();
+
+        try {
+            let resultData = null;
+
+            if (jobType === "SCRAPE_CHROMIUM") {
+                resultData = await handleChromiumScrape(payload);
+            } else if (jobType === "SCRAPE_HTTP") {
+                resultData = await handleHttpScrape(payload);
+            } else if (jobType === "PREVIEW_PAGE") {
+                resultData = await handlePreviewPage(payload);
+            } else if (jobType === "INSPECT_ELEMENT") {
+                resultData = await handleInspectElement(payload);
+            } else if (jobType === "SEND_EMAIL") {
+                resultData = await handleSendEmail(payload);
+            } else if (jobType === "AI_SMART_REPAIR") {
+                resultData = await handleAiSmartRepair(payload, config);
+            } else if (jobType === "WHATSAPP_SEND_MESSAGE") {
+                resultData = await waHandler.sendMessage(payload);
+            } else if (jobType === "WHATSAPP_SEND_DOCUMENT") {
+                resultData = await waHandler.sendDocument(payload);
+            } else if (jobType === "WHATSAPP_RECONCILE") {
+                resultData = await waHandler.reconcileDelivery(payload.recipientPhone, payload.idempotencyKey);
+            } else {
+                throw new Error(`Unsupported job type: ${jobType}`);
+            }
+
+            const durationMs = Date.now() - startTime;
+            logSuccess(`Completed ${jobType} for ${targetUrl} in ${durationMs}ms`);
+
+            sendJobResult({
+                jobId,
+                success: true,
+                data: resultData,
+                durationMs,
+            });
+        } catch (err) {
+            const durationMs = Date.now() - startTime;
+            logError(`Failed ${jobType} for ${targetUrl}: ${err.message}`);
+
+            sendJobResult({
+                jobId,
+                success: false,
+                error: {
+                    message: err.message,
+                    code: err.code || "JOB_EXECUTION_ERROR",
+                },
+                durationMs,
+            });
+        } finally {
+            activeJobs = Math.max(0, activeJobs - 1);
+        }
+    });
+
+    // Handle cluster heartbeat ping with enhanced capacity & memory telemetry
+    socket.on("ping", () => {
+        const telemetry = waHandler.getTelemetry();
+        socket.emit("pong", {
+            activeJobs,
+            memory: process.memoryUsage(),
+            uptime: process.uptime(),
+            freeSystemMemoryMb: telemetry.freeSystemMemoryMb,
+            totalSystemMemoryMb: telemetry.totalSystemMemoryMb,
+            processMemoryMb: telemetry.processMemoryMb,
+            memoryPressure: telemetry.memoryPressure,
+            whatsappStatus: telemetry.status,
+            whatsappUser: telemetry.user,
+        });
+    });
+
+    // Render Anti-Sleep Keep-Alive Ping
+    let keepAliveTimer = null;
+    let initialKeepAliveTimer = null;
+
+    const clearKeepAliveTimers = () => {
+        if (keepAliveTimer) {
+            clearInterval(keepAliveTimer);
+            keepAliveTimer = null;
+        }
+        if (initialKeepAliveTimer) {
+            clearTimeout(initialKeepAliveTimer);
+            initialKeepAliveTimer = null;
+        }
+    };
+
+    const pingKeepAlive = async () => {
+        try {
+            // Try the dedicated keep-alive endpoint first
+            const endpoint = `${serverUrl}/api/workers/keep-alive`;
+            const res = await axios.get(endpoint, {
+                timeout: 15000,
+                headers: {
+                    "x-worker-key": workerKey || "",
+                    "user-agent": "AppsManagerEdgeWorker/1.1.0",
+                },
+            });
+            logInfo(`[Keep-Alive] Server awake ping OK (${res.status}) - Render instance active`);
+        } catch (err) {
+            // Fallback to /health
+            try {
+                const fallbackUrl = `${serverUrl}/health`;
+                await axios.get(fallbackUrl, {
+                    timeout: 10000,
+                    headers: {
+                        "user-agent": "AppsManagerEdgeWorker/1.1.0",
+                    },
+                });
+                logInfo(`[Keep-Alive] Fallback health ping OK - Render instance active`);
+            } catch (fallbackErr) {
+                logWarn(`[Keep-Alive] Server awake ping warning (${err.message}) - will retry next cycle`);
+            }
+        }
+    };
+
+    const startKeepAlive = () => {
+        if (keepAliveMinutes <= 0) return;
+        clearKeepAliveTimers();
+
+        // Send initial ping 15s after startup to warm up/verify
+        initialKeepAliveTimer = setTimeout(pingKeepAlive, 15000);
+        if (initialKeepAliveTimer.unref) initialKeepAliveTimer.unref();
+
+        const intervalMs = Math.max(1, keepAliveMinutes) * 60 * 1000;
+        keepAliveTimer = setInterval(pingKeepAlive, intervalMs);
+        if (keepAliveTimer.unref) keepAliveTimer.unref();
+    };
+
+    if (keepAliveMinutes > 0) {
+        startKeepAlive();
+    }
+
+    // Clean up keep-alive timers on socket disconnect
+    socket.on("disconnect", (reason) => {
+        logWarn(`Disconnected from cluster: ${reason}`);
+        clearKeepAliveTimers();
+    });
+
+    socket.on("connect", () => {
+        if (keepAliveMinutes > 0 && !keepAliveTimer) {
+            startKeepAlive();
+        }
+    });
+
+    // Provide programmatic close for automated test harnesses
+    socket.closeClient = async () => {
+        clearKeepAliveTimers();
+        if (isTermux) {
+            try {
+                execSync("termux-wake-unlock", { stdio: "ignore" });
+            } catch {}
+        }
+        socket.disconnect();
+        await localBrowserManager.close();
+    };
+
+    // Graceful shutdown handling
+    const shutdown = async () => {
+        logInfo("\nGracefully shutting down edge worker...");
+        await socket.closeClient();
+        logSuccess("Edge worker stopped cleanly.");
+        process.exit(0);
+    };
+
+    process.on("SIGINT", shutdown);
+    process.on("SIGTERM", shutdown);
+
+    return socket;
+}