app-manager-edge-worker 1.1.3 → 1.1.5

package/README.md

@@ -203,6 +203,20 @@ Log levels: `[INFO]` · `[SUCCESS]` · `[WARN]` · `[ERROR]` · `[TASK]` (job di
 
 ## 📝 Changelog
 
+### v1.1.5
+- **Real-Time WhatsApp Task & Document Streaming Telemetry**:
+  - Added timestamped terminal logging for incoming WhatsApp text and document task receipts.
+  - Live progress logs for authenticated PDF document streaming, byte transfer size, and SHA-256 integrity validation.
+  - Real-time typing presence simulation status and delivery confirmations with official message IDs (`wamid`).
+
+### v1.1.4
+- **Persistent MongoDB Session Restoration for WhatsApp**:
+  - Automatically restores WhatsApp companion device credentials (`creds.json`, pre-keys, session keys) from cluster MongoDB on worker boot.
+  - Workers restart without losing WhatsApp connection state or requiring QR code re-pairing.
+  - Live 150ms debounced synchronization of key rotations and credentials to MongoDB via `bulkWrite`.
+  - Automatic backup of pre-existing local disk sessions to cluster database on initial connection.
+  - Coordinated session wiping on remote logout and phone unlinking.
+
 ### v1.1.0
 - **WhatsApp Multi-Device Companion Relay**:
   - Independent companion device pairing via live QR streaming to Admin Dashboard.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "app-manager-edge-worker",
-  "version": "1.1.3",
+  "version": "1.1.5",
   "description": "Distributed Scraper Edge Worker for Apps Manager. Run on any local machine, VPS, or Termux Android device to serve browser and HTTP scraping jobs.",
   "type": "module",
   "bin": {

package/src/banner.js

@@ -1,7 +1,7 @@
 import { createRequire } from "module";
 
 const require = createRequire(import.meta.url);
-let workerVersion = "1.1.0";
+let workerVersion = "1.1.5";
 try {
     const pkg = require("../package.json");
     if (pkg?.version) workerVersion = pkg.version;

package/src/client.js

@@ -8,9 +8,16 @@ import { handleChromiumScrape, handleHttpScrape } from "./handlers/scrapeHandler
 import { handlePreviewPage, handleInspectElement } from "./handlers/previewHandler.js";
 import { handleSendEmail } from "./handlers/mailHandler.js";
 import { handleAiSmartRepair } from "./handlers/aiRepairHandler.js";
+import fs from "fs";
 import { getWhatsAppHandler } from "./handlers/whatsappHandler.js";
 import { printBanner, logInfo, logSuccess, logWarn, logError, logJob } from "./banner.js";
 
+let workerPkgVersion = "1.1.5";
+try {
+    const pkg = JSON.parse(fs.readFileSync(new URL("../package.json", import.meta.url), "utf-8"));
+    if (pkg?.version) workerPkgVersion = pkg.version;
+} catch (_) {}
+
 export async function startWorkerClient(config) {
     const { workerKey, serverUrl, concurrency, workerName, keepAliveMinutes = 7 } = config;
     const isTermux = localBrowserManager.isTermux();
@@ -115,15 +122,53 @@ export async function startWorkerClient(config) {
         onQrGenerated: ({ qrDataUrl }) => {
             socket.emit("whatsapp:qr_stream", { qrDataUrl });
         },
+        onSessionBatchUpdate: (batchData) => {
+            socket.emit("whatsapp:session_batch_update", batchData);
+        },
+        onClearSession: () => {
+            socket.emit("whatsapp:clear_session");
+        },
     });
 
     socket.on("connect", () => {
         logSuccess(`Connected to cluster! (Socket ID: ${socket.id})`);
         logInfo(`Ready to accept scraping and visual preview jobs (Concurrency: ${activeConcurrency}).\n`);
-        
-        // Announce current WhatsApp companion device status
-        const waTelemetry = waHandler.getTelemetry();
-        socket.emit("whatsapp:status_update", waTelemetry);
+
+        // If Baileys is already CONNECTED in memory (e.g. quick network reconnection), announce immediately
+        if (waHandler.status === "CONNECTED" || waHandler.status === "STANDBY") {
+            const waTelemetry = waHandler.getTelemetry();
+            socket.emit("whatsapp:status_update", waTelemetry);
+        }
+        // If not connected yet, wait for whatsapp:session_sync from cluster server before announcing status
+    });
+
+    // Handle MongoDB WhatsApp Session restore from cluster server
+    socket.on("whatsapp:session_sync", async ({ hasSession, files }) => {
+        if (hasSession && files && Object.keys(files).length > 0) {
+            logInfo(`[WhatsApp] Cluster provided ${Object.keys(files).length} session credentials from MongoDB. Restoring session...`);
+            waHandler.hydrateFromSessionFiles(files);
+            try {
+                await waHandler.initialize(false);
+                logSuccess("[WhatsApp] Auto-connected successfully using restored MongoDB session!");
+            } catch (err) {
+                logWarn(`[WhatsApp] Auto-connect with MongoDB credentials failed: ${err.message}`);
+                socket.emit("whatsapp:status_update", waHandler.getTelemetry());
+            }
+        } else if (waHandler.hasCredentials()) {
+            logInfo("[WhatsApp] Found local credentials on disk not yet in MongoDB. Syncing to cluster database...");
+            const localFiles = waHandler.exportAllSessionFiles();
+            socket.emit("whatsapp:save_session_batch", { files: localFiles });
+            try {
+                await waHandler.initialize(false);
+                logSuccess("[WhatsApp] Connected using local credentials and backed up to cluster database.");
+            } catch (err) {
+                logWarn(`[WhatsApp] Auto-connect with local credentials failed: ${err.message}`);
+                socket.emit("whatsapp:status_update", waHandler.getTelemetry());
+            }
+        } else {
+            // No stored credentials on cluster or local disk - report clean disconnected status
+            socket.emit("whatsapp:status_update", waHandler.getTelemetry());
+        }
     });
 
     socket.on("worker:registered", (data) => {
@@ -274,7 +319,7 @@ export async function startWorkerClient(config) {
                 timeout: 15000,
                 headers: {
                     "x-worker-key": workerKey || "",
-                    "user-agent": "AppsManagerEdgeWorker/1.1.0",
+                    "user-agent": `AppsManagerEdgeWorker/${workerPkgVersion}`,
                 },
             });
             logInfo(`[Keep-Alive] Server awake ping OK (${res.status}) - Render instance active`);
@@ -285,7 +330,7 @@ export async function startWorkerClient(config) {
                 await axios.get(fallbackUrl, {
                     timeout: 10000,
                     headers: {
-                        "user-agent": "AppsManagerEdgeWorker/1.1.0",
+                        "user-agent": `AppsManagerEdgeWorker/${workerPkgVersion}`,
                     },
                 });
                 logInfo(`[Keep-Alive] Fallback health ping OK - Render instance active`);

package/src/config.js

@@ -111,7 +111,7 @@ export async function resolveWorkerConfig() {
     const cli = parseCliArgs();
 
     if (cli.help) {
-        let pkgVer = "1.1.0";
+        let pkgVer = "1.1.5";
         try {
             const pkg = JSON.parse(fs.readFileSync(new URL("../package.json", import.meta.url), "utf-8"));
             if (pkg?.version) pkgVer = pkg.version;

package/src/handlers/whatsappHandler.js

@@ -11,6 +11,7 @@ import path from "path";
 import os from "os";
 import crypto from "crypto";
 import axios from "axios";
+import { logInfo, logSuccess, logWarn } from "../banner.js";
 
 /**
  * WhatsApp Companion Device Engine for App Manager Edge Worker.
@@ -36,18 +37,116 @@ export class WhatsAppWorkerHandler {
         this.IDLE_COOLDOWN_MS = options.idleCooldownMs || 10 * 60 * 1000; // 10 minutes
         this.onStatusChange = options.onStatusChange || (() => {});
         this.onQrGenerated = options.onQrGenerated || (() => {});
+        this.onSessionBatchUpdate = options.onSessionBatchUpdate || (() => {});
+        this.onClearSession = options.onClearSession || (() => {});
         this.chatStore = new Map(); // JID -> Array of recent messages for reconciliation
         this.isInitializing = false;
+        this.pendingWrites = new Map(); // keyId -> string data
+        this.pendingDeletes = new Set(); // keyId
+        this.syncDebounceTimer = null;
 
         if (!fs.existsSync(this.authDir)) {
             fs.mkdirSync(this.authDir, { recursive: true });
         }
     }
 
+    /**
+     * Queues a key write to be batched and synced to the central MongoDB server.
+     */
+    queueSyncWrite(keyId, data) {
+        if (!keyId || typeof data !== "string") return;
+        this.pendingDeletes.delete(keyId);
+        this.pendingWrites.set(keyId, data);
+        this._scheduleBatchSync();
+    }
+
+    /**
+     * Queues a key removal to be batched and deleted from central MongoDB server.
+     */
+    queueSyncDelete(keyId) {
+        if (!keyId) return;
+        this.pendingWrites.delete(keyId);
+        this.pendingDeletes.add(keyId);
+        this._scheduleBatchSync();
+    }
+
+    /**
+     * Debounces and dispatches batched key updates to the cluster WebSocket.
+     */
+    _scheduleBatchSync() {
+        if (this.syncDebounceTimer) return;
+        this.syncDebounceTimer = setTimeout(() => {
+            this.syncDebounceTimer = null;
+            if (this.pendingWrites.size === 0 && this.pendingDeletes.size === 0) return;
+
+            const writes = [];
+            for (const [keyId, data] of this.pendingWrites.entries()) {
+                writes.push({ keyId, data });
+            }
+            const deletes = Array.from(this.pendingDeletes);
+
+            this.pendingWrites.clear();
+            this.pendingDeletes.clear();
+
+            this.onSessionBatchUpdate({ writes, deletes });
+        }, 150);
+    }
+
+    /**
+     * Hydrates local session disk cache with stored credentials from MongoDB.
+     */
+    hydrateFromSessionFiles(files) {
+        if (!files || typeof files !== "object") return;
+        if (!fs.existsSync(this.authDir)) {
+            fs.mkdirSync(this.authDir, { recursive: true });
+        }
+
+        let hydratedCount = 0;
+        for (const [filename, content] of Object.entries(files)) {
+            if (!filename || typeof content !== "string") continue;
+            const safeName = filename.replace(/\//g, "__").replace(/:/g, "-");
+            const targetPath = path.join(this.authDir, safeName);
+            try {
+                fs.writeFileSync(targetPath, content, "utf-8");
+                hydratedCount++;
+            } catch (err) {
+                console.error(`[WhatsApp Worker] Failed to hydrate ${filename}:`, err.message);
+            }
+        }
+        console.log(`[WhatsApp Worker] Successfully hydrated ${hydratedCount} session files from MongoDB.`);
+    }
+
+    /**
+     * Reads all local JSON credential files from disk for initial backup to MongoDB.
+     */
+    exportAllSessionFiles() {
+        if (!fs.existsSync(this.authDir)) return {};
+        const result = {};
+        try {
+            const files = fs.readdirSync(this.authDir);
+            for (const file of files) {
+                if (file.endsWith(".json")) {
+                    try {
+                        const content = fs.readFileSync(path.join(this.authDir, file), "utf-8");
+                        result[file] = content;
+                    } catch (_) {}
+                }
+            }
+        } catch (_) {}
+        return result;
+    }
+
     /**
      * Wipes local stored credentials.
      */
     clearAuth() {
+        this.pendingWrites.clear();
+        this.pendingDeletes.clear();
+        if (this.syncDebounceTimer) {
+            clearTimeout(this.syncDebounceTimer);
+            this.syncDebounceTimer = null;
+        }
+
         if (fs.existsSync(this.authDir)) {
             try {
                 fs.rmSync(this.authDir, { recursive: true, force: true });
@@ -142,6 +241,44 @@ export class WhatsAppWorkerHandler {
 
         try {
             const { state, saveCreds } = await useMultiFileAuthState(this.authDir);
+
+            // Wrap saveCreds to mirror credentials to central MongoDB
+            const originalSaveCreds = saveCreds;
+            const wrappedSaveCreds = async () => {
+                await originalSaveCreds();
+                try {
+                    const credsPath = path.join(this.authDir, "creds.json");
+                    if (fs.existsSync(credsPath)) {
+                        const credsStr = fs.readFileSync(credsPath, "utf-8");
+                        this.queueSyncWrite("creds.json", credsStr);
+                    }
+                } catch (_) {}
+            };
+
+            // Wrap state.keys.set to mirror key additions, updates and removals to central MongoDB
+            const originalKeysSet = state.keys.set;
+            state.keys.set = async (data) => {
+                await originalKeysSet(data);
+                for (const category in data) {
+                    for (const id in data[category]) {
+                        const value = data[category][id];
+                        const file = `${category}-${id}.json`;
+                        const safeName = file.replace(/\//g, "__").replace(/:/g, "-");
+                        if (value) {
+                            try {
+                                const filePath = path.join(this.authDir, safeName);
+                                if (fs.existsSync(filePath)) {
+                                    const content = fs.readFileSync(filePath, "utf-8");
+                                    this.queueSyncWrite(safeName, content);
+                                }
+                            } catch (_) {}
+                        } else {
+                            this.queueSyncDelete(safeName);
+                        }
+                    }
+                }
+            };
+
             let version = undefined;
             try {
                 const fetched = await fetchLatestBaileysVersion();
@@ -167,7 +304,7 @@ export class WhatsAppWorkerHandler {
 
             this.sock = makeWASocket(socketConfig);
 
-            this.sock.ev.on("creds.update", saveCreds);
+            this.sock.ev.on("creds.update", wrappedSaveCreds);
 
             // Message Upsert listener for companion store sync & pre-send delivery reconciliation
             this.sock.ev.on("messages.upsert", ({ messages }) => {
@@ -243,6 +380,7 @@ export class WhatsAppWorkerHandler {
                         this.user = null;
                         this.currentQrDataUrl = null;
                         this.clearAuth();
+                        this.onClearSession();
 
                         if (this.sock) {
                             try {
@@ -326,6 +464,7 @@ export class WhatsAppWorkerHandler {
      * Sends a plain text WhatsApp message with typing simulation and jitter.
      */
     async sendMessage({ recipientPhone, message, idempotencyKey = null }) {
+        logInfo(`[WhatsApp] Incoming message dispatch for ${recipientPhone}`);
         await this.initialize(false);
         if (!this.sock || this.status !== "CONNECTED") {
             throw new Error("WhatsApp companion device is not connected. Please pair via Admin Panel.");
@@ -337,6 +476,7 @@ export class WhatsAppWorkerHandler {
         if (idempotencyKey) {
             const alreadyDelivered = await this.reconcileDelivery(recipientPhone, idempotencyKey);
             if (alreadyDelivered) {
+                logInfo(`[WhatsApp] Reconciled delivery for ${recipientPhone}; duplicate aborted.`);
                 return {
                     success: true,
                     reconciled: true,
@@ -355,6 +495,7 @@ export class WhatsAppWorkerHandler {
         const targetJid = verified.jid || jid;
 
         // 3. Human Typing Simulation
+        logInfo(`[WhatsApp] Simulating human typing for ${digits}...`);
         await this.sock.sendPresenceUpdate("composing", targetJid).catch(() => {});
         const typingDurationMs = Math.min(Math.max((message.length || 20) * 35, 2000), 4500);
         await new Promise((r) => setTimeout(r, typingDurationMs));
@@ -370,10 +511,12 @@ export class WhatsAppWorkerHandler {
         // 5. Dispatch
         const result = await this.sock.sendMessage(targetJid, { text: finalMessage });
         this._resetIdleTimer();
+        const msgId = result?.key?.id || `wamid-${Date.now()}`;
+        logSuccess(`[WhatsApp] Text message successfully delivered to ${recipientPhone} (ID: ${msgId})`);
 
         return {
             success: true,
-            messageId: result?.key?.id || `wamid-${Date.now()}`,
+            messageId: msgId,
             timestamp: Date.now(),
         };
     }
@@ -391,6 +534,7 @@ export class WhatsAppWorkerHandler {
         caption = "",
         idempotencyKey = null,
     }) {
+        logInfo(`[WhatsApp] Incoming document task for ${recipientPhone} (${fileName})`);
         await this.initialize(false);
         if (!this.sock || this.status !== "CONNECTED") {
             throw new Error("WhatsApp companion device is not connected. Please pair via Admin Panel.");
@@ -402,6 +546,7 @@ export class WhatsAppWorkerHandler {
         if (idempotencyKey) {
             const alreadyDelivered = await this.reconcileDelivery(recipientPhone, idempotencyKey);
             if (alreadyDelivered) {
+                logInfo(`[WhatsApp] Reconciled delivery for ${recipientPhone}; duplicate aborted.`);
                 return {
                     success: true,
                     reconciled: true,
@@ -420,6 +565,7 @@ export class WhatsAppWorkerHandler {
         const targetJid = verified.jid || jid;
 
         // 3. Stream PDF from authenticated URL (Zero buffer in central WebSocket)
+        logInfo(`[WhatsApp] Streaming document attachment (${fileName})...`);
         const response = await axios.get(downloadUrl, {
             responseType: "arraybuffer",
             timeout: 25000,
@@ -444,8 +590,10 @@ export class WhatsAppWorkerHandler {
                 );
             }
         }
+        logInfo(`[WhatsApp] Document received (${(buffer.length / 1024).toFixed(1)} KB), SHA-256 integrity verified (${actualSha256.substring(0, 12)}...)`);
 
         // 5. Human Typing Simulation
+        logInfo(`[WhatsApp] Simulating human typing for ${digits}...`);
         await this.sock.sendPresenceUpdate("composing", targetJid).catch(() => {});
         await new Promise((r) => setTimeout(r, 2500));
         await this.sock.sendPresenceUpdate("paused", targetJid).catch(() => {});
@@ -466,10 +614,12 @@ export class WhatsAppWorkerHandler {
         });
 
         this._resetIdleTimer();
+        const msgId = result?.key?.id || `wamid-${Date.now()}`;
+        logSuccess(`[WhatsApp] Document successfully delivered to ${recipientPhone} (ID: ${msgId})`);
 
         return {
             success: true,
-            messageId: result?.key?.id || `wamid-${Date.now()}`,
+            messageId: msgId,
             timestamp: Date.now(),
         };
     }
@@ -489,6 +639,7 @@ export class WhatsAppWorkerHandler {
         }
 
         this.clearAuth();
+        this.onClearSession();
         this.status = "DISCONNECTED";
         this.user = null;
         this.currentQrDataUrl = null;