app-manager-edge-worker 1.2.5 → 1.3.0

package/README.md

@@ -201,6 +201,13 @@ Log levels: `[INFO]` · `[SUCCESS]` · `[WARN]` · `[ERROR]` · `[TASK]` (job di
 
 ---
 
+### v1.3.0
+- **Multi-Error Identifier Rules & Inspection Resilience Engine**:
+  - **Custom Error Identifier Rules**: Supported typed error detection rules (`errorDetection.rules`) allowing classification of multiple distinct errors (e.g. `INVALID_GSTIN`, `INVALID_CAPTCHA`, `NOT_FOUND`) with tailored actions (`RETRY_CAPTCHA`, `BUSINESS_ERROR`, `ABORT_SESSION`).
+  - **Inspection Mode Zero-Purge Resilience**: When `preservePageForInspection: true`, the worker never terminates the session on errors or timeouts. Instead, it settles the DOM, takes a full-page snapshot, and returns the live updated screenshot to the visual element inspector so users can inspect red error messages and map error selectors.
+  - **Broadened Intelligent Error Heuristics**: Added automatic recognition for standard Indian government and SPA error selectors (`p#alertMsg`, `#alertMsg`, `.alertMsg`, `.text-danger`, `.invalid-feedback`, `[aria-invalid='true']`), resolving outcome races in sub-second time without 15s timeouts.
+  - **Added `HITL_SESSION_SNAPSHOT`**: On-demand snapshot handler allowing the visual inspector to capture and synchronize current browser page state at any point.
+
 ### v1.2.5
 - **Interactive Session Visual Inspector Sync & Robust Captcha Refresh**:
   - `HITL_SESSION_INIT` now captures and returns full-page `pageScreenshotBase64`, dimensions, and title after typing initial form inputs, keeping the Visual Element Inspector synchronized with the active session.
@@ -256,6 +263,13 @@ Log levels: `[INFO]` · `[SUCCESS]` · `[WARN]` · `[ERROR]` · `[TASK]` (job di
   - Automatic backup of pre-existing local disk sessions to cluster database on initial connection.
   - Coordinated session wiping on remote logout and phone unlinking.
 
+### v1.2.5
+- **Interactive Human-In-The-Loop (HITL) Web Scraping**:
+  - Pinned stateful browser sessions for interactive portals (GSTIN, PAN, Parivahan, e-Courts).
+  - Multi-step execution protocol: `HITL_SESSION_INIT`, `HITL_SESSION_REFRESH`, `HITL_SESSION_SOLVE`, `HITL_SESSION_EXTRACT`, and `HITL_SESSION_ABORT`.
+  - Memory-safe binary captcha streaming and bounded 90s session TTL auto-termination.
+  - Server URL resolution propagation for Baileys companion document relay.
+
 ### v1.1.0
 - **WhatsApp Multi-Device Companion Relay**:
   - Independent companion device pairing via live QR streaming to Admin Dashboard.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "app-manager-edge-worker",
-  "version": "1.2.5",
+  "version": "1.3.0",
   "description": "Distributed Scraper Edge Worker for Apps Manager. Run on any local machine, VPS, or Termux Android device to serve browser and HTTP scraping jobs.",
   "type": "module",
   "bin": {

package/src/banner.js

@@ -1,7 +1,7 @@
 import { createRequire } from "module";
 
 const require = createRequire(import.meta.url);
-let workerVersion = "1.2.5";
+let workerVersion = "1.3.0";
 try {
     const pkg = require("../package.json");
     if (pkg?.version) workerVersion = pkg.version;

package/src/client.js

@@ -14,12 +14,13 @@ import {
     handleInteractiveSessionSolve,
     handleInteractiveSessionAbort,
     handleInteractiveSessionExtract,
+    handleInteractiveSessionSnapshot,
 } from "./handlers/interactiveHandler.js";
 import fs from "fs";
 import { getWhatsAppHandler } from "./handlers/whatsappHandler.js";
 import { printBanner, logInfo, logSuccess, logWarn, logError, logJob } from "./banner.js";
 
-let workerPkgVersion = "1.2.5";
+let workerPkgVersion = "1.3.0";
 try {
     const pkg = JSON.parse(fs.readFileSync(new URL("../package.json", import.meta.url), "utf-8"));
     if (pkg?.version) workerPkgVersion = pkg.version;
@@ -263,6 +264,8 @@ export async function startWorkerClient(config) {
                 resultData = await handleInteractiveSessionRefresh(payload);
             } else if (jobType === "HITL_SESSION_SOLVE") {
                 resultData = await handleInteractiveSessionSolve(payload);
+            } else if (jobType === "HITL_SESSION_SNAPSHOT") {
+                resultData = await handleInteractiveSessionSnapshot(payload);
             } else if (jobType === "HITL_SESSION_EXTRACT") {
                 resultData = await handleInteractiveSessionExtract(payload);
             } else if (jobType === "HITL_SESSION_ABORT") {

package/src/handlers/interactiveHandler.js

@@ -422,7 +422,24 @@ export async function handleInteractiveSessionSolve(payload) {
     const inputSelector = cfg.inputSelector || "#captcha, input[name='captcha']";
     const submitSelector = cfg.submitSelector || "button[type='submit']";
     const successSelector = cfg.waitForAfterSubmit?.selector || ".table-responsive, table, .result-pane";
-    const errorSelector = cfg.errorDetection?.selector || ".alert-danger, .has-error, #error-msg, .ui-messages-error";
+    
+    // Broad default error selectors covering Indian gov and modern SPA patterns
+    const defaultBroadErrorSelector =
+        "p#alertMsg, #alertMsg, .alertMsg, .alert-danger, .alert-warning, " +
+        ".has-error, .text-danger, #error-msg, .error-message, .err-msg, " +
+        ".ui-messages-error, [class*='error-msg'], [class*='alert-message'], " +
+        ".invalid-feedback, .help-block.error, .form-error, [aria-invalid='true']";
+
+    const customRules = cfg.errorDetection?.rules || [];
+    const ruleSelectors = customRules.map((r) => r.selector).filter(Boolean);
+    const combinedErrorSelectors = [
+        cfg.errorDetection?.selector,
+        ...ruleSelectors,
+        defaultBroadErrorSelector,
+    ]
+        .filter(Boolean)
+        .join(", ");
+
     const invalidPattern = cfg.errorDetection?.invalidCaptchaPattern || "captcha.*(incorrect|invalid|wrong|mismatch|enter text)";
 
     // 1. Type captcha solution
@@ -436,17 +453,32 @@ export async function handleInteractiveSessionSolve(payload) {
     await page.click(submitSelector);
 
     // 3. Post-Submit Race: Wait for Success Selector OR Error Selector
+    // In inspection mode, use a shorter safety window (5-7s) so users aren't stalled waiting on empty selectors
+    const safetyTimeoutMs = preservePageForInspection
+        ? Math.min(cfg.waitForAfterSubmit?.timeoutMs || 8000, 6000)
+        : (cfg.waitForAfterSubmit?.timeoutMs || 15000);
+
     const matchedOutcome = await page
         .waitForFunction(
             (succ, err) => {
                 if (document.querySelector(succ)) return "SUCCESS";
-                const errEl = document.querySelector(err);
-                if (errEl && errEl.innerText && errEl.innerText.trim().length > 0) return "ERROR";
+                try {
+                    const elements = document.querySelectorAll(err);
+                    for (const el of elements) {
+                        const txt = (el.innerText || el.textContent || "").trim();
+                        if (txt.length > 0) {
+                            const style = window.getComputedStyle(el);
+                            if (style.display !== "none" && style.visibility !== "hidden" && style.opacity !== "0") {
+                                return "ERROR";
+                            }
+                        }
+                    }
+                } catch (e) {}
                 return null;
             },
-            { timeout: cfg.waitForAfterSubmit?.timeoutMs || 15000 },
+            { timeout: safetyTimeoutMs },
             successSelector,
-            errorSelector
+            combinedErrorSelectors
         )
         .then((handle) => handle.jsonValue())
         .catch(() => "TIMEOUT");
@@ -482,28 +514,12 @@ export async function handleInteractiveSessionSolve(payload) {
             // Settle time for tables & styles
             await new Promise((r) => setTimeout(r, 600));
 
-            const screenshotBuffer = await page.screenshot({
-                encoding: "binary",
-                fullPage: false,
-                type: "jpeg",
-                quality: 85,
-            });
-            const screenshotBase64 = Buffer.from(screenshotBuffer).toString("base64");
-            const title = await page.title().catch(() => "");
-            const dimensions = await page
-                .evaluate(() => ({
-                    width: document.documentElement.scrollWidth || window.innerWidth,
-                    height: document.documentElement.scrollHeight || window.innerHeight,
-                }))
-                .catch(() => ({ width: 1366, height: 900 }));
-
+            const snap = await captureInspectionSnapshot(page);
             return {
                 status: "COMPLETED",
                 success: true,
                 data: raw,
-                screenshotBase64: `data:image/jpeg;base64,${screenshotBase64}`,
-                dimensions,
-                title: title || "Results Page",
+                ...snap,
                 sessionId,
                 isInspectionMode: true,
             };
@@ -519,55 +535,129 @@ export async function handleInteractiveSessionSolve(payload) {
         };
     }
 
-    // Case B: In-Page Error Banner Detected
-    if (matchedOutcome === "ERROR") {
-        const errorText = await page.evaluate((sel) => {
-            const el = document.querySelector(sel);
-            return el ? el.innerText.trim() : "";
-        }, errorSelector);
+    // Evaluate error rules and messages if matchedOutcome === "ERROR" or if we are inspecting
+    let matchedRule = null;
+    let errorText = "";
+
+    // 1. Check custom configured rules first
+    for (const rule of customRules) {
+        if (!rule.selector) continue;
+        const check = await page.evaluate((sel, pat) => {
+            try {
+                const els = document.querySelectorAll(sel);
+                for (const el of els) {
+                    const txt = (el.innerText || el.textContent || "").trim();
+                    if (txt) {
+                        const style = window.getComputedStyle(el);
+                        if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue;
+                        if (!pat) return { matched: true, text: txt };
+                        const re = new RegExp(pat, "i");
+                        if (re.test(txt)) return { matched: true, text: txt };
+                    }
+                }
+            } catch (e) {}
+            return null;
+        }, rule.selector, rule.pattern);
+
+        if (check && check.matched) {
+            matchedRule = rule;
+            errorText = check.text;
+            break;
+        }
+    }
 
-        const regex = new RegExp(invalidPattern, "i");
-        const isCaptchaError = regex.test(errorText);
+    // 2. If no custom rule matched, extract text from combined error selectors
+    if (!matchedRule) {
+        errorText = await page.evaluate((sel) => {
+            try {
+                const els = document.querySelectorAll(sel);
+                for (const el of els) {
+                    const txt = (el.innerText || el.textContent || "").trim();
+                    if (txt) {
+                        const style = window.getComputedStyle(el);
+                        if (style.display !== "none" && style.visibility !== "hidden" && style.opacity !== "0") {
+                            return txt;
+                        }
+                    }
+                }
+            } catch (e) {}
+            return "";
+        }, combinedErrorSelectors);
+    }
 
-        if (isCaptchaError) {
-            // Keep page alive! Reset TTL timer for retry
-            if (session.timeoutTimer) clearTimeout(session.timeoutTimer);
-            const retryTtlSeconds = cfg.sessionTimeoutSeconds || 90;
-            session.timeoutTimer = setTimeout(() => {
-                purgeSession(sessionId).catch(() => {});
-            }, retryTtlSeconds * 1000);
+    const regex = new RegExp(invalidPattern, "i");
+    const isCaptchaError = matchedRule
+        ? matchedRule.action === "RETRY_CAPTCHA"
+        : (Boolean(errorText) && regex.test(errorText));
 
-            // Give the target portal 400ms to re-render the refreshed captcha image
-            await new Promise((r) => setTimeout(r, 400));
-            const imageSelector = cfg.imageSelector || "img[src*='captcha'], #imgCaptcha";
-            const newCaptchaEl = await page.$(imageSelector);
-            let newImageBase64 = null;
+    // Check if new captcha image was rendered on page
+    const imageSelector = cfg.imageSelector || "img[src*='captcha'], #imgCaptcha";
+    let newImageBase64 = null;
+    try {
+        const newCaptchaEl = await page.$(imageSelector);
+        if (newCaptchaEl) {
+            const newBuffer = await newCaptchaEl.screenshot({ encoding: "binary", type: "png" });
+            newImageBase64 = `data:image/png;base64,${Buffer.from(newBuffer).toString("base64")}`;
+        }
+    } catch (e) {}
+
+    // CRITICAL FOR INSPECTOR: In inspection mode, NEVER purge the session on errors or timeout!
+    // Always snap the live DOM and return it so the user can inspect the red text and configure selectors
+    if (preservePageForInspection) {
+        if (session.timeoutTimer) clearTimeout(session.timeoutTimer);
+        const inspectTtl = 300;
+        session.timeoutTimer = setTimeout(() => {
+            purgeSession(sessionId).catch(() => {});
+        }, inspectTtl * 1000);
 
-            if (newCaptchaEl) {
-                const newBuffer = await newCaptchaEl.screenshot({ encoding: "binary", type: "png" });
-                newImageBase64 = `data:image/png;base64,${Buffer.from(newBuffer).toString("base64")}`;
-            }
+        const snap = await captureInspectionSnapshot(page);
+        const effectiveStatus = isCaptchaError
+            ? "INVALID_CAPTCHA"
+            : (errorText ? "BUSINESS_ERROR" : "INSPECTION_MODE");
 
-            return {
-                status: "INVALID_CAPTCHA",
-                success: false,
-                message: errorText || "The captcha entered was incorrect.",
-                canRetry: true,
-                newImageBase64,
-            };
-        } else {
-            // Business error (e.g. "No record found for this number")
-            await purgeSession(sessionId);
-            return {
-                status: "BUSINESS_ERROR",
-                success: false,
-                message: errorText || "Target portal returned an error.",
-                canRetry: false,
-            };
-        }
+        return {
+            status: effectiveStatus,
+            success: true, // Allows frontend to render the inspection screenshot
+            errorCode: (matchedRule && matchedRule.id) || (isCaptchaError ? "INVALID_CAPTCHA" : (errorText ? "BUSINESS_ERROR" : "INSPECTION_MODE")),
+            message: (matchedRule && matchedRule.customMessage) || errorText || (matchedOutcome === "TIMEOUT" ? "Page updated (no success table found). You can now inspect elements or error messages." : "Page outcome revealed for inspection."),
+            newImageBase64,
+            matchedRule: matchedRule ? { id: matchedRule.id, name: matchedRule.name, action: matchedRule.action } : null,
+            sessionId,
+            isInspectionMode: true,
+            ...snap,
+        };
+    }
+
+    // Runtime non-inspection mode:
+    if (isCaptchaError) {
+        if (session.timeoutTimer) clearTimeout(session.timeoutTimer);
+        const retryTtlSeconds = cfg.sessionTimeoutSeconds || 90;
+        session.timeoutTimer = setTimeout(() => {
+            purgeSession(sessionId).catch(() => {});
+        }, retryTtlSeconds * 1000);
+
+        return {
+            status: "INVALID_CAPTCHA",
+            success: false,
+            message: (matchedRule && matchedRule.customMessage) || errorText || "The captcha entered was incorrect.",
+            canRetry: true,
+            newImageBase64,
+            errorCode: (matchedRule && matchedRule.id) || "INVALID_CAPTCHA",
+        };
     }
 
-    // Case C: Timeout
+    if (errorText || matchedRule) {
+        await purgeSession(sessionId);
+        return {
+            status: "BUSINESS_ERROR",
+            success: false,
+            errorCode: (matchedRule && matchedRule.id) || "BUSINESS_ERROR",
+            message: (matchedRule && matchedRule.customMessage) || errorText || "Target portal returned an error.",
+            canRetry: false,
+        };
+    }
+
+    // Runtime Timeout
     await purgeSession(sessionId);
     return {
         status: "TIMEOUT",
@@ -577,6 +667,58 @@ export async function handleInteractiveSessionSolve(payload) {
     };
 }
 
+/**
+ * Capture full inspection snapshot helper
+ */
+async function captureInspectionSnapshot(page) {
+    await new Promise((r) => setTimeout(r, 500));
+    const screenshotBuffer = await page.screenshot({
+        encoding: "binary",
+        fullPage: false,
+        type: "jpeg",
+        quality: 85,
+    });
+    const screenshotBase64 = `data:image/jpeg;base64,${Buffer.from(screenshotBuffer).toString("base64")}`;
+    const title = await page.title().catch(() => "");
+    const dimensions = await page
+        .evaluate(() => ({
+            width: document.documentElement.scrollWidth || window.innerWidth,
+            height: document.documentElement.scrollHeight || window.innerHeight,
+        }))
+        .catch(() => ({ width: 1366, height: 900 }));
+
+    return {
+        screenshotBase64,
+        dimensions,
+        title: title || "Inspection Page",
+    };
+}
+
+/**
+ * Force capture snapshot of the active interactive session's page
+ */
+export async function handleInteractiveSessionSnapshot(payload) {
+    const { sessionId } = payload;
+    const session = activeSessions.get(sessionId);
+
+    if (!session || !session.page || session.page.isClosed()) {
+        throw new Error("Interactive session not found or already expired");
+    }
+
+    const { page } = session;
+    if (session.timeoutTimer) clearTimeout(session.timeoutTimer);
+    session.timeoutTimer = setTimeout(() => {
+        purgeSession(sessionId).catch(() => {});
+    }, 300 * 1000);
+
+    const snap = await captureInspectionSnapshot(page);
+    return {
+        success: true,
+        sessionId,
+        ...snap,
+    };
+}
+
 /**
  * 4. Handles HITL_SESSION_ABORT:
  *    Explicit client abort or session abandonment.