app-manager-edge-worker 1.0.2 → 1.0.4

package/README.md

@@ -177,6 +177,22 @@ No manual setup required!
 
 ---
 
+## 🕐 Log Output Format
+
+Every log line is prefixed with a timestamp in `YYYY-MM-DD HH:mm:ss` format:
+
+```
+[2026-09-15 14:32:07] [INFO] Connected to Apps Manager server: https://apps-manager.onrender.com
+[2026-09-15 14:32:07] [SUCCESS] Chromium detected: /usr/bin/chromium-browser
+[2026-09-15 14:32:10] [TASK] SCRAPE_CHROMIUM -> https://example.com/product (1243ms)
+[2026-09-15 14:32:11] [WARN] Scrape returned no data for selector: .price-value
+[2026-09-15 14:32:11] [ERROR] Socket connection lost, reconnecting in 5s...
+```
+
+Log levels: `[INFO]` · `[SUCCESS]` · `[WARN]` · `[ERROR]` · `[TASK]` (job dispatch with optional duration).
+
+---
+
 ## 🔒 Security & Sandboxing
 
 - **Outbound Only**: All socket tunnels are initiated outbound to the central Apps Manager server over TLS (`wss://`). No public IP, dynamic DNS, or port forwarding required.
@@ -185,6 +201,28 @@ No manual setup required!
 
 ---
 
+## 📝 Changelog
+
+### v1.0.4
+- **Timestamped logs**: every log line now starts with `[YYYY-MM-DD HH:mm:ss]` for debugging and production monitoring
+- **Version fix**: banner, help text, and user-agent string now correctly report `v1.0.4`
+
+### v1.0.3
+- Improved element inspector selector generation (CSS/XPath ranked candidates)
+
+### v1.0.2
+- Comprehensive CSS/XPath selector generation for visual element picker
+
+### v1.0.1
+- Chunked streaming for large payloads (256 KB chunks over Socket.IO)
+
+### v1.0.0
+- Initial release — SCRAPE_CHROMIUM, SCRAPE_HTTP, PREVIEW_PAGE, INSPECT_ELEMENT handlers
+- Multi-layer Chromium auto-detection + auto-install (Windows/macOS/Linux/Alpine/Termux)
+- Keep-alive ping, single-flight deduplication, PM2/Docker support
+
+---
+
 ## 👤 Author
 
 **Chetan Mohite**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "app-manager-edge-worker",
-  "version": "1.0.2",
+  "version": "1.0.4",
   "description": "Distributed Scraper Edge Worker for Apps Manager. Run on any local machine, VPS, or Termux Android device to serve browser and HTTP scraping jobs.",
   "type": "module",
   "bin": {

package/src/banner.js

@@ -11,6 +11,16 @@ const colors = {
     gray: "\x1b[90m",
 };
 
+/**
+ * Returns a formatted timestamp string: [YYYY-MM-DD HH:mm:ss]
+ */
+function timestamp() {
+    const now = new Date();
+    const date = now.toISOString().slice(0, 10);
+    const time = now.toISOString().slice(11, 19);
+    return `${colors.gray}[${date} ${time}]${colors.reset}`;
+}
+
 export function printBanner({ workerName, serverUrl, concurrency, platform, isTermux, keepAliveMinutes }) {
     console.clear();
     console.log(`${colors.cyan}${colors.bright}`);
@@ -21,7 +31,7 @@ export function printBanner({ workerName, serverUrl, concurrency, platform, isTe
     console.log("  ██║  ██║██║     ██║     ███████║██╗██║ ╚═╝ ██║██║  ██║██║ ╚████║██║  ██║╚██████╔╝███████╗██║  ██║");
     console.log("  ╚═╝  ╚═╝╚═╝     ╚═╝     ╚══════╝╚═╝╚═╝     ╚═╝╚═╝  ╚═╝╚═╝  ╚═══╝╚═╝  ╚═╝ ╚═════╝ ╚══════╝╚═╝  ╚═╝");
     console.log(`${colors.reset}`);
-    console.log(`${colors.bright}  🌐 Distributed Scraper Edge Worker Client v1.0.2${colors.reset}`);
+    console.log(`${colors.bright}  🌐 Distributed Scraper Edge Worker Client v1.0.4${colors.reset}`);
     console.log(`${colors.dim}  Zero-cost, persistent tunneling & residential edge scraping${colors.reset}`);
     console.log("--------------------------------------------------------------------------------");
     console.log(`  ${colors.bright}Device Name  :${colors.reset} ${colors.green}${workerName}${colors.reset}`);
@@ -33,24 +43,24 @@ export function printBanner({ workerName, serverUrl, concurrency, platform, isTe
 }
 
 export function logInfo(msg) {
-    console.log(`${colors.cyan}[INFO]${colors.reset} ${msg}`);
+    console.log(`${timestamp()} ${colors.cyan}[INFO]${colors.reset} ${msg}`);
 }
 
 export function logSuccess(msg) {
-    console.log(`${colors.green}[SUCCESS]${colors.reset} ${msg}`);
+    console.log(`${timestamp()} ${colors.green}[SUCCESS]${colors.reset} ${msg}`);
 }
 
 export function logWarn(msg) {
-    console.log(`${colors.yellow}[WARN]${colors.reset} ${msg}`);
+    console.log(`${timestamp()} ${colors.yellow}[WARN]${colors.reset} ${msg}`);
 }
 
 export function logError(msg) {
-    console.log(`${colors.red}[ERROR]${colors.reset} ${msg}`);
+    console.log(`${timestamp()} ${colors.red}[ERROR]${colors.reset} ${msg}`);
 }
 
 export function logJob(jobType, url, durationMs = null) {
     const timeStr = durationMs ? ` ${colors.gray}(${durationMs}ms)${colors.reset}` : "";
     console.log(
-        `${colors.magenta}[TASK]${colors.reset} ${colors.bright}${jobType}${colors.reset} -> ${url}${timeStr}`
+        `${timestamp()} ${colors.magenta}[TASK]${colors.reset} ${colors.bright}${jobType}${colors.reset} -> ${url}${timeStr}`
     );
 }

package/src/client.js

@@ -215,7 +215,7 @@ export async function startWorkerClient(config) {
                 timeout: 15000,
                 headers: {
                     "x-worker-key": workerKey || "",
-                    "user-agent": "AppsManagerEdgeWorker/1.0.2",
+                    "user-agent": "AppsManagerEdgeWorker/1.0.4",
                 },
             });
             logInfo(`[Keep-Alive] Server awake ping OK (${res.status}) - Render instance active`);
@@ -226,7 +226,7 @@ export async function startWorkerClient(config) {
                 await axios.get(fallbackUrl, {
                     timeout: 10000,
                     headers: {
-                        "user-agent": "AppsManagerEdgeWorker/1.0.2",
+                        "user-agent": "AppsManagerEdgeWorker/1.0.4",
                     },
                 });
                 logInfo(`[Keep-Alive] Fallback health ping OK - Render instance active`);

package/src/config.js

@@ -106,7 +106,7 @@ export async function resolveWorkerConfig() {
 
     if (cli.help) {
         console.log(`
-Apps Manager Scraper Edge Worker CLI v1.0.2
+Apps Manager Scraper Edge Worker CLI v1.0.4
 
 Usage:
   npx app-manager-edge-worker [options]

package/src/handlers/previewHandler.js

@@ -93,48 +93,210 @@ export async function handleInspectElement(payload) {
                     height: Math.round(rect.height),
                 };
 
+                // ─── Helper: count matching elements for a CSS selector ───
+                const countMatches = (sel) => {
+                    try { return document.querySelectorAll(sel).length; } catch { return 999; }
+                };
+
+                // ─── Helper: Full positional CSS path anchored at nearest ID ───
+                const buildCssPath = (element, maxDepth = 8) => {
+                    const parts = [];
+                    let curr = element;
+                    while (curr && curr.nodeType === 1 && curr !== document.documentElement) {
+                        if (curr === document.body) { parts.unshift("body"); break; }
+                        let seg = curr.tagName.toLowerCase();
+                        if (curr.id && /^[a-zA-Z][\w-]*$/.test(curr.id)) {
+                            parts.unshift(`#${curr.id}`);
+                            break;
+                        }
+                        if (curr.parentElement) {
+                            const pos = Array.from(curr.parentElement.children).indexOf(curr) + 1;
+                            seg += `:nth-child(${pos})`;
+                        }
+                        parts.unshift(seg);
+                        curr = curr.parentElement;
+                        if (parts.length >= maxDepth) break;
+                    }
+                    return parts.join(" > ");
+                };
+
+                // ─── Helper: Shortest unique CSS path using class combos + nth-child ───
+                const buildShortUniqueSelector = (element) => {
+                    const pathParts = [];
+                    let curr = element;
+                    while (curr && curr !== document.documentElement) {
+                        if (curr === document.body) { pathParts.unshift("body"); break; }
+                        let seg = curr.tagName.toLowerCase();
+                        if (curr.id && /^[a-zA-Z][\w-]*$/.test(curr.id)) {
+                            pathParts.unshift(`#${curr.id}`);
+                            break;
+                        }
+                        const classes = (curr.className && typeof curr.className === "string")
+                            ? curr.className.split(/\s+/).filter(c => c && /^[a-zA-Z_-][\w-]*$/.test(c))
+                            : [];
+                        let foundUnique = false;
+                        for (let len = Math.min(classes.length, 5); len >= 1; len--) {
+                            const combo = classes.slice(0, len);
+                            const candidate = `${seg}.${combo.join(".")}`;
+                            const full = [...pathParts, candidate].join(" > ");
+                            if (countMatches(full) === 1) {
+                                pathParts.unshift(candidate);
+                                foundUnique = true;
+                                break;
+                            }
+                        }
+                        if (!foundUnique) {
+                            if (curr.parentElement) {
+                                const pos = Array.from(curr.parentElement.children).indexOf(curr) + 1;
+                                seg += `:nth-child(${pos})`;
+                            }
+                            pathParts.unshift(seg);
+                        }
+                        const full = pathParts.join(" > ");
+                        if (countMatches(full) === 1) break;
+                        curr = curr.parentElement;
+                    }
+                    return pathParts.join(" > ");
+                };
+
+                // ─── Helper: Robust absolute XPath ───
+                const buildXPath = (element) => {
+                    if (!element || element === document) return "";
+                    if (element.id && /^[a-zA-Z][\w-]*$/.test(element.id)) {
+                        return `//*[@id="${element.id}"]`;
+                    }
+                    if (element === document.body) return "/html/body";
+                    const parts = [];
+                    let curr = element;
+                    while (curr && curr.nodeType === 1 && curr !== document.documentElement) {
+                        if (curr === document.body) { parts.unshift("body"); break; }
+                        if (curr.id && /^[a-zA-Z][\w-]*$/.test(curr.id)) {
+                            parts.unshift(`//*[@id="${curr.id}"]`);
+                            return parts.join("/");
+                        }
+                        const tag = curr.tagName.toLowerCase();
+                        let idx = 1;
+                        let sib = curr.previousElementSibling;
+                        while (sib) {
+                            if (sib.tagName.toLowerCase() === tag) idx++;
+                            sib = sib.previousElementSibling;
+                        }
+                        const total = curr.parentElement
+                            ? Array.from(curr.parentElement.children).filter(c => c.tagName.toLowerCase() === tag).length
+                            : 1;
+                        parts.unshift(total > 1 ? `${tag}[${idx}]` : tag);
+                        curr = curr.parentElement;
+                    }
+                    return "/" + parts.join("/");
+                };
+
+                // ─── Build ranked candidates ───
                 const candidates = [];
-                if (el.id) {
-                    candidates.push({ type: "css", value: `#${el.id}`, attribute: "text" });
+                const seen = new Set();
+                const addCandidate = (type, value, extra = {}) => {
+                    if (!value || seen.has(value)) return;
+                    seen.add(value);
+                    const matchCount = type === "css" ? countMatches(value) : null;
+                    candidates.push({ type, value, attribute: "text", matchCount, ...extra });
+                };
+
+                // 1. ID (most reliable, unique)
+                if (el.id && /^[a-zA-Z][\w-]*$/.test(el.id)) {
+                    addCandidate("css", `#${el.id}`, { specificity: "unique", label: "ID Selector" });
                 }
 
+                // 2. data-* and aria-* attributes
                 for (const attr of el.attributes) {
-                    if (attr.name.startsWith("data-") && attr.value) {
-                        candidates.push({
-                            type: "css",
-                            value: `[${attr.name}='${attr.value}']`,
-                            attribute: "text",
+                    if ((attr.name.startsWith("data-") || attr.name.startsWith("aria-")) && attr.value && attr.value.length < 80) {
+                        addCandidate("css", `${el.tagName.toLowerCase()}[${attr.name}="${attr.value}"]`, {
+                            specificity: "high", label: `Attribute: ${attr.name}`
                         });
                     }
                 }
 
-                if (el.className && typeof el.className === "string") {
-                    const classes = el.className
-                        .split(/\s+/)
-                        .filter((c) => c && !c.includes(":") && !c.includes("/") && !c.includes("["));
-                    if (classes.length > 0) {
-                        candidates.push({
-                            type: "css",
-                            value: `${el.tagName.toLowerCase()}.${classes.slice(0, 2).join(".")}`,
-                            attribute: "text",
+                // 3. Shortest unique CSS path
+                const shortUnique = buildShortUniqueSelector(el);
+                if (shortUnique) {
+                    const mc = countMatches(shortUnique);
+                    addCandidate("css", shortUnique, {
+                        specificity: mc === 1 ? "unique" : "medium",
+                        label: "Smart CSS Path",
+                        matchCount: mc,
+                    });
+                }
+
+                // 4. Class-based selectors (increasing combos until unique)
+                const classes = (el.className && typeof el.className === "string")
+                    ? el.className.split(/\s+/).filter(c => c && /^[a-zA-Z_-][\w-]*$/.test(c))
+                    : [];
+                for (let len = 1; len <= Math.min(classes.length, 5); len++) {
+                    const sel = `${el.tagName.toLowerCase()}.${classes.slice(0, len).join(".")}`;
+                    const mc = countMatches(sel);
+                    addCandidate("css", sel, {
+                        specificity: mc === 1 ? "unique" : mc <= 3 ? "medium" : "generic",
+                        label: "Class Selector",
+                        matchCount: mc,
+                    });
+                    if (mc === 1) break;
+                }
+
+                // 5. Table-specific selectors
+                const tr = el.closest("tr");
+                if (tr && tr.parentElement) {
+                    const rowIdx = Array.from(tr.parentElement.children).indexOf(tr) + 1;
+                    const td = el.closest("td, th");
+                    if (td) {
+                        const colIdx = Array.from(tr.children).indexOf(td) + 1;
+                        addCandidate("css", `table tr:nth-child(${rowIdx}) td:nth-child(${colIdx})`, {
+                            specificity: "medium", label: "Table Cell"
+                        });
+                        addCandidate("css", `table tbody tr:nth-child(${rowIdx}) td:nth-child(${colIdx})`, {
+                            specificity: "medium", label: "Table Cell (tbody)"
                         });
                     }
                 }
 
-                const tagName = el.tagName.toLowerCase();
-                candidates.push({ type: "css", value: tagName, attribute: "text" });
+                // 6. Full structural CSS path
+                const fullCss = buildCssPath(el, 8);
+                addCandidate("css", fullCss, { specificity: "unique", label: "Full CSS Path (Structural)" });
+
+                // 7. XPath — most reliable for scrapers
+                const xpath = buildXPath(el);
+                addCandidate("xpath", xpath, { specificity: "high", label: "XPath" });
+
+                // Sort by specificity: unique > high > medium > generic
+                const rank = { unique: 0, high: 1, medium: 2, generic: 3 };
+                candidates.sort((a, b) => (rank[a.specificity] ?? 4) - (rank[b.specificity] ?? 4));
+
+                // ─── Breadcrumb hierarchy for UI ───
+                const hierarchy = [];
+                let curr = el;
+                while (curr && curr !== document.body && hierarchy.length < 7) {
+                    let desc = curr.tagName.toLowerCase();
+                    if (curr.id) {
+                        desc += `#${curr.id}`;
+                    } else if (curr.classList && curr.classList.length > 0) {
+                        const cls = Array.from(curr.classList)
+                            .filter(c => /^[a-zA-Z_-][\w-]*$/.test(c))
+                            .slice(0, 2);
+                        if (cls.length) desc += `.${cls.join(".")}`;
+                    }
+                    hierarchy.unshift(desc);
+                    curr = curr.parentElement;
+                }
 
                 const text = el.innerText ? el.innerText.trim() : el.textContent?.trim() || "";
 
                 return {
                     found: true,
-                    tagName,
+                    tagName: el.tagName.toLowerCase(),
                     text,
                     attributes: Array.from(el.attributes).reduce((acc, a) => {
                         acc[a.name] = a.value;
                         return acc;
                     }, {}),
                     candidates,
+                    hierarchy,
                     boundingRect,
                 };
             }, x, y);