app-manager-edge-worker 1.0.8 → 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/worker.js +16 -16
- package/package.json +46 -46
- package/src/banner.js +80 -80
- package/src/client.js +300 -300
- package/src/config.js +1 -1
- package/src/handlers/mailHandler.js +74 -74
- package/src/handlers/previewHandler.js +647 -557
- package/src/handlers/scrapeHandler.js +887 -887
- package/src/handlers/transformUtils.js +340 -340
package/bin/worker.js
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
import { resolveWorkerConfig } from "../src/config.js";
|
|
4
|
-
import { startWorkerClient } from "../src/client.js";
|
|
5
|
-
|
|
6
|
-
async function main() {
|
|
7
|
-
try {
|
|
8
|
-
const config = await resolveWorkerConfig();
|
|
9
|
-
await startWorkerClient(config);
|
|
10
|
-
} catch (err) {
|
|
11
|
-
console.error("Fatal Worker Error:", err);
|
|
12
|
-
process.exit(1);
|
|
13
|
-
}
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
main();
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import { resolveWorkerConfig } from "../src/config.js";
|
|
4
|
+
import { startWorkerClient } from "../src/client.js";
|
|
5
|
+
|
|
6
|
+
async function main() {
|
|
7
|
+
try {
|
|
8
|
+
const config = await resolveWorkerConfig();
|
|
9
|
+
await startWorkerClient(config);
|
|
10
|
+
} catch (err) {
|
|
11
|
+
console.error("Fatal Worker Error:", err);
|
|
12
|
+
process.exit(1);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
main();
|
package/package.json
CHANGED
|
@@ -1,46 +1,46 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "app-manager-edge-worker",
|
|
3
|
-
"version": "1.0.
|
|
4
|
-
"description": "Distributed Scraper Edge Worker for Apps Manager. Run on any local machine, VPS, or Termux Android device to serve browser and HTTP scraping jobs.",
|
|
5
|
-
"type": "module",
|
|
6
|
-
"bin": {
|
|
7
|
-
"app-manager-edge-worker": "bin/worker.js",
|
|
8
|
-
"apps-manager-worker": "bin/worker.js"
|
|
9
|
-
},
|
|
10
|
-
"main": "./src/client.js",
|
|
11
|
-
"scripts": {
|
|
12
|
-
"start": "node ./bin/worker.js"
|
|
13
|
-
},
|
|
14
|
-
"keywords": [
|
|
15
|
-
"app-manager-edge-worker",
|
|
16
|
-
"apps-manager",
|
|
17
|
-
"scraper",
|
|
18
|
-
"edge-worker",
|
|
19
|
-
"puppeteer",
|
|
20
|
-
"residential-proxy",
|
|
21
|
-
"distributed-scraping",
|
|
22
|
-
"termux"
|
|
23
|
-
],
|
|
24
|
-
"author": "Chetan Mohite <chetanmohite2128@gmail.com> (https://github.com/codes-by-chetan)",
|
|
25
|
-
"license": "MIT",
|
|
26
|
-
"homepage": "https://apps-manager.onrender.com",
|
|
27
|
-
"repository": {
|
|
28
|
-
"type": "git",
|
|
29
|
-
"url": "git+https://github.com/codes-by-chetan/Apps_Manager.git",
|
|
30
|
-
"directory": "packages/worker"
|
|
31
|
-
},
|
|
32
|
-
"publishConfig": {
|
|
33
|
-
"access": "public"
|
|
34
|
-
},
|
|
35
|
-
"dependencies": {
|
|
36
|
-
"@puppeteer/browsers": "^3.2.2",
|
|
37
|
-
"axios": "^1.8.1",
|
|
38
|
-
"cheerio": "^1.0.0",
|
|
39
|
-
"puppeteer": "^24.3.0",
|
|
40
|
-
"puppeteer-core": "^24.3.0",
|
|
41
|
-
"puppeteer-extra": "^3.3.6",
|
|
42
|
-
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
|
43
|
-
"socket.io-client": "^4.8.1",
|
|
44
|
-
"nodemailer": "^6.9.16"
|
|
45
|
-
}
|
|
46
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"name": "app-manager-edge-worker",
|
|
3
|
+
"version": "1.0.9",
|
|
4
|
+
"description": "Distributed Scraper Edge Worker for Apps Manager. Run on any local machine, VPS, or Termux Android device to serve browser and HTTP scraping jobs.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"bin": {
|
|
7
|
+
"app-manager-edge-worker": "bin/worker.js",
|
|
8
|
+
"apps-manager-worker": "bin/worker.js"
|
|
9
|
+
},
|
|
10
|
+
"main": "./src/client.js",
|
|
11
|
+
"scripts": {
|
|
12
|
+
"start": "node ./bin/worker.js"
|
|
13
|
+
},
|
|
14
|
+
"keywords": [
|
|
15
|
+
"app-manager-edge-worker",
|
|
16
|
+
"apps-manager",
|
|
17
|
+
"scraper",
|
|
18
|
+
"edge-worker",
|
|
19
|
+
"puppeteer",
|
|
20
|
+
"residential-proxy",
|
|
21
|
+
"distributed-scraping",
|
|
22
|
+
"termux"
|
|
23
|
+
],
|
|
24
|
+
"author": "Chetan Mohite <chetanmohite2128@gmail.com> (https://github.com/codes-by-chetan)",
|
|
25
|
+
"license": "MIT",
|
|
26
|
+
"homepage": "https://apps-manager.onrender.com",
|
|
27
|
+
"repository": {
|
|
28
|
+
"type": "git",
|
|
29
|
+
"url": "git+https://github.com/codes-by-chetan/Apps_Manager.git",
|
|
30
|
+
"directory": "packages/worker"
|
|
31
|
+
},
|
|
32
|
+
"publishConfig": {
|
|
33
|
+
"access": "public"
|
|
34
|
+
},
|
|
35
|
+
"dependencies": {
|
|
36
|
+
"@puppeteer/browsers": "^3.2.2",
|
|
37
|
+
"axios": "^1.8.1",
|
|
38
|
+
"cheerio": "^1.0.0",
|
|
39
|
+
"puppeteer": "^24.3.0",
|
|
40
|
+
"puppeteer-core": "^24.3.0",
|
|
41
|
+
"puppeteer-extra": "^3.3.6",
|
|
42
|
+
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
|
43
|
+
"socket.io-client": "^4.8.1",
|
|
44
|
+
"nodemailer": "^6.9.16"
|
|
45
|
+
}
|
|
46
|
+
}
|
package/src/banner.js
CHANGED
|
@@ -1,80 +1,80 @@
|
|
|
1
|
-
import { createRequire } from "module";
|
|
2
|
-
|
|
3
|
-
const require = createRequire(import.meta.url);
|
|
4
|
-
let workerVersion = "1.0.
|
|
5
|
-
try {
|
|
6
|
-
const pkg = require("../package.json");
|
|
7
|
-
if (pkg?.version) workerVersion = pkg.version;
|
|
8
|
-
} catch (_) { }
|
|
9
|
-
|
|
10
|
-
// ANSI color helpers
|
|
11
|
-
const colors = {
|
|
12
|
-
reset: "\x1b[0m",
|
|
13
|
-
bright: "\x1b[1m",
|
|
14
|
-
dim: "\x1b[2m",
|
|
15
|
-
cyan: "\x1b[36m",
|
|
16
|
-
green: "\x1b[32m",
|
|
17
|
-
yellow: "\x1b[33m",
|
|
18
|
-
red: "\x1b[31m",
|
|
19
|
-
magenta: "\x1b[35m",
|
|
20
|
-
gray: "\x1b[90m",
|
|
21
|
-
};
|
|
22
|
-
|
|
23
|
-
/**
|
|
24
|
-
* Returns a formatted timestamp string using the device's local time: [YYYY-MM-DD HH:mm:ss]
|
|
25
|
-
*/
|
|
26
|
-
function timestamp() {
|
|
27
|
-
const now = new Date();
|
|
28
|
-
const pad = (n) => String(n).padStart(2, "0");
|
|
29
|
-
const year = now.getFullYear();
|
|
30
|
-
const month = pad(now.getMonth() + 1);
|
|
31
|
-
const day = pad(now.getDate());
|
|
32
|
-
const hours = pad(now.getHours());
|
|
33
|
-
const minutes = pad(now.getMinutes());
|
|
34
|
-
const seconds = pad(now.getSeconds());
|
|
35
|
-
return `${colors.gray}[${year}-${month}-${day} ${hours}:${minutes}:${seconds}]${colors.reset}`;
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
export function printBanner({ workerName, serverUrl, concurrency, platform, isTermux, keepAliveMinutes }) {
|
|
39
|
-
console.clear();
|
|
40
|
-
console.log(`${colors.cyan}${colors.bright}`);
|
|
41
|
-
console.log(" █████╗ ██████╗ ██████╗ ███████╗ ███╗ ███╗ █████╗ ███╗ ██╗ █████╗ ██████╗ ███████╗██████╗ ");
|
|
42
|
-
console.log(" ██╔══██╗██╔══██╗██╔══██╗██╔════╝ ████╗ ████║██╔══██╗████╗ ██║██╔══██╗██╔════╝ ██╔════╝██╔══██╗");
|
|
43
|
-
console.log(" ███████║██████╔╝██████╔╝███████╗ ██╔████╔██║███████║██╔██╗ ██║███████║██║ ███╗█████╗ ██████╔╝");
|
|
44
|
-
console.log(" ██╔══██║██╔═══╝ ██╔═══╝ ╚════██║ ██║╚██╔╝██║██╔══██║██║╚██╗██║██╔══██║██║ ██║██╔══╝ ██╔══██╗");
|
|
45
|
-
console.log(" ██║ ██║██║ ██║ ███████║██╗██║ ╚═╝ ██║██║ ██║██║ ╚████║██║ ██║╚██████╔╝███████╗██║ ██║");
|
|
46
|
-
console.log(" ╚═╝ ╚═╝╚═╝ ╚═╝ ╚══════╝╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═══╝╚═╝ ╚═╝ ╚═════╝ ╚══════╝╚═╝ ╚═╝");
|
|
47
|
-
console.log(`${colors.reset}`);
|
|
48
|
-
console.log(`${colors.bright} 🌐 Distributed Scraper Edge Worker Client v${workerVersion}${colors.reset}`);
|
|
49
|
-
console.log(`${colors.dim} Zero-cost, persistent tunneling & residential edge scraping${colors.reset}`);
|
|
50
|
-
console.log("--------------------------------------------------------------------------------");
|
|
51
|
-
console.log(` ${colors.bright}Device Name :${colors.reset} ${colors.green}${workerName}${colors.reset}`);
|
|
52
|
-
console.log(` ${colors.bright}Server URL :${colors.reset} ${colors.cyan}${serverUrl}${colors.reset}`);
|
|
53
|
-
console.log(` ${colors.bright}Concurrency :${colors.reset} ${colors.yellow}${concurrency} concurrent tabs${colors.reset}`);
|
|
54
|
-
console.log(` ${colors.bright}Environment :${colors.reset} ${platform} ${isTermux ? "(Termux Android)" : ""}`);
|
|
55
|
-
console.log(` ${colors.bright}Keep-Alive :${colors.reset} ${keepAliveMinutes > 0 ? `${colors.green}Every ${keepAliveMinutes}m (Render Anti-Sleep Active)${colors.reset}` : `${colors.gray}Disabled${colors.reset}`}`);
|
|
56
|
-
console.log("--------------------------------------------------------------------------------\n");
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
export function logInfo(msg) {
|
|
60
|
-
console.log(`${timestamp()} ${colors.cyan}[INFO]${colors.reset} ${msg}`);
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
export function logSuccess(msg) {
|
|
64
|
-
console.log(`${timestamp()} ${colors.green}[SUCCESS]${colors.reset} ${msg}`);
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
export function logWarn(msg) {
|
|
68
|
-
console.log(`${timestamp()} ${colors.yellow}[WARN]${colors.reset} ${msg}`);
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
export function logError(msg) {
|
|
72
|
-
console.log(`${timestamp()} ${colors.red}[ERROR]${colors.reset} ${msg}`);
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
export function logJob(jobType, url, durationMs = null) {
|
|
76
|
-
const timeStr = durationMs ? ` ${colors.gray}(${durationMs}ms)${colors.reset}` : "";
|
|
77
|
-
console.log(
|
|
78
|
-
`${timestamp()} ${colors.magenta}[TASK]${colors.reset} ${colors.bright}${jobType}${colors.reset} -> ${url}${timeStr}`
|
|
79
|
-
);
|
|
80
|
-
}
|
|
1
|
+
import { createRequire } from "module";
|
|
2
|
+
|
|
3
|
+
const require = createRequire(import.meta.url);
|
|
4
|
+
let workerVersion = "1.0.9";
|
|
5
|
+
try {
|
|
6
|
+
const pkg = require("../package.json");
|
|
7
|
+
if (pkg?.version) workerVersion = pkg.version;
|
|
8
|
+
} catch (_) { }
|
|
9
|
+
|
|
10
|
+
// ANSI color helpers
|
|
11
|
+
const colors = {
|
|
12
|
+
reset: "\x1b[0m",
|
|
13
|
+
bright: "\x1b[1m",
|
|
14
|
+
dim: "\x1b[2m",
|
|
15
|
+
cyan: "\x1b[36m",
|
|
16
|
+
green: "\x1b[32m",
|
|
17
|
+
yellow: "\x1b[33m",
|
|
18
|
+
red: "\x1b[31m",
|
|
19
|
+
magenta: "\x1b[35m",
|
|
20
|
+
gray: "\x1b[90m",
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Returns a formatted timestamp string using the device's local time: [YYYY-MM-DD HH:mm:ss]
|
|
25
|
+
*/
|
|
26
|
+
function timestamp() {
|
|
27
|
+
const now = new Date();
|
|
28
|
+
const pad = (n) => String(n).padStart(2, "0");
|
|
29
|
+
const year = now.getFullYear();
|
|
30
|
+
const month = pad(now.getMonth() + 1);
|
|
31
|
+
const day = pad(now.getDate());
|
|
32
|
+
const hours = pad(now.getHours());
|
|
33
|
+
const minutes = pad(now.getMinutes());
|
|
34
|
+
const seconds = pad(now.getSeconds());
|
|
35
|
+
return `${colors.gray}[${year}-${month}-${day} ${hours}:${minutes}:${seconds}]${colors.reset}`;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function printBanner({ workerName, serverUrl, concurrency, platform, isTermux, keepAliveMinutes }) {
|
|
39
|
+
console.clear();
|
|
40
|
+
console.log(`${colors.cyan}${colors.bright}`);
|
|
41
|
+
console.log(" █████╗ ██████╗ ██████╗ ███████╗ ███╗ ███╗ █████╗ ███╗ ██╗ █████╗ ██████╗ ███████╗██████╗ ");
|
|
42
|
+
console.log(" ██╔══██╗██╔══██╗██╔══██╗██╔════╝ ████╗ ████║██╔══██╗████╗ ██║██╔══██╗██╔════╝ ██╔════╝██╔══██╗");
|
|
43
|
+
console.log(" ███████║██████╔╝██████╔╝███████╗ ██╔████╔██║███████║██╔██╗ ██║███████║██║ ███╗█████╗ ██████╔╝");
|
|
44
|
+
console.log(" ██╔══██║██╔═══╝ ██╔═══╝ ╚════██║ ██║╚██╔╝██║██╔══██║██║╚██╗██║██╔══██║██║ ██║██╔══╝ ██╔══██╗");
|
|
45
|
+
console.log(" ██║ ██║██║ ██║ ███████║██╗██║ ╚═╝ ██║██║ ██║██║ ╚████║██║ ██║╚██████╔╝███████╗██║ ██║");
|
|
46
|
+
console.log(" ╚═╝ ╚═╝╚═╝ ╚═╝ ╚══════╝╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═══╝╚═╝ ╚═╝ ╚═════╝ ╚══════╝╚═╝ ╚═╝");
|
|
47
|
+
console.log(`${colors.reset}`);
|
|
48
|
+
console.log(`${colors.bright} 🌐 Distributed Scraper Edge Worker Client v${workerVersion}${colors.reset}`);
|
|
49
|
+
console.log(`${colors.dim} Zero-cost, persistent tunneling & residential edge scraping${colors.reset}`);
|
|
50
|
+
console.log("--------------------------------------------------------------------------------");
|
|
51
|
+
console.log(` ${colors.bright}Device Name :${colors.reset} ${colors.green}${workerName}${colors.reset}`);
|
|
52
|
+
console.log(` ${colors.bright}Server URL :${colors.reset} ${colors.cyan}${serverUrl}${colors.reset}`);
|
|
53
|
+
console.log(` ${colors.bright}Concurrency :${colors.reset} ${colors.yellow}${concurrency} concurrent tabs${colors.reset}`);
|
|
54
|
+
console.log(` ${colors.bright}Environment :${colors.reset} ${platform} ${isTermux ? "(Termux Android)" : ""}`);
|
|
55
|
+
console.log(` ${colors.bright}Keep-Alive :${colors.reset} ${keepAliveMinutes > 0 ? `${colors.green}Every ${keepAliveMinutes}m (Render Anti-Sleep Active)${colors.reset}` : `${colors.gray}Disabled${colors.reset}`}`);
|
|
56
|
+
console.log("--------------------------------------------------------------------------------\n");
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function logInfo(msg) {
|
|
60
|
+
console.log(`${timestamp()} ${colors.cyan}[INFO]${colors.reset} ${msg}`);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function logSuccess(msg) {
|
|
64
|
+
console.log(`${timestamp()} ${colors.green}[SUCCESS]${colors.reset} ${msg}`);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function logWarn(msg) {
|
|
68
|
+
console.log(`${timestamp()} ${colors.yellow}[WARN]${colors.reset} ${msg}`);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function logError(msg) {
|
|
72
|
+
console.log(`${timestamp()} ${colors.red}[ERROR]${colors.reset} ${msg}`);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function logJob(jobType, url, durationMs = null) {
|
|
76
|
+
const timeStr = durationMs ? ` ${colors.gray}(${durationMs}ms)${colors.reset}` : "";
|
|
77
|
+
console.log(
|
|
78
|
+
`${timestamp()} ${colors.magenta}[TASK]${colors.reset} ${colors.bright}${jobType}${colors.reset} -> ${url}${timeStr}`
|
|
79
|
+
);
|
|
80
|
+
}
|