app-manager-edge-worker 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/package.json +1 -1
- package/src/banner.js +3 -2
- package/src/client.js +48 -1
- package/src/config.js +18 -1
package/README.md
CHANGED
|
@@ -17,6 +17,7 @@ Cloud datacenter IPs (Render, AWS, DigitalOcean, Hetzner) are frequently blocked
|
|
|
17
17
|
- 🔒 **Zero Port Forwarding**: Persistent outbound WebSockets (`wss://.../workers`) effortlessly traverse home Wi-Fi routers, firewalls, and carrier-grade NAT (CGNAT).
|
|
18
18
|
- ⚡ **Autonomous Chromium Engine**: Checks your system for Chrome/Edge or automatically installs stable Chromium without any manual intervention.
|
|
19
19
|
- 📱 **Android Phone Support**: Transform an old spare Android phone into a 24/7 scraping powerhouse using Termux and Wake Lock.
|
|
20
|
+
- ⏰ **Automated Render Anti-Sleep Keep-Alive**: Periodically pings the central server (every 7 minutes by default) to keep Render free-tier instances warm and eliminate cold-start spindowns.
|
|
20
21
|
- 🚀 **Smart Single-Flight Deduplication**: Concurrent scrapes for the exact same target URL coalesce into a single execution flight.
|
|
21
22
|
|
|
22
23
|
---
|
|
@@ -142,6 +143,8 @@ No manual setup required!
|
|
|
142
143
|
| `-s, --server <url>` | Apps Manager Server URL | `https://apps-manager.onrender.com` |
|
|
143
144
|
| `-c, --concurrency <n>` | Max concurrent Chromium browser tabs | `3` |
|
|
144
145
|
| `-n, --name <name>` | Custom display name for this worker | Device hostname |
|
|
146
|
+
| `-p, --keep-alive <minutes>` | Keep-alive ping interval to prevent Render free-tier cold sleep | `7` |
|
|
147
|
+
| `--no-keep-alive` | Disable keep-alive ping loop | - |
|
|
145
148
|
| `--reset` | Clear stored credentials and prompt again | - |
|
|
146
149
|
| `-h, --help` | Display help screen | - |
|
|
147
150
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "app-manager-edge-worker",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.1",
|
|
4
4
|
"description": "Distributed Scraper Edge Worker for Apps Manager. Run on any local machine, VPS, or Termux Android device to serve browser and HTTP scraping jobs.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/src/banner.js
CHANGED
|
@@ -11,7 +11,7 @@ const colors = {
|
|
|
11
11
|
gray: "\x1b[90m",
|
|
12
12
|
};
|
|
13
13
|
|
|
14
|
-
export function printBanner({ workerName, serverUrl, concurrency, platform, isTermux }) {
|
|
14
|
+
export function printBanner({ workerName, serverUrl, concurrency, platform, isTermux, keepAliveMinutes }) {
|
|
15
15
|
console.clear();
|
|
16
16
|
console.log(`${colors.cyan}${colors.bright}`);
|
|
17
17
|
console.log(" █████╗ ██████╗ ██████╗ ███████╗ ███╗ ███╗ █████╗ ███╗ ██╗ █████╗ ██████╗ ███████╗██████╗ ");
|
|
@@ -21,13 +21,14 @@ export function printBanner({ workerName, serverUrl, concurrency, platform, isTe
|
|
|
21
21
|
console.log(" ██║ ██║██║ ██║ ███████║██╗██║ ╚═╝ ██║██║ ██║██║ ╚████║██║ ██║╚██████╔╝███████╗██║ ██║");
|
|
22
22
|
console.log(" ╚═╝ ╚═╝╚═╝ ╚═╝ ╚══════╝╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═══╝╚═╝ ╚═╝ ╚═════╝ ╚══════╝╚═╝ ╚═╝");
|
|
23
23
|
console.log(`${colors.reset}`);
|
|
24
|
-
console.log(`${colors.bright} 🌐 Distributed Scraper Edge Worker Client v1.0.
|
|
24
|
+
console.log(`${colors.bright} 🌐 Distributed Scraper Edge Worker Client v1.0.1${colors.reset}`);
|
|
25
25
|
console.log(`${colors.dim} Zero-cost, persistent tunneling & residential edge scraping${colors.reset}`);
|
|
26
26
|
console.log("--------------------------------------------------------------------------------");
|
|
27
27
|
console.log(` ${colors.bright}Device Name :${colors.reset} ${colors.green}${workerName}${colors.reset}`);
|
|
28
28
|
console.log(` ${colors.bright}Server URL :${colors.reset} ${colors.cyan}${serverUrl}${colors.reset}`);
|
|
29
29
|
console.log(` ${colors.bright}Concurrency :${colors.reset} ${colors.yellow}${concurrency} concurrent tabs${colors.reset}`);
|
|
30
30
|
console.log(` ${colors.bright}Environment :${colors.reset} ${platform} ${isTermux ? "(Termux Android)" : ""}`);
|
|
31
|
+
console.log(` ${colors.bright}Keep-Alive :${colors.reset} ${keepAliveMinutes > 0 ? `${colors.green}Every ${keepAliveMinutes}m (Render Anti-Sleep Active)${colors.reset}` : `${colors.gray}Disabled${colors.reset}`}`);
|
|
31
32
|
console.log("--------------------------------------------------------------------------------\n");
|
|
32
33
|
}
|
|
33
34
|
|
package/src/client.js
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import { io } from "socket.io-client";
|
|
2
2
|
import os from "os";
|
|
3
3
|
import path from "path";
|
|
4
|
+
import axios from "axios";
|
|
4
5
|
import localBrowserManager from "./browser.js";
|
|
5
6
|
import { handleChromiumScrape, handleHttpScrape } from "./handlers/scrapeHandler.js";
|
|
6
7
|
import { handlePreviewPage, handleInspectElement } from "./handlers/previewHandler.js";
|
|
7
8
|
import { printBanner, logInfo, logSuccess, logWarn, logError, logJob } from "./banner.js";
|
|
8
9
|
|
|
9
10
|
export async function startWorkerClient(config) {
|
|
10
|
-
const { workerKey, serverUrl, concurrency, workerName } = config;
|
|
11
|
+
const { workerKey, serverUrl, concurrency, workerName, keepAliveMinutes = 7 } = config;
|
|
11
12
|
const isTermux = localBrowserManager.isTermux();
|
|
12
13
|
|
|
13
14
|
printBanner({
|
|
@@ -16,6 +17,7 @@ export async function startWorkerClient(config) {
|
|
|
16
17
|
concurrency,
|
|
17
18
|
platform: `${os.platform()} ${os.arch()}`,
|
|
18
19
|
isTermux,
|
|
20
|
+
keepAliveMinutes,
|
|
19
21
|
});
|
|
20
22
|
|
|
21
23
|
logInfo("Checking Chromium browser availability...");
|
|
@@ -133,8 +135,53 @@ export async function startWorkerClient(config) {
|
|
|
133
135
|
});
|
|
134
136
|
});
|
|
135
137
|
|
|
138
|
+
// Render Anti-Sleep Keep-Alive Ping
|
|
139
|
+
let keepAliveTimer = null;
|
|
140
|
+
let initialKeepAliveTimer = null;
|
|
141
|
+
|
|
142
|
+
if (keepAliveMinutes > 0) {
|
|
143
|
+
const pingKeepAlive = async () => {
|
|
144
|
+
try {
|
|
145
|
+
// Try the dedicated keep-alive endpoint first
|
|
146
|
+
const endpoint = `${serverUrl}/api/workers/keep-alive`;
|
|
147
|
+
const res = await axios.get(endpoint, {
|
|
148
|
+
timeout: 15000,
|
|
149
|
+
headers: {
|
|
150
|
+
"x-worker-key": workerKey || "",
|
|
151
|
+
"user-agent": "AppsManagerEdgeWorker/1.0.1",
|
|
152
|
+
},
|
|
153
|
+
});
|
|
154
|
+
logInfo(`[Keep-Alive] Server awake ping OK (${res.status}) - Render instance active`);
|
|
155
|
+
} catch (err) {
|
|
156
|
+
// Fallback to /health
|
|
157
|
+
try {
|
|
158
|
+
const fallbackUrl = `${serverUrl}/health`;
|
|
159
|
+
await axios.get(fallbackUrl, {
|
|
160
|
+
timeout: 10000,
|
|
161
|
+
headers: {
|
|
162
|
+
"user-agent": "AppsManagerEdgeWorker/1.0.1",
|
|
163
|
+
},
|
|
164
|
+
});
|
|
165
|
+
logInfo(`[Keep-Alive] Fallback health ping OK - Render instance active`);
|
|
166
|
+
} catch (fallbackErr) {
|
|
167
|
+
logWarn(`[Keep-Alive] Server awake ping warning (${err.message}) - will retry next cycle`);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
};
|
|
171
|
+
|
|
172
|
+
// Send initial ping 15s after startup to warm up/verify
|
|
173
|
+
initialKeepAliveTimer = setTimeout(pingKeepAlive, 15000);
|
|
174
|
+
if (initialKeepAliveTimer.unref) initialKeepAliveTimer.unref();
|
|
175
|
+
|
|
176
|
+
const intervalMs = Math.max(1, keepAliveMinutes) * 60 * 1000;
|
|
177
|
+
keepAliveTimer = setInterval(pingKeepAlive, intervalMs);
|
|
178
|
+
if (keepAliveTimer.unref) keepAliveTimer.unref();
|
|
179
|
+
}
|
|
180
|
+
|
|
136
181
|
// Graceful shutdown handling
|
|
137
182
|
const shutdown = async () => {
|
|
183
|
+
if (keepAliveTimer) clearInterval(keepAliveTimer);
|
|
184
|
+
if (initialKeepAliveTimer) clearTimeout(initialKeepAliveTimer);
|
|
138
185
|
logInfo("\nGracefully shutting down edge worker...");
|
|
139
186
|
socket.disconnect();
|
|
140
187
|
await localBrowserManager.close();
|
package/src/config.js
CHANGED
|
@@ -65,6 +65,10 @@ export function parseCliArgs() {
|
|
|
65
65
|
parsed.concurrency = parseInt(args[++i], 10);
|
|
66
66
|
} else if (arg === "--name" || arg === "-n") {
|
|
67
67
|
parsed.name = args[++i];
|
|
68
|
+
} else if (arg === "--keep-alive" || arg === "-p") {
|
|
69
|
+
parsed.keepAlive = parseInt(args[++i], 10);
|
|
70
|
+
} else if (arg === "--no-keep-alive") {
|
|
71
|
+
parsed.keepAlive = false;
|
|
68
72
|
} else if (arg === "--reset") {
|
|
69
73
|
parsed.reset = true;
|
|
70
74
|
} else if (arg === "--help" || arg === "-h") {
|
|
@@ -102,7 +106,7 @@ export async function resolveWorkerConfig() {
|
|
|
102
106
|
|
|
103
107
|
if (cli.help) {
|
|
104
108
|
console.log(`
|
|
105
|
-
Apps Manager Scraper Edge Worker CLI
|
|
109
|
+
Apps Manager Scraper Edge Worker CLI v1.0.1
|
|
106
110
|
|
|
107
111
|
Usage:
|
|
108
112
|
npx app-manager-edge-worker [options]
|
|
@@ -112,6 +116,8 @@ Options:
|
|
|
112
116
|
-s, --server <url> Apps Manager Server URL (e.g. https://apps-manager.onrender.com)
|
|
113
117
|
-c, --concurrency <number> Maximum concurrent browser tabs (default: 3)
|
|
114
118
|
-n, --name <name> Custom display name for this worker device
|
|
119
|
+
-p, --keep-alive <minutes> Interval in minutes to ping server and prevent Render free-tier sleep (default: 7)
|
|
120
|
+
--no-keep-alive Disable keep-alive ping
|
|
115
121
|
--reset Clear stored credentials and prompt again
|
|
116
122
|
-h, --help Display this help message
|
|
117
123
|
`);
|
|
@@ -192,10 +198,21 @@ Options:
|
|
|
192
198
|
// Clean up trailing slash
|
|
193
199
|
serverUrl = serverUrl.replace(/\/+$/, "");
|
|
194
200
|
|
|
201
|
+
// Keep-alive ping interval (default 7 minutes, 0 to disable)
|
|
202
|
+
let keepAliveMinutes = 7;
|
|
203
|
+
if (cli.keepAlive === false || process.env.KEEP_ALIVE === "false") {
|
|
204
|
+
keepAliveMinutes = 0;
|
|
205
|
+
} else if (typeof cli.keepAlive === "number" && !isNaN(cli.keepAlive)) {
|
|
206
|
+
keepAliveMinutes = Math.max(0, cli.keepAlive);
|
|
207
|
+
} else if (process.env.KEEP_ALIVE_MINUTES) {
|
|
208
|
+
keepAliveMinutes = parseInt(process.env.KEEP_ALIVE_MINUTES, 10) || 7;
|
|
209
|
+
}
|
|
210
|
+
|
|
195
211
|
return {
|
|
196
212
|
workerKey: workerKey.trim(),
|
|
197
213
|
serverUrl,
|
|
198
214
|
concurrency,
|
|
199
215
|
workerName,
|
|
216
|
+
keepAliveMinutes,
|
|
200
217
|
};
|
|
201
218
|
}
|