app-manager-edge-worker 1.1.1 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/worker.js +16 -16
- package/package.json +5 -5
- package/src/banner.js +80 -80
- package/src/client.js +351 -351
- package/src/handlers/mailHandler.js +74 -74
- package/src/handlers/previewHandler.js +647 -647
- package/src/handlers/scrapeHandler.js +887 -887
- package/src/handlers/transformUtils.js +340 -340
- package/src/handlers/whatsappHandler.js +36 -6
- package/LICENSE.txt +0 -21
package/src/client.js
CHANGED
|
@@ -1,351 +1,351 @@
|
|
|
1
|
-
import { io } from "socket.io-client";
|
|
2
|
-
import os from "os";
|
|
3
|
-
import path from "path";
|
|
4
|
-
import axios from "axios";
|
|
5
|
-
import { execSync } from "child_process";
|
|
6
|
-
import localBrowserManager from "./browser.js";
|
|
7
|
-
import { handleChromiumScrape, handleHttpScrape } from "./handlers/scrapeHandler.js";
|
|
8
|
-
import { handlePreviewPage, handleInspectElement } from "./handlers/previewHandler.js";
|
|
9
|
-
import { handleSendEmail } from "./handlers/mailHandler.js";
|
|
10
|
-
import { handleAiSmartRepair } from "./handlers/aiRepairHandler.js";
|
|
11
|
-
import { getWhatsAppHandler } from "./handlers/whatsappHandler.js";
|
|
12
|
-
import { printBanner, logInfo, logSuccess, logWarn, logError, logJob } from "./banner.js";
|
|
13
|
-
|
|
14
|
-
export async function startWorkerClient(config) {
|
|
15
|
-
const { workerKey, serverUrl, concurrency, workerName, keepAliveMinutes = 7 } = config;
|
|
16
|
-
const isTermux = localBrowserManager.isTermux();
|
|
17
|
-
|
|
18
|
-
printBanner({
|
|
19
|
-
workerName,
|
|
20
|
-
serverUrl,
|
|
21
|
-
concurrency,
|
|
22
|
-
platform: `${os.platform()} ${os.arch()}`,
|
|
23
|
-
isTermux,
|
|
24
|
-
keepAliveMinutes,
|
|
25
|
-
});
|
|
26
|
-
|
|
27
|
-
if (isTermux) {
|
|
28
|
-
try {
|
|
29
|
-
execSync("termux-wake-lock", { stdio: "ignore" });
|
|
30
|
-
logSuccess("[Android Termux] Wake lock acquired to keep CPU awake in background.");
|
|
31
|
-
} catch {
|
|
32
|
-
// termux-tools might not be installed yet
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
logInfo("Checking Chromium browser availability...");
|
|
37
|
-
let chromiumExecutable = null;
|
|
38
|
-
try {
|
|
39
|
-
chromiumExecutable = await localBrowserManager.ensureChromiumExecutable();
|
|
40
|
-
logSuccess(`Chromium ready: ${chromiumExecutable}`);
|
|
41
|
-
} catch (browserErr) {
|
|
42
|
-
logWarn(`Chromium initialization warning: ${browserErr.message}`);
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
logInfo("Connecting to Apps Manager cluster...");
|
|
46
|
-
|
|
47
|
-
const socket = io(`${serverUrl}/workers`, {
|
|
48
|
-
transports: ["websocket", "polling"],
|
|
49
|
-
reconnection: true,
|
|
50
|
-
reconnectionAttempts: Infinity,
|
|
51
|
-
reconnectionDelay: 1000,
|
|
52
|
-
reconnectionDelayMax: 5000,
|
|
53
|
-
auth: {
|
|
54
|
-
workerKey,
|
|
55
|
-
metadata: {
|
|
56
|
-
hostname: workerName || os.hostname(),
|
|
57
|
-
platform: os.platform(),
|
|
58
|
-
arch: os.arch(),
|
|
59
|
-
osVersion: os.release(),
|
|
60
|
-
nodeVersion: process.version,
|
|
61
|
-
chromiumVersion: chromiumExecutable ? path.basename(chromiumExecutable) : "",
|
|
62
|
-
chromiumPath: chromiumExecutable || "",
|
|
63
|
-
isTermux,
|
|
64
|
-
},
|
|
65
|
-
},
|
|
66
|
-
});
|
|
67
|
-
|
|
68
|
-
const CHUNK_SIZE = 256 * 1024; // 256 KB chunks to prevent any WebSocket frame buffer overflow
|
|
69
|
-
|
|
70
|
-
const sendJobResult = (payload) => {
|
|
71
|
-
let payloadStr = "";
|
|
72
|
-
try {
|
|
73
|
-
payloadStr = JSON.stringify(payload);
|
|
74
|
-
} catch {
|
|
75
|
-
socket.emit("job:result", payload);
|
|
76
|
-
return;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
// Small payload: send directly
|
|
80
|
-
if (payloadStr.length <= CHUNK_SIZE) {
|
|
81
|
-
socket.emit("job:result", payload);
|
|
82
|
-
return;
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
// Large payload (e.g. screenshot or large scrape): stream in chunks
|
|
86
|
-
const totalChunks = Math.ceil(payloadStr.length / CHUNK_SIZE);
|
|
87
|
-
const jobId = payload.jobId;
|
|
88
|
-
|
|
89
|
-
socket.emit("job:result:stream:start", {
|
|
90
|
-
jobId,
|
|
91
|
-
totalChunks,
|
|
92
|
-
totalLength: payloadStr.length,
|
|
93
|
-
});
|
|
94
|
-
|
|
95
|
-
for (let i = 0; i < totalChunks; i++) {
|
|
96
|
-
const chunk = payloadStr.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE);
|
|
97
|
-
socket.emit("job:result:stream:chunk", {
|
|
98
|
-
jobId,
|
|
99
|
-
index: i,
|
|
100
|
-
chunk,
|
|
101
|
-
});
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
socket.emit("job:result:stream:end", { jobId });
|
|
105
|
-
};
|
|
106
|
-
|
|
107
|
-
let activeJobs = 0;
|
|
108
|
-
let activeConcurrency = concurrency;
|
|
109
|
-
|
|
110
|
-
// Initialize WhatsApp Companion Device Engine
|
|
111
|
-
const waHandler = getWhatsAppHandler({
|
|
112
|
-
onStatusChange: (statusData) => {
|
|
113
|
-
socket.emit("whatsapp:status_update", statusData);
|
|
114
|
-
},
|
|
115
|
-
onQrGenerated: ({ qrDataUrl }) => {
|
|
116
|
-
socket.emit("whatsapp:qr_stream", { qrDataUrl });
|
|
117
|
-
},
|
|
118
|
-
});
|
|
119
|
-
|
|
120
|
-
socket.on("connect", () => {
|
|
121
|
-
logSuccess(`Connected to cluster! (Socket ID: ${socket.id})`);
|
|
122
|
-
logInfo(`Ready to accept scraping and visual preview jobs (Concurrency: ${activeConcurrency}).\n`);
|
|
123
|
-
|
|
124
|
-
// Announce current WhatsApp companion device status
|
|
125
|
-
const waTelemetry = waHandler.getTelemetry();
|
|
126
|
-
socket.emit("whatsapp:status_update", waTelemetry);
|
|
127
|
-
});
|
|
128
|
-
|
|
129
|
-
socket.on("worker:registered", (data) => {
|
|
130
|
-
if (data?.concurrencyLimit && typeof data.concurrencyLimit === "number") {
|
|
131
|
-
activeConcurrency = data.concurrencyLimit;
|
|
132
|
-
logInfo(`[Cluster Config] Worker registered as '${data.name || data.workerId}'. Active concurrency: ${activeConcurrency} concurrent tabs.`);
|
|
133
|
-
}
|
|
134
|
-
});
|
|
135
|
-
|
|
136
|
-
// Handle Admin Panel WhatsApp QR stream request
|
|
137
|
-
socket.on("whatsapp:request_qr", async () => {
|
|
138
|
-
logInfo("[WhatsApp] QR pairing requested via Admin Panel...");
|
|
139
|
-
try {
|
|
140
|
-
await waHandler.initialize(true);
|
|
141
|
-
} catch (err) {
|
|
142
|
-
logError(`[WhatsApp] Failed to initialize QR pairing: ${err.message}`);
|
|
143
|
-
}
|
|
144
|
-
});
|
|
145
|
-
|
|
146
|
-
// Handle Admin Panel Remote Logout
|
|
147
|
-
socket.on("whatsapp:remote_logout", async () => {
|
|
148
|
-
logInfo("[WhatsApp] Remote unpair requested via Admin Panel...");
|
|
149
|
-
try {
|
|
150
|
-
await waHandler.logout();
|
|
151
|
-
logSuccess("[WhatsApp] Companion device unpaired and local session cleared.");
|
|
152
|
-
} catch (err) {
|
|
153
|
-
logError(`[WhatsApp] Logout error: ${err.message}`);
|
|
154
|
-
}
|
|
155
|
-
});
|
|
156
|
-
|
|
157
|
-
socket.on("connect_error", (err) => {
|
|
158
|
-
if (err.message && err.message.includes("Authentication failed")) {
|
|
159
|
-
logError(`Authentication Failed: ${err.message}`);
|
|
160
|
-
logWarn("Please verify your worker key or run with '--reset' to enter a new key.");
|
|
161
|
-
} else {
|
|
162
|
-
logWarn(`Connection failed (${err.message}). Retrying in a few seconds...`);
|
|
163
|
-
}
|
|
164
|
-
});
|
|
165
|
-
|
|
166
|
-
socket.on("disconnect", (reason) => {
|
|
167
|
-
logWarn(`Disconnected from cluster: ${reason}`);
|
|
168
|
-
});
|
|
169
|
-
|
|
170
|
-
// Handle incoming job dispatch from server
|
|
171
|
-
socket.on("job:dispatch", async (job) => {
|
|
172
|
-
const jobId = job.jobId;
|
|
173
|
-
const jobType = job.jobType || job.type;
|
|
174
|
-
const payload = job.payload || {};
|
|
175
|
-
const targetUrl = jobType === "SEND_EMAIL"
|
|
176
|
-
? (payload.mailOptions?.to ? `to: ${payload.mailOptions.to}` : "email")
|
|
177
|
-
: jobType.startsWith("WHATSAPP_")
|
|
178
|
-
? (payload.recipientPhone ? `WA: ${payload.recipientPhone}` : "whatsapp")
|
|
179
|
-
: (payload.url || "(no url)");
|
|
180
|
-
|
|
181
|
-
logJob(jobType, targetUrl);
|
|
182
|
-
activeJobs++;
|
|
183
|
-
|
|
184
|
-
const startTime = Date.now();
|
|
185
|
-
|
|
186
|
-
try {
|
|
187
|
-
let resultData = null;
|
|
188
|
-
|
|
189
|
-
if (jobType === "SCRAPE_CHROMIUM") {
|
|
190
|
-
resultData = await handleChromiumScrape(payload);
|
|
191
|
-
} else if (jobType === "SCRAPE_HTTP") {
|
|
192
|
-
resultData = await handleHttpScrape(payload);
|
|
193
|
-
} else if (jobType === "PREVIEW_PAGE") {
|
|
194
|
-
resultData = await handlePreviewPage(payload);
|
|
195
|
-
} else if (jobType === "INSPECT_ELEMENT") {
|
|
196
|
-
resultData = await handleInspectElement(payload);
|
|
197
|
-
} else if (jobType === "SEND_EMAIL") {
|
|
198
|
-
resultData = await handleSendEmail(payload);
|
|
199
|
-
} else if (jobType === "AI_SMART_REPAIR") {
|
|
200
|
-
resultData = await handleAiSmartRepair(payload, config);
|
|
201
|
-
} else if (jobType === "WHATSAPP_SEND_MESSAGE") {
|
|
202
|
-
resultData = await waHandler.sendMessage(payload);
|
|
203
|
-
} else if (jobType === "WHATSAPP_SEND_DOCUMENT") {
|
|
204
|
-
resultData = await waHandler.sendDocument(payload);
|
|
205
|
-
} else if (jobType === "WHATSAPP_RECONCILE") {
|
|
206
|
-
resultData = await waHandler.reconcileDelivery(payload.recipientPhone, payload.idempotencyKey);
|
|
207
|
-
} else {
|
|
208
|
-
throw new Error(`Unsupported job type: ${jobType}`);
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
const durationMs = Date.now() - startTime;
|
|
212
|
-
logSuccess(`Completed ${jobType} for ${targetUrl} in ${durationMs}ms`);
|
|
213
|
-
|
|
214
|
-
sendJobResult({
|
|
215
|
-
jobId,
|
|
216
|
-
success: true,
|
|
217
|
-
data: resultData,
|
|
218
|
-
durationMs,
|
|
219
|
-
});
|
|
220
|
-
} catch (err) {
|
|
221
|
-
const durationMs = Date.now() - startTime;
|
|
222
|
-
logError(`Failed ${jobType} for ${targetUrl}: ${err.message}`);
|
|
223
|
-
|
|
224
|
-
sendJobResult({
|
|
225
|
-
jobId,
|
|
226
|
-
success: false,
|
|
227
|
-
error: {
|
|
228
|
-
message: err.message,
|
|
229
|
-
code: err.code || "JOB_EXECUTION_ERROR",
|
|
230
|
-
},
|
|
231
|
-
durationMs,
|
|
232
|
-
});
|
|
233
|
-
} finally {
|
|
234
|
-
activeJobs = Math.max(0, activeJobs - 1);
|
|
235
|
-
}
|
|
236
|
-
});
|
|
237
|
-
|
|
238
|
-
// Handle cluster heartbeat ping with enhanced capacity & memory telemetry
|
|
239
|
-
socket.on("ping", () => {
|
|
240
|
-
const telemetry = waHandler.getTelemetry();
|
|
241
|
-
socket.emit("pong", {
|
|
242
|
-
activeJobs,
|
|
243
|
-
memory: process.memoryUsage(),
|
|
244
|
-
uptime: process.uptime(),
|
|
245
|
-
freeSystemMemoryMb: telemetry.freeSystemMemoryMb,
|
|
246
|
-
totalSystemMemoryMb: telemetry.totalSystemMemoryMb,
|
|
247
|
-
processMemoryMb: telemetry.processMemoryMb,
|
|
248
|
-
memoryPressure: telemetry.memoryPressure,
|
|
249
|
-
whatsappStatus: telemetry.status,
|
|
250
|
-
whatsappUser: telemetry.user,
|
|
251
|
-
});
|
|
252
|
-
});
|
|
253
|
-
|
|
254
|
-
// Render Anti-Sleep Keep-Alive Ping
|
|
255
|
-
let keepAliveTimer = null;
|
|
256
|
-
let initialKeepAliveTimer = null;
|
|
257
|
-
|
|
258
|
-
const clearKeepAliveTimers = () => {
|
|
259
|
-
if (keepAliveTimer) {
|
|
260
|
-
clearInterval(keepAliveTimer);
|
|
261
|
-
keepAliveTimer = null;
|
|
262
|
-
}
|
|
263
|
-
if (initialKeepAliveTimer) {
|
|
264
|
-
clearTimeout(initialKeepAliveTimer);
|
|
265
|
-
initialKeepAliveTimer = null;
|
|
266
|
-
}
|
|
267
|
-
};
|
|
268
|
-
|
|
269
|
-
const pingKeepAlive = async () => {
|
|
270
|
-
try {
|
|
271
|
-
// Try the dedicated keep-alive endpoint first
|
|
272
|
-
const endpoint = `${serverUrl}/api/workers/keep-alive`;
|
|
273
|
-
const res = await axios.get(endpoint, {
|
|
274
|
-
timeout: 15000,
|
|
275
|
-
headers: {
|
|
276
|
-
"x-worker-key": workerKey || "",
|
|
277
|
-
"user-agent": "AppsManagerEdgeWorker/1.1.0",
|
|
278
|
-
},
|
|
279
|
-
});
|
|
280
|
-
logInfo(`[Keep-Alive] Server awake ping OK (${res.status}) - Render instance active`);
|
|
281
|
-
} catch (err) {
|
|
282
|
-
// Fallback to /health
|
|
283
|
-
try {
|
|
284
|
-
const fallbackUrl = `${serverUrl}/health`;
|
|
285
|
-
await axios.get(fallbackUrl, {
|
|
286
|
-
timeout: 10000,
|
|
287
|
-
headers: {
|
|
288
|
-
"user-agent": "AppsManagerEdgeWorker/1.1.0",
|
|
289
|
-
},
|
|
290
|
-
});
|
|
291
|
-
logInfo(`[Keep-Alive] Fallback health ping OK - Render instance active`);
|
|
292
|
-
} catch (fallbackErr) {
|
|
293
|
-
logWarn(`[Keep-Alive] Server awake ping warning (${err.message}) - will retry next cycle`);
|
|
294
|
-
}
|
|
295
|
-
}
|
|
296
|
-
};
|
|
297
|
-
|
|
298
|
-
const startKeepAlive = () => {
|
|
299
|
-
if (keepAliveMinutes <= 0) return;
|
|
300
|
-
clearKeepAliveTimers();
|
|
301
|
-
|
|
302
|
-
// Send initial ping 15s after startup to warm up/verify
|
|
303
|
-
initialKeepAliveTimer = setTimeout(pingKeepAlive, 15000);
|
|
304
|
-
if (initialKeepAliveTimer.unref) initialKeepAliveTimer.unref();
|
|
305
|
-
|
|
306
|
-
const intervalMs = Math.max(1, keepAliveMinutes) * 60 * 1000;
|
|
307
|
-
keepAliveTimer = setInterval(pingKeepAlive, intervalMs);
|
|
308
|
-
if (keepAliveTimer.unref) keepAliveTimer.unref();
|
|
309
|
-
};
|
|
310
|
-
|
|
311
|
-
if (keepAliveMinutes > 0) {
|
|
312
|
-
startKeepAlive();
|
|
313
|
-
}
|
|
314
|
-
|
|
315
|
-
// Clean up keep-alive timers on socket disconnect
|
|
316
|
-
socket.on("disconnect", (reason) => {
|
|
317
|
-
logWarn(`Disconnected from cluster: ${reason}`);
|
|
318
|
-
clearKeepAliveTimers();
|
|
319
|
-
});
|
|
320
|
-
|
|
321
|
-
socket.on("connect", () => {
|
|
322
|
-
if (keepAliveMinutes > 0 && !keepAliveTimer) {
|
|
323
|
-
startKeepAlive();
|
|
324
|
-
}
|
|
325
|
-
});
|
|
326
|
-
|
|
327
|
-
// Provide programmatic close for automated test harnesses
|
|
328
|
-
socket.closeClient = async () => {
|
|
329
|
-
clearKeepAliveTimers();
|
|
330
|
-
if (isTermux) {
|
|
331
|
-
try {
|
|
332
|
-
execSync("termux-wake-unlock", { stdio: "ignore" });
|
|
333
|
-
} catch {}
|
|
334
|
-
}
|
|
335
|
-
socket.disconnect();
|
|
336
|
-
await localBrowserManager.close();
|
|
337
|
-
};
|
|
338
|
-
|
|
339
|
-
// Graceful shutdown handling
|
|
340
|
-
const shutdown = async () => {
|
|
341
|
-
logInfo("\nGracefully shutting down edge worker...");
|
|
342
|
-
await socket.closeClient();
|
|
343
|
-
logSuccess("Edge worker stopped cleanly.");
|
|
344
|
-
process.exit(0);
|
|
345
|
-
};
|
|
346
|
-
|
|
347
|
-
process.on("SIGINT", shutdown);
|
|
348
|
-
process.on("SIGTERM", shutdown);
|
|
349
|
-
|
|
350
|
-
return socket;
|
|
351
|
-
}
|
|
1
|
+
import { io } from "socket.io-client";
|
|
2
|
+
import os from "os";
|
|
3
|
+
import path from "path";
|
|
4
|
+
import axios from "axios";
|
|
5
|
+
import { execSync } from "child_process";
|
|
6
|
+
import localBrowserManager from "./browser.js";
|
|
7
|
+
import { handleChromiumScrape, handleHttpScrape } from "./handlers/scrapeHandler.js";
|
|
8
|
+
import { handlePreviewPage, handleInspectElement } from "./handlers/previewHandler.js";
|
|
9
|
+
import { handleSendEmail } from "./handlers/mailHandler.js";
|
|
10
|
+
import { handleAiSmartRepair } from "./handlers/aiRepairHandler.js";
|
|
11
|
+
import { getWhatsAppHandler } from "./handlers/whatsappHandler.js";
|
|
12
|
+
import { printBanner, logInfo, logSuccess, logWarn, logError, logJob } from "./banner.js";
|
|
13
|
+
|
|
14
|
+
export async function startWorkerClient(config) {
|
|
15
|
+
const { workerKey, serverUrl, concurrency, workerName, keepAliveMinutes = 7 } = config;
|
|
16
|
+
const isTermux = localBrowserManager.isTermux();
|
|
17
|
+
|
|
18
|
+
printBanner({
|
|
19
|
+
workerName,
|
|
20
|
+
serverUrl,
|
|
21
|
+
concurrency,
|
|
22
|
+
platform: `${os.platform()} ${os.arch()}`,
|
|
23
|
+
isTermux,
|
|
24
|
+
keepAliveMinutes,
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
if (isTermux) {
|
|
28
|
+
try {
|
|
29
|
+
execSync("termux-wake-lock", { stdio: "ignore" });
|
|
30
|
+
logSuccess("[Android Termux] Wake lock acquired to keep CPU awake in background.");
|
|
31
|
+
} catch {
|
|
32
|
+
// termux-tools might not be installed yet
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
logInfo("Checking Chromium browser availability...");
|
|
37
|
+
let chromiumExecutable = null;
|
|
38
|
+
try {
|
|
39
|
+
chromiumExecutable = await localBrowserManager.ensureChromiumExecutable();
|
|
40
|
+
logSuccess(`Chromium ready: ${chromiumExecutable}`);
|
|
41
|
+
} catch (browserErr) {
|
|
42
|
+
logWarn(`Chromium initialization warning: ${browserErr.message}`);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
logInfo("Connecting to Apps Manager cluster...");
|
|
46
|
+
|
|
47
|
+
const socket = io(`${serverUrl}/workers`, {
|
|
48
|
+
transports: ["websocket", "polling"],
|
|
49
|
+
reconnection: true,
|
|
50
|
+
reconnectionAttempts: Infinity,
|
|
51
|
+
reconnectionDelay: 1000,
|
|
52
|
+
reconnectionDelayMax: 5000,
|
|
53
|
+
auth: {
|
|
54
|
+
workerKey,
|
|
55
|
+
metadata: {
|
|
56
|
+
hostname: workerName || os.hostname(),
|
|
57
|
+
platform: os.platform(),
|
|
58
|
+
arch: os.arch(),
|
|
59
|
+
osVersion: os.release(),
|
|
60
|
+
nodeVersion: process.version,
|
|
61
|
+
chromiumVersion: chromiumExecutable ? path.basename(chromiumExecutable) : "",
|
|
62
|
+
chromiumPath: chromiumExecutable || "",
|
|
63
|
+
isTermux,
|
|
64
|
+
},
|
|
65
|
+
},
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
const CHUNK_SIZE = 256 * 1024; // 256 KB chunks to prevent any WebSocket frame buffer overflow
|
|
69
|
+
|
|
70
|
+
const sendJobResult = (payload) => {
|
|
71
|
+
let payloadStr = "";
|
|
72
|
+
try {
|
|
73
|
+
payloadStr = JSON.stringify(payload);
|
|
74
|
+
} catch {
|
|
75
|
+
socket.emit("job:result", payload);
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// Small payload: send directly
|
|
80
|
+
if (payloadStr.length <= CHUNK_SIZE) {
|
|
81
|
+
socket.emit("job:result", payload);
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Large payload (e.g. screenshot or large scrape): stream in chunks
|
|
86
|
+
const totalChunks = Math.ceil(payloadStr.length / CHUNK_SIZE);
|
|
87
|
+
const jobId = payload.jobId;
|
|
88
|
+
|
|
89
|
+
socket.emit("job:result:stream:start", {
|
|
90
|
+
jobId,
|
|
91
|
+
totalChunks,
|
|
92
|
+
totalLength: payloadStr.length,
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
for (let i = 0; i < totalChunks; i++) {
|
|
96
|
+
const chunk = payloadStr.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE);
|
|
97
|
+
socket.emit("job:result:stream:chunk", {
|
|
98
|
+
jobId,
|
|
99
|
+
index: i,
|
|
100
|
+
chunk,
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
socket.emit("job:result:stream:end", { jobId });
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
let activeJobs = 0;
|
|
108
|
+
let activeConcurrency = concurrency;
|
|
109
|
+
|
|
110
|
+
// Initialize WhatsApp Companion Device Engine
|
|
111
|
+
const waHandler = getWhatsAppHandler({
|
|
112
|
+
onStatusChange: (statusData) => {
|
|
113
|
+
socket.emit("whatsapp:status_update", statusData);
|
|
114
|
+
},
|
|
115
|
+
onQrGenerated: ({ qrDataUrl }) => {
|
|
116
|
+
socket.emit("whatsapp:qr_stream", { qrDataUrl });
|
|
117
|
+
},
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
socket.on("connect", () => {
|
|
121
|
+
logSuccess(`Connected to cluster! (Socket ID: ${socket.id})`);
|
|
122
|
+
logInfo(`Ready to accept scraping and visual preview jobs (Concurrency: ${activeConcurrency}).\n`);
|
|
123
|
+
|
|
124
|
+
// Announce current WhatsApp companion device status
|
|
125
|
+
const waTelemetry = waHandler.getTelemetry();
|
|
126
|
+
socket.emit("whatsapp:status_update", waTelemetry);
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
socket.on("worker:registered", (data) => {
|
|
130
|
+
if (data?.concurrencyLimit && typeof data.concurrencyLimit === "number") {
|
|
131
|
+
activeConcurrency = data.concurrencyLimit;
|
|
132
|
+
logInfo(`[Cluster Config] Worker registered as '${data.name || data.workerId}'. Active concurrency: ${activeConcurrency} concurrent tabs.`);
|
|
133
|
+
}
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
// Handle Admin Panel WhatsApp QR stream request
|
|
137
|
+
socket.on("whatsapp:request_qr", async () => {
|
|
138
|
+
logInfo("[WhatsApp] QR pairing requested via Admin Panel...");
|
|
139
|
+
try {
|
|
140
|
+
await waHandler.initialize(true);
|
|
141
|
+
} catch (err) {
|
|
142
|
+
logError(`[WhatsApp] Failed to initialize QR pairing: ${err.message}`);
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
// Handle Admin Panel Remote Logout
|
|
147
|
+
socket.on("whatsapp:remote_logout", async () => {
|
|
148
|
+
logInfo("[WhatsApp] Remote unpair requested via Admin Panel...");
|
|
149
|
+
try {
|
|
150
|
+
await waHandler.logout();
|
|
151
|
+
logSuccess("[WhatsApp] Companion device unpaired and local session cleared.");
|
|
152
|
+
} catch (err) {
|
|
153
|
+
logError(`[WhatsApp] Logout error: ${err.message}`);
|
|
154
|
+
}
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
socket.on("connect_error", (err) => {
|
|
158
|
+
if (err.message && err.message.includes("Authentication failed")) {
|
|
159
|
+
logError(`Authentication Failed: ${err.message}`);
|
|
160
|
+
logWarn("Please verify your worker key or run with '--reset' to enter a new key.");
|
|
161
|
+
} else {
|
|
162
|
+
logWarn(`Connection failed (${err.message}). Retrying in a few seconds...`);
|
|
163
|
+
}
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
socket.on("disconnect", (reason) => {
|
|
167
|
+
logWarn(`Disconnected from cluster: ${reason}`);
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
// Handle incoming job dispatch from server
|
|
171
|
+
socket.on("job:dispatch", async (job) => {
|
|
172
|
+
const jobId = job.jobId;
|
|
173
|
+
const jobType = job.jobType || job.type;
|
|
174
|
+
const payload = job.payload || {};
|
|
175
|
+
const targetUrl = jobType === "SEND_EMAIL"
|
|
176
|
+
? (payload.mailOptions?.to ? `to: ${payload.mailOptions.to}` : "email")
|
|
177
|
+
: jobType.startsWith("WHATSAPP_")
|
|
178
|
+
? (payload.recipientPhone ? `WA: ${payload.recipientPhone}` : "whatsapp")
|
|
179
|
+
: (payload.url || "(no url)");
|
|
180
|
+
|
|
181
|
+
logJob(jobType, targetUrl);
|
|
182
|
+
activeJobs++;
|
|
183
|
+
|
|
184
|
+
const startTime = Date.now();
|
|
185
|
+
|
|
186
|
+
try {
|
|
187
|
+
let resultData = null;
|
|
188
|
+
|
|
189
|
+
if (jobType === "SCRAPE_CHROMIUM") {
|
|
190
|
+
resultData = await handleChromiumScrape(payload);
|
|
191
|
+
} else if (jobType === "SCRAPE_HTTP") {
|
|
192
|
+
resultData = await handleHttpScrape(payload);
|
|
193
|
+
} else if (jobType === "PREVIEW_PAGE") {
|
|
194
|
+
resultData = await handlePreviewPage(payload);
|
|
195
|
+
} else if (jobType === "INSPECT_ELEMENT") {
|
|
196
|
+
resultData = await handleInspectElement(payload);
|
|
197
|
+
} else if (jobType === "SEND_EMAIL") {
|
|
198
|
+
resultData = await handleSendEmail(payload);
|
|
199
|
+
} else if (jobType === "AI_SMART_REPAIR") {
|
|
200
|
+
resultData = await handleAiSmartRepair(payload, config);
|
|
201
|
+
} else if (jobType === "WHATSAPP_SEND_MESSAGE") {
|
|
202
|
+
resultData = await waHandler.sendMessage(payload);
|
|
203
|
+
} else if (jobType === "WHATSAPP_SEND_DOCUMENT") {
|
|
204
|
+
resultData = await waHandler.sendDocument(payload);
|
|
205
|
+
} else if (jobType === "WHATSAPP_RECONCILE") {
|
|
206
|
+
resultData = await waHandler.reconcileDelivery(payload.recipientPhone, payload.idempotencyKey);
|
|
207
|
+
} else {
|
|
208
|
+
throw new Error(`Unsupported job type: ${jobType}`);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
const durationMs = Date.now() - startTime;
|
|
212
|
+
logSuccess(`Completed ${jobType} for ${targetUrl} in ${durationMs}ms`);
|
|
213
|
+
|
|
214
|
+
sendJobResult({
|
|
215
|
+
jobId,
|
|
216
|
+
success: true,
|
|
217
|
+
data: resultData,
|
|
218
|
+
durationMs,
|
|
219
|
+
});
|
|
220
|
+
} catch (err) {
|
|
221
|
+
const durationMs = Date.now() - startTime;
|
|
222
|
+
logError(`Failed ${jobType} for ${targetUrl}: ${err.message}`);
|
|
223
|
+
|
|
224
|
+
sendJobResult({
|
|
225
|
+
jobId,
|
|
226
|
+
success: false,
|
|
227
|
+
error: {
|
|
228
|
+
message: err.message,
|
|
229
|
+
code: err.code || "JOB_EXECUTION_ERROR",
|
|
230
|
+
},
|
|
231
|
+
durationMs,
|
|
232
|
+
});
|
|
233
|
+
} finally {
|
|
234
|
+
activeJobs = Math.max(0, activeJobs - 1);
|
|
235
|
+
}
|
|
236
|
+
});
|
|
237
|
+
|
|
238
|
+
// Handle cluster heartbeat ping with enhanced capacity & memory telemetry
|
|
239
|
+
socket.on("ping", () => {
|
|
240
|
+
const telemetry = waHandler.getTelemetry();
|
|
241
|
+
socket.emit("pong", {
|
|
242
|
+
activeJobs,
|
|
243
|
+
memory: process.memoryUsage(),
|
|
244
|
+
uptime: process.uptime(),
|
|
245
|
+
freeSystemMemoryMb: telemetry.freeSystemMemoryMb,
|
|
246
|
+
totalSystemMemoryMb: telemetry.totalSystemMemoryMb,
|
|
247
|
+
processMemoryMb: telemetry.processMemoryMb,
|
|
248
|
+
memoryPressure: telemetry.memoryPressure,
|
|
249
|
+
whatsappStatus: telemetry.status,
|
|
250
|
+
whatsappUser: telemetry.user,
|
|
251
|
+
});
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
// Render Anti-Sleep Keep-Alive Ping
|
|
255
|
+
let keepAliveTimer = null;
|
|
256
|
+
let initialKeepAliveTimer = null;
|
|
257
|
+
|
|
258
|
+
const clearKeepAliveTimers = () => {
|
|
259
|
+
if (keepAliveTimer) {
|
|
260
|
+
clearInterval(keepAliveTimer);
|
|
261
|
+
keepAliveTimer = null;
|
|
262
|
+
}
|
|
263
|
+
if (initialKeepAliveTimer) {
|
|
264
|
+
clearTimeout(initialKeepAliveTimer);
|
|
265
|
+
initialKeepAliveTimer = null;
|
|
266
|
+
}
|
|
267
|
+
};
|
|
268
|
+
|
|
269
|
+
const pingKeepAlive = async () => {
|
|
270
|
+
try {
|
|
271
|
+
// Try the dedicated keep-alive endpoint first
|
|
272
|
+
const endpoint = `${serverUrl}/api/workers/keep-alive`;
|
|
273
|
+
const res = await axios.get(endpoint, {
|
|
274
|
+
timeout: 15000,
|
|
275
|
+
headers: {
|
|
276
|
+
"x-worker-key": workerKey || "",
|
|
277
|
+
"user-agent": "AppsManagerEdgeWorker/1.1.0",
|
|
278
|
+
},
|
|
279
|
+
});
|
|
280
|
+
logInfo(`[Keep-Alive] Server awake ping OK (${res.status}) - Render instance active`);
|
|
281
|
+
} catch (err) {
|
|
282
|
+
// Fallback to /health
|
|
283
|
+
try {
|
|
284
|
+
const fallbackUrl = `${serverUrl}/health`;
|
|
285
|
+
await axios.get(fallbackUrl, {
|
|
286
|
+
timeout: 10000,
|
|
287
|
+
headers: {
|
|
288
|
+
"user-agent": "AppsManagerEdgeWorker/1.1.0",
|
|
289
|
+
},
|
|
290
|
+
});
|
|
291
|
+
logInfo(`[Keep-Alive] Fallback health ping OK - Render instance active`);
|
|
292
|
+
} catch (fallbackErr) {
|
|
293
|
+
logWarn(`[Keep-Alive] Server awake ping warning (${err.message}) - will retry next cycle`);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
};
|
|
297
|
+
|
|
298
|
+
const startKeepAlive = () => {
|
|
299
|
+
if (keepAliveMinutes <= 0) return;
|
|
300
|
+
clearKeepAliveTimers();
|
|
301
|
+
|
|
302
|
+
// Send initial ping 15s after startup to warm up/verify
|
|
303
|
+
initialKeepAliveTimer = setTimeout(pingKeepAlive, 15000);
|
|
304
|
+
if (initialKeepAliveTimer.unref) initialKeepAliveTimer.unref();
|
|
305
|
+
|
|
306
|
+
const intervalMs = Math.max(1, keepAliveMinutes) * 60 * 1000;
|
|
307
|
+
keepAliveTimer = setInterval(pingKeepAlive, intervalMs);
|
|
308
|
+
if (keepAliveTimer.unref) keepAliveTimer.unref();
|
|
309
|
+
};
|
|
310
|
+
|
|
311
|
+
if (keepAliveMinutes > 0) {
|
|
312
|
+
startKeepAlive();
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
// Clean up keep-alive timers on socket disconnect
|
|
316
|
+
socket.on("disconnect", (reason) => {
|
|
317
|
+
logWarn(`Disconnected from cluster: ${reason}`);
|
|
318
|
+
clearKeepAliveTimers();
|
|
319
|
+
});
|
|
320
|
+
|
|
321
|
+
socket.on("connect", () => {
|
|
322
|
+
if (keepAliveMinutes > 0 && !keepAliveTimer) {
|
|
323
|
+
startKeepAlive();
|
|
324
|
+
}
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
// Provide programmatic close for automated test harnesses
|
|
328
|
+
socket.closeClient = async () => {
|
|
329
|
+
clearKeepAliveTimers();
|
|
330
|
+
if (isTermux) {
|
|
331
|
+
try {
|
|
332
|
+
execSync("termux-wake-unlock", { stdio: "ignore" });
|
|
333
|
+
} catch {}
|
|
334
|
+
}
|
|
335
|
+
socket.disconnect();
|
|
336
|
+
await localBrowserManager.close();
|
|
337
|
+
};
|
|
338
|
+
|
|
339
|
+
// Graceful shutdown handling
|
|
340
|
+
const shutdown = async () => {
|
|
341
|
+
logInfo("\nGracefully shutting down edge worker...");
|
|
342
|
+
await socket.closeClient();
|
|
343
|
+
logSuccess("Edge worker stopped cleanly.");
|
|
344
|
+
process.exit(0);
|
|
345
|
+
};
|
|
346
|
+
|
|
347
|
+
process.on("SIGINT", shutdown);
|
|
348
|
+
process.on("SIGTERM", shutdown);
|
|
349
|
+
|
|
350
|
+
return socket;
|
|
351
|
+
}
|