diggylab 0.1.50 → 0.1.51

package/README.md

@@ -4,6 +4,9 @@ The Desktop Agent also runs headless as a CLI worker, for customers who scan fro
 instead of the desktop app. It pairs like any device, receives `DESKTOP` scan jobs, and scans with
 headless Chromium — the Electron app is not involved.
 
+For the platform-run scan worker (admin-managed Proxmox/Linux nodes with one Facebook account and an
+optional proxy), see [worker-agent.md](worker-agent.md).
+
 ## Requirements
 
 - Node.js 20+

package/cli.js

@@ -8,6 +8,93 @@ import { hostname, homedir } from "node:os";
 import { dirname as dirname3, join as join4 } from "node:path";
 import { fileURLToPath } from "node:url";
 
+// ../../packages/shared/src/index.ts
+var DESKTOP_AGENT_PROTOCOL_VERSION = 1;
+var DESKTOP_POST_COMMENTS_MAX_LIMIT = 200;
+var DESKTOP_POST_COMMENTS_REQUEST_TIMEOUT_MS = 3 * 60 * 1e3;
+var FACEBOOK_DOMAIN = ".facebook.com";
+var FACEBOOK_HOST = /(^|\.)facebook\.com$/i;
+function isFacebookDomain(domain) {
+  return Boolean(domain && FACEBOOK_HOST.test(domain.replace(/^\./, "")));
+}
+function normalizeSameSite(value) {
+  if (typeof value !== "string") return void 0;
+  const lower = value.toLowerCase().replace(/[^a-z]/g, "");
+  if (lower === "lax") return "Lax";
+  if (lower === "strict") return "Strict";
+  if (lower === "none" || lower === "norestriction") return "None";
+  return void 0;
+}
+function toCookie(value) {
+  const name = typeof value.name === "string" ? value.name.trim() : "";
+  const cookieValue = typeof value.value === "string" ? value.value : "";
+  if (!name || !cookieValue) return null;
+  return {
+    domain: typeof value.domain === "string" ? value.domain : void 0,
+    expires: typeof value.expires === "number" ? value.expires : typeof value.expirationDate === "number" ? value.expirationDate : void 0,
+    httpOnly: typeof value.httpOnly === "boolean" ? value.httpOnly : void 0,
+    name,
+    path: typeof value.path === "string" ? value.path : void 0,
+    sameSite: normalizeSameSite(value.sameSite),
+    secure: typeof value.secure === "boolean" ? value.secure : void 0,
+    value: cookieValue
+  };
+}
+function parseCookieHeader(input) {
+  const header = input.replace(/^\s*cookie\s*:/i, "");
+  return header.split(/[;\n]/).map((part) => part.trim()).filter(Boolean).map((part) => {
+    const separator = part.indexOf("=");
+    if (separator <= 0) return null;
+    return { name: part.slice(0, separator).trim(), value: part.slice(separator + 1) };
+  }).filter((cookie) => cookie !== null);
+}
+function parseCookieTable(input) {
+  const cookies = [];
+  for (const line of input.split("\n")) {
+    const columns = line.split("	");
+    const name = columns[0]?.trim() ?? "";
+    const value = columns[1];
+    if (!value || !name || name === "Name") continue;
+    cookies.push({ domain: columns[2]?.trim(), name, value });
+  }
+  return cookies;
+}
+function parseCookieInput(input) {
+  const trimmed = input.trim();
+  if (!trimmed) return [];
+  if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) {
+    return trimmed.includes("	") ? parseCookieTable(trimmed) : parseCookieHeader(trimmed);
+  }
+  try {
+    const parsed = JSON.parse(trimmed);
+    if (Array.isArray(parsed)) {
+      return parsed.map((entry) => entry && typeof entry === "object" ? toCookie(entry) : null).filter((cookie) => cookie !== null);
+    }
+    if (parsed && typeof parsed === "object") {
+      const cookies = parsed.cookies;
+      if (Array.isArray(cookies)) {
+        return cookies.map((entry) => entry && typeof entry === "object" ? toCookie(entry) : null).filter((cookie) => cookie !== null);
+      }
+    }
+  } catch {
+    return [];
+  }
+  return [];
+}
+function toFacebookStorageState(cookies) {
+  const usable = cookies.filter((cookie) => isFacebookDomain(cookie.domain) || cookie.domain === void 0).map((cookie) => ({
+    domain: isFacebookDomain(cookie.domain) ? cookie.domain : FACEBOOK_DOMAIN,
+    expires: typeof cookie.expires === "number" && cookie.expires > 0 ? cookie.expires : -1,
+    httpOnly: cookie.httpOnly ?? true,
+    name: cookie.name,
+    path: cookie.path?.startsWith("/") ? cookie.path : "/",
+    sameSite: cookie.sameSite ?? "Lax",
+    secure: cookie.secure ?? true,
+    value: cookie.value
+  }));
+  return { cookies: usable, origins: [] };
+}
+
 // ../../packages/facebook-scanner/dist/parser.js
 var COMMENT_TEXT_LIMIT = 2e3;
 var COMMENT_AUTHOR_LIMIT = 200;
@@ -708,6 +795,7 @@ var FacebookScannerError = class extends Error {
 };
 var FacebookScanner = class {
   browserType;
+  proxy;
   extractDomItems;
   extractDomComments;
   parseGraphqlBody;
@@ -717,6 +805,7 @@ var FacebookScanner = class {
   running = false;
   constructor(options) {
     this.browserType = options.browserType;
+    this.proxy = options.proxy;
     this.extractDomItems = options.extractDomItems ?? extractFacebookDomItems;
     this.extractDomComments = options.extractDomComments ?? extractFacebookDomComments;
     this.parseGraphqlBody = options.parseGraphqlBody ?? parseFacebookGraphqlBody;
@@ -730,7 +819,10 @@ var FacebookScanner = class {
     this.running = true;
     let browser = null;
     try {
-      browser = await this.browserType.launch({ headless: true });
+      browser = await this.browserType.launch({
+        headless: true,
+        ...this.proxy ? { proxy: this.proxy } : {}
+      });
       const results = [];
       for (const [index, groupUrl] of options.groupUrls.entries()) {
         assertNotAborted(options.signal);
@@ -753,7 +845,10 @@ var FacebookScanner = class {
     this.running = true;
     let browser = null;
     try {
-      browser = await this.browserType.launch({ headless: true });
+      browser = await this.browserType.launch({
+        headless: true,
+        ...this.proxy ? { proxy: this.proxy } : {}
+      });
       const context = await browser.newContext({ storageState: options.storageState });
       let page = null;
       const graphqlComments = [];
@@ -1360,13 +1455,6 @@ function isDeviceSummary(value) {
 
 // src/main/desktop-agent-session.ts
 import { randomUUID } from "node:crypto";
-
-// ../../packages/shared/src/index.ts
-var DESKTOP_AGENT_PROTOCOL_VERSION = 1;
-var DESKTOP_POST_COMMENTS_MAX_LIMIT = 200;
-var DESKTOP_POST_COMMENTS_REQUEST_TIMEOUT_MS = 3 * 60 * 1e3;
-
-// src/main/desktop-agent-session.ts
 var MESSAGE_FRESHNESS_MS = 5 * 60 * 1e3;
 var MAX_REPLAY_IDS = 512;
 var DesktopAgentSession = class {
@@ -1527,6 +1615,7 @@ var DesktopAgentSession = class {
           envelope.payload.challenge
         ),
         agentVersion: this.options.agentVersion,
+        ...this.options.clientType ? { clientType: this.options.clientType } : {},
         platform: this.options.platform,
         state: this.reportedState,
         facebookReady: this.options.getFacebookReady(),
@@ -1594,10 +1683,12 @@ var DesktopAgentSession = class {
     this.clearHeartbeat();
     this.heartbeatTimer = this.options.setTimer(() => {
       if (generation !== this.generation || socket !== this.socket) return;
+      const metrics = this.options.metrics?.();
       this.send(socket, deviceId, "HEARTBEAT", {
         state: this.reportedState,
         facebookReady: this.options.getFacebookReady(),
-        currentJobId: this.reportedJobId
+        currentJobId: this.reportedJobId,
+        ...metrics ? { metrics } : {}
       });
       this.scheduleHeartbeat(socket, deviceId, interval, generation);
     }, interval);
@@ -1703,9 +1794,9 @@ var DesktopAgentSession = class {
     if (this.accessRefreshTimer) this.options.clearTimer(this.accessRefreshTimer);
     this.accessRefreshTimer = null;
   }
-  update(snapshot) {
-    this.snapshot = snapshot;
-    for (const listener of this.listeners) listener(snapshot);
+  update(snapshot2) {
+    this.snapshot = snapshot2;
+    for (const listener of this.listeners) listener(snapshot2);
   }
 };
 function parseServerEnvelope(data) {
@@ -2542,9 +2633,9 @@ var DesktopScanRunner = class {
   leaseExpiresAt(job) {
     return this.activeAttempt?.jobId === job.jobId && this.activeAttempt.attemptToken === job.attemptToken ? this.activeAttempt.leaseExpiresAt : job.leaseExpiresAt;
   }
-  update(snapshot) {
-    this.snapshot = snapshot;
-    for (const listener of this.listeners) listener(snapshot);
+  update(snapshot2) {
+    this.snapshot = snapshot2;
+    for (const listener of this.listeners) listener(snapshot2);
   }
 };
 function toDesktopPost(item) {
@@ -2870,7 +2961,7 @@ var FacebookLoginService = class {
     await this.finish({ state: "READY", errorCode: null, accountId });
     return true;
   }
-  async finish(snapshot) {
+  async finish(snapshot2) {
     if (this.timer) this.options.clearTimer(this.timer);
     this.timer = null;
     const page = this.page;
@@ -2882,12 +2973,12 @@ var FacebookLoginService = class {
     page?.off("framenavigated", this.navigationHandler);
     await context?.close();
     await browser?.close();
-    this.update(snapshot);
+    this.update(snapshot2);
   }
-  update(snapshot) {
-    if (snapshot.state === this.snapshot.state && snapshot.errorCode === this.snapshot.errorCode && snapshot.accountId === this.snapshot.accountId) return;
-    this.snapshot = snapshot;
-    for (const listener of this.listeners) listener(snapshot);
+  update(snapshot2) {
+    if (snapshot2.state === this.snapshot.state && snapshot2.errorCode === this.snapshot.errorCode && snapshot2.accountId === this.snapshot.accountId) return;
+    this.snapshot = snapshot2;
+    for (const listener of this.listeners) listener(snapshot2);
   }
   serialize(operation) {
     const result = this.pending.catch(() => void 0).then(operation);
@@ -2943,6 +3034,41 @@ function getFacebookAccountId(state) {
   return value && /^\d{1,32}$/.test(value) ? value : null;
 }
 
+// src/shared/resource-metrics.ts
+import os from "node:os";
+function snapshot(cpus) {
+  let idle = 0;
+  let total = 0;
+  for (const cpu of cpus) {
+    const times = cpu.times;
+    idle += times.idle;
+    total += times.user + times.nice + times.sys + times.idle + times.irq;
+  }
+  return { idle, total };
+}
+function usageBetween(previous, current) {
+  const totalDelta = current.total - previous.total;
+  const idleDelta = current.idle - previous.idle;
+  if (totalDelta <= 0) return 0;
+  return Math.min(100, Math.max(0, (1 - idleDelta / totalDelta) * 100));
+}
+function createMetricsCollector() {
+  let previous = null;
+  return () => {
+    const cpus = os.cpus();
+    const current = snapshot(cpus);
+    const cpuPercent = previous ? usageBetween(previous, current) : usageBetween({ idle: 0, total: 0 }, current);
+    previous = current;
+    const memTotalBytes = os.totalmem();
+    return {
+      cpuCount: cpus.length,
+      cpuPercent: Math.round(cpuPercent * 10) / 10,
+      memTotalBytes,
+      memUsedBytes: Math.max(0, memTotalBytes - os.freemem())
+    };
+  };
+}
+
 // src/main/graphql-dump.ts
 import { writeFile as writeFile4 } from "node:fs/promises";
 import { join as join3 } from "node:path";
@@ -3047,90 +3173,6 @@ function browserInstallCommands(platform) {
   return platform === "linux" ? [["install", "--with-deps", "chromium"], install] : [install];
 }
 
-// src/cli/cookie-import.ts
-var FACEBOOK_DOMAIN = ".facebook.com";
-var FACEBOOK_HOST = /(^|\.)facebook\.com$/i;
-function isFacebookDomain(domain) {
-  return Boolean(domain && FACEBOOK_HOST.test(domain.replace(/^\./, "")));
-}
-function normalizeSameSite(value) {
-  if (typeof value !== "string") return void 0;
-  const lower = value.toLowerCase().replace(/[^a-z]/g, "");
-  if (lower === "lax") return "Lax";
-  if (lower === "strict") return "Strict";
-  if (lower === "none" || lower === "norestriction") return "None";
-  return void 0;
-}
-function toCookie(value) {
-  const name = typeof value.name === "string" ? value.name.trim() : "";
-  const cookieValue = typeof value.value === "string" ? value.value : "";
-  if (!name || !cookieValue) return null;
-  return {
-    domain: typeof value.domain === "string" ? value.domain : void 0,
-    expires: typeof value.expires === "number" ? value.expires : typeof value.expirationDate === "number" ? value.expirationDate : void 0,
-    httpOnly: typeof value.httpOnly === "boolean" ? value.httpOnly : void 0,
-    name,
-    path: typeof value.path === "string" ? value.path : void 0,
-    sameSite: normalizeSameSite(value.sameSite),
-    secure: typeof value.secure === "boolean" ? value.secure : void 0,
-    value: cookieValue
-  };
-}
-function parseCookieHeader(input) {
-  const header = input.replace(/^\s*cookie\s*:/i, "");
-  return header.split(/[;\n]/).map((part) => part.trim()).filter(Boolean).map((part) => {
-    const separator = part.indexOf("=");
-    if (separator <= 0) return null;
-    return { name: part.slice(0, separator).trim(), value: part.slice(separator + 1) };
-  }).filter((cookie) => cookie !== null);
-}
-function parseCookieTable(input) {
-  const cookies = [];
-  for (const line of input.split("\n")) {
-    const columns = line.split("	");
-    const name = columns[0]?.trim() ?? "";
-    const value = columns[1];
-    if (!value || !name || name === "Name") continue;
-    cookies.push({ domain: columns[2]?.trim(), name, value });
-  }
-  return cookies;
-}
-function parseCookieInput(input) {
-  const trimmed = input.trim();
-  if (!trimmed) return [];
-  if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) {
-    return trimmed.includes("	") ? parseCookieTable(trimmed) : parseCookieHeader(trimmed);
-  }
-  try {
-    const parsed = JSON.parse(trimmed);
-    if (Array.isArray(parsed)) {
-      return parsed.map((entry) => entry && typeof entry === "object" ? toCookie(entry) : null).filter((cookie) => cookie !== null);
-    }
-    if (parsed && typeof parsed === "object") {
-      const cookies = parsed.cookies;
-      if (Array.isArray(cookies)) {
-        return cookies.map((entry) => entry && typeof entry === "object" ? toCookie(entry) : null).filter((cookie) => cookie !== null);
-      }
-    }
-  } catch {
-    return [];
-  }
-  return [];
-}
-function toFacebookStorageState(cookies) {
-  const usable = cookies.filter((cookie) => isFacebookDomain(cookie.domain) || cookie.domain === void 0).map((cookie) => ({
-    domain: isFacebookDomain(cookie.domain) ? cookie.domain : FACEBOOK_DOMAIN,
-    expires: typeof cookie.expires === "number" && cookie.expires > 0 ? cookie.expires : -1,
-    httpOnly: cookie.httpOnly ?? true,
-    name: cookie.name,
-    path: cookie.path?.startsWith("/") ? cookie.path : "/",
-    sameSite: cookie.sameSite ?? "Lax",
-    secure: cookie.secure ?? true,
-    value: cookie.value
-  }));
-  return { cookies: usable, origins: [] };
-}
-
 // src/cli/file-crypto.ts
 import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
 import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
@@ -3184,15 +3226,15 @@ var SERVICE_PATH = `/etc/systemd/system/${SERVICE_NAME}.service`;
 function quote(value) {
   return /\s/.test(value) ? `"${value.replace(/"/g, '\\"')}"` : value;
 }
-function buildSystemdUnit(options) {
+function buildServiceUnit(options) {
   return `[Unit]
-Description=DiggyLab Desktop Agent
+Description=${options.description}
 After=network-online.target
 Wants=network-online.target
 
 [Service]
 ExecStart=${options.execStart.map(quote).join(" ")} run
-Environment="DIGGYLAB_AGENT_DIR=${options.configDirectory}"
+Environment="${options.environmentName}=${options.configDirectory}"
 Environment="DIGGYLAB_API_URL=${options.apiOrigin}"
 Restart=on-failure
 RestartSec=10
@@ -3201,6 +3243,13 @@ RestartSec=10
 WantedBy=multi-user.target
 `;
 }
+function buildSystemdUnit(options) {
+  return buildServiceUnit({
+    ...options,
+    description: "DiggyLab Desktop Agent",
+    environmentName: "DIGGYLAB_AGENT_DIR"
+  });
+}
 
 // src/cli/cli.ts
 var LOGIN_TIMEOUT_MS = 10 * 60 * 1e3;
@@ -3289,17 +3338,17 @@ function waitForFacebookLogin(login, timeoutMs) {
       reject(new Error("Timed out waiting for the Facebook login."));
     }, timeoutMs);
     timer.unref();
-    const unsubscribe = login.subscribe((snapshot) => {
-      if (snapshot.state === "READY") {
+    const unsubscribe = login.subscribe((snapshot2) => {
+      if (snapshot2.state === "READY") {
         clearTimeout(timer);
         unsubscribe();
         resolve();
         return;
       }
-      if (snapshot.state === "NOT_READY" && snapshot.errorCode) {
+      if (snapshot2.state === "NOT_READY" && snapshot2.errorCode) {
         clearTimeout(timer);
         unsubscribe();
-        reject(new Error(`Facebook login failed (${snapshot.errorCode}).`));
+        reject(new Error(`Facebook login failed (${snapshot2.errorCode}).`));
       }
     });
     void login.open().catch((error) => {
@@ -3609,6 +3658,8 @@ async function commandRun(runtime, io) {
   const facebookReady = await runtime.facebook.isReady();
   const session = new DesktopAgentSession({
     agentVersion: agentVersion(),
+    clientType: "CLI",
+    metrics: createMetricsCollector(),
     clearTimer: (timer) => clearTimeout(timer),
     createSocket: (url, accessToken) => {
       const socket = new WebSocket(url, {
@@ -3685,18 +3736,18 @@ async function commandRun(runtime, io) {
   const replaced = new Promise((resolve) => {
     onReplaced = resolve;
   });
-  session.subscribe((snapshot) => {
-    io.log(`${stamp()} session: ${snapshot.state}${snapshot.errorCode ? ` (${snapshot.errorCode})` : ""}`);
-    recordActivity(`Session state changed to ${snapshot.state}.`, snapshot.state);
-    if (snapshot.errorCode === "CONNECTION_REPLACED") {
+  session.subscribe((snapshot2) => {
+    io.log(`${stamp()} session: ${snapshot2.state}${snapshot2.errorCode ? ` (${snapshot2.errorCode})` : ""}`);
+    recordActivity(`Session state changed to ${snapshot2.state}.`, snapshot2.state);
+    if (snapshot2.errorCode === "CONNECTION_REPLACED") {
       io.error("Another worker is already running with this device, so this one stopped.");
       io.error("Stop the other worker (for example: systemctl stop diggylab), then start this one again.");
       onReplaced();
     }
   });
-  scanRunner.subscribe((snapshot) => {
-    io.log(`scanner: ${snapshot.state}${snapshot.jobId ? ` (job ${snapshot.jobId})` : ""}`);
-    recordActivity(`Scanner state changed to ${snapshot.state}.`, snapshot.state);
+  scanRunner.subscribe((snapshot2) => {
+    io.log(`scanner: ${snapshot2.state}${snapshot2.jobId ? ` (job ${snapshot2.jobId})` : ""}`);
+    recordActivity(`Scanner state changed to ${snapshot2.state}.`, snapshot2.state);
   });
   scanRunner.start();
   commentRunner.start();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "diggylab",
-  "version": "0.1.50",
+  "version": "0.1.51",
   "description": "DiggyLab Desktop Agent collects posts and information from Facebook Groups on a paired computer. (headless CLI)",
   "type": "module",
   "bin": {