diggylab 0.1.43 → 0.1.45

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +16 -7
  2. package/cli.js +430 -27
  3. package/package.json +1 -1
package/README.md CHANGED
@@ -111,8 +111,12 @@ The equivalents: `diggylab login --cookies 'name=value; ...'` for a quoted one-l
111
111
  `diggylab login --cookies cookies.json` for a saved file.
112
112
 
113
113
  Only `name` and `value` are required; the agent fills in the rest (`.facebook.com`, `/`, session
114
- expiry). Every `facebook.com` cookie is kept, and at least `c_user` plus `xs` must be present — those
115
- two carry the session, and `datr` is worth keeping too.
114
+ expiry). Every `facebook.com` cookie is kept. The export must contain `c_user` (the agent rejects it
115
+ otherwise) and `xs`, which carries the session — without `xs` Facebook refuses the session during the
116
+ check, and `datr` is worth keeping too.
117
+
118
+ If you paste cookies, prefer the file form or `--cookies -` over `--cookies '...'`: a value on the
119
+ command line stays in your shell history, and pasted text stays in the terminal scrollback.
116
120
 
117
121
  Do not use `document.cookie` from the Console: Facebook's `c_user` and `xs` are HttpOnly, so they
118
122
  never appear there. The DevTools cookie table and the extension do show them.
@@ -126,12 +130,17 @@ happens the command says so — approve the login from your phone and import aga
126
130
 
127
131
  ## Troubleshooting
128
132
 
133
+ - `socket closed: 1005` followed by `session: ONLINE` about every 14 minutes is **normal**: the access
134
+ token lives 15 minutes, so the worker refreshes it one minute early and reconnects with the new one
135
+ (the disconnect lasts a second or two, and a running scan keeps going).
129
136
  - `socket closed: 4009` right after `session: ONLINE` means **two workers use the same device** — for
130
137
  example the systemd service plus a manual `diggylab run`. The worker that connected last keeps the
131
138
  device and the older one stops with a message. Check `pgrep -af "diggylab run"` and
132
139
  `systemctl status diggylab --no-pager`, then keep only one.
133
- - `socket closed: 1006` with reconnects is a dropped connection; the worker keeps retrying with
134
- backoff. If it never settles, check that the server can reach the API origin.
140
+ - `socket error:` plus `socket closed: 1006` and `RECONNECTING (NETWORK_ERROR)` is a dropped
141
+ connection; the worker retries with backoff. If it never settles, check `curl -sS -o /dev/null -w
142
+ '%{http_code}\n' https://api.diggylab.com/health` and that the machine clock is right (`date -u`),
143
+ because a clock off by more than five minutes makes the server's messages look stale.
135
144
  - `Facebook: missing` after a login means the session did not survive; run `diggylab login` again.
136
145
 
137
146
  ## Security notes
@@ -167,9 +176,9 @@ Wants=network-online.target
167
176
 
168
177
  [Service]
169
178
  ExecStart=/usr/local/bin/diggylab run
170
- Environment=DIGGYLAB_AGENT_DIR=/root/.diggylab
171
- Environment=DIGGYLAB_API_URL=https://api.diggylab.com
172
- Restart=always
179
+ Environment="DIGGYLAB_AGENT_DIR=/root/.diggylab"
180
+ Environment="DIGGYLAB_API_URL=https://api.diggylab.com"
181
+ Restart=on-failure
173
182
  RestartSec=10
174
183
 
175
184
  [Install]
package/cli.js CHANGED
@@ -9,6 +9,8 @@ import { dirname as dirname3, join as join3 } from "node:path";
9
9
  import { fileURLToPath } from "node:url";
10
10
 
11
11
  // ../../packages/facebook-scanner/dist/parser.js
12
+ var COMMENT_TEXT_LIMIT = 2e3;
13
+ var COMMENT_AUTHOR_LIMIT = 200;
12
14
  var paths = {
13
15
  text: [["message", "text"], ["comet_sections", "content", "story", "message", "text"], ["text"]],
14
16
  url: [["url"], ["permalink_url"], ["wwwURL"], ["comet_sections", "content", "story", "url"]],
@@ -36,6 +38,21 @@ var textFormatPaths = [
36
38
  "text_format_metadata"
37
39
  ]
38
40
  ];
41
+ var commentTextPaths = [
42
+ ["body", "text"],
43
+ ["comment_body", "text"],
44
+ ["preferred_body", "text"]
45
+ ];
46
+ var commentAuthorPaths = [
47
+ ["author", "name"],
48
+ ["author_name"],
49
+ ["author", "short_name"]
50
+ ];
51
+ var commentTimePaths = [
52
+ ["created_time"],
53
+ ["creation_time"],
54
+ ["publish_time"]
55
+ ];
39
56
  var facebookHosts = /* @__PURE__ */ new Set([
40
57
  "facebook.com",
41
58
  "fb.com",
@@ -327,6 +344,68 @@ function parseFacebookGraphqlBody(body, groupUrl) {
327
344
  }
328
345
  return mergeFacebookScanItems(items, Number.POSITIVE_INFINITY);
329
346
  }
347
+ function toComment(node) {
348
+ const typename = typeof node.__typename === "string" ? node.__typename : void 0;
349
+ const text = firstString(node, commentTextPaths);
350
+ if (!text || typename && !/comment$/i.test(typename))
351
+ return void 0;
352
+ const authorName = firstString(node, commentAuthorPaths);
353
+ const publishedAt = toIso(firstString(node, commentTimePaths));
354
+ return {
355
+ text: text.trim(),
356
+ ...authorName ? { authorName } : {},
357
+ ...publishedAt ? { publishedAt } : {}
358
+ };
359
+ }
360
+ function visitComments(value, items) {
361
+ if (Array.isArray(value)) {
362
+ for (const item of value)
363
+ visitComments(item, items);
364
+ return;
365
+ }
366
+ if (!isData(value))
367
+ return;
368
+ const comment = toComment(value);
369
+ if (comment)
370
+ items.push(comment);
371
+ for (const child of Object.values(value))
372
+ visitComments(child, items);
373
+ }
374
+ function parseFacebookCommentBodies(body) {
375
+ const items = [];
376
+ for (const line of body.split(/\r?\n/)) {
377
+ const json = line.trim().replace(/^for\s*\(;;\);/, "");
378
+ if (!json.startsWith("{") && !json.startsWith("["))
379
+ continue;
380
+ try {
381
+ visitComments(JSON.parse(json), items);
382
+ } catch {
383
+ }
384
+ }
385
+ return mergeFacebookComments(items, Number.POSITIVE_INFINITY);
386
+ }
387
+ function mergeFacebookComments(items, limit) {
388
+ const seen = /* @__PURE__ */ new Set();
389
+ const merged = [];
390
+ for (const item of items) {
391
+ const text = item.text.trim();
392
+ if (!text)
393
+ continue;
394
+ const authorName = item.authorName?.trim();
395
+ const key = `${authorName ?? ""}\0${text}\0${item.publishedAt ?? ""}`;
396
+ if (seen.has(key))
397
+ continue;
398
+ if (merged.length >= limit)
399
+ break;
400
+ seen.add(key);
401
+ merged.push({
402
+ text: text.slice(0, COMMENT_TEXT_LIMIT),
403
+ ...authorName ? { authorName: authorName.slice(0, COMMENT_AUTHOR_LIMIT) } : {},
404
+ ...item.publishedAt ? { publishedAt: item.publishedAt } : {}
405
+ });
406
+ }
407
+ return merged;
408
+ }
330
409
  async function extractFacebookDomItems(page, groupUrl) {
331
410
  const requestedGroup = requestedGroupIdentifier(groupUrl);
332
411
  if (!requestedGroup)
@@ -368,6 +447,28 @@ async function extractFacebookDomItems(page, groupUrl) {
368
447
  }] : [];
369
448
  }), Number.POSITIVE_INFINITY);
370
449
  }
450
+ async function extractFacebookDomComments(page) {
451
+ const raw = await page.locator('[role="article"][aria-label]').evaluateAll((articles) => articles.map((article) => {
452
+ const label = (article.getAttribute("aria-label") ?? "").trim();
453
+ if (!/^(?:bình luận|comment|trả lời|reply)\b/i.test(label) || !article.checkVisibility()) {
454
+ return void 0;
455
+ }
456
+ const belongsToArticle = (element) => element.closest('[role="article"]') === article;
457
+ const text = Array.from(article.querySelectorAll('[dir="auto"]')).filter((element) => belongsToArticle(element) && element.checkVisibility()).map((element) => element.innerText.trim()).filter(Boolean).sort((left, right) => right.length - left.length)[0] ?? "";
458
+ const authorMatch = /(?:dưới tên|by)\s+(.+?)(?:\s+(?:vào|on)\s+|$)/i.exec(label);
459
+ const time = Array.from(article.querySelectorAll("time[datetime], abbr[data-utime]")).find((element) => belongsToArticle(element) && element.checkVisibility());
460
+ return {
461
+ text,
462
+ authorName: authorMatch?.[1]?.trim(),
463
+ publishedAt: time?.getAttribute("datetime") ?? time?.getAttribute("data-utime") ?? void 0
464
+ };
465
+ }));
466
+ return mergeFacebookComments(raw.flatMap((item) => item?.text.trim() ? [{
467
+ text: item.text,
468
+ ...item.authorName ? { authorName: item.authorName } : {},
469
+ ...toIso(item.publishedAt) ? { publishedAt: toIso(item.publishedAt) } : {}
470
+ }] : []), Number.POSITIVE_INFINITY);
471
+ }
371
472
 
372
473
  // ../../packages/facebook-scanner/dist/session.js
373
474
  async function detectFacebookSession(page, context) {
@@ -403,13 +504,17 @@ var FacebookScannerError = class extends Error {
403
504
  var FacebookScanner = class {
404
505
  browserType;
405
506
  extractDomItems;
507
+ extractDomComments;
406
508
  parseGraphqlBody;
509
+ parseGraphqlComments;
407
510
  graphqlResponseTimeoutMs;
408
511
  running = false;
409
512
  constructor(options) {
410
513
  this.browserType = options.browserType;
411
514
  this.extractDomItems = options.extractDomItems ?? extractFacebookDomItems;
515
+ this.extractDomComments = options.extractDomComments ?? extractFacebookDomComments;
412
516
  this.parseGraphqlBody = options.parseGraphqlBody ?? parseFacebookGraphqlBody;
517
+ this.parseGraphqlComments = options.parseGraphqlComments ?? parseFacebookCommentBodies;
413
518
  this.graphqlResponseTimeoutMs = options.graphqlResponseTimeoutMs ?? 1e4;
414
519
  }
415
520
  async scanMany(options) {
@@ -431,6 +536,76 @@ var FacebookScanner = class {
431
536
  this.running = false;
432
537
  }
433
538
  }
539
+ async fetchComments(options) {
540
+ if (this.running)
541
+ throw new FacebookScannerError("SCANNER_BUSY");
542
+ const target = normalizePostUrl(options.postUrl);
543
+ if (!target)
544
+ throw new FacebookScannerError("POST_URL_INVALID");
545
+ assertNotAborted(options.signal);
546
+ this.running = true;
547
+ let browser = null;
548
+ try {
549
+ browser = await this.browserType.launch({ headless: true });
550
+ const context = await browser.newContext({ storageState: options.storageState });
551
+ let page = null;
552
+ const graphqlComments = [];
553
+ const responseTasks = [];
554
+ const responseHandler = (response) => {
555
+ if (!response.url().includes("/api/graphql/"))
556
+ return;
557
+ responseTasks.push(withTimeout(response.text(), this.graphqlResponseTimeoutMs).then((body) => graphqlComments.push(...this.parseGraphqlComments(body))).then(() => void 0).catch(() => void 0));
558
+ };
559
+ try {
560
+ page = await context.newPage();
561
+ page.on("response", responseHandler);
562
+ await page.route("**/*", (route) => ["font", "image", "media"].includes(route.request().resourceType()) ? route.abort() : route.continue());
563
+ try {
564
+ await page.goto(target, { waitUntil: "domcontentloaded", timeout: 45e3 });
565
+ } catch {
566
+ throw new FacebookScannerError("NAVIGATION_FAILED");
567
+ }
568
+ const condition = await detectFacebookSession(page, context);
569
+ if (condition !== "READY")
570
+ throw new FacebookScannerError(condition);
571
+ await options.onProgress?.({ phase: "FETCHING", commentCount: 0 });
572
+ let comments = [];
573
+ let idleRounds = 0;
574
+ let rounds = 0;
575
+ while (comments.length < options.limit && idleRounds < 3 && rounds < 20) {
576
+ assertNotAborted(options.signal);
577
+ rounds += 1;
578
+ const previousCount = comments.length;
579
+ await page.waitForTimeout(1200);
580
+ await drain(responseTasks, this.graphqlResponseTimeoutMs);
581
+ const domComments2 = await this.extractDomComments(page);
582
+ comments = mergeFacebookComments([...graphqlComments, ...domComments2], options.limit);
583
+ if (comments.length !== previousCount) {
584
+ await options.onProgress?.({ phase: "FETCHING", commentCount: comments.length });
585
+ }
586
+ const expanded = await clickCommentExpanders(page);
587
+ idleRounds = comments.length === previousCount && !expanded ? idleRounds + 1 : 0;
588
+ if (comments.length < options.limit)
589
+ await page.mouse.wheel(0, 2e3);
590
+ }
591
+ assertNotAborted(options.signal);
592
+ await drain(responseTasks, this.graphqlResponseTimeoutMs);
593
+ const domComments = await this.extractDomComments(page);
594
+ comments = mergeFacebookComments([...graphqlComments, ...domComments], options.limit);
595
+ await options.onProgress?.({ phase: "COMPLETE", commentCount: comments.length });
596
+ return comments;
597
+ } finally {
598
+ page?.off("response", responseHandler);
599
+ await page?.unroute("**/*").catch(() => void 0);
600
+ await Promise.allSettled(responseTasks);
601
+ await page?.close();
602
+ await context.close();
603
+ }
604
+ } finally {
605
+ await browser?.close();
606
+ this.running = false;
607
+ }
608
+ }
434
609
  async scanGroup(browser, options, groupUrl, groupIndex) {
435
610
  const context = await browser.newContext({ storageState: options.storageState });
436
611
  let page = null;
@@ -510,6 +685,29 @@ function withTimeout(promise, timeoutMs) {
510
685
  });
511
686
  return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
512
687
  }
688
+ var COMMENT_EXPANDER_PATTERNS = [
689
+ /^xem thêm bình luận$/i,
690
+ /^xem \d+ bình luận$/i,
691
+ /^xem các bình luận khác$/i,
692
+ /^view more comments$/i,
693
+ /^view \d+ more comments$/i,
694
+ /^xem thêm câu trả lời$/i,
695
+ /^view more replies$/i
696
+ ];
697
+ async function clickCommentExpanders(page) {
698
+ let clicked = 0;
699
+ for (const pattern of COMMENT_EXPANDER_PATTERNS) {
700
+ for (let attempt = 0; attempt < 2; attempt += 1) {
701
+ try {
702
+ await page.getByRole("button", { name: pattern }).first().click({ timeout: 1500 });
703
+ clicked += 1;
704
+ } catch {
705
+ break;
706
+ }
707
+ }
708
+ }
709
+ return clicked;
710
+ }
513
711
 
514
712
  // src/cli/cli.ts
515
713
  import { chromium } from "playwright";
@@ -926,6 +1124,7 @@ import { randomUUID } from "node:crypto";
926
1124
 
927
1125
  // ../../packages/shared/src/index.ts
928
1126
  var DESKTOP_AGENT_PROTOCOL_VERSION = 1;
1127
+ var DESKTOP_POST_COMMENTS_REQUEST_TIMEOUT_MS = 3 * 60 * 1e3;
929
1128
 
930
1129
  // src/main/desktop-agent-session.ts
931
1130
  var MESSAGE_FRESHNESS_MS = 5 * 60 * 1e3;
@@ -1091,7 +1290,8 @@ var DesktopAgentSession = class {
1091
1290
  platform: this.options.platform,
1092
1291
  state: this.reportedState,
1093
1292
  facebookReady: this.options.getFacebookReady(),
1094
- currentJobId: this.reportedJobId
1293
+ currentJobId: this.reportedJobId,
1294
+ capabilities: ["POST_COMMENTS"]
1095
1295
  });
1096
1296
  this.handshake = "WELCOME";
1097
1297
  return;
@@ -1135,8 +1335,15 @@ var DesktopAgentSession = class {
1135
1335
  this.emitJob({ type: "STOP", ...envelope.payload });
1136
1336
  return;
1137
1337
  }
1338
+ if (envelope.type === "POST_COMMENTS_FETCH") {
1339
+ if (!isPostCommentsFetch(envelope.payload)) {
1340
+ this.protocolFailure(socket, stored, generation);
1341
+ return;
1342
+ }
1343
+ this.emitJob({ type: "COMMENTS", ...envelope.payload });
1344
+ return;
1345
+ }
1138
1346
  if (envelope.type !== "SCAN_JOB_START") {
1139
- this.protocolFailure(socket, stored, generation);
1140
1347
  return;
1141
1348
  }
1142
1349
  this.update({ ...this.snapshot, currentJobId: envelope.payload.jobId, pendingJobId: null });
@@ -1280,6 +1487,11 @@ function describeEnvelopeProblem(envelope, deviceId, replayIds) {
1280
1487
  if (replayIds.has(envelope.messageId)) return `replayed message ${envelope.messageId}`;
1281
1488
  return "unknown";
1282
1489
  }
1490
+ function isPostCommentsFetch(value) {
1491
+ if (!value || typeof value !== "object") return false;
1492
+ const payload = value;
1493
+ return typeof payload.requestId === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(payload.requestId) && typeof payload.postUrl === "string" && payload.postUrl.length <= 2e3 && /^https:\/\/[\w.-]*facebook\.com\//i.test(payload.postUrl) && typeof payload.limit === "number" && Number.isInteger(payload.limit) && payload.limit > 0 && payload.limit <= 50;
1494
+ }
1283
1495
  function emptySnapshot(state) {
1284
1496
  return {
1285
1497
  state,
@@ -1309,6 +1521,127 @@ function readErrorCode(error) {
1309
1521
  return error && typeof error === "object" && "code" in error && typeof error.code === "string" ? error.code : "UNKNOWN_ERROR";
1310
1522
  }
1311
1523
 
1524
+ // src/main/desktop-comment-runner.ts
1525
+ var DEFAULT_FETCH_TIMEOUT_MS = 2 * 60 * 1e3;
1526
+ var DEFAULT_MAX_QUEUE = 5;
1527
+ var DesktopCommentRunner = class {
1528
+ options;
1529
+ queue = [];
1530
+ stopController = new AbortController();
1531
+ active = null;
1532
+ currentController = null;
1533
+ unsubscribe = null;
1534
+ stopped = false;
1535
+ constructor(options) {
1536
+ this.options = options;
1537
+ }
1538
+ start() {
1539
+ if (this.unsubscribe) return;
1540
+ this.unsubscribe = this.options.session.subscribeJobs((event) => this.handleEvent(event));
1541
+ }
1542
+ stop() {
1543
+ this.stopped = true;
1544
+ this.queue.length = 0;
1545
+ this.stopController.abort();
1546
+ this.currentController?.abort();
1547
+ this.unsubscribe?.();
1548
+ this.unsubscribe = null;
1549
+ }
1550
+ isRunning() {
1551
+ return this.active !== null;
1552
+ }
1553
+ async whenIdle() {
1554
+ await this.active;
1555
+ }
1556
+ async handleEvent(event) {
1557
+ if (event.type !== "COMMENTS" || this.stopped) return;
1558
+ if (this.queue.length >= (this.options.maxQueue ?? DEFAULT_MAX_QUEUE)) {
1559
+ await this.upload(event.requestId, { comments: [], errorCode: "QUEUE_FULL" });
1560
+ return;
1561
+ }
1562
+ this.queue.push(event);
1563
+ await this.drain();
1564
+ }
1565
+ drain() {
1566
+ if (!this.active) {
1567
+ this.active = this.run().finally(() => {
1568
+ this.active = null;
1569
+ });
1570
+ }
1571
+ return this.active;
1572
+ }
1573
+ async run() {
1574
+ while (this.queue.length > 0 && !this.stopped) {
1575
+ const request = this.queue.shift();
1576
+ if (request) await this.process(request);
1577
+ }
1578
+ }
1579
+ async process(request) {
1580
+ const controller = new AbortController();
1581
+ const timeout = this.options.setTimer(
1582
+ () => controller.abort(),
1583
+ this.options.fetchTimeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS
1584
+ );
1585
+ this.currentController = controller;
1586
+ try {
1587
+ if (this.options.isScanActive()) {
1588
+ await this.upload(request.requestId, { comments: [], errorCode: "SCANNER_BUSY" });
1589
+ return;
1590
+ }
1591
+ if (this.options.isLoginActive()) {
1592
+ await this.upload(request.requestId, { comments: [], errorCode: "LOGIN_ACTIVE" });
1593
+ return;
1594
+ }
1595
+ const storageState = await this.options.facebookState.load();
1596
+ if (!storageState) {
1597
+ await this.upload(request.requestId, { comments: [], errorCode: "FACEBOOK_SESSION_MISSING" });
1598
+ return;
1599
+ }
1600
+ let comments;
1601
+ try {
1602
+ comments = await this.options.scanner.fetchComments({
1603
+ limit: request.limit,
1604
+ postUrl: request.postUrl,
1605
+ signal: controller.signal,
1606
+ storageState
1607
+ });
1608
+ } catch (error) {
1609
+ await this.upload(request.requestId, { comments: [], errorCode: readErrorCode2(error) });
1610
+ return;
1611
+ }
1612
+ await this.upload(request.requestId, { comments });
1613
+ } finally {
1614
+ this.options.clearTimer(timeout);
1615
+ if (this.currentController === controller) this.currentController = null;
1616
+ }
1617
+ }
1618
+ async upload(requestId, payload) {
1619
+ const signal = this.stopController.signal;
1620
+ if (signal.aborted) return;
1621
+ for (let attempt = 0; ; attempt += 1) {
1622
+ try {
1623
+ await this.options.http.uploadComments(requestId, payload, signal);
1624
+ return;
1625
+ } catch (error) {
1626
+ if (!isRecoverable(error) || attempt >= 2 || signal.aborted) return;
1627
+ try {
1628
+ await this.options.wait(1e3 * 2 ** attempt, signal);
1629
+ } catch {
1630
+ return;
1631
+ }
1632
+ }
1633
+ }
1634
+ }
1635
+ };
1636
+ function readErrorCode2(error) {
1637
+ const code = error && typeof error === "object" && "code" in error && typeof error.code === "string" ? error.code : "";
1638
+ const sanitized = code.replace(/[^A-Z0-9_]/g, "_").slice(0, 100);
1639
+ return sanitized || "COMMENT_FETCH_FAILED";
1640
+ }
1641
+ function isRecoverable(error) {
1642
+ return Boolean(error && typeof error === "object" && "recoverable" in error && error.recoverable === true);
1643
+ }
1644
+
1312
1645
  // src/main/desktop-scan-http.ts
1313
1646
  import { createHash, createPrivateKey, randomUUID as randomUUID2, sign } from "node:crypto";
1314
1647
  var RESPONSE_LIMIT2 = 64 * 1024;
@@ -1344,6 +1677,13 @@ var DesktopScanHttp = class {
1344
1677
  cancel(jobId, attemptToken) {
1345
1678
  return this.signedPost(`/leads/desktop-agent/jobs/${jobId}/cancel`, { attemptToken });
1346
1679
  }
1680
+ uploadComments(requestId, payload, signal) {
1681
+ return this.signedPost(
1682
+ `/leads/desktop-agent/post-comments/${requestId}`,
1683
+ normalizeComments(payload),
1684
+ signal
1685
+ );
1686
+ }
1347
1687
  async signedPost(path, businessPayload, signal) {
1348
1688
  const credential = this.getCredential();
1349
1689
  if (!credential) throw new DesktopScanHttpError("CREDENTIAL_UNAVAILABLE");
@@ -1485,6 +1825,15 @@ function normalizeFormatToken(value) {
1485
1825
  const normalized = value?.trim();
1486
1826
  return normalized && normalized.length <= 50 ? normalized : void 0;
1487
1827
  }
1828
+ function normalizeComments(payload) {
1829
+ const comments = payload.comments.slice(0, 50).map((comment) => ({
1830
+ text: comment.text.trim().slice(0, 2e3),
1831
+ ...comment.authorName?.trim() ? { authorName: comment.authorName.trim().slice(0, 200) } : {},
1832
+ ...comment.publishedAt ? { publishedAt: comment.publishedAt } : {}
1833
+ })).filter((comment) => comment.text.length > 0);
1834
+ const errorCode = payload.errorCode?.replace(/[^A-Z0-9_]/g, "_").slice(0, 100);
1835
+ return { comments, ...errorCode ? { errorCode } : {} };
1836
+ }
1488
1837
  function normalizeFacebookPostUrl(value) {
1489
1838
  try {
1490
1839
  const url = new URL(value);
@@ -1525,7 +1874,7 @@ var DesktopScanRunner = class {
1525
1874
  records = await this.options.spool.loadAll();
1526
1875
  } catch (error) {
1527
1876
  this.recoveryBlocked = true;
1528
- this.update({ ...idleSnapshot(), state: "SPOOL_FULL", errorCode: readErrorCode2(error) });
1877
+ this.update({ ...idleSnapshot(), state: "SPOOL_FULL", errorCode: readErrorCode3(error) });
1529
1878
  return;
1530
1879
  }
1531
1880
  const current = records.find((record) => Date.parse(record.job.leaseExpiresAt) > this.options.now()) ?? null;
@@ -1587,6 +1936,7 @@ var DesktopScanRunner = class {
1587
1936
  return true;
1588
1937
  }
1589
1938
  async handleEvent(event) {
1939
+ if (event.type === "COMMENTS") return;
1590
1940
  if (event.type === "AVAILABLE") {
1591
1941
  await this.claimIfReady(event.jobId);
1592
1942
  return;
@@ -1653,6 +2003,7 @@ var DesktopScanRunner = class {
1653
2003
  async claimIfReady(jobId) {
1654
2004
  const session = this.options.session.getSnapshot();
1655
2005
  if (this.recoveryBlocked || this.active || this.snapshot.jobId || session.currentJobId || session.state !== "ONLINE" || session.accessMode !== "FULL" || this.options.isLoginActive() || !await this.options.facebookState.isReady()) return;
2006
+ if (this.options.isCommentFetchActive?.()) return;
1656
2007
  this.options.session.claimJob(jobId);
1657
2008
  }
1658
2009
  async run(job, resume) {
@@ -1783,7 +2134,7 @@ var DesktopScanRunner = class {
1783
2134
  this.recovered = null;
1784
2135
  this.options.session.finishJob(job.jobId);
1785
2136
  } catch (error) {
1786
- const code = readErrorCode2(error);
2137
+ const code = readErrorCode3(error);
1787
2138
  const checkpoint = code === "LOGIN_REQUIRED" || code === "CHECKPOINT_REQUIRED";
1788
2139
  if (code === "OFFLINE_PENDING") {
1789
2140
  this.recovered = record;
@@ -1820,7 +2171,7 @@ var DesktopScanRunner = class {
1820
2171
  await this.options.spool.put(record);
1821
2172
  confirmed = await this.completeWithRetry(job, completion, controller.signal);
1822
2173
  } catch (completionError) {
1823
- const completionCode = readErrorCode2(completionError);
2174
+ const completionCode = readErrorCode3(completionError);
1824
2175
  terminalPersistenceFailed = true;
1825
2176
  this.recovered = record;
1826
2177
  this.update({
@@ -1856,7 +2207,7 @@ var DesktopScanRunner = class {
1856
2207
  await this.options.http.uploadBatch(job.jobId, batch, signal);
1857
2208
  return true;
1858
2209
  } catch (error) {
1859
- if (!isRecoverable(error)) throw error;
2210
+ if (!isRecoverable2(error)) throw error;
1860
2211
  if (attempt >= 4) return false;
1861
2212
  attempt += 1;
1862
2213
  await this.options.wait(Math.min(8e3, 500 * 2 ** attempt), signal);
@@ -1871,7 +2222,7 @@ var DesktopScanRunner = class {
1871
2222
  await this.options.http.complete(job.jobId, payload, signal);
1872
2223
  return true;
1873
2224
  } catch (error) {
1874
- if (!isRecoverable(error)) throw error;
2225
+ if (!isRecoverable2(error)) throw error;
1875
2226
  if (attempt >= 4) return false;
1876
2227
  attempt += 1;
1877
2228
  await this.options.wait(Math.min(8e3, 500 * 2 ** attempt), signal);
@@ -1960,10 +2311,10 @@ function createSpoolRecord(job, now) {
1960
2311
  function nowIso(now) {
1961
2312
  return new Date(now).toISOString();
1962
2313
  }
1963
- function readErrorCode2(error) {
2314
+ function readErrorCode3(error) {
1964
2315
  return error && typeof error === "object" && "code" in error && typeof error.code === "string" ? error.code.slice(0, 500) : "SCAN_FAILED";
1965
2316
  }
1966
- function isRecoverable(error) {
2317
+ function isRecoverable2(error) {
1967
2318
  return Boolean(error && typeof error === "object" && "recoverable" in error && error.recoverable === true);
1968
2319
  }
1969
2320
  function abortableDelay(milliseconds, signal) {
@@ -2434,11 +2785,10 @@ function parseCookieTable(input) {
2434
2785
  const cookies = [];
2435
2786
  for (const line of input.split("\n")) {
2436
2787
  const columns = line.split(" ");
2437
- if (columns.length < 2) return [];
2438
- const name = columns[0].trim();
2788
+ const name = columns[0]?.trim() ?? "";
2439
2789
  const value = columns[1];
2440
- if (!name || name === "Name" || !value) continue;
2441
- cookies.push({ name, value });
2790
+ if (!value || !name || name === "Name") continue;
2791
+ cookies.push({ domain: columns[2]?.trim(), name, value });
2442
2792
  }
2443
2793
  return cookies;
2444
2794
  }
@@ -2528,6 +2878,9 @@ function createFileCrypto(keyPath) {
2528
2878
  // src/cli/systemd.ts
2529
2879
  var SERVICE_NAME = "diggylab";
2530
2880
  var SERVICE_PATH = `/etc/systemd/system/${SERVICE_NAME}.service`;
2881
+ function quote(value) {
2882
+ return /\s/.test(value) ? `"${value.replace(/"/g, '\\"')}"` : value;
2883
+ }
2531
2884
  function buildSystemdUnit(options) {
2532
2885
  return `[Unit]
2533
2886
  Description=DiggyLab Desktop Agent
@@ -2535,9 +2888,9 @@ After=network-online.target
2535
2888
  Wants=network-online.target
2536
2889
 
2537
2890
  [Service]
2538
- ExecStart=${options.execStart} run
2539
- Environment=DIGGYLAB_AGENT_DIR=${options.configDirectory}
2540
- Environment=DIGGYLAB_API_URL=${options.apiOrigin}
2891
+ ExecStart=${options.execStart.map(quote).join(" ")} run
2892
+ Environment="DIGGYLAB_AGENT_DIR=${options.configDirectory}"
2893
+ Environment="DIGGYLAB_API_URL=${options.apiOrigin}"
2541
2894
  Restart=on-failure
2542
2895
  RestartSec=10
2543
2896
 
@@ -2612,6 +2965,14 @@ function createRuntime(options) {
2612
2965
  function stamp() {
2613
2966
  return `[${(/* @__PURE__ */ new Date()).toTimeString().slice(0, 8)}]`;
2614
2967
  }
2968
+ function describeSocketError(error) {
2969
+ if (error instanceof Error) {
2970
+ const code = error.code;
2971
+ return `${error.message || error.name}${typeof code === "string" ? ` (${code})` : ""}`;
2972
+ }
2973
+ if (error && typeof error === "object" && "type" in error) return `event ${String(error.type)}`;
2974
+ return String(error);
2975
+ }
2615
2976
  function waitForFacebookLogin(login, timeoutMs) {
2616
2977
  return new Promise((resolve, reject) => {
2617
2978
  const timer = setTimeout(() => {
@@ -2711,17 +3072,22 @@ async function commandRename(runtime, options, io) {
2711
3072
  return 1;
2712
3073
  }
2713
3074
  const requested = options.name?.trim() || await promptForName(io);
2714
- const name = sanitizeDeviceName(requested);
2715
- if (!name) {
3075
+ if (!requested) {
2716
3076
  io.error('A device name is required, for example `diggylab rename --name "VPS H\xE0 N\u1ED9i"`.');
2717
3077
  return 2;
2718
3078
  }
3079
+ const name = sanitizeDeviceName(requested);
2719
3080
  try {
2720
3081
  const device = await runtime.manager.rename(name);
2721
3082
  io.log(`Renamed to "${device.name}".`);
2722
3083
  await printReadiness(runtime, io);
2723
3084
  return 0;
2724
3085
  } catch (error) {
3086
+ if (error instanceof DesktopAgentClientError && error.code === "DESKTOP_RESPONSE_INVALID") {
3087
+ io.error("The API answered the rename with something unexpected \u2014 usually an API that does not have this endpoint yet.");
3088
+ io.error(`Deploy the API, then check it with: curl -s -o /dev/null -w '%{http_code}\\n' -X PATCH https://api.diggylab.com/desktop-agent-auth/me -H 'content-type: application/json' -d '{"name":"x"}' (401 = the endpoint exists)`);
3089
+ return 1;
3090
+ }
2725
3091
  io.error(`Could not rename this device: ${error instanceof Error ? error.message : String(error)}`);
2726
3092
  return 1;
2727
3093
  }
@@ -2798,8 +3164,10 @@ async function importFacebookCookies(runtime, source, io) {
2798
3164
  io.error("Open facebook.com in your browser, make sure you are logged in, then copy the `cookie:` request header again.");
2799
3165
  return 1;
2800
3166
  }
3167
+ let saved = false;
2801
3168
  try {
2802
3169
  const accountId = await runtime.facebook.save(state);
3170
+ saved = true;
2803
3171
  io.log(`Facebook session saved for account ${accountId}. Asking Facebook to confirm it...`);
2804
3172
  const condition = await validateImportedSession(runtime, state);
2805
3173
  if (condition !== "READY") {
@@ -2809,6 +3177,7 @@ async function importFacebookCookies(runtime, source, io) {
2809
3177
  }
2810
3178
  } catch (error) {
2811
3179
  const message = error instanceof Error ? error.message : String(error);
3180
+ if (saved) await runtime.facebook.clear();
2812
3181
  if (/Executable doesn't exist|playwright install/i.test(message)) {
2813
3182
  io.error(MISSING_BROWSER_HINT);
2814
3183
  } else if (message === "FACEBOOK_SESSION_INVALID") {
@@ -2867,7 +3236,9 @@ function readAnswer(options = {}) {
2867
3236
  return;
2868
3237
  }
2869
3238
  if (idle) clearTimeout(idle);
2870
- idle = setTimeout(finish, 350);
3239
+ idle = setTimeout(() => {
3240
+ if (!options.until || options.until(value)) finish();
3241
+ }, 350);
2871
3242
  };
2872
3243
  process.stdin.setEncoding("utf8");
2873
3244
  process.stdin.on("data", handleData);
@@ -2895,10 +3266,14 @@ async function commandInstallBrowser(io) {
2895
3266
  child.once("exit", (value) => resolve(value ?? 1));
2896
3267
  });
2897
3268
  let code = 1;
2898
- for (const [index, args] of browserInstallCommands(process.platform).entries()) {
2899
- if (index > 0) io.log("Installing the system libraries failed; downloading the browser only...");
2900
- code = await run(args);
2901
- if (code === 0) break;
3269
+ try {
3270
+ for (const [index, args] of browserInstallCommands(process.platform).entries()) {
3271
+ if (index > 0) io.log("Installing the system libraries failed; downloading the browser only...");
3272
+ code = await run(args);
3273
+ if (code === 0) break;
3274
+ }
3275
+ } catch {
3276
+ code = 1;
2902
3277
  }
2903
3278
  if (code !== 0) {
2904
3279
  io.error(soft ? "Chromium download failed. Run `diggylab install-browser` later." : "Chromium download failed. Check the network and try again.");
@@ -2925,7 +3300,7 @@ async function commandRun(runtime, io) {
2925
3300
  headers: { authorization: `Bearer ${accessToken}` },
2926
3301
  maxPayload: 16 * 1024
2927
3302
  });
2928
- socket.on("error", (error) => io.log(`${stamp()} socket error: ${error.message}`));
3303
+ socket.on("error", (error) => io.log(`${stamp()} socket error: ${describeSocketError(error)}`));
2929
3304
  socket.on("close", (code, reason) => {
2930
3305
  io.log(`${stamp()} socket closed: ${code}${reason?.length ? ` (${reason.toString()})` : ""}`);
2931
3306
  });
@@ -2951,17 +3326,34 @@ async function commandRun(runtime, io) {
2951
3326
  },
2952
3327
  origin: runtime.endpoint.httpOrigin
2953
3328
  });
3329
+ const scanner = new FacebookScanner({ browserType: runtime.browserType });
3330
+ const commentRunner = new DesktopCommentRunner({
3331
+ clearTimer: (timer) => clearTimeout(timer),
3332
+ facebookState: runtime.facebook,
3333
+ http: scanHttp,
3334
+ isLoginActive: () => false,
3335
+ isScanActive: () => scanRunner?.isRunning() ?? false,
3336
+ scanner,
3337
+ session,
3338
+ setTimer: (callback, delay) => {
3339
+ const timer = setTimeout(callback, delay);
3340
+ timer.unref();
3341
+ return timer;
3342
+ },
3343
+ wait: abortableDelay
3344
+ });
2954
3345
  const scanRunner = new DesktopScanRunner({
2955
3346
  clearTimer: (timer) => clearTimeout(timer),
2956
3347
  facebookState: runtime.facebook,
2957
3348
  http: scanHttp,
3349
+ isCommentFetchActive: () => commentRunner.isRunning(),
2958
3350
  isLoginActive: () => false,
2959
3351
  now: Date.now,
2960
3352
  randomUuid: () => randomUUID3(),
2961
3353
  requestFacebookLogin: async () => {
2962
3354
  io.error("The Facebook session expired. Stop this worker, run `diggylab login`, then start it again.");
2963
3355
  },
2964
- scanner: new FacebookScanner({ browserType: runtime.browserType }),
3356
+ scanner,
2965
3357
  session,
2966
3358
  setTimer: (callback, delay) => {
2967
3359
  const timer = setTimeout(callback, delay);
@@ -2989,6 +3381,7 @@ async function commandRun(runtime, io) {
2989
3381
  recordActivity(`Scanner state changed to ${snapshot.state}.`, snapshot.state);
2990
3382
  });
2991
3383
  scanRunner.start();
3384
+ commentRunner.start();
2992
3385
  await scanRunner.recover();
2993
3386
  await session.start();
2994
3387
  io.log(`Worker running as "${stored.device.name}" (API ${runtime.endpoint.httpOrigin}). Press Ctrl+C to stop.`);
@@ -3004,6 +3397,7 @@ async function commandRun(runtime, io) {
3004
3397
  } finally {
3005
3398
  clearInterval(keepAlive);
3006
3399
  session.stop();
3400
+ commentRunner.stop();
3007
3401
  scanRunner.stop();
3008
3402
  }
3009
3403
  return 0;
@@ -3036,7 +3430,7 @@ function runCommand(command, args) {
3036
3430
  function resolveServiceExecStart() {
3037
3431
  const candidates = [join3(dirname3(process.execPath), "diggylab"), "/usr/local/bin/diggylab", "/usr/bin/diggylab"];
3038
3432
  const link = candidates.find((candidate) => existsSync2(candidate));
3039
- return link ?? `${process.execPath} ${fileURLToPath(import.meta.url)}`;
3433
+ return link ? [link] : [process.execPath, fileURLToPath(import.meta.url)];
3040
3434
  }
3041
3435
  async function commandInstallService(runtime, io) {
3042
3436
  if (process.platform !== "linux") {
@@ -3059,6 +3453,11 @@ async function commandInstallService(runtime, io) {
3059
3453
  return 1;
3060
3454
  }
3061
3455
  io.log(`Wrote ${SERVICE_PATH}.`);
3456
+ io.log(`State directory: ${runtime.directory}`);
3457
+ if (process.env.SUDO_USER && process.env.SUDO_USER !== "root" && runtime.directory === join3(homedir(), CONFIG_DIRECTORY)) {
3458
+ io.error(`Heads up: this ran as root, so the service will use ${runtime.directory} (root's home).`);
3459
+ io.error(`If you paired as "${process.env.SUDO_USER}", pass --config to point at that user's ~/${CONFIG_DIRECTORY} instead.`);
3460
+ }
3062
3461
  await runCommand("systemctl", ["daemon-reload"]).catch(() => 1);
3063
3462
  const code = await runCommand("systemctl", ["enable", "--now", SERVICE_NAME]);
3064
3463
  if (code !== 0) {
@@ -3126,8 +3525,12 @@ async function main(argv, io = console) {
3126
3525
  return commandRun(runtime, io);
3127
3526
  case "status":
3128
3527
  return commandStatus(runtime, io);
3129
- default:
3528
+ case "unpair":
3130
3529
  return commandUnpair(runtime, io);
3530
+ default:
3531
+ io.error(`Unknown command "${command}".`);
3532
+ io.error(USAGE);
3533
+ return 2;
3131
3534
  }
3132
3535
  }
3133
3536
  function isEntryPoint() {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "diggylab",
3
- "version": "0.1.43",
3
+ "version": "0.1.45",
4
4
  "description": "DiggyLab Desktop Agent collects posts and information from Facebook Groups on a paired computer. (headless CLI)",
5
5
  "type": "module",
6
6
  "bin": {