diggylab 0.1.45 → 0.1.46

package/README.md

@@ -52,6 +52,19 @@ npm publish apps/desktop-agent/dist-npm --access public
 
 `CLI_PACKAGE_VERSION=0.1.31` works as well.
 
+## Local development
+
+Run the CLI from source against the local API without touching a real installation:
+
+```bash
+pnpm dev:cli                  # prints status against http://localhost:4000 and ~/.diggylab-dev
+pnpm dev:cli pair             # forwards any command to the CLI
+pnpm dev:cli --build run      # rebuilds dist-cli first, then runs the worker
+```
+
+The wrapper points `DIGGYLAB_API_URL` at `http://localhost:4000` and `DIGGYLAB_AGENT_DIR` at
+`~/.diggylab-dev`; set either variable to override the defaults.
+
 ## Commands
 
 | Command | What it does |

package/cli.js

@@ -5,7 +5,7 @@ import { randomUUID as randomUUID3 } from "node:crypto";
 import { spawn } from "node:child_process";
 import { existsSync as existsSync2, readFileSync as readFileSync2, realpathSync, rmSync, writeFileSync as writeFileSync2 } from "node:fs";
 import { hostname, homedir } from "node:os";
-import { dirname as dirname3, join as join3 } from "node:path";
+import { dirname as dirname3, join as join4 } from "node:path";
 import { fileURLToPath } from "node:url";
 
 // ../../packages/facebook-scanner/dist/parser.js
@@ -48,11 +48,25 @@ var commentAuthorPaths = [
   ["author_name"],
   ["author", "short_name"]
 ];
+var commentAvatarPaths = [
+  ["author", "profile_picture_depth_0", "uri"],
+  ["author", "profile_picture_depth_1", "uri"],
+  ["author", "profile_picture_depth_0_increased", "uri"],
+  ["author", "profile_picture", "uri"]
+];
 var commentTimePaths = [
   ["created_time"],
   ["creation_time"],
   ["publish_time"]
 ];
+var commentCountPaths = [
+  ["feedback", "comment_count", "total_count"],
+  ["feedback", "comments", "total_count"],
+  ["feedback", "total_comment_count"],
+  ["comment_count", "total_count"],
+  ["comments", "total_count"],
+  ["total_comment_count"]
+];
 var facebookHosts = /* @__PURE__ */ new Set([
   "facebook.com",
   "fb.com",
@@ -214,6 +228,55 @@ function getTextFormat(node) {
   };
   return Object.values(textFormat).some(Boolean) ? textFormat : void 0;
 }
+function getCommentCount(node) {
+  return firstCount(node, commentCountPaths) ?? sweepCommentCount(node, 0);
+}
+function firstCount(root, candidates) {
+  for (const path of candidates) {
+    const count = toCount(get(root, path));
+    if (count !== void 0)
+      return count;
+  }
+  return void 0;
+}
+function sweepCommentCount(value, depth) {
+  if (!value || depth > 10)
+    return void 0;
+  if (Array.isArray(value)) {
+    for (const item of value) {
+      const count = sweepCommentCount(item, depth + 1);
+      if (count !== void 0)
+        return count;
+    }
+    return void 0;
+  }
+  if (!isData(value))
+    return void 0;
+  if ("comment_id" in value || "parent_comment" in value)
+    return void 0;
+  const typename = typeof value.__typename === "string" ? value.__typename : void 0;
+  if (typename && /comment$/i.test(typename))
+    return void 0;
+  const direct = toCount(value.comment_count) ?? toCount(value.total_comment_count) ?? toCount(value.comments);
+  if (direct !== void 0)
+    return direct;
+  for (const child of Object.values(value)) {
+    const count = sweepCommentCount(child, depth + 1);
+    if (count !== void 0)
+      return count;
+  }
+  return void 0;
+}
+function toCount(value) {
+  if (typeof value === "number") {
+    return Number.isSafeInteger(value) && value >= 0 ? value : void 0;
+  }
+  if (typeof value === "string" && /^\d{1,9}$/.test(value))
+    return Number(value);
+  if (isData(value))
+    return toCount(value.total_count);
+  return void 0;
+}
 function collectImageUrls(value, urls, depth) {
   if (!value || depth > 12 || urls.size >= 12)
     return;
@@ -275,12 +338,14 @@ function toItem(node, requestedGroup) {
       attachmentImages = dedupeImageUrls([...fallbackUrls]);
   }
   const textFormat = getTextFormat(node);
+  const commentCount = getCommentCount(node);
   return {
     text,
     url,
     authorName: firstString(node, paths.author),
     ...authorAvatarUrl ? { authorAvatarUrl } : {},
     ...attachmentImages.length ? { attachmentImages } : {},
+    ...commentCount !== void 0 ? { commentCount } : {},
     ...textFormat ? { textFormat } : {},
     publishedAt: toIso(firstString(node, paths.time)),
     groupName: firstString(node, paths.group)
@@ -315,6 +380,9 @@ function mergeFacebookScanItems(items, limit) {
       if (!existing.authorAvatarUrl && item.authorAvatarUrl) {
         existing.authorAvatarUrl = item.authorAvatarUrl;
       }
+      if (existing.commentCount === void 0 && item.commentCount !== void 0) {
+        existing.commentCount = item.commentCount;
+      }
       const attachmentImages = dedupeImageUrls([
         ...existing.attachmentImages ?? [],
         ...item.attachmentImages ?? []
@@ -350,13 +418,25 @@ function toComment(node) {
   if (!text || typename && !/comment$/i.test(typename))
     return void 0;
   const authorName = firstString(node, commentAuthorPaths);
+  const authorAvatarUrl = firstString(node, commentAvatarPaths, false);
+  const depth = toDepth(node);
   const publishedAt = toIso(firstString(node, commentTimePaths));
+  const id = firstString(node, [["id"]], false);
+  const parentId = firstString(node, [["comment_direct_parent", "id"]], false);
   return {
     text: text.trim(),
     ...authorName ? { authorName } : {},
+    ...authorAvatarUrl ? { authorAvatarUrl } : {},
+    ...depth !== void 0 ? { depth } : {},
+    ...id ? { id } : {},
+    ...parentId ? { parentId } : {},
     ...publishedAt ? { publishedAt } : {}
   };
 }
+function toDepth(node) {
+  const value = node.depth;
+  return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 5 ? value : void 0;
+}
 function visitComments(value, items) {
   if (Array.isArray(value)) {
     for (const item of value)
@@ -384,27 +464,81 @@ function parseFacebookCommentBodies(body) {
   }
   return mergeFacebookComments(items, Number.POSITIVE_INFINITY);
 }
+function parseFacebookCommentJson(value) {
+  const items = [];
+  visitComments(value, items);
+  return mergeFacebookComments(items, Number.POSITIVE_INFINITY);
+}
 function mergeFacebookComments(items, limit) {
-  const seen = /* @__PURE__ */ new Set();
-  const merged = [];
+  const merged = /* @__PURE__ */ new Map();
   for (const item of items) {
     const text = item.text.trim();
     if (!text)
       continue;
     const authorName = item.authorName?.trim();
-    const key = `${authorName ?? ""}\0${text}\0${item.publishedAt ?? ""}`;
-    if (seen.has(key))
+    const key = `${authorName ?? ""}\0${commentKeyText(text)}`;
+    const existing = merged.get(key);
+    if (existing) {
+      if (!existing.publishedAt && item.publishedAt)
+        existing.publishedAt = item.publishedAt;
+      if (!existing.authorAvatarUrl && item.authorAvatarUrl) {
+        existing.authorAvatarUrl = item.authorAvatarUrl;
+      }
+      if (!existing.id && item.id)
+        existing.id = item.id;
+      if (!existing.parentId && item.parentId)
+        existing.parentId = item.parentId;
+      if (item.depth !== void 0) {
+        existing.depth = existing.depth === void 0 ? item.depth : Math.min(existing.depth, item.depth);
+      }
       continue;
-    if (merged.length >= limit)
-      break;
-    seen.add(key);
-    merged.push({
+    }
+    if (merged.size >= limit)
+      continue;
+    merged.set(key, {
       text: text.slice(0, COMMENT_TEXT_LIMIT),
       ...authorName ? { authorName: authorName.slice(0, COMMENT_AUTHOR_LIMIT) } : {},
+      ...item.authorAvatarUrl ? { authorAvatarUrl: item.authorAvatarUrl } : {},
+      ...item.depth !== void 0 ? { depth: item.depth } : {},
+      ...item.id ? { id: item.id } : {},
+      ...item.parentId ? { parentId: item.parentId } : {},
       ...item.publishedAt ? { publishedAt: item.publishedAt } : {}
     });
   }
-  return merged;
+  return orderFacebookCommentThreads([...merged.values()]);
+}
+function orderFacebookCommentThreads(items) {
+  const byId = /* @__PURE__ */ new Map();
+  for (const item of items) {
+    if (item.id && !byId.has(item.id))
+      byId.set(item.id, item);
+  }
+  const children = /* @__PURE__ */ new Map();
+  const roots = [];
+  for (const item of items) {
+    if (item.parentId && byId.has(item.parentId)) {
+      children.set(item.parentId, [...children.get(item.parentId) ?? [], item]);
+    } else {
+      roots.push(item);
+    }
+  }
+  const ordered = [];
+  const visited = /* @__PURE__ */ new Set();
+  const append = (item, depth) => {
+    if (visited.has(item))
+      return;
+    visited.add(item);
+    ordered.push({ ...item, depth: Math.max(depth, item.depth ?? 0) });
+    for (const child of item.id ? children.get(item.id) ?? [] : [])
+      append(child, depth + 1);
+  };
+  for (const root of roots)
+    append(root, 0);
+  for (const item of items) {
+    if (!visited.has(item))
+      append(item, item.depth ?? 0);
+  }
+  return ordered;
 }
 async function extractFacebookDomItems(page, groupUrl) {
   const requestedGroup = requestedGroupIdentifier(groupUrl);
@@ -427,12 +561,16 @@ async function extractFacebookDomItems(page, groupUrl) {
     const message = Array.from(article.querySelectorAll('[data-ad-preview="message"]')).find((element) => belongsToArticle(element) && element.checkVisibility());
     const author = Array.from(article.querySelectorAll("h2 a, h3 a, strong a")).find((element) => belongsToArticle(element) && element.checkVisibility());
     const time = Array.from(article.querySelectorAll("time[datetime], abbr[data-utime]")).find((element) => belongsToArticle(element) && element.checkVisibility());
+    const countText = Array.from(article.querySelectorAll('a, [role="button"]')).filter((element) => belongsToArticle(element) && element.checkVisibility()).map((element) => element.innerText.trim()).find((text) => /(?:^|\s)\d+\s*(?:bình luận|comments?)$/i.test(text));
+    const countMatch = countText ? /(\d+)\s*(?:bình luận|comments?)$/i.exec(countText) : null;
+    const commentCount = countMatch ? Number(countMatch[1]) : void 0;
     const attachmentImages = Array.from(article.querySelectorAll("img")).filter((image) => belongsToArticle(image) && image.naturalWidth >= 200).map((image) => image.currentSrc || image.src).filter((src) => /^https?:\/\//i.test(src) && /(scontent\.|fbcdn\.net)/i.test(src));
     return {
       text: message?.innerText ?? "",
       urls,
       authorName: author?.innerText,
       publishedAt: time?.getAttribute("datetime") ?? time?.getAttribute("data-utime") ?? void 0,
+      ...commentCount !== void 0 ? { commentCount } : {},
       ...attachmentImages.length ? { attachmentImages } : {}
     };
   }));
@@ -443,6 +581,7 @@ async function extractFacebookDomItems(page, groupUrl) {
       url,
       authorName: item.authorName,
       publishedAt: item.publishedAt,
+      ...item.commentCount !== void 0 ? { commentCount: item.commentCount } : {},
       ...item.attachmentImages?.length ? { attachmentImages: dedupeImageUrls(item.attachmentImages) } : {}
     }] : [];
   }), Number.POSITIVE_INFINITY);
@@ -454,20 +593,67 @@ async function extractFacebookDomComments(page) {
       return void 0;
     }
     const belongsToArticle = (element) => element.closest('[role="article"]') === article;
-    const text = Array.from(article.querySelectorAll('[dir="auto"]')).filter((element) => belongsToArticle(element) && element.checkVisibility()).map((element) => element.innerText.trim()).filter(Boolean).sort((left, right) => right.length - left.length)[0] ?? "";
-    const authorMatch = /(?:dưới tên|by)\s+(.+?)(?:\s+(?:vào|on)\s+|$)/i.exec(label);
+    const readText = (element) => {
+      let text = "";
+      for (const node of Array.from(element.childNodes)) {
+        if (node.nodeType === Node.TEXT_NODE) {
+          text += node.textContent ?? "";
+        } else if (node instanceof HTMLImageElement) {
+          text += node.getAttribute("alt") ?? "";
+        } else if (node instanceof HTMLElement) {
+          text += readText(node);
+        }
+      }
+      return text.trim();
+    };
+    const texts = Array.from(article.querySelectorAll('[dir="auto"]')).filter((element) => belongsToArticle(element) && element.checkVisibility()).map((element) => readText(element)).filter(Boolean);
     const time = Array.from(article.querySelectorAll("time[datetime], abbr[data-utime]")).find((element) => belongsToArticle(element) && element.checkVisibility());
     return {
-      text,
-      authorName: authorMatch?.[1]?.trim(),
+      texts: Array.from(new Set(texts)),
+      authorLabel: label,
       publishedAt: time?.getAttribute("datetime") ?? time?.getAttribute("data-utime") ?? void 0
     };
   }));
-  return mergeFacebookComments(raw.flatMap((item) => item?.text.trim() ? [{
-    text: item.text,
-    ...item.authorName ? { authorName: item.authorName } : {},
-    ...toIso(item.publishedAt) ? { publishedAt: toIso(item.publishedAt) } : {}
-  }] : []), Number.POSITIVE_INFINITY);
+  return mergeFacebookComments(raw.flatMap((item) => {
+    if (!item)
+      return [];
+    const authorName = parseCommentAuthor(item.authorLabel);
+    const text = pickCommentText(item.texts, authorName);
+    if (!text)
+      return [];
+    const depth = parseCommentDepth(item.authorLabel);
+    return [{
+      text,
+      ...authorName ? { authorName } : {},
+      ...depth !== void 0 ? { depth } : {},
+      ...toIso(item.publishedAt) ? { publishedAt: toIso(item.publishedAt) } : {}
+    }];
+  }), Number.POSITIVE_INFINITY);
+}
+function pickCommentText(texts, authorName) {
+  if (!Array.isArray(texts) || !texts.length)
+    return "";
+  const candidates = texts.filter((text) => text !== authorName && !COMMENT_UI_LABELS.test(text));
+  return candidates.reduce((longest, text) => text.length > longest.length ? text : longest, "");
+}
+var COMMENT_UI_LABELS = /^(?:follow|theo dõi|like|thích|reply|trả lời|share|chia sẻ|see more|xem thêm|hide|ẩn|report|báo cáo)$/i;
+function commentKeyText(text) {
+  const normalized = text.replace(/[\p{Extended_Pictographic}\u200d\uFE0F]/gu, "").replace(/[:;=8xX]-?[)(\]\[dDpP/\\|]+/g, "").replace(/\s+/g, " ").trim();
+  return normalized || text.replace(/\s+/g, " ");
+}
+function parseCommentAuthor(label) {
+  const match = /^(?:bình luận|comment|trả lời|reply)\s+(?:dưới tên|by)\s+(.+)$/i.exec(label.trim());
+  if (!match)
+    return void 0;
+  const name = match[1].replace(/\s+vào\s+.*$/i, "").replace(/\s+on\s+.*$/i, "").replace(/\s+to\s+.*$/i, "").replace(/\s+\d+\s*(?:giây|phút|giờ|ngày|tuần|tháng|năm|seconds?|minutes?|hours?|days?|weeks?|months?|years?)\s*(?:ago|trước)?\s*$/i, "").trim();
+  return name || void 0;
+}
+function parseCommentDepth(label) {
+  const trimmed = label.trim();
+  if (/^(?:bình luận|comment)\b/i.test(trimmed))
+    return 0;
+  const parentPart = trimmed.split(/\s+(?:to|về|của)\s+/i)[1] ?? "";
+  return /\brepl(?:y|ies)\b|trả lời|phản hồi/i.test(parentPart) ? 2 : 1;
 }
 
 // ../../packages/facebook-scanner/dist/session.js
@@ -507,6 +693,7 @@ var FacebookScanner = class {
   extractDomComments;
   parseGraphqlBody;
   parseGraphqlComments;
+  parseCommentJson;
   graphqlResponseTimeoutMs;
   running = false;
   constructor(options) {
@@ -515,6 +702,7 @@ var FacebookScanner = class {
     this.extractDomComments = options.extractDomComments ?? extractFacebookDomComments;
     this.parseGraphqlBody = options.parseGraphqlBody ?? parseFacebookGraphqlBody;
     this.parseGraphqlComments = options.parseGraphqlComments ?? parseFacebookCommentBodies;
+    this.parseCommentJson = options.parseCommentJson ?? parseFacebookCommentJson;
     this.graphqlResponseTimeoutMs = options.graphqlResponseTimeoutMs ?? 1e4;
   }
   async scanMany(options) {
@@ -569,6 +757,7 @@ var FacebookScanner = class {
         if (condition !== "READY")
           throw new FacebookScannerError(condition);
         await options.onProgress?.({ phase: "FETCHING", commentCount: 0 });
+        const embeddedComments = await readEmbeddedComments(page, this.parseCommentJson);
         let comments = [];
         let idleRounds = 0;
         let rounds = 0;
@@ -579,19 +768,21 @@ var FacebookScanner = class {
           await page.waitForTimeout(1200);
           await drain(responseTasks, this.graphqlResponseTimeoutMs);
           const domComments2 = await this.extractDomComments(page);
-          comments = mergeFacebookComments([...graphqlComments, ...domComments2], options.limit);
+          comments = mergeFacebookComments([...embeddedComments, ...graphqlComments, ...domComments2], options.limit);
           if (comments.length !== previousCount) {
             await options.onProgress?.({ phase: "FETCHING", commentCount: comments.length });
           }
           const expanded = await clickCommentExpanders(page);
           idleRounds = comments.length === previousCount && !expanded ? idleRounds + 1 : 0;
-          if (comments.length < options.limit)
+          if (comments.length < options.limit) {
+            await scrollCommentArea(page);
             await page.mouse.wheel(0, 2e3);
+          }
         }
         assertNotAborted(options.signal);
         await drain(responseTasks, this.graphqlResponseTimeoutMs);
         const domComments = await this.extractDomComments(page);
-        comments = mergeFacebookComments([...graphqlComments, ...domComments], options.limit);
+        comments = mergeFacebookComments([...embeddedComments, ...graphqlComments, ...domComments], options.limit);
         await options.onProgress?.({ phase: "COMPLETE", commentCount: comments.length });
         return comments;
       } finally {
@@ -685,29 +876,58 @@ function withTimeout(promise, timeoutMs) {
   });
   return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
 }
-var COMMENT_EXPANDER_PATTERNS = [
-  /^xem thêm bình luận$/i,
-  /^xem \d+ bình luận$/i,
-  /^xem các bình luận khác$/i,
-  /^view more comments$/i,
-  /^view \d+ more comments$/i,
-  /^xem thêm câu trả lời$/i,
-  /^view more replies$/i
-];
+var COMMENT_EXPANDER_PATTERN = /^(?:xem|view)\b.*(?:bình luận|comments?|câu trả lời|repl(?:y|ies))/i;
+var MAX_EXPANDER_CLICKS = 4;
 async function clickCommentExpanders(page) {
   let clicked = 0;
-  for (const pattern of COMMENT_EXPANDER_PATTERNS) {
-    for (let attempt = 0; attempt < 2; attempt += 1) {
-      try {
-        await page.getByRole("button", { name: pattern }).first().click({ timeout: 1500 });
-        clicked += 1;
-      } catch {
-        break;
-      }
+  for (let attempt = 0; attempt < MAX_EXPANDER_CLICKS; attempt += 1) {
+    const buttons = page.getByRole("button", { name: COMMENT_EXPANDER_PATTERN });
+    let count = 0;
+    try {
+      count = await buttons.count();
+    } catch {
+      break;
+    }
+    if (!count)
+      break;
+    try {
+      await buttons.first().click({ timeout: 1e3 });
+      clicked += 1;
+    } catch {
+      break;
     }
   }
   return clicked;
 }
+async function scrollCommentArea(page) {
+  await page.evaluate(() => {
+    const article = document.querySelector('[role="article"][aria-label]');
+    let node = article instanceof HTMLElement ? article.parentElement : null;
+    while (node) {
+      if (node.scrollHeight > node.clientHeight + 80) {
+        node.scrollTop = node.scrollHeight;
+        return;
+      }
+      node = node.parentElement;
+    }
+    window.scrollTo(0, document.body.scrollHeight);
+  }).catch(() => void 0);
+}
+async function readEmbeddedComments(page, parse) {
+  const blocks = await page.evaluate(() => Array.from(document.querySelectorAll('script[type="application/json"]')).map((script) => script.textContent ?? "").filter((text) => text.includes('"Comment"'))).catch(() => []);
+  if (!Array.isArray(blocks))
+    return [];
+  const items = [];
+  for (const block of blocks) {
+    if (typeof block !== "string")
+      continue;
+    try {
+      items.push(...parse(JSON.parse(block)));
+    } catch {
+    }
+  }
+  return mergeFacebookComments(items, Number.POSITIVE_INFINITY);
+}
 
 // src/cli/cli.ts
 import { chromium } from "playwright";
@@ -823,7 +1043,7 @@ function sanitize(value) {
   return value.replace(/Bearer\s+\S+/gi, "Bearer [REDACTED]").replace(/\b(authorization|cookie|set-cookie|token|access[ _-]?token|refresh[ _-]?token|api[ _-]?key|private[ _-]?key|signature|pairing[ _-]?code|storage[ _-]?state)\b\s*[:=]\s*[^\s;]+/gi, "$1=[REDACTED]").replace(/https?:\/\/[^\s/@:]+:[^\s/@]+@/gi, "https://[REDACTED]@").replace(/\b[A-F0-9]{32}\b/gi, "[REDACTED]").replace(/[\r\n\t]+/g, " ").trim();
 }
 function isEntry(value) {
-  return typeof value?.id === "string" && typeof value.timestamp === "string" && ["INFO", "WARN", "ERROR"].includes(value.level) && ["CONNECTION", "PAIRING", "FACEBOOK", "SCAN", "UPLOAD", "SYSTEM"].includes(value.category) && typeof value.code === "string" && typeof value.message === "string";
+  return typeof value?.id === "string" && typeof value.timestamp === "string" && ["INFO", "WARN", "ERROR"].includes(value.level) && ["CONNECTION", "PAIRING", "FACEBOOK", "SCAN", "COMMENTS", "UPLOAD", "SYSTEM"].includes(value.category) && typeof value.code === "string" && typeof value.message === "string";
 }
 
 // src/main/browser-runtime.ts
@@ -1124,6 +1344,7 @@ import { randomUUID } from "node:crypto";
 
 // ../../packages/shared/src/index.ts
 var DESKTOP_AGENT_PROTOCOL_VERSION = 1;
+var DESKTOP_POST_COMMENTS_MAX_LIMIT = 200;
 var DESKTOP_POST_COMMENTS_REQUEST_TIMEOUT_MS = 3 * 60 * 1e3;
 
 // src/main/desktop-agent-session.ts
@@ -1291,7 +1512,7 @@ var DesktopAgentSession = class {
         state: this.reportedState,
         facebookReady: this.options.getFacebookReady(),
         currentJobId: this.reportedJobId,
-        capabilities: ["POST_COMMENTS"]
+        capabilities: ["POST_COMMENTS", "POST_COMMENTS_MORE"]
       });
       this.handshake = "WELCOME";
       return;
@@ -1490,7 +1711,7 @@ function describeEnvelopeProblem(envelope, deviceId, replayIds) {
 function isPostCommentsFetch(value) {
   if (!value || typeof value !== "object") return false;
   const payload = value;
-  return typeof payload.requestId === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(payload.requestId) && typeof payload.postUrl === "string" && payload.postUrl.length <= 2e3 && /^https:\/\/[\w.-]*facebook\.com\//i.test(payload.postUrl) && typeof payload.limit === "number" && Number.isInteger(payload.limit) && payload.limit > 0 && payload.limit <= 50;
+  return typeof payload.requestId === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(payload.requestId) && typeof payload.postUrl === "string" && payload.postUrl.length <= 2e3 && /^https:\/\/[\w.-]*facebook\.com\//i.test(payload.postUrl) && typeof payload.limit === "number" && Number.isInteger(payload.limit) && payload.limit > 0 && payload.limit <= DESKTOP_POST_COMMENTS_MAX_LIMIT;
 }
 function emptySnapshot(state) {
   return {
@@ -1527,6 +1748,7 @@ var DEFAULT_MAX_QUEUE = 5;
 var DesktopCommentRunner = class {
   options;
   queue = [];
+  listeners = /* @__PURE__ */ new Set();
   stopController = new AbortController();
   active = null;
   currentController = null;
@@ -1550,13 +1772,17 @@ var DesktopCommentRunner = class {
   isRunning() {
     return this.active !== null;
   }
+  subscribe(listener) {
+    this.listeners.add(listener);
+    return () => this.listeners.delete(listener);
+  }
   async whenIdle() {
     await this.active;
   }
   async handleEvent(event) {
     if (event.type !== "COMMENTS" || this.stopped) return;
     if (this.queue.length >= (this.options.maxQueue ?? DEFAULT_MAX_QUEUE)) {
-      await this.upload(event.requestId, { comments: [], errorCode: "QUEUE_FULL" });
+      await this.fail(event, "QUEUE_FULL");
       return;
     }
     this.queue.push(event);
@@ -1583,18 +1809,19 @@ var DesktopCommentRunner = class {
       this.options.fetchTimeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS
     );
     this.currentController = controller;
+    this.emit({ type: "STARTED", requestId: request.requestId, postUrl: request.postUrl });
     try {
       if (this.options.isScanActive()) {
-        await this.upload(request.requestId, { comments: [], errorCode: "SCANNER_BUSY" });
+        await this.fail(request, "SCANNER_BUSY");
         return;
       }
       if (this.options.isLoginActive()) {
-        await this.upload(request.requestId, { comments: [], errorCode: "LOGIN_ACTIVE" });
+        await this.fail(request, "LOGIN_ACTIVE");
         return;
       }
       const storageState = await this.options.facebookState.load();
       if (!storageState) {
-        await this.upload(request.requestId, { comments: [], errorCode: "FACEBOOK_SESSION_MISSING" });
+        await this.fail(request, "FACEBOOK_SESSION_MISSING");
         return;
       }
       let comments;
@@ -1606,28 +1833,56 @@ var DesktopCommentRunner = class {
           storageState
         });
       } catch (error) {
-        await this.upload(request.requestId, { comments: [], errorCode: readErrorCode2(error) });
+        await this.fail(request, readErrorCode2(error));
+        return;
+      }
+      if (!await this.upload(request.requestId, { comments })) {
+        if (!this.stopped) {
+          this.emit({
+            type: "FAILED",
+            requestId: request.requestId,
+            postUrl: request.postUrl,
+            errorCode: "UPLOAD_FAILED"
+          });
+        }
         return;
       }
-      await this.upload(request.requestId, { comments });
+      this.emit({
+        type: "SUCCEEDED",
+        requestId: request.requestId,
+        postUrl: request.postUrl,
+        commentCount: comments.length
+      });
     } finally {
       this.options.clearTimer(timeout);
       if (this.currentController === controller) this.currentController = null;
     }
   }
+  async fail(request, errorCode) {
+    await this.upload(request.requestId, { comments: [], errorCode });
+    this.emit({
+      type: "FAILED",
+      requestId: request.requestId,
+      postUrl: request.postUrl,
+      errorCode
+    });
+  }
+  emit(event) {
+    for (const listener of this.listeners) listener(event);
+  }
   async upload(requestId, payload) {
     const signal = this.stopController.signal;
-    if (signal.aborted) return;
+    if (signal.aborted) return false;
     for (let attempt = 0; ; attempt += 1) {
       try {
         await this.options.http.uploadComments(requestId, payload, signal);
-        return;
+        return true;
       } catch (error) {
-        if (!isRecoverable(error) || attempt >= 2 || signal.aborted) return;
+        if (!isRecoverable(error) || attempt >= 2 || signal.aborted) return false;
         try {
           await this.options.wait(1e3 * 2 ** attempt, signal);
         } catch {
-          return;
+          return false;
         }
       }
     }
@@ -1765,10 +2020,12 @@ function normalizeBatch(payload) {
     const authorAvatarUrl = normalizeMediaUrl(item.authorAvatarUrl);
     const attachmentImages = item.attachmentImages?.map(normalizeMediaUrl).filter((url) => Boolean(url)).slice(0, 6);
     const groupName = typeof item.groupName === "string" && item.groupName.trim() ? item.groupName.trim().slice(0, 200) : void 0;
+    const commentCount = normalizeCount(item.commentCount);
     const textFormat = normalizeTextFormat(item.textFormat);
     return {
       ...item,
       ...groupName ? { groupName } : {},
+      ...commentCount !== void 0 ? { commentCount } : {},
       sourceUrl: normalizeFacebookPostUrl(item.sourceUrl),
       authorAvatarUrl,
       attachmentImages: attachmentImages?.length ? attachmentImages : void 0,
@@ -1825,12 +2082,21 @@ function normalizeFormatToken(value) {
   const normalized = value?.trim();
   return normalized && normalized.length <= 50 ? normalized : void 0;
 }
+function normalizeCount(value) {
+  return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 && value <= 1e7 ? value : void 0;
+}
 function normalizeComments(payload) {
-  const comments = payload.comments.slice(0, 50).map((comment) => ({
-    text: comment.text.trim().slice(0, 2e3),
-    ...comment.authorName?.trim() ? { authorName: comment.authorName.trim().slice(0, 200) } : {},
-    ...comment.publishedAt ? { publishedAt: comment.publishedAt } : {}
-  })).filter((comment) => comment.text.length > 0);
+  const comments = payload.comments.slice(0, DESKTOP_POST_COMMENTS_MAX_LIMIT).map((comment) => {
+    const authorAvatarUrl = normalizeMediaUrl(comment.authorAvatarUrl);
+    const depth = normalizeCount(comment.depth);
+    return {
+      text: comment.text.trim().slice(0, 2e3),
+      ...comment.authorName?.trim() ? { authorName: comment.authorName.trim().slice(0, 200) } : {},
+      ...authorAvatarUrl ? { authorAvatarUrl } : {},
+      ...depth !== void 0 && depth <= 5 ? { depth } : {},
+      ...comment.publishedAt ? { publishedAt: comment.publishedAt } : {}
+    };
+  }).filter((comment) => comment.text.length > 0);
   const errorCode = payload.errorCode?.replace(/[^A-Z0-9_]/g, "_").slice(0, 100);
   return { comments, ...errorCode ? { errorCode } : {} };
 }
@@ -2267,6 +2533,7 @@ function toDesktopPost(item) {
     ...item.authorName ? { authorName: item.authorName } : {},
     ...item.authorAvatarUrl ? { authorAvatarUrl: item.authorAvatarUrl } : {},
     ...item.attachmentImages?.length ? { attachmentImages: item.attachmentImages } : {},
+    ...item.commentCount !== void 0 ? { commentCount: item.commentCount } : {},
     ...item.textFormat ? { textFormat: item.textFormat } : {},
     ...item.publishedAt ? { publishedAt: item.publishedAt } : {},
     ...item.groupName ? { groupName: item.groupName } : {}
@@ -2655,6 +2922,21 @@ function getFacebookAccountId(state) {
   return value && /^\d{1,32}$/.test(value) ? value : null;
 }
 
+// src/main/graphql-dump.ts
+import { writeFile as writeFile4 } from "node:fs/promises";
+import { join as join3 } from "node:path";
+function createGraphqlDumpParser(directory) {
+  if (!directory) return void 0;
+  return (body, groupUrl) => {
+    void writeFile4(
+      join3(directory, `graphql-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.txt`),
+      `// group: ${groupUrl}
+${body}`
+    ).catch(() => void 0);
+    return parseFacebookGraphqlBody(body, groupUrl);
+  };
+}
+
 // src/main/reconnect-backoff.ts
 var BASE_DELAY_MS = 1e3;
 var MAX_DELAY_MS = 3e4;
@@ -2925,7 +3207,7 @@ var MISSING_BROWSER_HINT = "Chromium for Playwright is not installed yet. Run `d
 function agentVersion() {
   let directory = fileURLToPath(new URL(".", import.meta.url));
   for (let depth = 0; depth < 4; depth += 1) {
-    const manifest = join3(directory, "package.json");
+    const manifest = join4(directory, "package.json");
     if (existsSync2(manifest)) {
       try {
         const version = JSON.parse(readFileSync2(manifest, "utf8")).version;
@@ -2942,7 +3224,7 @@ function resolvePlatform() {
   return process.platform === "linux" ? "LINUX" : "WINDOWS";
 }
 function resolveConfigDirectory(options) {
-  return options.config?.trim() || process.env.DIGGYLAB_AGENT_DIR?.trim() || join3(homedir(), CONFIG_DIRECTORY);
+  return options.config?.trim() || process.env.DIGGYLAB_AGENT_DIR?.trim() || join4(homedir(), CONFIG_DIRECTORY);
 }
 function resolveEndpoint(options) {
   const configured = (options.api ?? process.env.DIGGYLAB_API_URL)?.trim() || "https://api.diggylab.com";
@@ -2952,12 +3234,12 @@ function resolveEndpoint(options) {
 function createRuntime(options) {
   const directory = resolveConfigDirectory(options);
   const endpoint = resolveEndpoint(options);
-  const crypto2 = createFileCrypto(join3(directory, "key"));
-  const credentials = new CredentialStore(join3(directory, "credentials.json"), crypto2);
+  const crypto2 = createFileCrypto(join4(directory, "key"));
+  const credentials = new CredentialStore(join4(directory, "credentials.json"), crypto2);
   const sessions = new DeviceSessionStore(credentials);
   const facebook = new FacebookSessionStore(credentials);
-  const spool = new EncryptedScanSpool(join3(directory, "scan-spool"), crypto2);
-  const activity = new ActivityLog({ directory: join3(directory, "activity") });
+  const spool = new EncryptedScanSpool(join4(directory, "scan-spool"), crypto2);
+  const activity = new ActivityLog({ directory: join4(directory, "activity") });
   const http = new DesktopAgentHttpClient(endpoint.httpOrigin);
   const manager = new DeviceCredentialManager(sessions, http, createDeviceIdentity);
   return { activity, browserType: createBrowserLauncher(chromium), directory, endpoint, facebook, manager, sessions, spool };
@@ -3288,8 +3570,13 @@ async function commandRun(runtime, io) {
     io.error("This agent is not paired yet. Run `diggylab pair --code <code>` first.");
     return 2;
   }
-  const recordActivity = (message, code) => {
-    void runtime.activity.append({ category: "CONNECTION", code, level: "INFO", message }).catch(() => void 0);
+  const recordActivity = (message, code, options = {}) => {
+    void runtime.activity.append({
+      category: options.category ?? "CONNECTION",
+      code,
+      level: options.level ?? "INFO",
+      message
+    }).catch(() => void 0);
   };
   const facebookReady = await runtime.facebook.isReady();
   const session = new DesktopAgentSession({
@@ -3326,7 +3613,10 @@ async function commandRun(runtime, io) {
     },
     origin: runtime.endpoint.httpOrigin
   });
-  const scanner = new FacebookScanner({ browserType: runtime.browserType });
+  const scanner = new FacebookScanner({
+    browserType: runtime.browserType,
+    parseGraphqlBody: createGraphqlDumpParser(process.env.FACEBOOK_SCAN_DUMP_DIR)
+  });
   const commentRunner = new DesktopCommentRunner({
     clearTimer: (timer) => clearTimeout(timer),
     facebookState: runtime.facebook,
@@ -3382,6 +3672,29 @@ async function commandRun(runtime, io) {
   });
   scanRunner.start();
   commentRunner.start();
+  const unsubscribeComments = commentRunner.subscribe((event) => {
+    if (event.type === "STARTED") {
+      io.log(`${stamp()} comments: fetching ${event.postUrl}`);
+      recordActivity(`Fetching comments for ${event.postUrl}.`, "COMMENT_FETCH_STARTED", {
+        category: "COMMENTS"
+      });
+      return;
+    }
+    if (event.type === "SUCCEEDED") {
+      io.log(`${stamp()} comments: fetched ${event.commentCount} comment(s)`);
+      recordActivity(
+        `Fetched ${event.commentCount} comment(s) for ${event.postUrl}.`,
+        "COMMENT_FETCH_SUCCEEDED",
+        { category: "COMMENTS" }
+      );
+      return;
+    }
+    io.log(`${stamp()} comments: failed (${event.errorCode})`);
+    recordActivity(`Comment fetch failed (${event.errorCode}) for ${event.postUrl}.`, event.errorCode, {
+      category: "COMMENTS",
+      level: "WARN"
+    });
+  });
   await scanRunner.recover();
   await session.start();
   io.log(`Worker running as "${stored.device.name}" (API ${runtime.endpoint.httpOrigin}). Press Ctrl+C to stop.`);
@@ -3398,6 +3711,7 @@ async function commandRun(runtime, io) {
     clearInterval(keepAlive);
     session.stop();
     commentRunner.stop();
+    unsubscribeComments();
     scanRunner.stop();
   }
   return 0;
@@ -3428,7 +3742,7 @@ function runCommand(command, args) {
   });
 }
 function resolveServiceExecStart() {
-  const candidates = [join3(dirname3(process.execPath), "diggylab"), "/usr/local/bin/diggylab", "/usr/bin/diggylab"];
+  const candidates = [join4(dirname3(process.execPath), "diggylab"), "/usr/local/bin/diggylab", "/usr/bin/diggylab"];
   const link = candidates.find((candidate) => existsSync2(candidate));
   return link ? [link] : [process.execPath, fileURLToPath(import.meta.url)];
 }
@@ -3454,7 +3768,7 @@ async function commandInstallService(runtime, io) {
   }
   io.log(`Wrote ${SERVICE_PATH}.`);
   io.log(`State directory: ${runtime.directory}`);
-  if (process.env.SUDO_USER && process.env.SUDO_USER !== "root" && runtime.directory === join3(homedir(), CONFIG_DIRECTORY)) {
+  if (process.env.SUDO_USER && process.env.SUDO_USER !== "root" && runtime.directory === join4(homedir(), CONFIG_DIRECTORY)) {
     io.error(`Heads up: this ran as root, so the service will use ${runtime.directory} (root's home).`);
     io.error(`If you paired as "${process.env.SUDO_USER}", pass --config to point at that user's ~/${CONFIG_DIRECTORY} instead.`);
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "diggylab",
-  "version": "0.1.45",
+  "version": "0.1.46",
   "description": "DiggyLab Desktop Agent collects posts and information from Facebook Groups on a paired computer. (headless CLI)",
   "type": "module",
   "bin": {