diggylab 0.1.44 → 0.1.45

package/cli.js

@@ -9,6 +9,8 @@ import { dirname as dirname3, join as join3 } from "node:path";
 import { fileURLToPath } from "node:url";
 
 // ../../packages/facebook-scanner/dist/parser.js
+var COMMENT_TEXT_LIMIT = 2e3;
+var COMMENT_AUTHOR_LIMIT = 200;
 var paths = {
   text: [["message", "text"], ["comet_sections", "content", "story", "message", "text"], ["text"]],
   url: [["url"], ["permalink_url"], ["wwwURL"], ["comet_sections", "content", "story", "url"]],
@@ -36,6 +38,21 @@ var textFormatPaths = [
     "text_format_metadata"
   ]
 ];
+var commentTextPaths = [
+  ["body", "text"],
+  ["comment_body", "text"],
+  ["preferred_body", "text"]
+];
+var commentAuthorPaths = [
+  ["author", "name"],
+  ["author_name"],
+  ["author", "short_name"]
+];
+var commentTimePaths = [
+  ["created_time"],
+  ["creation_time"],
+  ["publish_time"]
+];
 var facebookHosts = /* @__PURE__ */ new Set([
   "facebook.com",
   "fb.com",
@@ -327,6 +344,68 @@ function parseFacebookGraphqlBody(body, groupUrl) {
   }
   return mergeFacebookScanItems(items, Number.POSITIVE_INFINITY);
 }
+function toComment(node) {
+  const typename = typeof node.__typename === "string" ? node.__typename : void 0;
+  const text = firstString(node, commentTextPaths);
+  if (!text || typename && !/comment$/i.test(typename))
+    return void 0;
+  const authorName = firstString(node, commentAuthorPaths);
+  const publishedAt = toIso(firstString(node, commentTimePaths));
+  return {
+    text: text.trim(),
+    ...authorName ? { authorName } : {},
+    ...publishedAt ? { publishedAt } : {}
+  };
+}
+function visitComments(value, items) {
+  if (Array.isArray(value)) {
+    for (const item of value)
+      visitComments(item, items);
+    return;
+  }
+  if (!isData(value))
+    return;
+  const comment = toComment(value);
+  if (comment)
+    items.push(comment);
+  for (const child of Object.values(value))
+    visitComments(child, items);
+}
+function parseFacebookCommentBodies(body) {
+  const items = [];
+  for (const line of body.split(/\r?\n/)) {
+    const json = line.trim().replace(/^for\s*\(;;\);/, "");
+    if (!json.startsWith("{") && !json.startsWith("["))
+      continue;
+    try {
+      visitComments(JSON.parse(json), items);
+    } catch {
+    }
+  }
+  return mergeFacebookComments(items, Number.POSITIVE_INFINITY);
+}
+function mergeFacebookComments(items, limit) {
+  const seen = /* @__PURE__ */ new Set();
+  const merged = [];
+  for (const item of items) {
+    const text = item.text.trim();
+    if (!text)
+      continue;
+    const authorName = item.authorName?.trim();
+    const key = `${authorName ?? ""}\0${text}\0${item.publishedAt ?? ""}`;
+    if (seen.has(key))
+      continue;
+    if (merged.length >= limit)
+      break;
+    seen.add(key);
+    merged.push({
+      text: text.slice(0, COMMENT_TEXT_LIMIT),
+      ...authorName ? { authorName: authorName.slice(0, COMMENT_AUTHOR_LIMIT) } : {},
+      ...item.publishedAt ? { publishedAt: item.publishedAt } : {}
+    });
+  }
+  return merged;
+}
 async function extractFacebookDomItems(page, groupUrl) {
   const requestedGroup = requestedGroupIdentifier(groupUrl);
   if (!requestedGroup)
@@ -368,6 +447,28 @@ async function extractFacebookDomItems(page, groupUrl) {
     }] : [];
   }), Number.POSITIVE_INFINITY);
 }
+async function extractFacebookDomComments(page) {
+  const raw = await page.locator('[role="article"][aria-label]').evaluateAll((articles) => articles.map((article) => {
+    const label = (article.getAttribute("aria-label") ?? "").trim();
+    if (!/^(?:bình luận|comment|trả lời|reply)\b/i.test(label) || !article.checkVisibility()) {
+      return void 0;
+    }
+    const belongsToArticle = (element) => element.closest('[role="article"]') === article;
+    const text = Array.from(article.querySelectorAll('[dir="auto"]')).filter((element) => belongsToArticle(element) && element.checkVisibility()).map((element) => element.innerText.trim()).filter(Boolean).sort((left, right) => right.length - left.length)[0] ?? "";
+    const authorMatch = /(?:dưới tên|by)\s+(.+?)(?:\s+(?:vào|on)\s+|$)/i.exec(label);
+    const time = Array.from(article.querySelectorAll("time[datetime], abbr[data-utime]")).find((element) => belongsToArticle(element) && element.checkVisibility());
+    return {
+      text,
+      authorName: authorMatch?.[1]?.trim(),
+      publishedAt: time?.getAttribute("datetime") ?? time?.getAttribute("data-utime") ?? void 0
+    };
+  }));
+  return mergeFacebookComments(raw.flatMap((item) => item?.text.trim() ? [{
+    text: item.text,
+    ...item.authorName ? { authorName: item.authorName } : {},
+    ...toIso(item.publishedAt) ? { publishedAt: toIso(item.publishedAt) } : {}
+  }] : []), Number.POSITIVE_INFINITY);
+}
 
 // ../../packages/facebook-scanner/dist/session.js
 async function detectFacebookSession(page, context) {
@@ -403,13 +504,17 @@ var FacebookScannerError = class extends Error {
 var FacebookScanner = class {
   browserType;
   extractDomItems;
+  extractDomComments;
   parseGraphqlBody;
+  parseGraphqlComments;
   graphqlResponseTimeoutMs;
   running = false;
   constructor(options) {
     this.browserType = options.browserType;
     this.extractDomItems = options.extractDomItems ?? extractFacebookDomItems;
+    this.extractDomComments = options.extractDomComments ?? extractFacebookDomComments;
     this.parseGraphqlBody = options.parseGraphqlBody ?? parseFacebookGraphqlBody;
+    this.parseGraphqlComments = options.parseGraphqlComments ?? parseFacebookCommentBodies;
     this.graphqlResponseTimeoutMs = options.graphqlResponseTimeoutMs ?? 1e4;
   }
   async scanMany(options) {
@@ -431,6 +536,76 @@ var FacebookScanner = class {
       this.running = false;
     }
   }
+  async fetchComments(options) {
+    if (this.running)
+      throw new FacebookScannerError("SCANNER_BUSY");
+    const target = normalizePostUrl(options.postUrl);
+    if (!target)
+      throw new FacebookScannerError("POST_URL_INVALID");
+    assertNotAborted(options.signal);
+    this.running = true;
+    let browser = null;
+    try {
+      browser = await this.browserType.launch({ headless: true });
+      const context = await browser.newContext({ storageState: options.storageState });
+      let page = null;
+      const graphqlComments = [];
+      const responseTasks = [];
+      const responseHandler = (response) => {
+        if (!response.url().includes("/api/graphql/"))
+          return;
+        responseTasks.push(withTimeout(response.text(), this.graphqlResponseTimeoutMs).then((body) => graphqlComments.push(...this.parseGraphqlComments(body))).then(() => void 0).catch(() => void 0));
+      };
+      try {
+        page = await context.newPage();
+        page.on("response", responseHandler);
+        await page.route("**/*", (route) => ["font", "image", "media"].includes(route.request().resourceType()) ? route.abort() : route.continue());
+        try {
+          await page.goto(target, { waitUntil: "domcontentloaded", timeout: 45e3 });
+        } catch {
+          throw new FacebookScannerError("NAVIGATION_FAILED");
+        }
+        const condition = await detectFacebookSession(page, context);
+        if (condition !== "READY")
+          throw new FacebookScannerError(condition);
+        await options.onProgress?.({ phase: "FETCHING", commentCount: 0 });
+        let comments = [];
+        let idleRounds = 0;
+        let rounds = 0;
+        while (comments.length < options.limit && idleRounds < 3 && rounds < 20) {
+          assertNotAborted(options.signal);
+          rounds += 1;
+          const previousCount = comments.length;
+          await page.waitForTimeout(1200);
+          await drain(responseTasks, this.graphqlResponseTimeoutMs);
+          const domComments2 = await this.extractDomComments(page);
+          comments = mergeFacebookComments([...graphqlComments, ...domComments2], options.limit);
+          if (comments.length !== previousCount) {
+            await options.onProgress?.({ phase: "FETCHING", commentCount: comments.length });
+          }
+          const expanded = await clickCommentExpanders(page);
+          idleRounds = comments.length === previousCount && !expanded ? idleRounds + 1 : 0;
+          if (comments.length < options.limit)
+            await page.mouse.wheel(0, 2e3);
+        }
+        assertNotAborted(options.signal);
+        await drain(responseTasks, this.graphqlResponseTimeoutMs);
+        const domComments = await this.extractDomComments(page);
+        comments = mergeFacebookComments([...graphqlComments, ...domComments], options.limit);
+        await options.onProgress?.({ phase: "COMPLETE", commentCount: comments.length });
+        return comments;
+      } finally {
+        page?.off("response", responseHandler);
+        await page?.unroute("**/*").catch(() => void 0);
+        await Promise.allSettled(responseTasks);
+        await page?.close();
+        await context.close();
+      }
+    } finally {
+      await browser?.close();
+      this.running = false;
+    }
+  }
   async scanGroup(browser, options, groupUrl, groupIndex) {
     const context = await browser.newContext({ storageState: options.storageState });
     let page = null;
@@ -510,6 +685,29 @@ function withTimeout(promise, timeoutMs) {
   });
   return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
 }
+var COMMENT_EXPANDER_PATTERNS = [
+  /^xem thêm bình luận$/i,
+  /^xem \d+ bình luận$/i,
+  /^xem các bình luận khác$/i,
+  /^view more comments$/i,
+  /^view \d+ more comments$/i,
+  /^xem thêm câu trả lời$/i,
+  /^view more replies$/i
+];
+async function clickCommentExpanders(page) {
+  let clicked = 0;
+  for (const pattern of COMMENT_EXPANDER_PATTERNS) {
+    for (let attempt = 0; attempt < 2; attempt += 1) {
+      try {
+        await page.getByRole("button", { name: pattern }).first().click({ timeout: 1500 });
+        clicked += 1;
+      } catch {
+        break;
+      }
+    }
+  }
+  return clicked;
+}
 
 // src/cli/cli.ts
 import { chromium } from "playwright";
@@ -926,6 +1124,7 @@ import { randomUUID } from "node:crypto";
 
 // ../../packages/shared/src/index.ts
 var DESKTOP_AGENT_PROTOCOL_VERSION = 1;
+var DESKTOP_POST_COMMENTS_REQUEST_TIMEOUT_MS = 3 * 60 * 1e3;
 
 // src/main/desktop-agent-session.ts
 var MESSAGE_FRESHNESS_MS = 5 * 60 * 1e3;
@@ -1091,7 +1290,8 @@ var DesktopAgentSession = class {
         platform: this.options.platform,
         state: this.reportedState,
         facebookReady: this.options.getFacebookReady(),
-        currentJobId: this.reportedJobId
+        currentJobId: this.reportedJobId,
+        capabilities: ["POST_COMMENTS"]
       });
       this.handshake = "WELCOME";
       return;
@@ -1135,8 +1335,15 @@ var DesktopAgentSession = class {
       this.emitJob({ type: "STOP", ...envelope.payload });
       return;
     }
+    if (envelope.type === "POST_COMMENTS_FETCH") {
+      if (!isPostCommentsFetch(envelope.payload)) {
+        this.protocolFailure(socket, stored, generation);
+        return;
+      }
+      this.emitJob({ type: "COMMENTS", ...envelope.payload });
+      return;
+    }
     if (envelope.type !== "SCAN_JOB_START") {
-      this.protocolFailure(socket, stored, generation);
       return;
     }
     this.update({ ...this.snapshot, currentJobId: envelope.payload.jobId, pendingJobId: null });
@@ -1280,6 +1487,11 @@ function describeEnvelopeProblem(envelope, deviceId, replayIds) {
   if (replayIds.has(envelope.messageId)) return `replayed message ${envelope.messageId}`;
   return "unknown";
 }
+function isPostCommentsFetch(value) {
+  if (!value || typeof value !== "object") return false;
+  const payload = value;
+  return typeof payload.requestId === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(payload.requestId) && typeof payload.postUrl === "string" && payload.postUrl.length <= 2e3 && /^https:\/\/[\w.-]*facebook\.com\//i.test(payload.postUrl) && typeof payload.limit === "number" && Number.isInteger(payload.limit) && payload.limit > 0 && payload.limit <= 50;
+}
 function emptySnapshot(state) {
   return {
     state,
@@ -1309,6 +1521,127 @@ function readErrorCode(error) {
   return error && typeof error === "object" && "code" in error && typeof error.code === "string" ? error.code : "UNKNOWN_ERROR";
 }
 
+// src/main/desktop-comment-runner.ts
+var DEFAULT_FETCH_TIMEOUT_MS = 2 * 60 * 1e3;
+var DEFAULT_MAX_QUEUE = 5;
+var DesktopCommentRunner = class {
+  options;
+  queue = [];
+  stopController = new AbortController();
+  active = null;
+  currentController = null;
+  unsubscribe = null;
+  stopped = false;
+  constructor(options) {
+    this.options = options;
+  }
+  start() {
+    if (this.unsubscribe) return;
+    this.unsubscribe = this.options.session.subscribeJobs((event) => this.handleEvent(event));
+  }
+  stop() {
+    this.stopped = true;
+    this.queue.length = 0;
+    this.stopController.abort();
+    this.currentController?.abort();
+    this.unsubscribe?.();
+    this.unsubscribe = null;
+  }
+  isRunning() {
+    return this.active !== null;
+  }
+  async whenIdle() {
+    await this.active;
+  }
+  async handleEvent(event) {
+    if (event.type !== "COMMENTS" || this.stopped) return;
+    if (this.queue.length >= (this.options.maxQueue ?? DEFAULT_MAX_QUEUE)) {
+      await this.upload(event.requestId, { comments: [], errorCode: "QUEUE_FULL" });
+      return;
+    }
+    this.queue.push(event);
+    await this.drain();
+  }
+  drain() {
+    if (!this.active) {
+      this.active = this.run().finally(() => {
+        this.active = null;
+      });
+    }
+    return this.active;
+  }
+  async run() {
+    while (this.queue.length > 0 && !this.stopped) {
+      const request = this.queue.shift();
+      if (request) await this.process(request);
+    }
+  }
+  async process(request) {
+    const controller = new AbortController();
+    const timeout = this.options.setTimer(
+      () => controller.abort(),
+      this.options.fetchTimeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS
+    );
+    this.currentController = controller;
+    try {
+      if (this.options.isScanActive()) {
+        await this.upload(request.requestId, { comments: [], errorCode: "SCANNER_BUSY" });
+        return;
+      }
+      if (this.options.isLoginActive()) {
+        await this.upload(request.requestId, { comments: [], errorCode: "LOGIN_ACTIVE" });
+        return;
+      }
+      const storageState = await this.options.facebookState.load();
+      if (!storageState) {
+        await this.upload(request.requestId, { comments: [], errorCode: "FACEBOOK_SESSION_MISSING" });
+        return;
+      }
+      let comments;
+      try {
+        comments = await this.options.scanner.fetchComments({
+          limit: request.limit,
+          postUrl: request.postUrl,
+          signal: controller.signal,
+          storageState
+        });
+      } catch (error) {
+        await this.upload(request.requestId, { comments: [], errorCode: readErrorCode2(error) });
+        return;
+      }
+      await this.upload(request.requestId, { comments });
+    } finally {
+      this.options.clearTimer(timeout);
+      if (this.currentController === controller) this.currentController = null;
+    }
+  }
+  async upload(requestId, payload) {
+    const signal = this.stopController.signal;
+    if (signal.aborted) return;
+    for (let attempt = 0; ; attempt += 1) {
+      try {
+        await this.options.http.uploadComments(requestId, payload, signal);
+        return;
+      } catch (error) {
+        if (!isRecoverable(error) || attempt >= 2 || signal.aborted) return;
+        try {
+          await this.options.wait(1e3 * 2 ** attempt, signal);
+        } catch {
+          return;
+        }
+      }
+    }
+  }
+};
+function readErrorCode2(error) {
+  const code = error && typeof error === "object" && "code" in error && typeof error.code === "string" ? error.code : "";
+  const sanitized = code.replace(/[^A-Z0-9_]/g, "_").slice(0, 100);
+  return sanitized || "COMMENT_FETCH_FAILED";
+}
+function isRecoverable(error) {
+  return Boolean(error && typeof error === "object" && "recoverable" in error && error.recoverable === true);
+}
+
 // src/main/desktop-scan-http.ts
 import { createHash, createPrivateKey, randomUUID as randomUUID2, sign } from "node:crypto";
 var RESPONSE_LIMIT2 = 64 * 1024;
@@ -1344,6 +1677,13 @@ var DesktopScanHttp = class {
   cancel(jobId, attemptToken) {
     return this.signedPost(`/leads/desktop-agent/jobs/${jobId}/cancel`, { attemptToken });
   }
+  uploadComments(requestId, payload, signal) {
+    return this.signedPost(
+      `/leads/desktop-agent/post-comments/${requestId}`,
+      normalizeComments(payload),
+      signal
+    );
+  }
   async signedPost(path, businessPayload, signal) {
     const credential = this.getCredential();
     if (!credential) throw new DesktopScanHttpError("CREDENTIAL_UNAVAILABLE");
@@ -1485,6 +1825,15 @@ function normalizeFormatToken(value) {
   const normalized = value?.trim();
   return normalized && normalized.length <= 50 ? normalized : void 0;
 }
+function normalizeComments(payload) {
+  const comments = payload.comments.slice(0, 50).map((comment) => ({
+    text: comment.text.trim().slice(0, 2e3),
+    ...comment.authorName?.trim() ? { authorName: comment.authorName.trim().slice(0, 200) } : {},
+    ...comment.publishedAt ? { publishedAt: comment.publishedAt } : {}
+  })).filter((comment) => comment.text.length > 0);
+  const errorCode = payload.errorCode?.replace(/[^A-Z0-9_]/g, "_").slice(0, 100);
+  return { comments, ...errorCode ? { errorCode } : {} };
+}
 function normalizeFacebookPostUrl(value) {
   try {
     const url = new URL(value);
@@ -1525,7 +1874,7 @@ var DesktopScanRunner = class {
       records = await this.options.spool.loadAll();
     } catch (error) {
       this.recoveryBlocked = true;
-      this.update({ ...idleSnapshot(), state: "SPOOL_FULL", errorCode: readErrorCode2(error) });
+      this.update({ ...idleSnapshot(), state: "SPOOL_FULL", errorCode: readErrorCode3(error) });
       return;
     }
     const current = records.find((record) => Date.parse(record.job.leaseExpiresAt) > this.options.now()) ?? null;
@@ -1587,6 +1936,7 @@ var DesktopScanRunner = class {
     return true;
   }
   async handleEvent(event) {
+    if (event.type === "COMMENTS") return;
     if (event.type === "AVAILABLE") {
       await this.claimIfReady(event.jobId);
       return;
@@ -1653,6 +2003,7 @@ var DesktopScanRunner = class {
   async claimIfReady(jobId) {
     const session = this.options.session.getSnapshot();
     if (this.recoveryBlocked || this.active || this.snapshot.jobId || session.currentJobId || session.state !== "ONLINE" || session.accessMode !== "FULL" || this.options.isLoginActive() || !await this.options.facebookState.isReady()) return;
+    if (this.options.isCommentFetchActive?.()) return;
     this.options.session.claimJob(jobId);
   }
   async run(job, resume) {
@@ -1783,7 +2134,7 @@ var DesktopScanRunner = class {
       this.recovered = null;
       this.options.session.finishJob(job.jobId);
     } catch (error) {
-      const code = readErrorCode2(error);
+      const code = readErrorCode3(error);
       const checkpoint = code === "LOGIN_REQUIRED" || code === "CHECKPOINT_REQUIRED";
       if (code === "OFFLINE_PENDING") {
         this.recovered = record;
@@ -1820,7 +2171,7 @@ var DesktopScanRunner = class {
           await this.options.spool.put(record);
           confirmed = await this.completeWithRetry(job, completion, controller.signal);
         } catch (completionError) {
-          const completionCode = readErrorCode2(completionError);
+          const completionCode = readErrorCode3(completionError);
           terminalPersistenceFailed = true;
           this.recovered = record;
           this.update({
@@ -1856,7 +2207,7 @@ var DesktopScanRunner = class {
         await this.options.http.uploadBatch(job.jobId, batch, signal);
         return true;
       } catch (error) {
-        if (!isRecoverable(error)) throw error;
+        if (!isRecoverable2(error)) throw error;
         if (attempt >= 4) return false;
         attempt += 1;
         await this.options.wait(Math.min(8e3, 500 * 2 ** attempt), signal);
@@ -1871,7 +2222,7 @@ var DesktopScanRunner = class {
         await this.options.http.complete(job.jobId, payload, signal);
         return true;
       } catch (error) {
-        if (!isRecoverable(error)) throw error;
+        if (!isRecoverable2(error)) throw error;
         if (attempt >= 4) return false;
         attempt += 1;
         await this.options.wait(Math.min(8e3, 500 * 2 ** attempt), signal);
@@ -1960,10 +2311,10 @@ function createSpoolRecord(job, now) {
 function nowIso(now) {
   return new Date(now).toISOString();
 }
-function readErrorCode2(error) {
+function readErrorCode3(error) {
   return error && typeof error === "object" && "code" in error && typeof error.code === "string" ? error.code.slice(0, 500) : "SCAN_FAILED";
 }
-function isRecoverable(error) {
+function isRecoverable2(error) {
   return Boolean(error && typeof error === "object" && "recoverable" in error && error.recoverable === true);
 }
 function abortableDelay(milliseconds, signal) {
@@ -2975,17 +3326,34 @@ async function commandRun(runtime, io) {
     },
     origin: runtime.endpoint.httpOrigin
   });
+  const scanner = new FacebookScanner({ browserType: runtime.browserType });
+  const commentRunner = new DesktopCommentRunner({
+    clearTimer: (timer) => clearTimeout(timer),
+    facebookState: runtime.facebook,
+    http: scanHttp,
+    isLoginActive: () => false,
+    isScanActive: () => scanRunner?.isRunning() ?? false,
+    scanner,
+    session,
+    setTimer: (callback, delay) => {
+      const timer = setTimeout(callback, delay);
+      timer.unref();
+      return timer;
+    },
+    wait: abortableDelay
+  });
   const scanRunner = new DesktopScanRunner({
     clearTimer: (timer) => clearTimeout(timer),
     facebookState: runtime.facebook,
     http: scanHttp,
+    isCommentFetchActive: () => commentRunner.isRunning(),
     isLoginActive: () => false,
     now: Date.now,
     randomUuid: () => randomUUID3(),
     requestFacebookLogin: async () => {
       io.error("The Facebook session expired. Stop this worker, run `diggylab login`, then start it again.");
     },
-    scanner: new FacebookScanner({ browserType: runtime.browserType }),
+    scanner,
     session,
     setTimer: (callback, delay) => {
       const timer = setTimeout(callback, delay);
@@ -3013,6 +3381,7 @@ async function commandRun(runtime, io) {
     recordActivity(`Scanner state changed to ${snapshot.state}.`, snapshot.state);
   });
   scanRunner.start();
+  commentRunner.start();
   await scanRunner.recover();
   await session.start();
   io.log(`Worker running as "${stored.device.name}" (API ${runtime.endpoint.httpOrigin}). Press Ctrl+C to stop.`);
@@ -3028,6 +3397,7 @@ async function commandRun(runtime, io) {
   } finally {
     clearInterval(keepAlive);
     session.stop();
+    commentRunner.stop();
     scanRunner.stop();
   }
   return 0;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "diggylab",
-  "version": "0.1.44",
+  "version": "0.1.45",
   "description": "DiggyLab Desktop Agent collects posts and information from Facebook Groups on a paired computer. (headless CLI)",
   "type": "module",
   "bin": {