diggylab 0.1.47 → 0.1.48

package/cli.js

@@ -180,8 +180,14 @@ function dedupeImageUrls(urls) {
   return order.map((pathname) => best.get(pathname).url).slice(0, 6);
 }
 function imageQuality(url) {
-  const match = /[?&]stp=[^&]*?_p(\d+)x(\d+)/.exec(url);
-  return match ? Math.max(Number(match[1]), Number(match[2])) : Number.MAX_SAFE_INTEGER;
+  if (/_fb30/i.test(url))
+    return 0;
+  const dimensions = [
+    ...url.matchAll(/(?:[?&](?:stp|cstp|ctp)=[^&]*?_s|_p)(\d+)x(\d+)/gi)
+  ];
+  if (!dimensions.length)
+    return Number.MAX_SAFE_INTEGER;
+  return Math.max(...dimensions.map((match) => Number(match[1]) * Number(match[2])));
 }
 function collectFallbackImageUrls(value, urls, depth) {
   if (!value || depth > 12 || urls.size >= 12)
@@ -423,10 +429,12 @@ function toComment(node) {
   const publishedAt = toIso(firstString(node, commentTimePaths));
   const id = firstString(node, [["id"]], false);
   const parentId = firstString(node, [["comment_direct_parent", "id"]], false);
+  const attachmentImages = getAttachmentImages(node);
   return {
     text: text.trim(),
     ...authorName ? { authorName } : {},
     ...authorAvatarUrl ? { authorAvatarUrl } : {},
+    ...attachmentImages.length ? { attachmentImages } : {},
     ...depth !== void 0 ? { depth } : {},
     ...id ? { id } : {},
     ...parentId ? { parentId } : {},
@@ -488,6 +496,12 @@ function mergeFacebookComments(items, limit) {
         existing.id = item.id;
       if (!existing.parentId && item.parentId)
         existing.parentId = item.parentId;
+      const attachmentImages = dedupeImageUrls([
+        ...existing.attachmentImages ?? [],
+        ...item.attachmentImages ?? []
+      ]);
+      if (attachmentImages.length)
+        existing.attachmentImages = attachmentImages;
       if (item.depth !== void 0) {
         existing.depth = existing.depth === void 0 ? item.depth : Math.min(existing.depth, item.depth);
       }
@@ -499,6 +513,7 @@ function mergeFacebookComments(items, limit) {
       text: text.slice(0, COMMENT_TEXT_LIMIT),
       ...authorName ? { authorName: authorName.slice(0, COMMENT_AUTHOR_LIMIT) } : {},
       ...item.authorAvatarUrl ? { authorAvatarUrl: item.authorAvatarUrl } : {},
+      ...item.attachmentImages?.length ? { attachmentImages: item.attachmentImages } : {},
       ...item.depth !== void 0 ? { depth: item.depth } : {},
       ...item.id ? { id: item.id } : {},
       ...item.parentId ? { parentId: item.parentId } : {},
@@ -608,10 +623,12 @@ async function extractFacebookDomComments(page) {
     };
     const texts = Array.from(article.querySelectorAll('[dir="auto"]')).filter((element) => belongsToArticle(element) && element.checkVisibility()).map((element) => readText(element)).filter(Boolean);
     const time = Array.from(article.querySelectorAll("time[datetime], abbr[data-utime]")).find((element) => belongsToArticle(element) && element.checkVisibility());
+    const attachmentImages = Array.from(article.querySelectorAll("img")).filter((image) => belongsToArticle(image) && image.checkVisibility()).map((image) => image.currentSrc || image.getAttribute("src") || "").filter((src) => /^https?:\/\//i.test(src) && /(scontent\.|fbcdn\.net)/i.test(src)).filter((src) => !/static\.xx\.fbcdn\.net|emoji\.php|rsrc\.php/i.test(src)).filter((src) => !/\/v\/t1\.30497-1\//i.test(src) && !/[?&]stp=cp0_/i.test(src));
     return {
       texts: Array.from(new Set(texts)),
       authorLabel: label,
-      publishedAt: time?.getAttribute("datetime") ?? time?.getAttribute("data-utime") ?? void 0
+      publishedAt: time?.getAttribute("datetime") ?? time?.getAttribute("data-utime") ?? void 0,
+      ...attachmentImages.length ? { attachmentImages } : {}
     };
   }));
   return mergeFacebookComments(raw.flatMap((item) => {
@@ -622,9 +639,11 @@ async function extractFacebookDomComments(page) {
     if (!text)
       return [];
     const depth = parseCommentDepth(item.authorLabel);
+    const attachmentImages = dedupeImageUrls(item.attachmentImages ?? []);
     return [{
       text,
       ...authorName ? { authorName } : {},
+      ...attachmentImages.length ? { attachmentImages } : {},
       ...depth !== void 0 ? { depth } : {},
       ...toIso(item.publishedAt) ? { publishedAt: toIso(item.publishedAt) } : {}
     }];
@@ -2088,11 +2107,13 @@ function normalizeCount(value) {
 function normalizeComments(payload) {
   const comments = payload.comments.slice(0, DESKTOP_POST_COMMENTS_MAX_LIMIT).map((comment) => {
     const authorAvatarUrl = normalizeMediaUrl(comment.authorAvatarUrl);
+    const attachmentImages = (comment.attachmentImages ?? []).map(normalizeMediaUrl).filter((url) => Boolean(url)).slice(0, 6);
     const depth = normalizeCount(comment.depth);
     return {
       text: comment.text.trim().slice(0, 2e3),
       ...comment.authorName?.trim() ? { authorName: comment.authorName.trim().slice(0, 200) } : {},
       ...authorAvatarUrl ? { authorAvatarUrl } : {},
+      ...attachmentImages.length ? { attachmentImages } : {},
       ...depth !== void 0 && depth <= 5 ? { depth } : {},
       ...comment.publishedAt ? { publishedAt: comment.publishedAt } : {}
     };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "diggylab",
-  "version": "0.1.47",
+  "version": "0.1.48",
   "description": "DiggyLab Desktop Agent collects posts and information from Facebook Groups on a paired computer. (headless CLI)",
   "type": "module",
   "bin": {