@sockethub/platform-metadata 1.0.1-beta.3 → 1.0.1-beta.5

package/README.md

@@ -98,6 +98,11 @@ The platform extracts the following metadata when available:
   flattened to one string, or the virtual location URL), `url`, and `status`
   (`scheduled`, `cancelled`, `postponed`, `rescheduled`). Only present when
   both a name and a start date are declared.
+* **Description fallback**: pages with no meta description use a JSON-LD
+  `description`, else a `headline` not already in the title (Wikipedia's
+  short description, e.g. "1979 studio album by Brian Eno").
+* **Non-HTML links** (PDFs and other files) return a page titled with the
+  file name instead of an error.
 
 ## Site-Specific Handling
 
@@ -108,7 +113,9 @@ so those URLs are resolved through preview services built for this purpose:
   are fetched from the [FxTwitter API](https://github.com/FxEmbed/FxEmbed)
   instead of scraped — X serves only a generic site banner to scrapers. The
   response carries the post's text, its actual photo (or video + poster
-  thumbnail), and the canonical `x.com` URL. If the API is unavailable or the
+  thumbnail), and the canonical `x.com` URL. Posts without their own media
+  use the quoted post's media, then the link card image, then the author's
+  avatar. If the API is unavailable or the
   post can't be resolved, the platform falls back to the regular scrape
   (which still yields the post text).
 * **Reddit post URLs** use `old.reddit.com`'s structured JSON response, avoiding
@@ -124,6 +131,13 @@ so those URLs are resolved through preview services built for this purpose:
   datacenter address. Full media still requires authentication. Facebook
   publishes no site name, so `actor.name` and `object.name` fall back to
   `Facebook`.
+* **Amazon product URLs** (storefronts such as `amazon.de` and `amazon.com`,
+  plus `amzn.to`/`a.co` short links) are scraped with WhatsApp's link-preview
+  user agent. Amazon answers other crawlers with a captcha or a generic
+  "Amazon" share card instead of the product's title and image. Mobile-app
+  links (`/gp/aw/d/<ASIN>`) are scraped as `/dp/<ASIN>`, which is the only
+  form that gets the product card. When Amazon serves its full desktop page
+  instead, the image comes from the page's main product image.
 * **Facebook engagement prefixes**: video titles and image alt text often
   start with a localized view or reaction count, such as
   `2.2M views · 21K reactions | …` or `347K views | …`. When the prefix is

package/dist/index.js

@@ -70777,7 +70777,7 @@ var import_undici3 = __toESM(require_undici(), 1);
 // package.json
 var package_default = {
   name: "@sockethub/platform-metadata",
-  version: "1.0.1-beta.3",
+  version: "1.0.1-beta.5",
   description: "Fetch metadata from a given URL",
   author: "Nick Jennings <nick@silverbucket.net>",
   license: "LGPL-3.0+",
@@ -70819,17 +70819,17 @@ var package_default = {
     "clean:deps": "rm -rf node_modules"
   },
   dependencies: {
-    "@sockethub/util": "^1.0.0-beta.3",
+    "@sockethub/util": "^1.0.0-beta.5",
     "open-graph-scraper": "^6.11.0",
     undici: "^7.30.0"
   },
   devDependencies: {
-    "@sockethub/schemas": "^3.0.0-beta.3"
+    "@sockethub/schemas": "^3.0.0-beta.5"
   },
   peerDependencies: {
-    "@sockethub/server": "^5.0.0-beta.3"
+    "@sockethub/server": "^5.0.0-beta.5"
   },
-  gitHead: "bc04522d60d1e2fa1b3098b339897de62c3b0549",
+  gitHead: "cc4ffdd9714ebbd8372a88c4f2d795974e04029e",
   types: "./dist/index.d.ts"
 };
 
@@ -71074,6 +71074,68 @@ function isFacebookUrl(url) {
   const host = parsed.hostname.toLowerCase();
   return FACEBOOK_HOSTS.has(host) || host === "fb.watch";
 }
+var AMAZON_STORE_TLDS = new Set([
+  "com",
+  "ca",
+  "com.mx",
+  "com.br",
+  "co.uk",
+  "de",
+  "fr",
+  "it",
+  "es",
+  "nl",
+  "se",
+  "pl",
+  "com.be",
+  "com.tr",
+  "ie",
+  "ae",
+  "sa",
+  "eg",
+  "in",
+  "co.jp",
+  "sg",
+  "com.au",
+  "co.za"
+]);
+var AMAZON_SHORT_HOSTS = new Set(["amzn.to", "amzn.eu", "amzn.asia", "a.co"]);
+function isAmazonUrl(url) {
+  let parsed;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return false;
+  }
+  const host = parsed.hostname.toLowerCase().replace(/^www\./, "");
+  if (AMAZON_SHORT_HOSTS.has(host))
+    return true;
+  return host.startsWith("amazon.") && AMAZON_STORE_TLDS.has(host.slice(7));
+}
+function resolveAmazonAppLink(url) {
+  if (!isAmazonUrl(url))
+    return null;
+  const parsed = new URL(url);
+  const asin = parsed.pathname.match(/^\/gp\/aw\/d\/([A-Z0-9]{10})(?:\/|$)/i)?.[1];
+  return asin ? `${parsed.origin}/dp/${asin}` : null;
+}
+function extractAmazonImage(html) {
+  const tag = html?.match(/<img[^>]*\bid="landingImage"[^>]*>/)?.[0];
+  return tag?.match(/\bdata-old-hires="(https:[^"]+)"/)?.[1];
+}
+function schemaOrgSummary(jsonLD, title = "") {
+  if (!Array.isArray(jsonLD))
+    return;
+  const items = jsonLD.flatMap((item) => item?.["@graph"] ?? item);
+  const description = items.find((item) => typeof item?.description === "string" && item.description.trim())?.description;
+  if (description)
+    return description;
+  for (const item of items) {
+    const headline = typeof item?.headline === "string" ? item.headline.trim() : "";
+    if (headline && !title.includes(headline))
+      return headline;
+  }
+}
 function stripFacebookEngagement(title) {
   const bar = title?.indexOf("|") ?? -1;
   if (title === undefined || bar === -1) {
@@ -71189,7 +71251,26 @@ function redditPostImage(post) {
       }
     ];
   }
-  return;
+  return redditGalleryImages(post);
+}
+function redditGalleryImages(post) {
+  const items = post.gallery_data?.items;
+  if (!Array.isArray(items))
+    return;
+  const images = [];
+  for (const item of items) {
+    const id = item?.media_id;
+    const source = typeof id === "string" ? post.media_metadata?.[id]?.s : undefined;
+    const url = redditMediaUrl(source?.u ?? source?.gif, new Set(["i.redd.it", "preview.redd.it"]));
+    if (url) {
+      images.push({
+        url,
+        width: finiteNonNegative(source?.x),
+        height: finiteNonNegative(source?.y)
+      });
+    }
+  }
+  return images.length ? images : undefined;
 }
 function redditPostVideo(post) {
   const video = post.secure_media?.reddit_video;
@@ -71281,10 +71362,12 @@ function tweetToPageObject(status) {
 
 `) : undefined;
   const title = typeof article?.title === "string" && article.title || (author.name && author.screen_name ? `${author.name} (@${author.screen_name}) on X` : author.name ?? "Post on X");
-  const photo = tweet.media?.photos?.[0];
-  const video = tweet.media?.videos?.[0];
+  const media = tweet.media?.photos?.length || tweet.media?.videos?.length ? tweet.media : tweet.quote?.media;
+  const photo = media?.photos?.[0];
+  const video = media?.videos?.[0];
   const articleImage = article?.cover_media?.media_info;
-  const imageUrl = articleImage?.original_img_url ?? photo?.url ?? video?.thumbnail_url;
+  const card = tweet.card?.image;
+  const imageUrl = articleImage?.original_img_url ?? photo?.url ?? video?.thumbnail_url ?? card?.url ?? author.avatar_url;
   const page = {
     type: "page",
     title,
@@ -71297,8 +71380,8 @@ function tweetToPageObject(status) {
     page.image = [
       {
         url: imageUrl,
-        width: articleImage?.original_img_width ?? photo?.width ?? video?.width,
-        height: articleImage?.original_img_height ?? photo?.height ?? video?.height
+        width: articleImage?.original_img_width ?? photo?.width ?? video?.width ?? card?.width,
+        height: articleImage?.original_img_height ?? photo?.height ?? video?.height ?? card?.height
       }
     ];
   }
@@ -71425,13 +71508,15 @@ var PlatformMetadataSchema = {
 // src/index.ts
 var DEFAULT_USER_AGENT = `Mozilla/5.0 (compatible; SockethubBot/${package_default.version}; +https://sockethub.org)`;
 var COMPAT_USER_AGENT = "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)";
+var AMAZON_USER_AGENT = "WhatsApp/2.23.20.0 A";
+var AMAZON_SCRAPE_TIMEOUT_MS = 1e4;
 var TWEET_API_TIMEOUT_MS = 1e4;
 var REDDIT_OEMBED_URL = "https://www.reddit.com/oembed";
 var REDDIT_OEMBED_TIMEOUT_MS = 4000;
 var YOUTUBE_OEMBED_TIMEOUT_MS = 4000;
 var FACEBOOK_EMBED_TIMEOUT_MS = 4000;
 var SCRAPE_TIMEOUT_MS = 5000;
-var REDDIT_JSON_TIMEOUT_MS = 2500;
+var REDDIT_JSON_TIMEOUT_MS = 5000;
 var REDDIT_JSON_MAX_BYTES = 1e6;
 var DIRECT_IMAGE_PROBE_TIMEOUT_MS = 5000;
 var FACEBOOK_TRACKING_PARAMS = ["_fb_noscript", "fbclid", "mibextid"];
@@ -71551,7 +71636,8 @@ class Metadata {
       this.fetchDirectImage(job, image, cb);
       return;
     }
-    this.scrape(job, cb);
+    const amazonUrl = resolveAmazonAppLink(job.actor.id);
+    this.scrape(job, cb, undefined, amazonUrl ?? job.actor.id);
   }
   async fetchDirectImage(job, candidate, cb, timeoutMs = DIRECT_IMAGE_PROBE_TIMEOUT_MS) {
     try {
@@ -71793,13 +71879,15 @@ class Metadata {
   scrape(job, cb, redditEmbed, scrapeUrl = job.actor.id, youtubeEmbed) {
     const youtubeVideo = Boolean(resolveYouTubeOEmbed(job.actor.id));
     const useCompatUserAgent = isRedditUrl(job.actor.id) || isFacebookUrl(job.actor.id) || youtubeVideo;
-    const userAgent = useCompatUserAgent ? this.compatUserAgent() : this.userAgent();
+    const amazon = isAmazonUrl(job.actor.id);
+    const scrapeTimeoutMs = amazon ? AMAZON_SCRAPE_TIMEOUT_MS : SCRAPE_TIMEOUT_MS;
+    const userAgent = amazon ? AMAZON_USER_AGENT : useCompatUserAgent ? this.compatUserAgent() : this.userAgent();
     const dispatcher = this.getDispatcher();
     let youtubeDescription;
     this.log.debug(`scrape started for ${job.actor.id} via ${scrapeUrl}`);
     const options = {
       url: scrapeUrl,
-      timeout: SCRAPE_TIMEOUT_MS / 1000,
+      timeout: scrapeTimeoutMs / 1000,
       urlValidatorSettings: {
         allow_fragments: true,
         allow_protocol_relative_urls: false,
@@ -71833,7 +71921,7 @@ class Metadata {
       }
       return import_open_graph_scraper.default({ html });
     }) : import_open_graph_scraper.default(options);
-    withDeadline(scrape, SCRAPE_TIMEOUT_MS).then(async (data) => {
+    withDeadline(scrape, scrapeTimeoutMs).then(async (data) => {
       let { result } = data;
       this.log.debug(`scrape completed for ${job.actor.id}`);
       const reddit = isRedditUrl(job.actor.id);
@@ -71874,6 +71962,10 @@ class Metadata {
           }
         }
       }
+      const amazonImage = amazon && !result.ogImage?.[0]?.url?.includes("/images/I/") && extractAmazonImage(data.html);
+      if (amazonImage) {
+        result = { ...result, ogImage: [{ url: amazonImage }] };
+      }
       const embed = reddit ? await redditEmbed : undefined;
       const youtube = await youtubeEmbed;
       if (!reddit)
@@ -71889,7 +71981,7 @@ class Metadata {
         language: result.ogLocale,
         title: embed?.title ?? youtube?.title ?? (facebook ? stripFacebookEngagement(result.ogTitle) : result.ogTitle),
         name: embed?.provider_name ?? youtube?.provider_name ?? siteName,
-        description: normalizeDescription(youtubeDescription || result.ogDescription || ""),
+        description: normalizeDescription(youtubeDescription || result.ogDescription || schemaOrgSummary(result.jsonLD, result.ogTitle) || ""),
         image: reddit ? redditPostImages(result.ogImage) ?? redditOEmbedImage(embed ?? {}) : scrapedImages?.length ? scrapedImages : youtubeOEmbedImage(youtube),
         url: job.actor.id,
         favicon: result.favicon || "/favicon.ico",
@@ -71934,6 +72026,25 @@ class Metadata {
         cb(null, job);
         return;
       }
+      let filename = "";
+      if (/must scrape an html page/i.test(err.message)) {
+        try {
+          filename = decodeURIComponent(new URL(job.actor.id).pathname.split("/").at(-1) || "");
+        } catch {
+          filename = job.actor.id.split(/[?#]/, 1)[0].split("/").at(-1) || "";
+        }
+      }
+      if (filename) {
+        job.object = {
+          type: "page",
+          title: filename,
+          description: "",
+          url: job.actor.id,
+          favicon: "/favicon.ico"
+        };
+        cb(null, job);
+        return;
+      }
       cb(markExpectedError(new Error(`metadata scrape failed for ${job.actor.id}: ${err.message}`)));
     });
   }
@@ -71950,4 +72061,4 @@ export {
   withDeadline
 };
 
-//# debugId=E83893E42765EF6064756E2164756E21
+//# debugId=256C32E14BF4C7F264756E2164756E21