@sockethub/platform-metadata 1.0.1-beta.2 → 1.0.1-beta.4

package/README.md

@@ -124,6 +124,10 @@ so those URLs are resolved through preview services built for this purpose:
   datacenter address. Full media still requires authentication. Facebook
   publishes no site name, so `actor.name` and `object.name` fall back to
   `Facebook`.
+* **Amazon product URLs** (storefronts such as `amazon.de` and `amazon.com`,
+  plus `amzn.to`/`a.co` short links) are scraped with WhatsApp's link-preview
+  user agent. Amazon answers other crawlers with a captcha or a generic
+  "Amazon" share card instead of the product's title and image.
 * **Facebook engagement prefixes**: video titles and image alt text often
   start with a localized view or reaction count, such as
   `2.2M views · 21K reactions | …` or `347K views | …`. When the prefix is

package/dist/index.js

@@ -70777,7 +70777,7 @@ var import_undici3 = __toESM(require_undici(), 1);
 // package.json
 var package_default = {
   name: "@sockethub/platform-metadata",
-  version: "1.0.1-beta.2",
+  version: "1.0.1-beta.4",
   description: "Fetch metadata from a given URL",
   author: "Nick Jennings <nick@silverbucket.net>",
   license: "LGPL-3.0+",
@@ -70819,17 +70819,17 @@ var package_default = {
     "clean:deps": "rm -rf node_modules"
   },
   dependencies: {
-    "@sockethub/util": "^1.0.0-beta.2",
+    "@sockethub/util": "^1.0.0-beta.4",
     "open-graph-scraper": "^6.11.0",
     undici: "^7.30.0"
   },
   devDependencies: {
-    "@sockethub/schemas": "^3.0.0-beta.2"
+    "@sockethub/schemas": "^3.0.0-beta.4"
   },
   peerDependencies: {
-    "@sockethub/server": "^5.0.0-beta.2"
+    "@sockethub/server": "^5.0.0-beta.4"
   },
-  gitHead: "1f74d4e07311496f0acc021855137bc197d79eab",
+  gitHead: "639c296a7332f192a1b3f7e86ef6603b5b67bbaa",
   types: "./dist/index.d.ts"
 };
 
@@ -71074,6 +71074,44 @@ function isFacebookUrl(url) {
   const host = parsed.hostname.toLowerCase();
   return FACEBOOK_HOSTS.has(host) || host === "fb.watch";
 }
+var AMAZON_STORE_TLDS = new Set([
+  "com",
+  "ca",
+  "com.mx",
+  "com.br",
+  "co.uk",
+  "de",
+  "fr",
+  "it",
+  "es",
+  "nl",
+  "se",
+  "pl",
+  "com.be",
+  "com.tr",
+  "ie",
+  "ae",
+  "sa",
+  "eg",
+  "in",
+  "co.jp",
+  "sg",
+  "com.au",
+  "co.za"
+]);
+var AMAZON_SHORT_HOSTS = new Set(["amzn.to", "amzn.eu", "amzn.asia", "a.co"]);
+function isAmazonUrl(url) {
+  let parsed;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return false;
+  }
+  const host = parsed.hostname.toLowerCase().replace(/^www\./, "");
+  if (AMAZON_SHORT_HOSTS.has(host))
+    return true;
+  return host.startsWith("amazon.") && AMAZON_STORE_TLDS.has(host.slice(7));
+}
 function stripFacebookEngagement(title) {
   const bar = title?.indexOf("|") ?? -1;
   if (title === undefined || bar === -1) {
@@ -71189,7 +71227,26 @@ function redditPostImage(post) {
       }
     ];
   }
-  return;
+  return redditGalleryImages(post);
+}
+function redditGalleryImages(post) {
+  const items = post.gallery_data?.items;
+  if (!Array.isArray(items))
+    return;
+  const images = [];
+  for (const item of items) {
+    const id = item?.media_id;
+    const source = typeof id === "string" ? post.media_metadata?.[id]?.s : undefined;
+    const url = redditMediaUrl(source?.u ?? source?.gif, new Set(["i.redd.it", "preview.redd.it"]));
+    if (url) {
+      images.push({
+        url,
+        width: finiteNonNegative(source?.x),
+        height: finiteNonNegative(source?.y)
+      });
+    }
+  }
+  return images.length ? images : undefined;
 }
 function redditPostVideo(post) {
   const video = post.secure_media?.reddit_video;
@@ -71425,13 +71482,15 @@ var PlatformMetadataSchema = {
 // src/index.ts
 var DEFAULT_USER_AGENT = `Mozilla/5.0 (compatible; SockethubBot/${package_default.version}; +https://sockethub.org)`;
 var COMPAT_USER_AGENT = "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)";
+var AMAZON_USER_AGENT = "WhatsApp/2.23.20.0 A";
+var AMAZON_SCRAPE_TIMEOUT_MS = 1e4;
 var TWEET_API_TIMEOUT_MS = 1e4;
 var REDDIT_OEMBED_URL = "https://www.reddit.com/oembed";
 var REDDIT_OEMBED_TIMEOUT_MS = 4000;
 var YOUTUBE_OEMBED_TIMEOUT_MS = 4000;
 var FACEBOOK_EMBED_TIMEOUT_MS = 4000;
 var SCRAPE_TIMEOUT_MS = 5000;
-var REDDIT_JSON_TIMEOUT_MS = 2500;
+var REDDIT_JSON_TIMEOUT_MS = 5000;
 var REDDIT_JSON_MAX_BYTES = 1e6;
 var DIRECT_IMAGE_PROBE_TIMEOUT_MS = 5000;
 var FACEBOOK_TRACKING_PARAMS = ["_fb_noscript", "fbclid", "mibextid"];
@@ -71793,13 +71852,15 @@ class Metadata {
   scrape(job, cb, redditEmbed, scrapeUrl = job.actor.id, youtubeEmbed) {
     const youtubeVideo = Boolean(resolveYouTubeOEmbed(job.actor.id));
     const useCompatUserAgent = isRedditUrl(job.actor.id) || isFacebookUrl(job.actor.id) || youtubeVideo;
-    const userAgent = useCompatUserAgent ? this.compatUserAgent() : this.userAgent();
+    const amazon = isAmazonUrl(job.actor.id);
+    const scrapeTimeoutMs = amazon ? AMAZON_SCRAPE_TIMEOUT_MS : SCRAPE_TIMEOUT_MS;
+    const userAgent = amazon ? AMAZON_USER_AGENT : useCompatUserAgent ? this.compatUserAgent() : this.userAgent();
     const dispatcher = this.getDispatcher();
     let youtubeDescription;
     this.log.debug(`scrape started for ${job.actor.id} via ${scrapeUrl}`);
     const options = {
       url: scrapeUrl,
-      timeout: SCRAPE_TIMEOUT_MS / 1000,
+      timeout: scrapeTimeoutMs / 1000,
       urlValidatorSettings: {
         allow_fragments: true,
         allow_protocol_relative_urls: false,
@@ -71833,7 +71894,7 @@ class Metadata {
       }
       return import_open_graph_scraper.default({ html });
     }) : import_open_graph_scraper.default(options);
-    withDeadline(scrape, SCRAPE_TIMEOUT_MS).then(async (data) => {
+    withDeadline(scrape, scrapeTimeoutMs).then(async (data) => {
       let { result } = data;
       this.log.debug(`scrape completed for ${job.actor.id}`);
       const reddit = isRedditUrl(job.actor.id);
@@ -71950,4 +72011,4 @@ export {
   withDeadline
 };
 
-//# debugId=B83EC33716FB37C264756E2164756E21
+//# debugId=391BA00E724CA7D764756E2164756E21