@sockethub/platform-metadata 1.0.1-beta.4 → 1.0.1-beta.5

package/README.md

@@ -98,6 +98,11 @@ The platform extracts the following metadata when available:
   flattened to one string, or the virtual location URL), `url`, and `status`
   (`scheduled`, `cancelled`, `postponed`, `rescheduled`). Only present when
   both a name and a start date are declared.
+* **Description fallback**: pages with no meta description use a JSON-LD
+  `description`, else a `headline` not already in the title (Wikipedia's
+  short description, e.g. "1979 studio album by Brian Eno").
+* **Non-HTML links** (PDFs and other files) return a page titled with the
+  file name instead of an error.
 
 ## Site-Specific Handling
 
@@ -108,7 +113,9 @@ so those URLs are resolved through preview services built for this purpose:
   are fetched from the [FxTwitter API](https://github.com/FxEmbed/FxEmbed)
   instead of scraped — X serves only a generic site banner to scrapers. The
   response carries the post's text, its actual photo (or video + poster
-  thumbnail), and the canonical `x.com` URL. If the API is unavailable or the
+  thumbnail), and the canonical `x.com` URL. Posts without their own media
+  use the quoted post's media, then the link card image, then the author's
+  avatar. If the API is unavailable or the
   post can't be resolved, the platform falls back to the regular scrape
   (which still yields the post text).
 * **Reddit post URLs** use `old.reddit.com`'s structured JSON response, avoiding
@@ -127,7 +134,10 @@ so those URLs are resolved through preview services built for this purpose:
 * **Amazon product URLs** (storefronts such as `amazon.de` and `amazon.com`,
   plus `amzn.to`/`a.co` short links) are scraped with WhatsApp's link-preview
   user agent. Amazon answers other crawlers with a captcha or a generic
-  "Amazon" share card instead of the product's title and image.
+  "Amazon" share card instead of the product's title and image. Mobile-app
+  links (`/gp/aw/d/<ASIN>`) are scraped as `/dp/<ASIN>`, which is the only
+  form that gets the product card. When Amazon serves its full desktop page
+  instead, the image comes from the page's main product image.
 * **Facebook engagement prefixes**: video titles and image alt text often
   start with a localized view or reaction count, such as
   `2.2M views · 21K reactions | …` or `347K views | …`. When the prefix is

package/dist/index.js

@@ -70777,7 +70777,7 @@ var import_undici3 = __toESM(require_undici(), 1);
 // package.json
 var package_default = {
   name: "@sockethub/platform-metadata",
-  version: "1.0.1-beta.4",
+  version: "1.0.1-beta.5",
   description: "Fetch metadata from a given URL",
   author: "Nick Jennings <nick@silverbucket.net>",
   license: "LGPL-3.0+",
@@ -70819,17 +70819,17 @@ var package_default = {
     "clean:deps": "rm -rf node_modules"
   },
   dependencies: {
-    "@sockethub/util": "^1.0.0-beta.4",
+    "@sockethub/util": "^1.0.0-beta.5",
     "open-graph-scraper": "^6.11.0",
     undici: "^7.30.0"
   },
   devDependencies: {
-    "@sockethub/schemas": "^3.0.0-beta.4"
+    "@sockethub/schemas": "^3.0.0-beta.5"
   },
   peerDependencies: {
-    "@sockethub/server": "^5.0.0-beta.4"
+    "@sockethub/server": "^5.0.0-beta.5"
   },
-  gitHead: "639c296a7332f192a1b3f7e86ef6603b5b67bbaa",
+  gitHead: "cc4ffdd9714ebbd8372a88c4f2d795974e04029e",
   types: "./dist/index.d.ts"
 };
 
@@ -71112,6 +71112,30 @@ function isAmazonUrl(url) {
     return true;
   return host.startsWith("amazon.") && AMAZON_STORE_TLDS.has(host.slice(7));
 }
+function resolveAmazonAppLink(url) {
+  if (!isAmazonUrl(url))
+    return null;
+  const parsed = new URL(url);
+  const asin = parsed.pathname.match(/^\/gp\/aw\/d\/([A-Z0-9]{10})(?:\/|$)/i)?.[1];
+  return asin ? `${parsed.origin}/dp/${asin}` : null;
+}
+function extractAmazonImage(html) {
+  const tag = html?.match(/<img[^>]*\bid="landingImage"[^>]*>/)?.[0];
+  return tag?.match(/\bdata-old-hires="(https:[^"]+)"/)?.[1];
+}
+function schemaOrgSummary(jsonLD, title = "") {
+  if (!Array.isArray(jsonLD))
+    return;
+  const items = jsonLD.flatMap((item) => item?.["@graph"] ?? item);
+  const description = items.find((item) => typeof item?.description === "string" && item.description.trim())?.description;
+  if (description)
+    return description;
+  for (const item of items) {
+    const headline = typeof item?.headline === "string" ? item.headline.trim() : "";
+    if (headline && !title.includes(headline))
+      return headline;
+  }
+}
 function stripFacebookEngagement(title) {
   const bar = title?.indexOf("|") ?? -1;
   if (title === undefined || bar === -1) {
@@ -71338,10 +71362,12 @@ function tweetToPageObject(status) {
 
 `) : undefined;
   const title = typeof article?.title === "string" && article.title || (author.name && author.screen_name ? `${author.name} (@${author.screen_name}) on X` : author.name ?? "Post on X");
-  const photo = tweet.media?.photos?.[0];
-  const video = tweet.media?.videos?.[0];
+  const media = tweet.media?.photos?.length || tweet.media?.videos?.length ? tweet.media : tweet.quote?.media;
+  const photo = media?.photos?.[0];
+  const video = media?.videos?.[0];
   const articleImage = article?.cover_media?.media_info;
-  const imageUrl = articleImage?.original_img_url ?? photo?.url ?? video?.thumbnail_url;
+  const card = tweet.card?.image;
+  const imageUrl = articleImage?.original_img_url ?? photo?.url ?? video?.thumbnail_url ?? card?.url ?? author.avatar_url;
   const page = {
     type: "page",
     title,
@@ -71354,8 +71380,8 @@ function tweetToPageObject(status) {
     page.image = [
       {
         url: imageUrl,
-        width: articleImage?.original_img_width ?? photo?.width ?? video?.width,
-        height: articleImage?.original_img_height ?? photo?.height ?? video?.height
+        width: articleImage?.original_img_width ?? photo?.width ?? video?.width ?? card?.width,
+        height: articleImage?.original_img_height ?? photo?.height ?? video?.height ?? card?.height
       }
     ];
   }
@@ -71610,7 +71636,8 @@ class Metadata {
       this.fetchDirectImage(job, image, cb);
       return;
     }
-    this.scrape(job, cb);
+    const amazonUrl = resolveAmazonAppLink(job.actor.id);
+    this.scrape(job, cb, undefined, amazonUrl ?? job.actor.id);
   }
   async fetchDirectImage(job, candidate, cb, timeoutMs = DIRECT_IMAGE_PROBE_TIMEOUT_MS) {
     try {
@@ -71935,6 +71962,10 @@ class Metadata {
           }
         }
       }
+      const amazonImage = amazon && !result.ogImage?.[0]?.url?.includes("/images/I/") && extractAmazonImage(data.html);
+      if (amazonImage) {
+        result = { ...result, ogImage: [{ url: amazonImage }] };
+      }
       const embed = reddit ? await redditEmbed : undefined;
       const youtube = await youtubeEmbed;
       if (!reddit)
@@ -71950,7 +71981,7 @@ class Metadata {
         language: result.ogLocale,
         title: embed?.title ?? youtube?.title ?? (facebook ? stripFacebookEngagement(result.ogTitle) : result.ogTitle),
         name: embed?.provider_name ?? youtube?.provider_name ?? siteName,
-        description: normalizeDescription(youtubeDescription || result.ogDescription || ""),
+        description: normalizeDescription(youtubeDescription || result.ogDescription || schemaOrgSummary(result.jsonLD, result.ogTitle) || ""),
         image: reddit ? redditPostImages(result.ogImage) ?? redditOEmbedImage(embed ?? {}) : scrapedImages?.length ? scrapedImages : youtubeOEmbedImage(youtube),
         url: job.actor.id,
         favicon: result.favicon || "/favicon.ico",
@@ -71995,6 +72026,25 @@ class Metadata {
         cb(null, job);
         return;
       }
+      let filename = "";
+      if (/must scrape an html page/i.test(err.message)) {
+        try {
+          filename = decodeURIComponent(new URL(job.actor.id).pathname.split("/").at(-1) || "");
+        } catch {
+          filename = job.actor.id.split(/[?#]/, 1)[0].split("/").at(-1) || "";
+        }
+      }
+      if (filename) {
+        job.object = {
+          type: "page",
+          title: filename,
+          description: "",
+          url: job.actor.id,
+          favicon: "/favicon.ico"
+        };
+        cb(null, job);
+        return;
+      }
       cb(markExpectedError(new Error(`metadata scrape failed for ${job.actor.id}: ${err.message}`)));
     });
   }
@@ -72011,4 +72061,4 @@ export {
   withDeadline
 };
 
-//# debugId=391BA00E724CA7D764756E2164756E21
+//# debugId=256C32E14BF4C7F264756E2164756E21