@sockethub/platform-metadata 1.0.1-alpha.17 → 1.0.1-alpha.19

package/dist/index.d.ts

@@ -160,6 +160,8 @@ export default class Metadata implements PlatformInterface {
     private userAgent;
     private compatUserAgent;
     fetch(job: ActivityStream, cb: PlatformCallback): void;
+    /** Fetch and validate YouTube's official preview metadata. */
+    private fetchYouTubeEmbed;
     private fetchTweet;
     private fetchReddit;
     /**

package/dist/index.js

@@ -71661,6 +71661,11 @@ function errorMessage(value) {
   }
   return String(value);
 }
+var EXPECTED_ERROR = Symbol.for("sockethub.expectedError");
+function markExpectedError(err) {
+  err[EXPECTED_ERROR] = true;
+  return err;
+}
 
 // ../util/dist/net/index.js
 var import_undici = __toESM(require_undici(), 1);
@@ -71813,7 +71818,7 @@ var import_undici2 = __toESM(require_undici(), 1);
 // package.json
 var package_default = {
   name: "@sockethub/platform-metadata",
-  version: "1.0.1-alpha.17",
+  version: "1.0.1-alpha.19",
   description: "Fetch metadata from a given URL",
   author: "Nick Jennings <nick@silverbucket.net>",
   license: "LGPL-3.0+",
@@ -71855,17 +71860,17 @@ var package_default = {
     "clean:deps": "rm -rf node_modules"
   },
   dependencies: {
-    "@sockethub/util": "^1.0.0-alpha.10",
+    "@sockethub/util": "^1.0.0-alpha.12",
     "open-graph-scraper": "^6.11.0",
     undici: "^7.16.0"
   },
   devDependencies: {
-    "@sockethub/schemas": "^3.0.0-alpha.22"
+    "@sockethub/schemas": "^3.0.0-alpha.24"
   },
   peerDependencies: {
-    "@sockethub/server": "^5.0.0-alpha.22"
+    "@sockethub/server": "^5.0.0-alpha.24"
   },
-  gitHead: "1630010f605e85cc548e3323cb9694fbe3a7a18a",
+  gitHead: "e778256d7067e2743cd6807a15d39a0cc65a358d",
   types: "./dist/index.d.ts"
 };
 
@@ -71878,6 +71883,21 @@ var TWITTER_HOSTS = new Set([
   "www.x.com",
   "mobile.x.com"
 ]);
+var YOUTUBE_HOSTS = new Set([
+  "youtube.com",
+  "www.youtube.com",
+  "m.youtube.com",
+  "music.youtube.com"
+]);
+var FACEBOOK_HOSTS = new Set([
+  "facebook.com",
+  "www.facebook.com",
+  "m.facebook.com",
+  "web.facebook.com",
+  "mbasic.facebook.com",
+  "fb.com",
+  "www.fb.com"
+]);
 var REDDIT_HOSTS = new Set([
   "reddit.com",
   "www.reddit.com",
@@ -71901,6 +71921,79 @@ function resolveTwitterStatus(url) {
   }
   return `https://api.fxtwitter.com/status/${match[1]}`;
 }
+function resolveYouTubeOEmbed(url) {
+  let parsed;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return null;
+  }
+  const host = parsed.hostname.toLowerCase();
+  let videoId;
+  if (YOUTUBE_HOSTS.has(host)) {
+    if (parsed.pathname === "/watch") {
+      videoId = parsed.searchParams.get("v");
+    } else {
+      videoId = parsed.pathname.match(/^\/(?:shorts|live|embed)\/([A-Za-z0-9_-]{11})(?:\/|$)/)?.[1];
+    }
+  } else if (host === "youtu.be") {
+    videoId = parsed.pathname.match(/^\/([A-Za-z0-9_-]{11})(?:\/|$)/)?.[1];
+  }
+  if (!videoId || !/^[A-Za-z0-9_-]{11}$/.test(videoId))
+    return null;
+  const canonical = new URL("https://www.youtube.com/watch");
+  canonical.searchParams.set("v", videoId);
+  const endpoint = new URL("https://www.youtube.com/oembed");
+  endpoint.searchParams.set("url", canonical.href);
+  endpoint.searchParams.set("format", "json");
+  return endpoint.href;
+}
+function parseYouTubeOEmbed(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value))
+    return null;
+  const embed = value;
+  if (typeof embed.title !== "string" || !embed.title || typeof embed.thumbnail_url !== "string" || !embed.thumbnail_url) {
+    return null;
+  }
+  if (embed.provider_name !== undefined && typeof embed.provider_name !== "string") {
+    return null;
+  }
+  for (const key of ["thumbnail_width", "thumbnail_height"]) {
+    if (embed[key] !== undefined && (typeof embed[key] !== "number" || !Number.isFinite(embed[key]) || embed[key] < 0)) {
+      return null;
+    }
+  }
+  try {
+    const thumbnail = new URL(embed.thumbnail_url);
+    if (!["http:", "https:"].includes(thumbnail.protocol))
+      return null;
+  } catch {
+    return null;
+  }
+  return embed;
+}
+function youtubeOEmbedImage(embed) {
+  return embed ? [
+    {
+      url: embed.thumbnail_url,
+      width: embed.thumbnail_width,
+      height: embed.thumbnail_height
+    }
+  ] : undefined;
+}
+function isFacebookUrl(url) {
+  let parsed;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return false;
+  }
+  const host = parsed.hostname.toLowerCase();
+  return FACEBOOK_HOSTS.has(host) || host === "fb.watch";
+}
+function stripFacebookEngagement(title) {
+  return title?.replace(/^[^|]*\p{Nd}[^|]*·[^|]*\|\s*/u, "");
+}
 function isRedditUrl(url) {
   let parsed;
   try {
@@ -71927,6 +72020,19 @@ function resolveRedditJson(url) {
   json.searchParams.set("raw_json", "1");
   return json.href;
 }
+function resolveRedditMedia(url) {
+  let parsed;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return null;
+  }
+  if (!REDDIT_HOSTS.has(parsed.hostname.toLowerCase()))
+    return null;
+  if (parsed.pathname.replace(/\/+$/, "") !== "/media")
+    return null;
+  return redditMediaUrl(parsed.searchParams.get("url"), new Set(["i.redd.it", "preview.redd.it", "external-preview.redd.it"]));
+}
 function parseRedditPost(value) {
   if (!Array.isArray(value))
     return null;
@@ -71942,19 +72048,51 @@ function parseRedditPost(value) {
   }
   return data;
 }
+function redditMediaUrl(value, allowedHosts) {
+  if (typeof value !== "string")
+    return null;
+  try {
+    const url = new URL(value);
+    return url.protocol === "https:" && allowedHosts.has(url.hostname.toLowerCase()) ? url.href : null;
+  } catch {
+    return null;
+  }
+}
+function finiteNonNegative(value) {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined;
+}
 function redditPostImage(post) {
   for (const candidate of [post.url_overridden_by_dest, post.url]) {
-    if (!candidate)
-      continue;
-    try {
-      const url = new URL(candidate);
-      if (["i.redd.it", "preview.redd.it"].includes(url.hostname.toLowerCase())) {
-        return [{ url: url.href }];
+    const url = redditMediaUrl(candidate, new Set(["i.redd.it", "preview.redd.it"]));
+    if (url)
+      return [{ url }];
+  }
+  const source = post.preview?.images?.[0]?.source;
+  const previewUrl = redditMediaUrl(source?.url, new Set(["preview.redd.it", "external-preview.redd.it"]));
+  if (previewUrl) {
+    return [
+      {
+        url: previewUrl,
+        width: finiteNonNegative(source?.width),
+        height: finiteNonNegative(source?.height)
       }
-    } catch {}
+    ];
   }
   return;
 }
+function redditPostVideo(post) {
+  const video = post.secure_media?.reddit_video;
+  const url = redditMediaUrl(video?.fallback_url, new Set(["v.redd.it"]));
+  if (!url)
+    return;
+  return {
+    url,
+    thumbnail: redditPostImage(post)?.[0]?.url,
+    width: finiteNonNegative(video?.width),
+    height: finiteNonNegative(video?.height),
+    duration: finiteNonNegative(video?.duration)
+  };
+}
 function parseRedditOEmbed(value) {
   if (!value || typeof value !== "object" || Array.isArray(value))
     return null;
@@ -72026,15 +72164,21 @@ function tweetToPageObject(status) {
     return null;
   }
   const author = tweet.author ?? {};
-  const title = author.name && author.screen_name ? `${author.name} (@${author.screen_name}) on X` : author.name ?? "Post on X";
+  const article = tweet.article;
+  const articleBlocks = article?.content?.blocks;
+  const articleBody = Array.isArray(articleBlocks) ? articleBlocks.map((block) => block?.type !== "atomic" && typeof block?.text === "string" ? block.text.trim() : "").filter(Boolean).join(`
+
+`) : undefined;
+  const title = typeof article?.title === "string" && article.title || (author.name && author.screen_name ? `${author.name} (@${author.screen_name}) on X` : author.name ?? "Post on X");
   const photo = tweet.media?.photos?.[0];
   const video = tweet.media?.videos?.[0];
-  const imageUrl = photo?.url ?? video?.thumbnail_url;
+  const articleImage = article?.cover_media?.media_info;
+  const imageUrl = articleImage?.original_img_url ?? photo?.url ?? video?.thumbnail_url;
   const page = {
     type: "page",
     title,
     name: "X (formerly Twitter)",
-    description: normalizeDescription(tweet.text ?? ""),
+    description: normalizeDescription(articleBody || (typeof article?.preview_text === "string" && article.preview_text.trim() ? article.preview_text : "") || tweet.text || ""),
     url: tweet.url,
     favicon: "https://x.com/favicon.ico"
   };
@@ -72042,8 +72186,8 @@ function tweetToPageObject(status) {
     page.image = [
       {
         url: imageUrl,
-        width: photo?.width ?? video?.width,
-        height: photo?.height ?? video?.height
+        width: articleImage?.original_img_width ?? photo?.width ?? video?.width,
+        height: articleImage?.original_img_height ?? photo?.height ?? video?.height
       }
     ];
   }
@@ -72153,6 +72297,7 @@ var COMPAT_USER_AGENT = "Mozilla/5.0 (compatible; Discordbot/2.0; +https://disco
 var TWEET_API_TIMEOUT_MS = 1e4;
 var REDDIT_OEMBED_URL = "https://www.reddit.com/oembed";
 var REDDIT_OEMBED_TIMEOUT_MS = 4000;
+var YOUTUBE_OEMBED_TIMEOUT_MS = 4000;
 var SCRAPE_TIMEOUT_MS = 5000;
 var REDDIT_JSON_TIMEOUT_MS = 2500;
 var REDDIT_JSON_MAX_BYTES = 1e6;
@@ -72210,8 +72355,36 @@ class Metadata {
       this.fetchReddit(job, cb);
       return;
     }
+    const youtubeOEmbedUrl = resolveYouTubeOEmbed(job.actor.id);
+    if (youtubeOEmbedUrl) {
+      const embed = withDeadline(this.fetchYouTubeEmbed(youtubeOEmbedUrl), YOUTUBE_OEMBED_TIMEOUT_MS).catch(() => {
+        return;
+      });
+      this.scrape(job, cb, undefined, job.actor.id, embed);
+      return;
+    }
     this.scrape(job, cb);
   }
+  async fetchYouTubeEmbed(embedUrl) {
+    try {
+      const res = await this.fetchImpl(embedUrl, {
+        dispatcher: this.getDispatcher(),
+        headers: { "user-agent": this.userAgent() },
+        signal: AbortSignal.timeout(YOUTUBE_OEMBED_TIMEOUT_MS)
+      });
+      if (!res.ok) {
+        throw new Error(`YouTube oEmbed returned HTTP ${res.status}`);
+      }
+      const embed = parseYouTubeOEmbed(await res.json());
+      if (!embed) {
+        throw new Error("YouTube oEmbed returned an invalid payload");
+      }
+      return embed;
+    } catch (err) {
+      this.log.debug(`youtube oEmbed fetch failed for ${embedUrl}: ${String(err)}; using scraped metadata`);
+      return;
+    }
+  }
   async fetchTweet(apiUrl, job, cb) {
     try {
       const res = await this.fetchImpl(apiUrl, {
@@ -72235,6 +72408,29 @@ class Metadata {
     this.scrape(job, cb);
   }
   async fetchReddit(job, cb) {
+    const mediaImage = resolveRedditMedia(job.actor.id);
+    if (mediaImage) {
+      const imagePath = new URL(mediaImage).pathname;
+      let title;
+      try {
+        title = decodeURIComponent(imagePath).slice(1);
+      } catch {
+        title = imagePath.slice(1);
+      }
+      job.actor.name = "reddit";
+      job.object = {
+        type: "page",
+        title,
+        name: "reddit",
+        description: "",
+        image: [{ url: mediaImage }],
+        url: job.actor.id,
+        favicon: "/favicon.ico"
+      };
+      this.log.debug(`reddit media link unwrapped for ${job.actor.id}`);
+      cb(null, job);
+      return;
+    }
     const jsonUrl = resolveRedditJson(job.actor.id);
     const embedPromise = withDeadline(this.fetchRedditEmbed(job.actor.id), REDDIT_JSON_TIMEOUT_MS).catch(() => {
       return;
@@ -72252,6 +72448,7 @@ class Metadata {
           name: "reddit",
           description: normalizeDescription(post.selftext ?? ""),
           image: redditPostImage(post),
+          video: redditPostVideo(post),
           url: job.actor.id,
           favicon: "/favicon.ico"
         };
@@ -72277,7 +72474,7 @@ class Metadata {
       cb(null, job);
       return;
     }
-    cb(new Error(`No Reddit metadata available for ${job.actor.id}`));
+    cb(markExpectedError(new Error(`No Reddit metadata available for ${job.actor.id}`)));
   }
   fetchRedditJson(url) {
     return new Promise((resolve, reject) => {
@@ -72339,8 +72536,9 @@ class Metadata {
       return;
     }
   }
-  scrape(job, cb, redditEmbed, scrapeUrl = job.actor.id) {
-    const userAgent = isRedditUrl(job.actor.id) ? this.compatUserAgent() : this.userAgent();
+  scrape(job, cb, redditEmbed, scrapeUrl = job.actor.id, youtubeEmbed) {
+    const useCompatUserAgent = isRedditUrl(job.actor.id) || isFacebookUrl(job.actor.id) || Boolean(resolveYouTubeOEmbed(job.actor.id));
+    const userAgent = useCompatUserAgent ? this.compatUserAgent() : this.userAgent();
     const dispatcher = this.getDispatcher();
     this.log.debug(`scrape started for ${job.actor.id} via ${scrapeUrl}`);
     const options = {
@@ -72379,17 +72577,24 @@ class Metadata {
       const { result } = data;
       this.log.debug(`scrape completed for ${job.actor.id}`);
       const reddit = isRedditUrl(job.actor.id);
+      const facebook = isFacebookUrl(job.actor.id);
       const embed = reddit ? await redditEmbed : undefined;
+      const youtube = await youtubeEmbed;
       if (!reddit)
         job.actor.id = result.ogUrl || job.actor.id;
-      job.actor.name = reddit ? embed?.provider_name ?? "reddit" : result.ogSiteName ?? job.actor.name ?? "";
+      const siteName = result.ogSiteName ?? (facebook ? "Facebook" : undefined);
+      const scrapedImages = facebook ? result.ogImage?.map((image) => ({
+        ...image,
+        alt: stripFacebookEngagement(image.alt)
+      })) : result.ogImage;
+      job.actor.name = reddit ? embed?.provider_name ?? "reddit" : siteName ?? job.actor.name ?? "";
       job.object = {
         type: "page",
         language: result.ogLocale,
-        title: embed?.title ?? result.ogTitle,
-        name: embed?.provider_name ?? result.ogSiteName,
+        title: embed?.title ?? youtube?.title ?? (facebook ? stripFacebookEngagement(result.ogTitle) : result.ogTitle),
+        name: embed?.provider_name ?? youtube?.provider_name ?? siteName,
         description: normalizeDescription(result.ogDescription || ""),
-        image: reddit ? redditPostImages(result.ogImage) ?? redditOEmbedImage(embed ?? {}) : result.ogImage,
+        image: reddit ? redditPostImages(result.ogImage) ?? redditOEmbedImage(embed ?? {}) : scrapedImages?.length ? scrapedImages : youtubeOEmbedImage(youtube),
         url: reddit ? job.actor.id : result.ogUrl,
         favicon: result.favicon || "/favicon.ico",
         charset: result.charset
@@ -72416,7 +72621,23 @@ class Metadata {
           return;
         }
       }
-      cb(err);
+      const youtube = await youtubeEmbed;
+      if (youtube) {
+        job.actor.name = youtube.provider_name || "YouTube";
+        job.object = {
+          type: "page",
+          title: youtube.title,
+          name: youtube.provider_name || "YouTube",
+          description: "",
+          image: youtubeOEmbedImage(youtube),
+          url: job.actor.id,
+          favicon: "/favicon.ico"
+        };
+        this.log.debug(`using youtube oEmbed fallback for ${job.actor.id}`);
+        cb(null, job);
+        return;
+      }
+      cb(markExpectedError(new Error(`metadata scrape failed for ${job.actor.id}: ${err.message}`)));
     });
   }
   cleanup(cb) {
@@ -72432,4 +72653,4 @@ export {
   withDeadline
 };
 
-//# debugId=D50095C23221680664756E2164756E21
+//# debugId=136FC9DCA9AFCE9364756E2164756E21