@sockethub/platform-metadata 1.0.1-alpha.17 → 1.0.1-alpha.18

package/dist/index.d.ts

@@ -160,6 +160,8 @@ export default class Metadata implements PlatformInterface {
     private userAgent;
     private compatUserAgent;
     fetch(job: ActivityStream, cb: PlatformCallback): void;
+    /** Fetch and validate YouTube's official preview metadata. */
+    private fetchYouTubeEmbed;
     private fetchTweet;
     private fetchReddit;
     /**

package/dist/index.js

@@ -71813,7 +71813,7 @@ var import_undici2 = __toESM(require_undici(), 1);
 // package.json
 var package_default = {
   name: "@sockethub/platform-metadata",
-  version: "1.0.1-alpha.17",
+  version: "1.0.1-alpha.18",
   description: "Fetch metadata from a given URL",
   author: "Nick Jennings <nick@silverbucket.net>",
   license: "LGPL-3.0+",
@@ -71855,17 +71855,17 @@ var package_default = {
     "clean:deps": "rm -rf node_modules"
   },
   dependencies: {
-    "@sockethub/util": "^1.0.0-alpha.10",
+    "@sockethub/util": "^1.0.0-alpha.11",
     "open-graph-scraper": "^6.11.0",
     undici: "^7.16.0"
   },
   devDependencies: {
-    "@sockethub/schemas": "^3.0.0-alpha.22"
+    "@sockethub/schemas": "^3.0.0-alpha.23"
   },
   peerDependencies: {
-    "@sockethub/server": "^5.0.0-alpha.22"
+    "@sockethub/server": "^5.0.0-alpha.23"
   },
-  gitHead: "1630010f605e85cc548e3323cb9694fbe3a7a18a",
+  gitHead: "a6d303097640e82c7d1a7cef6dea2e6dae5f2cac",
   types: "./dist/index.d.ts"
 };
 
@@ -71878,6 +71878,12 @@ var TWITTER_HOSTS = new Set([
   "www.x.com",
   "mobile.x.com"
 ]);
+var YOUTUBE_HOSTS = new Set([
+  "youtube.com",
+  "www.youtube.com",
+  "m.youtube.com",
+  "music.youtube.com"
+]);
 var REDDIT_HOSTS = new Set([
   "reddit.com",
   "www.reddit.com",
@@ -71901,6 +71907,66 @@ function resolveTwitterStatus(url) {
   }
   return `https://api.fxtwitter.com/status/${match[1]}`;
 }
+function resolveYouTubeOEmbed(url) {
+  let parsed;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return null;
+  }
+  const host = parsed.hostname.toLowerCase();
+  let videoId;
+  if (YOUTUBE_HOSTS.has(host)) {
+    if (parsed.pathname === "/watch") {
+      videoId = parsed.searchParams.get("v");
+    } else {
+      videoId = parsed.pathname.match(/^\/(?:shorts|live|embed)\/([A-Za-z0-9_-]{11})(?:\/|$)/)?.[1];
+    }
+  } else if (host === "youtu.be") {
+    videoId = parsed.pathname.match(/^\/([A-Za-z0-9_-]{11})(?:\/|$)/)?.[1];
+  }
+  if (!videoId || !/^[A-Za-z0-9_-]{11}$/.test(videoId))
+    return null;
+  const canonical = new URL("https://www.youtube.com/watch");
+  canonical.searchParams.set("v", videoId);
+  const endpoint = new URL("https://www.youtube.com/oembed");
+  endpoint.searchParams.set("url", canonical.href);
+  endpoint.searchParams.set("format", "json");
+  return endpoint.href;
+}
+function parseYouTubeOEmbed(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value))
+    return null;
+  const embed = value;
+  if (typeof embed.title !== "string" || !embed.title || typeof embed.thumbnail_url !== "string" || !embed.thumbnail_url) {
+    return null;
+  }
+  if (embed.provider_name !== undefined && typeof embed.provider_name !== "string") {
+    return null;
+  }
+  for (const key of ["thumbnail_width", "thumbnail_height"]) {
+    if (embed[key] !== undefined && (typeof embed[key] !== "number" || !Number.isFinite(embed[key]) || embed[key] < 0)) {
+      return null;
+    }
+  }
+  try {
+    const thumbnail = new URL(embed.thumbnail_url);
+    if (!["http:", "https:"].includes(thumbnail.protocol))
+      return null;
+  } catch {
+    return null;
+  }
+  return embed;
+}
+function youtubeOEmbedImage(embed) {
+  return embed ? [
+    {
+      url: embed.thumbnail_url,
+      width: embed.thumbnail_width,
+      height: embed.thumbnail_height
+    }
+  ] : undefined;
+}
 function isRedditUrl(url) {
   let parsed;
   try {
@@ -72026,15 +72092,21 @@ function tweetToPageObject(status) {
     return null;
   }
   const author = tweet.author ?? {};
-  const title = author.name && author.screen_name ? `${author.name} (@${author.screen_name}) on X` : author.name ?? "Post on X";
+  const article = tweet.article;
+  const articleBlocks = article?.content?.blocks;
+  const articleBody = Array.isArray(articleBlocks) ? articleBlocks.map((block) => block?.type !== "atomic" && typeof block?.text === "string" ? block.text.trim() : "").filter(Boolean).join(`
+
+`) : undefined;
+  const title = typeof article?.title === "string" && article.title || (author.name && author.screen_name ? `${author.name} (@${author.screen_name}) on X` : author.name ?? "Post on X");
   const photo = tweet.media?.photos?.[0];
   const video = tweet.media?.videos?.[0];
-  const imageUrl = photo?.url ?? video?.thumbnail_url;
+  const articleImage = article?.cover_media?.media_info;
+  const imageUrl = articleImage?.original_img_url ?? photo?.url ?? video?.thumbnail_url;
   const page = {
     type: "page",
     title,
     name: "X (formerly Twitter)",
-    description: normalizeDescription(tweet.text ?? ""),
+    description: normalizeDescription(articleBody || (typeof article?.preview_text === "string" && article.preview_text.trim() ? article.preview_text : "") || tweet.text || ""),
     url: tweet.url,
     favicon: "https://x.com/favicon.ico"
   };
@@ -72042,8 +72114,8 @@ function tweetToPageObject(status) {
     page.image = [
       {
         url: imageUrl,
-        width: photo?.width ?? video?.width,
-        height: photo?.height ?? video?.height
+        width: articleImage?.original_img_width ?? photo?.width ?? video?.width,
+        height: articleImage?.original_img_height ?? photo?.height ?? video?.height
       }
     ];
   }
@@ -72153,6 +72225,7 @@ var COMPAT_USER_AGENT = "Mozilla/5.0 (compatible; Discordbot/2.0; +https://disco
 var TWEET_API_TIMEOUT_MS = 1e4;
 var REDDIT_OEMBED_URL = "https://www.reddit.com/oembed";
 var REDDIT_OEMBED_TIMEOUT_MS = 4000;
+var YOUTUBE_OEMBED_TIMEOUT_MS = 4000;
 var SCRAPE_TIMEOUT_MS = 5000;
 var REDDIT_JSON_TIMEOUT_MS = 2500;
 var REDDIT_JSON_MAX_BYTES = 1e6;
@@ -72210,8 +72283,36 @@ class Metadata {
       this.fetchReddit(job, cb);
       return;
     }
+    const youtubeOEmbedUrl = resolveYouTubeOEmbed(job.actor.id);
+    if (youtubeOEmbedUrl) {
+      const embed = withDeadline(this.fetchYouTubeEmbed(youtubeOEmbedUrl), YOUTUBE_OEMBED_TIMEOUT_MS).catch(() => {
+        return;
+      });
+      this.scrape(job, cb, undefined, job.actor.id, embed);
+      return;
+    }
     this.scrape(job, cb);
   }
+  async fetchYouTubeEmbed(embedUrl) {
+    try {
+      const res = await this.fetchImpl(embedUrl, {
+        dispatcher: this.getDispatcher(),
+        headers: { "user-agent": this.userAgent() },
+        signal: AbortSignal.timeout(YOUTUBE_OEMBED_TIMEOUT_MS)
+      });
+      if (!res.ok) {
+        throw new Error(`YouTube oEmbed returned HTTP ${res.status}`);
+      }
+      const embed = parseYouTubeOEmbed(await res.json());
+      if (!embed) {
+        throw new Error("YouTube oEmbed returned an invalid payload");
+      }
+      return embed;
+    } catch (err) {
+      this.log.debug(`youtube oEmbed fetch failed for ${embedUrl}: ${String(err)}; using scraped metadata`);
+      return;
+    }
+  }
   async fetchTweet(apiUrl, job, cb) {
     try {
       const res = await this.fetchImpl(apiUrl, {
@@ -72339,8 +72440,9 @@ class Metadata {
       return;
     }
   }
-  scrape(job, cb, redditEmbed, scrapeUrl = job.actor.id) {
-    const userAgent = isRedditUrl(job.actor.id) ? this.compatUserAgent() : this.userAgent();
+  scrape(job, cb, redditEmbed, scrapeUrl = job.actor.id, youtubeEmbed) {
+    const useCompatUserAgent = isRedditUrl(job.actor.id) || Boolean(resolveYouTubeOEmbed(job.actor.id));
+    const userAgent = useCompatUserAgent ? this.compatUserAgent() : this.userAgent();
     const dispatcher = this.getDispatcher();
     this.log.debug(`scrape started for ${job.actor.id} via ${scrapeUrl}`);
     const options = {
@@ -72380,16 +72482,17 @@ class Metadata {
       this.log.debug(`scrape completed for ${job.actor.id}`);
       const reddit = isRedditUrl(job.actor.id);
       const embed = reddit ? await redditEmbed : undefined;
+      const youtube = await youtubeEmbed;
       if (!reddit)
         job.actor.id = result.ogUrl || job.actor.id;
       job.actor.name = reddit ? embed?.provider_name ?? "reddit" : result.ogSiteName ?? job.actor.name ?? "";
       job.object = {
         type: "page",
         language: result.ogLocale,
-        title: embed?.title ?? result.ogTitle,
-        name: embed?.provider_name ?? result.ogSiteName,
+        title: embed?.title ?? youtube?.title ?? result.ogTitle,
+        name: embed?.provider_name ?? youtube?.provider_name ?? result.ogSiteName,
         description: normalizeDescription(result.ogDescription || ""),
-        image: reddit ? redditPostImages(result.ogImage) ?? redditOEmbedImage(embed ?? {}) : result.ogImage,
+        image: reddit ? redditPostImages(result.ogImage) ?? redditOEmbedImage(embed ?? {}) : result.ogImage?.length ? result.ogImage : youtubeOEmbedImage(youtube),
         url: reddit ? job.actor.id : result.ogUrl,
         favicon: result.favicon || "/favicon.ico",
         charset: result.charset
@@ -72416,6 +72519,22 @@ class Metadata {
           return;
         }
       }
+      const youtube = await youtubeEmbed;
+      if (youtube) {
+        job.actor.name = youtube.provider_name || "YouTube";
+        job.object = {
+          type: "page",
+          title: youtube.title,
+          name: youtube.provider_name || "YouTube",
+          description: "",
+          image: youtubeOEmbedImage(youtube),
+          url: job.actor.id,
+          favicon: "/favicon.ico"
+        };
+        this.log.debug(`using youtube oEmbed fallback for ${job.actor.id}`);
+        cb(null, job);
+        return;
+      }
       cb(err);
     });
   }
@@ -72432,4 +72551,4 @@ export {
   withDeadline
 };
 
-//# debugId=D50095C23221680664756E2164756E21
+//# debugId=98F3FC3874006CC764756E2164756E21