@sockethub/platform-metadata 1.0.1-alpha.14 → 1.0.1-alpha.15

package/README.md

@@ -96,11 +96,11 @@ so those URLs are resolved through preview services built for this purpose:
   thumbnail), and the canonical `x.com` URL. If the API is unavailable or the
   post can't be resolved, the platform falls back to the regular scrape
   (which still yields the post text).
-* **Reddit URLs** (`reddit.com` hosts and `redd.it` short links) are scraped
-  directly, but with a *compatibility user agent* — Reddit serves its Open
-  Graph data (including the post's real preview image) only to recognized
-  embed crawlers, and returns a page with no OG tags (or a 403) to anything
-  else. See `compatUserAgent` below.
+* **Reddit post URLs** use `old.reddit.com`'s structured JSON response, avoiding
+  Reddit's unreliable HTML and Open Graph crawler behavior. The post title,
+  self-text, and direct `i.redd.it` image are mapped into the preview; text posts
+  intentionally have no image. oEmbed remains a title-only failure fallback.
+  See `compatUserAgent` below.
 * **Facebook** remains best-effort: post text usually comes through, but
   media requires authentication and no public preview service exists.
 
@@ -124,7 +124,7 @@ default UA. Override per deployment via `packageConfig`:
 
 ### `compatUserAgent`
 
-Sites that only serve Open Graph data to *recognized* embed crawlers (Reddit)
+Sites that only serve metadata to *recognized* embed crawlers (Reddit)
 are fetched with a link-preview crawler user agent instead — the established
 practice for self-hosted preview fetchers. Defaults to
 `Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)`;

package/dist/index.d.ts

@@ -1,6 +1,9 @@
 import type { ActivityStream, PlatformCallback, PlatformConfig, PlatformInterface, PlatformSession } from "@sockethub/schemas";
+/** Enforce a deadline independently of a dependency's AbortSignal handling. */
+export declare function withDeadline<T>(promise: Promise<T>, timeoutMs: number): Promise<T>;
 export default class Metadata implements PlatformInterface {
     private readonly log;
+    private readonly fetchImpl;
     private dispatcher?;
     config: PlatformConfig;
     constructor(session: PlatformSession);
@@ -158,6 +161,15 @@ export default class Metadata implements PlatformInterface {
     private compatUserAgent;
     fetch(job: ActivityStream, cb: PlatformCallback): void;
     private fetchTweet;
+    private fetchReddit;
+    /**
+     * Fetch Reddit's fixed-host JSON endpoint without the Undici dispatcher.
+     * The platform child runtime has exhibited stuck Undici requests even when
+     * AbortSignal fires. node:https gives us an independently enforced socket
+     * timeout; the outer withDeadline still guarantees callback completion.
+     */
+    private fetchRedditJson;
+    private fetchRedditEmbed;
     private scrape;
     cleanup(cb: PlatformCallback): void;
 }

package/dist/index.js

@@ -66294,11 +66294,11 @@ var require_isUrl = __commonJS((exports) => {
   var IPv4AddressRegExp = new RegExp(`^${IPv4AddressFormat}$`);
   var IPv6SegmentFormat = "(?:[0-9a-fA-F]{1,4})";
   var IPv6AddressRegExp = new RegExp("^(" + `(?:${IPv6SegmentFormat}:){7}(?:${IPv6SegmentFormat}|:)|` + `(?:${IPv6SegmentFormat}:){6}(?:${IPv4AddressFormat}|:${IPv6SegmentFormat}|:)|` + `(?:${IPv6SegmentFormat}:){5}(?::${IPv4AddressFormat}|(:${IPv6SegmentFormat}){1,2}|:)|` + `(?:${IPv6SegmentFormat}:){4}(?:(:${IPv6SegmentFormat}){0,1}:${IPv4AddressFormat}|(:${IPv6SegmentFormat}){1,3}|:)|` + `(?:${IPv6SegmentFormat}:){3}(?:(:${IPv6SegmentFormat}){0,2}:${IPv4AddressFormat}|(:${IPv6SegmentFormat}){1,4}|:)|` + `(?:${IPv6SegmentFormat}:){2}(?:(:${IPv6SegmentFormat}){0,3}:${IPv4AddressFormat}|(:${IPv6SegmentFormat}){1,5}|:)|` + `(?:${IPv6SegmentFormat}:){1}(?:(:${IPv6SegmentFormat}){0,4}:${IPv4AddressFormat}|(:${IPv6SegmentFormat}){1,6}|:)|` + `(?::((?::${IPv6SegmentFormat}){0,5}:${IPv4AddressFormat}|(?::${IPv6SegmentFormat}){1,7}|:))` + ")(%[0-9a-zA-Z-.:]{1,})?$");
-  function isIP(str, version = "") {
+  function isIP2(str, version = "") {
     assertString(str);
     version = String(version);
     if (!version) {
-      return isIP(str, 4) || isIP(str, 6);
+      return isIP2(str, 4) || isIP2(str, 6);
     }
     if (version === "4") {
       return IPv4AddressRegExp.test(str);
@@ -66433,7 +66433,7 @@ var require_isUrl = __commonJS((exports) => {
     if (host === "" && !options.require_host) {
       return true;
     }
-    if (!isIP(host) && !isFQDN(host, options) && (!ipv6 || !isIP(ipv6, 6))) {
+    if (!isIP2(host) && !isFQDN(host, options) && (!ipv6 || !isIP2(ipv6, 6))) {
       return false;
     }
     host = host || ipv6;
@@ -71643,6 +71643,9 @@ var require_esm = __commonJS((exports, module) => {
   module.exports = run;
 });
 
+// src/index.ts
+import { request as httpsRequest } from "node:https";
+
 // ../util/dist/error/index.js
 function toError(value) {
   if (value instanceof Error) {
@@ -71660,6 +71663,7 @@ function errorMessage(value) {
 // ../util/dist/net/index.js
 var import_undici = __toESM(require_undici(), 1);
 import { lookup as dnsLookup } from "node:dns";
+import { isIP } from "node:net";
 function isBlockedAddress(ip) {
   const addr = ip.trim().toLowerCase();
   if (addr.includes(":")) {
@@ -71778,6 +71782,18 @@ function createGuardedLookup(allowPrivateAddresses) {
     });
   };
 }
+function createGuardedConnector(allowPrivateAddresses, connector = import_undici.buildConnector({
+  lookup: createGuardedLookup(allowPrivateAddresses)
+})) {
+  return (options, callback) => {
+    const hostname = options.hostname.replace(/^\[|\]$/g, "");
+    if (!allowPrivateAddresses && isIP(hostname) !== 0 && isBlockedAddress(hostname)) {
+      callback(new Error(`blocked non-public address ${hostname}`), null);
+      return;
+    }
+    connector(options, callback);
+  };
+}
 function createGuardedDispatcher(options = {}) {
   const {
     allowPrivateAddresses = false,
@@ -71785,18 +71801,17 @@ function createGuardedDispatcher(options = {}) {
   } = options;
   return new import_undici.Agent({
     maxResponseSize: maxResponseBytes,
-    connect: {
-      lookup: createGuardedLookup(allowPrivateAddresses)
-    }
+    connect: createGuardedConnector(allowPrivateAddresses)
   });
 }
 
 // src/index.ts
 var import_open_graph_scraper = __toESM(require_esm(), 1);
+var import_undici2 = __toESM(require_undici(), 1);
 // package.json
 var package_default = {
   name: "@sockethub/platform-metadata",
-  version: "1.0.1-alpha.14",
+  version: "1.0.1-alpha.15",
   description: "Fetch metadata from a given URL",
   author: "Nick Jennings <nick@silverbucket.net>",
   license: "LGPL-3.0+",
@@ -71838,16 +71853,17 @@ var package_default = {
     "clean:deps": "rm -rf node_modules"
   },
   dependencies: {
-    "@sockethub/util": "^1.0.0-alpha.7",
-    "open-graph-scraper": "^6.11.0"
+    "@sockethub/util": "^1.0.0-alpha.8",
+    "open-graph-scraper": "^6.11.0",
+    undici: "^7.16.0"
   },
   devDependencies: {
-    "@sockethub/schemas": "^3.0.0-alpha.19"
+    "@sockethub/schemas": "^3.0.0-alpha.20"
   },
   peerDependencies: {
-    "@sockethub/server": "^5.0.0-alpha.19"
+    "@sockethub/server": "^5.0.0-alpha.20"
   },
-  gitHead: "9dc63d790aabfbfe894b1d1b0754013e40fd25ec",
+  gitHead: "194422de2ffe1a95ffff45063bc988da8594756a",
   types: "./dist/index.d.ts"
 };
 
@@ -71893,6 +71909,115 @@ function isRedditUrl(url) {
   const host = parsed.hostname.toLowerCase();
   return REDDIT_HOSTS.has(host) || host === "redd.it";
 }
+function resolveRedditJson(url) {
+  let parsed;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return null;
+  }
+  if (!REDDIT_HOSTS.has(parsed.hostname.toLowerCase()))
+    return null;
+  const match = parsed.pathname.match(/^\/(?:r|user)\/[^/]+\/comments\/[^/]+(?:\/[^/]*)?\/?$/);
+  if (!match)
+    return null;
+  const json = new URL(`${parsed.pathname.replace(/\/$/, "")}.json`, "https://old.reddit.com");
+  json.searchParams.set("raw_json", "1");
+  return json.href;
+}
+function parseRedditPost(value) {
+  if (!Array.isArray(value))
+    return null;
+  const post = value[0]?.data?.children?.[0]?.data;
+  if (!post || typeof post !== "object" || Array.isArray(post))
+    return null;
+  const data = post;
+  if (typeof data.title !== "string" || !data.title)
+    return null;
+  for (const key of ["selftext", "url", "url_overridden_by_dest"]) {
+    if (data[key] !== undefined && typeof data[key] !== "string")
+      return null;
+  }
+  return data;
+}
+function redditPostImage(post) {
+  for (const candidate of [post.url_overridden_by_dest, post.url]) {
+    if (!candidate)
+      continue;
+    try {
+      const url = new URL(candidate);
+      if (["i.redd.it", "preview.redd.it"].includes(url.hostname.toLowerCase())) {
+        return [{ url: url.href }];
+      }
+    } catch {}
+  }
+  return;
+}
+function parseRedditOEmbed(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value))
+    return null;
+  const embed = value;
+  for (const key of [
+    "title",
+    "author_name",
+    "provider_name",
+    "thumbnail_url"
+  ]) {
+    if (embed[key] !== undefined && typeof embed[key] !== "string") {
+      return null;
+    }
+  }
+  for (const key of ["thumbnail_width", "thumbnail_height"]) {
+    if (embed[key] !== undefined && (typeof embed[key] !== "number" || !Number.isFinite(embed[key]) || embed[key] < 0)) {
+      return null;
+    }
+  }
+  return embed;
+}
+function redditOEmbedImage(embed) {
+  if (!embed?.thumbnail_url) {
+    return;
+  }
+  let url;
+  try {
+    url = new URL(embed.thumbnail_url);
+  } catch {
+    return;
+  }
+  if (url.protocol !== "http:" && url.protocol !== "https:") {
+    return;
+  }
+  return [
+    {
+      url: url.href,
+      width: embed.thumbnail_width,
+      height: embed.thumbnail_height
+    }
+  ];
+}
+function redditPostImages(images) {
+  const allowedHosts = new Set([
+    "i.redd.it",
+    "preview.redd.it",
+    "external-preview.redd.it"
+  ]);
+  const filtered = images?.filter((image) => {
+    try {
+      return allowedHosts.has(new URL(image.url).hostname.toLowerCase());
+    } catch {
+      return false;
+    }
+  });
+  return filtered?.length ? filtered : undefined;
+}
+function normalizeDescription(value) {
+  return value.replace(/\r\n?/g, `
+`).replace(/\u00a0/g, " ").split(`
+`).map((line) => line.replace(/[\t\p{Zs}]+/gu, " ").trim()).join(`
+`).replace(/^\n+|\n+$/g, "").replace(/\n{3,}/g, `
+
+`);
+}
 function tweetToPageObject(status) {
   const tweet = status?.tweet;
   if (status?.code !== 200 || !tweet) {
@@ -71907,7 +72032,7 @@ function tweetToPageObject(status) {
     type: "page",
     title,
     name: "X (formerly Twitter)",
-    description: tweet.text ?? "",
+    description: normalizeDescription(tweet.text ?? ""),
     url: tweet.url,
     favicon: "https://x.com/favicon.ico"
   };
@@ -72024,9 +72149,27 @@ var PlatformMetadataSchema = {
 var DEFAULT_USER_AGENT = `Mozilla/5.0 (compatible; SockethubBot/${package_default.version}; +https://sockethub.org)`;
 var COMPAT_USER_AGENT = "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)";
 var TWEET_API_TIMEOUT_MS = 1e4;
+var REDDIT_OEMBED_URL = "https://www.reddit.com/oembed";
+var REDDIT_OEMBED_TIMEOUT_MS = 4000;
+var SCRAPE_TIMEOUT_MS = 5000;
+var REDDIT_JSON_TIMEOUT_MS = 2500;
+var REDDIT_JSON_MAX_BYTES = 1e6;
+function withDeadline(promise, timeoutMs) {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => reject(new Error(`metadata scrape timed out after ${timeoutMs}ms`)), timeoutMs);
+    promise.then((value) => {
+      clearTimeout(timer);
+      resolve(value);
+    }, (err) => {
+      clearTimeout(timer);
+      reject(err);
+    });
+  });
+}
 
 class Metadata {
   log;
+  fetchImpl = import_undici2.fetch;
   dispatcher;
   config = {
     persist: false
@@ -72061,11 +72204,15 @@ class Metadata {
       this.fetchTweet(tweetApiUrl, job, cb);
       return;
     }
+    if (isRedditUrl(job.actor.id)) {
+      this.fetchReddit(job, cb);
+      return;
+    }
     this.scrape(job, cb);
   }
   async fetchTweet(apiUrl, job, cb) {
     try {
-      const res = await globalThis.fetch(apiUrl, {
+      const res = await this.fetchImpl(apiUrl, {
         dispatcher: this.getDispatcher(),
         headers: { "user-agent": this.userAgent() },
         signal: AbortSignal.timeout(TWEET_API_TIMEOUT_MS)
@@ -72085,11 +72232,118 @@ class Metadata {
     }
     this.scrape(job, cb);
   }
-  scrape(job, cb) {
+  async fetchReddit(job, cb) {
+    const jsonUrl = resolveRedditJson(job.actor.id);
+    const embedPromise = withDeadline(this.fetchRedditEmbed(job.actor.id), REDDIT_JSON_TIMEOUT_MS).catch(() => {
+      return;
+    });
+    if (jsonUrl) {
+      try {
+        this.log.debug(`reddit JSON started for ${job.actor.id}`);
+        const post = parseRedditPost(await withDeadline(this.fetchRedditJson(jsonUrl), REDDIT_JSON_TIMEOUT_MS));
+        if (!post)
+          throw new Error("Reddit JSON returned an invalid payload");
+        job.actor.name = "reddit";
+        job.object = {
+          type: "page",
+          title: post.title,
+          name: "reddit",
+          description: normalizeDescription(post.selftext ?? ""),
+          image: redditPostImage(post),
+          url: job.actor.id,
+          favicon: "/favicon.ico"
+        };
+        this.log.debug(`reddit JSON completed for ${job.actor.id}`);
+        cb(null, job);
+        return;
+      } catch (err) {
+        this.log.debug(`reddit JSON failed for ${job.actor.id}: ${String(err)}; falling back to oEmbed`);
+      }
+    }
+    const embed = await embedPromise;
+    if (embed?.title) {
+      job.actor.name = embed.provider_name || "reddit";
+      job.object = {
+        type: "page",
+        title: embed.title,
+        name: embed.provider_name || "reddit",
+        description: "",
+        image: redditOEmbedImage(embed),
+        url: job.actor.id,
+        favicon: "/favicon.ico"
+      };
+      cb(null, job);
+      return;
+    }
+    cb(new Error(`No Reddit metadata available for ${job.actor.id}`));
+  }
+  fetchRedditJson(url) {
+    return new Promise((resolve, reject) => {
+      const req = httpsRequest(url, {
+        headers: { "user-agent": this.compatUserAgent() }
+      }, (res) => {
+        if (res.statusCode !== 200) {
+          res.resume();
+          reject(new Error(`Reddit JSON returned HTTP ${res.statusCode}`));
+          return;
+        }
+        let bytes = 0;
+        const chunks = [];
+        res.on("data", (chunk) => {
+          bytes += chunk.length;
+          if (bytes > REDDIT_JSON_MAX_BYTES) {
+            req.destroy(new Error("Reddit JSON response too large"));
+            return;
+          }
+          chunks.push(chunk);
+        });
+        res.on("end", () => {
+          try {
+            resolve(JSON.parse(Buffer.concat(chunks).toString()));
+          } catch (err) {
+            reject(err);
+          }
+        });
+        res.on("error", reject);
+      });
+      req.setTimeout(REDDIT_JSON_TIMEOUT_MS, () => {
+        req.destroy(new Error("Reddit JSON request timed out"));
+      });
+      req.on("error", reject);
+      req.end();
+    });
+  }
+  async fetchRedditEmbed(postUrl) {
+    try {
+      const apiUrl = new URL(REDDIT_OEMBED_URL);
+      apiUrl.searchParams.set("url", postUrl);
+      this.log.debug(`reddit oEmbed started for ${postUrl}`);
+      const res = await this.fetchImpl(apiUrl, {
+        dispatcher: this.getDispatcher(),
+        headers: { "user-agent": this.userAgent() },
+        signal: AbortSignal.timeout(REDDIT_OEMBED_TIMEOUT_MS)
+      });
+      if (!res.ok) {
+        throw new Error(`Reddit oEmbed returned HTTP ${res.status}`);
+      }
+      const embed = parseRedditOEmbed(await res.json());
+      if (!embed)
+        throw new Error("Reddit oEmbed returned an invalid payload");
+      const image = redditOEmbedImage(embed);
+      this.log.debug(`reddit oEmbed completed for ${postUrl} (${image ? "thumbnail" : "no thumbnail"})`);
+      return embed;
+    } catch (err) {
+      this.log.debug(`reddit oEmbed fetch failed for ${postUrl}: ${String(err)}; returning a text-only scrape`);
+      return;
+    }
+  }
+  scrape(job, cb, redditEmbed, scrapeUrl = job.actor.id) {
     const userAgent = isRedditUrl(job.actor.id) ? this.compatUserAgent() : this.userAgent();
     const dispatcher = this.getDispatcher();
-    import_open_graph_scraper.default({
-      url: job.actor.id,
+    this.log.debug(`scrape started for ${job.actor.id} via ${scrapeUrl}`);
+    const options = {
+      url: scrapeUrl,
+      timeout: SCRAPE_TIMEOUT_MS / 1000,
       urlValidatorSettings: {
         allow_fragments: true,
         allow_protocol_relative_urls: false,
@@ -72108,33 +72362,72 @@ class Metadata {
         dispatcher,
         headers: { "user-agent": userAgent }
       }
-    }).then((data) => {
+    };
+    const scrape = isRedditUrl(job.actor.id) ? this.fetchImpl(scrapeUrl, {
+      dispatcher,
+      headers: { "user-agent": userAgent },
+      signal: AbortSignal.timeout(SCRAPE_TIMEOUT_MS)
+    }).then(async (res) => {
+      if (!res.ok) {
+        throw new Error(`metadata scrape returned HTTP ${res.status}`);
+      }
+      return import_open_graph_scraper.default({ html: await res.text() });
+    }) : import_open_graph_scraper.default(options);
+    withDeadline(scrape, SCRAPE_TIMEOUT_MS).then(async (data) => {
       const { result } = data;
-      job.actor.id = result.ogUrl || job.actor.id;
-      job.actor.name = result.ogSiteName || job.actor.name || "";
+      this.log.debug(`scrape completed for ${job.actor.id}`);
+      const reddit = isRedditUrl(job.actor.id);
+      const embed = reddit ? await redditEmbed : undefined;
+      if (!reddit)
+        job.actor.id = result.ogUrl || job.actor.id;
+      job.actor.name = reddit ? embed?.provider_name ?? "reddit" : result.ogSiteName ?? job.actor.name ?? "";
       job.object = {
         type: "page",
         language: result.ogLocale,
-        title: result.ogTitle,
-        name: result.ogSiteName,
-        description: result.ogDescription || "",
-        image: result.ogImage,
-        url: result.ogUrl,
+        title: embed?.title ?? result.ogTitle,
+        name: embed?.provider_name ?? result.ogSiteName,
+        description: normalizeDescription(result.ogDescription || ""),
+        image: reddit ? redditPostImages(result.ogImage) ?? redditOEmbedImage(embed ?? {}) : result.ogImage,
+        url: reddit ? job.actor.id : result.ogUrl,
         favicon: result.favicon || "/favicon.ico",
         charset: result.charset
       };
       cb(null, job);
-    }).catch((data) => {
-      cb(toError(data?.result?.error ?? data));
+    }).catch(async (data) => {
+      const err = toError(data?.result?.error ?? data);
+      this.log.debug(`scrape failed for ${job.actor.id}: ${String(err)}`);
+      if (isRedditUrl(job.actor.id)) {
+        const embed = await redditEmbed;
+        if (embed?.title) {
+          job.actor.name = embed.provider_name || "reddit";
+          job.object = {
+            type: "page",
+            title: embed.title,
+            name: embed.provider_name || "reddit",
+            description: "",
+            image: redditOEmbedImage(embed),
+            url: job.actor.id,
+            favicon: "/favicon.ico"
+          };
+          this.log.debug(`using reddit oEmbed fallback for ${job.actor.id}`);
+          cb(null, job);
+          return;
+        }
+      }
+      cb(err);
     });
   }
   cleanup(cb) {
-    this.dispatcher?.close().catch(() => {});
+    const close = this.dispatcher?.close;
+    if (typeof close === "function") {
+      close.call(this.dispatcher).catch(() => {});
+    }
     cb();
   }
 }
 export {
+  withDeadline,
   Metadata as default
 };
 
-//# debugId=A271557344D4528064756E2164756E21
+//# debugId=119F74A1FCE6AC3B64756E2164756E21