@sockethub/platform-metadata 1.0.1-alpha.9 → 1.0.1-beta.1

package/README.md

@@ -77,9 +77,72 @@ The platform returns an ActivityStream object with extracted metadata:
 The platform extracts the following metadata when available:
 
 * **Open Graph data**: Title, description, images, site name, URL, locale
-* **Site information**: Favicon, character encoding
+* **Site information**: Favicon, character encoding — when a page declares no
+  icon, `favicon` falls back to the conventional relative `/favicon.ico` so
+  clients can resolve it against the page URL and decorate previews
 * **Canonical URLs**: Resolved and canonical page URLs
-* **Media**: Images with dimensions and type information
+* **Media**: Images with dimensions and type information; for X/Twitter posts,
+  a `video` object (direct media URL, poster `thumbnail`, dimensions, duration)
+
+## Site-Specific Handling
+
+Some large platforms don't expose a post's own media to Open Graph scrapers,
+so those URLs are resolved through preview services built for this purpose:
+
+* **X / Twitter status URLs** (`x.com`, `twitter.com`, incl. `/i/web/status/…`)
+  are fetched from the [FxTwitter API](https://github.com/FxEmbed/FxEmbed)
+  instead of scraped — X serves only a generic site banner to scrapers. The
+  response carries the post's text, its actual photo (or video + poster
+  thumbnail), and the canonical `x.com` URL. If the API is unavailable or the
+  post can't be resolved, the platform falls back to the regular scrape
+  (which still yields the post text).
+* **Reddit post URLs** use `old.reddit.com`'s structured JSON response, avoiding
+  Reddit's unreliable HTML and Open Graph crawler behavior. The post title,
+  self-text, and direct `i.redd.it` image are mapped into the preview; text posts
+  intentionally have no image. oEmbed remains a title-only failure fallback.
+  See `compatUserAgent` below.
+* **Facebook** remains best-effort: post text usually comes through. The
+  video or photo thumbnail is taken from the post's Open Graph image, then
+  its Twitter card image, and finally the image rendered by Facebook's public
+  embed plugin (`/plugins/video.php` or `/plugins/post.php`), which Facebook
+  serves to third parties when it withholds Open Graph media from a
+  datacenter address. Full media still requires authentication.
+
+### `userAgent`
+
+All outbound requests identify as
+`Mozilla/5.0 (compatible; SockethubBot/<version>; +https://sockethub.org)`.
+Sites gate scraper-facing behavior on the user agent — preview services serve
+their Open Graph payload to bot-like UAs, and many hosts reject undici's
+default UA. Override per deployment via `packageConfig`:
+
+```json
+{
+  "packageConfig": {
+    "@sockethub/platform-metadata": {
+      "userAgent": "MyDeployment/1.0 (+https://my.example)"
+    }
+  }
+}
+```
+
+### `compatUserAgent`
+
+Sites that only serve metadata to *recognized* embed crawlers (Reddit)
+are fetched with a link-preview crawler user agent instead — the established
+practice for self-hosted preview fetchers. Defaults to
+`Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)`;
+override via `packageConfig`:
+
+```json
+{
+  "packageConfig": {
+    "@sockethub/platform-metadata": {
+      "compatUserAgent": "Mozilla/5.0 (compatible; TelegramBot/1.0)"
+    }
+  }
+}
+```
 
 ## Dependencies
 
@@ -92,9 +155,14 @@ Because the server fetches whatever URL a client supplies as `actor.id`, metadat
 requests are hardened against server-side request forgery (SSRF) and resource
 exhaustion. The fetch runs through a guarded connection layer that:
 
+* rejects URL schemes other than `http` and `https`;
 * refuses to connect to loopback, private, link-local, carrier-grade NAT, or
-  cloud-metadata addresses (e.g. `169.254.169.254`), in every IPv4/IPv6 spelling,
-  validated at connection time so it also covers redirect hops;
+  cloud-metadata addresses (e.g. `169.254.169.254`), validated at connection
+  time so it also covers redirect hops. IPv4-mapped, IPv4-compatible, and
+  well-known NAT64 (`64:ff9b::/96`) embeddings are judged as the embedded IPv4
+  address. The local-use NAT64 prefix `64:ff9b:1::/48` is refused entirely.
+  The ranges are listed under
+  [Outbound URL guard](../../docs/configuration.md#outbound-url-guard);
 * caps the response body size.
 
 ### `allowPrivateAddresses`

package/dist/index.d.ts

@@ -1,6 +1,9 @@
 import type { ActivityStream, PlatformCallback, PlatformConfig, PlatformInterface, PlatformSession } from "@sockethub/schemas";
+/** Enforce a deadline independently of a dependency's AbortSignal handling. */
+export declare function withDeadline<T>(promise: Promise<T>, timeoutMs: number): Promise<T>;
 export default class Metadata implements PlatformInterface {
     private readonly log;
+    private readonly fetchImpl;
     private dispatcher?;
     config: PlatformConfig;
     constructor(session: PlatformSession);
@@ -88,6 +91,28 @@ export default class Metadata implements PlatformInterface {
                             image: {
                                 $ref: string;
                             };
+                            video: {
+                                type: string;
+                                required: string[];
+                                additionalProperties: boolean;
+                                properties: {
+                                    url: {
+                                        type: string;
+                                    };
+                                    thumbnail: {
+                                        type: string;
+                                    };
+                                    width: {
+                                        type: string;
+                                    };
+                                    height: {
+                                        type: string;
+                                    };
+                                    duration: {
+                                        type: string;
+                                    };
+                                };
+                            };
                             url: {
                                 type: string;
                             };
@@ -118,6 +143,9 @@ export default class Metadata implements PlatformInterface {
                                 height: {
                                     type: string[];
                                 };
+                                alt: {
+                                    type: string;
+                                };
                             };
                         };
                     };
@@ -129,6 +157,33 @@ export default class Metadata implements PlatformInterface {
      * Stateless platforms are always ready to handle jobs.
      */
     isInitialized(): boolean;
+    private userAgent;
+    private compatUserAgent;
     fetch(job: ActivityStream, cb: PlatformCallback): void;
+    /**
+     * Verify likely direct-image links when possible. Origins commonly block
+     * server-side preview agents, so a strong file extension remains a useful
+     * fallback: the client can still request the original resource itself.
+     */
+    private fetchDirectImage;
+    /** Fetch and validate YouTube's official preview metadata. */
+    private fetchYouTubeEmbed;
+    /**
+     * Last-resort thumbnail for a Facebook post whose page carried no
+     * Open Graph image: the media image rendered by Facebook's public embed
+     * plugin, which is served to third parties without a login wall.
+     */
+    private fetchFacebookEmbedImage;
+    private fetchTweet;
+    private fetchReddit;
+    /**
+     * Fetch Reddit's fixed-host JSON endpoint without the Undici dispatcher.
+     * The platform child runtime has exhibited stuck Undici requests even when
+     * AbortSignal fires. node:https gives us an independently enforced socket
+     * timeout; the outer withDeadline still guarantees callback completion.
+     */
+    private fetchRedditJson;
+    private fetchRedditEmbed;
+    private scrape;
     cleanup(cb: PlatformCallback): void;
 }