@sockethub/platform-metadata 1.0.1-alpha.8 → 1.0.1-beta.0

package/README.md

@@ -77,9 +77,68 @@ The platform returns an ActivityStream object with extracted metadata:
 The platform extracts the following metadata when available:
 
 * **Open Graph data**: Title, description, images, site name, URL, locale
-* **Site information**: Favicon, character encoding
+* **Site information**: Favicon, character encoding — when a page declares no
+  icon, `favicon` falls back to the conventional relative `/favicon.ico` so
+  clients can resolve it against the page URL and decorate previews
 * **Canonical URLs**: Resolved and canonical page URLs
-* **Media**: Images with dimensions and type information
+* **Media**: Images with dimensions and type information; for X/Twitter posts,
+  a `video` object (direct media URL, poster `thumbnail`, dimensions, duration)
+
+## Site-Specific Handling
+
+Some large platforms don't expose a post's own media to Open Graph scrapers,
+so those URLs are resolved through preview services built for this purpose:
+
+* **X / Twitter status URLs** (`x.com`, `twitter.com`, incl. `/i/web/status/…`)
+  are fetched from the [FxTwitter API](https://github.com/FxEmbed/FxEmbed)
+  instead of scraped — X serves only a generic site banner to scrapers. The
+  response carries the post's text, its actual photo (or video + poster
+  thumbnail), and the canonical `x.com` URL. If the API is unavailable or the
+  post can't be resolved, the platform falls back to the regular scrape
+  (which still yields the post text).
+* **Reddit post URLs** use `old.reddit.com`'s structured JSON response, avoiding
+  Reddit's unreliable HTML and Open Graph crawler behavior. The post title,
+  self-text, and direct `i.redd.it` image are mapped into the preview; text posts
+  intentionally have no image. oEmbed remains a title-only failure fallback.
+  See `compatUserAgent` below.
+* **Facebook** remains best-effort: post text usually comes through, but
+  media requires authentication and no public preview service exists.
+
+### `userAgent`
+
+All outbound requests identify as
+`Mozilla/5.0 (compatible; SockethubBot/<version>; +https://sockethub.org)`.
+Sites gate scraper-facing behavior on the user agent — preview services serve
+their Open Graph payload to bot-like UAs, and many hosts reject undici's
+default UA. Override per deployment via `packageConfig`:
+
+```json
+{
+  "packageConfig": {
+    "@sockethub/platform-metadata": {
+      "userAgent": "MyDeployment/1.0 (+https://my.example)"
+    }
+  }
+}
+```
+
+### `compatUserAgent`
+
+Sites that only serve metadata to *recognized* embed crawlers (Reddit)
+are fetched with a link-preview crawler user agent instead — the established
+practice for self-hosted preview fetchers. Defaults to
+`Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)`;
+override via `packageConfig`:
+
+```json
+{
+  "packageConfig": {
+    "@sockethub/platform-metadata": {
+      "compatUserAgent": "Mozilla/5.0 (compatible; TelegramBot/1.0)"
+    }
+  }
+}
+```
 
 ## Dependencies
 

package/dist/index.d.ts

@@ -1,6 +1,9 @@
 import type { ActivityStream, PlatformCallback, PlatformConfig, PlatformInterface, PlatformSession } from "@sockethub/schemas";
+/** Enforce a deadline independently of a dependency's AbortSignal handling. */
+export declare function withDeadline<T>(promise: Promise<T>, timeoutMs: number): Promise<T>;
 export default class Metadata implements PlatformInterface {
     private readonly log;
+    private readonly fetchImpl;
     private dispatcher?;
     config: PlatformConfig;
     constructor(session: PlatformSession);
@@ -88,6 +91,28 @@ export default class Metadata implements PlatformInterface {
                             image: {
                                 $ref: string;
                             };
+                            video: {
+                                type: string;
+                                required: string[];
+                                additionalProperties: boolean;
+                                properties: {
+                                    url: {
+                                        type: string;
+                                    };
+                                    thumbnail: {
+                                        type: string;
+                                    };
+                                    width: {
+                                        type: string;
+                                    };
+                                    height: {
+                                        type: string;
+                                    };
+                                    duration: {
+                                        type: string;
+                                    };
+                                };
+                            };
                             url: {
                                 type: string;
                             };
@@ -118,6 +143,9 @@ export default class Metadata implements PlatformInterface {
                                 height: {
                                     type: string[];
                                 };
+                                alt: {
+                                    type: string;
+                                };
                             };
                         };
                     };
@@ -129,6 +157,27 @@ export default class Metadata implements PlatformInterface {
      * Stateless platforms are always ready to handle jobs.
      */
     isInitialized(): boolean;
+    private userAgent;
+    private compatUserAgent;
     fetch(job: ActivityStream, cb: PlatformCallback): void;
+    /**
+     * Verify likely direct-image links when possible. Origins commonly block
+     * server-side preview agents, so a strong file extension remains a useful
+     * fallback: the client can still request the original resource itself.
+     */
+    private fetchDirectImage;
+    /** Fetch and validate YouTube's official preview metadata. */
+    private fetchYouTubeEmbed;
+    private fetchTweet;
+    private fetchReddit;
+    /**
+     * Fetch Reddit's fixed-host JSON endpoint without the Undici dispatcher.
+     * The platform child runtime has exhibited stuck Undici requests even when
+     * AbortSignal fires. node:https gives us an independently enforced socket
+     * timeout; the outer withDeadline still guarantees callback completion.
+     */
+    private fetchRedditJson;
+    private fetchRedditEmbed;
+    private scrape;
     cleanup(cb: PlatformCallback): void;
 }