@sockethub/platform-metadata 1.0.1-alpha.9 → 1.0.1-beta.0

package/dist/resolvers.d.ts

@@ -0,0 +1,209 @@
+/**
+ * Site-specific URL resolution for pages that don't serve useful Open Graph
+ * data to scrapers.
+ *
+ * Some large platforms return a generic banner image (X/Twitter), serve
+ * their Open Graph data only to recognized embed crawlers (Reddit), or
+ * hide post media behind a login (Facebook):
+ *
+ * - X/Twitter → FxTwitter's JSON API (api.fxtwitter.com), built for embed
+ *   previews, returns the tweet text plus direct photo/video URLs.
+ * - Reddit → the regular scrape works, but only when presented with an
+ *   embed-crawler user agent (see COMPAT_USER_AGENT in index.ts).
+ * - Facebook → same crawler-UA gating as Reddit, plus its video titles
+ *   arrive polluted with localized engagement stats that need stripping.
+ *
+ * These are pure URL matchers/mappers — the platform decides what to do
+ * with the resolution (call a JSON API vs. pick a scrape user agent).
+ */
+/**
+ * Match an X/Twitter status URL and return the FxTwitter API URL for it,
+ * or null when the URL isn't a status page (profiles, search, home feed —
+ * those go through the normal OG scrape).
+ *
+ * Handles `/<user>/status/<id>` and the `/i/web/status/<id>` share form.
+ */
+export declare function resolveTwitterStatus(url: string): string | null;
+/** Resolve supported YouTube video URLs to the official oEmbed endpoint. */
+export declare function resolveYouTubeOEmbed(url: string): string | null;
+export interface YouTubeOEmbed {
+    title: string;
+    provider_name?: string;
+    thumbnail_url: string;
+    thumbnail_width?: number;
+    thumbnail_height?: number;
+}
+/** Validate the subset of YouTube's external oEmbed payload we consume. */
+export declare function parseYouTubeOEmbed(value: unknown): YouTubeOEmbed | null;
+/** Map a validated YouTube oEmbed thumbnail to the platform image shape. */
+export declare function youtubeOEmbedImage(embed?: YouTubeOEmbed): PageObject["image"] | undefined;
+/**
+ * Extract YouTube's full video description from the player bootstrap JSON.
+ * This data is embedded in the watch HTML while the Open Graph description is
+ * intentionally truncated. Return undefined when the undocumented payload
+ * shape changes so callers can safely fall back to Open Graph metadata.
+ */
+export declare function extractYouTubeDescription(html: string): string | undefined;
+/**
+ * True for Facebook URLs (including fb.watch short links). Facebook serves
+ * an unrecognized scraper a login interstitial instead of the post, but
+ * serves recognized link-preview crawlers the post's Open Graph data —
+ * title, caption, and the video/photo thumbnail — so these URLs scrape
+ * with COMPAT_USER_AGENT (see index.ts), like Reddit.
+ */
+export declare function isFacebookUrl(url: string): boolean;
+/**
+ * Remove the engagement-stats segment Facebook prepends to video titles:
+ * "2.2M views · 21K reactions | <video title> | <author>". The stats are
+ * localized to the *scraping server's* geo-IP language, so match their
+ * shape — a leading pipe-delimited segment containing a digit and the "·"
+ * separator — rather than any particular wording.
+ */
+export declare function stripFacebookEngagement(title: string | undefined): string | undefined;
+/**
+ * True for Reddit URLs (including redd.it short links). Reddit serves its
+ * Open Graph tags — with the post's real preview image — only to
+ * recognized embed-crawler user agents; everything else gets a page with
+ * no OG data (or an outright 403 from datacenter addresses).
+ */
+export declare function isRedditUrl(url: string): boolean;
+/** Map a canonical Reddit post URL to Reddit's official media-enabled embed. */
+export declare function resolveRedditEmbed(url: string): string | null;
+/** Map a canonical Reddit post URL to old.reddit.com's public JSON endpoint. */
+export declare function resolveRedditJson(url: string): string | null;
+/**
+ * Unwrap Reddit's media-viewer URL (`reddit.com/media?url=<image>`) to the
+ * image it displays. Reddit serves neither post JSON nor oEmbed for these
+ * links (its oEmbed endpoint returns HTTP 400), but the target image is
+ * embedded in the link itself, so no network request is needed at all.
+ */
+export declare function resolveRedditMedia(url: string): string | null;
+export interface RedditPost {
+    title: string;
+    selftext?: string;
+    url?: string;
+    url_overridden_by_dest?: string;
+    secure_media?: {
+        reddit_video?: {
+            fallback_url?: string;
+            width?: number;
+            height?: number;
+            duration?: number;
+        };
+    };
+    preview?: {
+        images?: Array<{
+            source?: {
+                url?: string;
+                width?: number;
+                height?: number;
+            };
+        }>;
+    };
+}
+/** Validate and extract the post object from Reddit's listing response. */
+export declare function parseRedditPost(value: unknown): RedditPost | null;
+/** Select only a direct Reddit-hosted image belonging to the post. */
+export declare function redditPostImage(post: RedditPost): PageObject["image"];
+/** Map Reddit's hosted-video payload to the common page video shape. */
+export declare function redditPostVideo(post: RedditPost): PageObject["video"];
+/** Subset of Reddit's oEmbed response used for link previews. */
+export interface RedditOEmbed {
+    title?: string;
+    author_name?: string;
+    provider_name?: string;
+    thumbnail_url?: string;
+    thumbnail_width?: number;
+    thumbnail_height?: number;
+}
+/** Validate the subset of Reddit's untrusted oEmbed JSON that we consume. */
+export declare function parseRedditOEmbed(value: unknown): RedditOEmbed | null;
+/**
+ * Convert Reddit's optional oEmbed thumbnail into the platform image shape.
+ * A missing or malformed thumbnail deliberately means "no image": Reddit's
+ * scraped Open Graph image may be a generic site hero rather than post media.
+ */
+export declare function redditOEmbedImage(embed: RedditOEmbed): PageObject["image"] | undefined;
+/** Keep only Reddit-hosted post media, excluding branding/share-card images. */
+export declare function redditPostImages(images: PageObject["image"]): PageObject["image"] | undefined;
+/**
+ * Make scraped plain text readable without flattening intentional structure.
+ * Preserve line breaks and paragraphs, but remove HTML indentation and cap
+ * excessive vertical whitespace at one blank line.
+ */
+export declare function normalizeDescription(value: string): string;
+/** Subset of the FxTwitter status API response the platform consumes. */
+export interface FxTwitterStatus {
+    code: number;
+    message?: string;
+    tweet?: {
+        url?: string;
+        text?: string;
+        author?: {
+            name?: string;
+            screen_name?: string;
+        };
+        media?: {
+            photos?: Array<{
+                url?: string;
+                width?: number;
+                height?: number;
+            }>;
+            videos?: Array<{
+                url?: string;
+                thumbnail_url?: string;
+                width?: number;
+                height?: number;
+                duration?: number;
+            }>;
+        };
+        article?: {
+            title?: string;
+            preview_text?: string;
+            cover_media?: {
+                media_info?: {
+                    original_img_url?: string;
+                    original_img_width?: number;
+                    original_img_height?: number;
+                };
+            };
+            content?: {
+                blocks?: Array<{
+                    text?: string;
+                    type?: string;
+                }>;
+            };
+        };
+    };
+}
+/** The `object` payload shape the metadata platform responds with. */
+export interface PageObject {
+    type: "page";
+    language?: string;
+    title?: string;
+    name?: string;
+    description?: string;
+    image?: Array<{
+        url: string;
+        type?: string;
+        width?: string | number;
+        height?: string | number;
+    }>;
+    video?: {
+        url: string;
+        thumbnail?: string;
+        width?: number;
+        height?: number;
+        duration?: number;
+    };
+    url?: string;
+    favicon?: string;
+    charset?: string;
+}
+/**
+ * Map an FxTwitter API response onto the platform's page object. The
+ * preview image is the post's own media — first photo, else the video
+ * thumbnail — never the generic site banner. Returns null when the API
+ * reports anything but success (the caller falls back to the OG scrape).
+ */
+export declare function tweetToPageObject(status: FxTwitterStatus): PageObject | null;

package/dist/schema.d.ts

@@ -76,6 +76,28 @@ export declare const PlatformMetadataSchema: {
                         image: {
                             $ref: string;
                         };
+                        video: {
+                            type: string;
+                            required: string[];
+                            additionalProperties: boolean;
+                            properties: {
+                                url: {
+                                    type: string;
+                                };
+                                thumbnail: {
+                                    type: string;
+                                };
+                                width: {
+                                    type: string;
+                                };
+                                height: {
+                                    type: string;
+                                };
+                                duration: {
+                                    type: string;
+                                };
+                            };
+                        };
                         url: {
                             type: string;
                         };
@@ -106,6 +128,9 @@ export declare const PlatformMetadataSchema: {
                             height: {
                                 type: string[];
                             };
+                            alt: {
+                                type: string;
+                            };
                         };
                     };
                 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sockethub/platform-metadata",
-  "version": "1.0.1-alpha.9",
+  "version": "1.0.1-beta.0",
   "description": "Fetch metadata from a given URL",
   "author": "Nick Jennings <nick@silverbucket.net>",
   "license": "LGPL-3.0+",
@@ -42,15 +42,16 @@
     "clean:deps": "rm -rf node_modules"
   },
   "dependencies": {
-    "@sockethub/util": "^1.0.0-alpha.2",
-    "open-graph-scraper": "^6.11.0"
+    "@sockethub/util": "^1.0.0-beta.0",
+    "open-graph-scraper": "^6.11.0",
+    "undici": "^7.30.0"
   },
   "devDependencies": {
-    "@sockethub/schemas": "^3.0.0-alpha.14"
+    "@sockethub/schemas": "^3.0.0-beta.0"
   },
   "peerDependencies": {
-    "@sockethub/server": "^5.0.0-alpha.14"
+    "@sockethub/server": "^5.0.0-beta.0"
   },
-  "gitHead": "4f99b07fc19c9606875621b3f96323fa6e62ae3f",
+  "gitHead": "a17e337ee7d2dd496d225c0d6e8ae48f4f4ce73c",
   "types": "./dist/index.d.ts"
 }