@sockethub/platform-metadata 1.0.1-alpha.2 → 1.0.1-alpha.20

package/dist/resolvers.d.ts

@@ -0,0 +1,202 @@
+/**
+ * Site-specific URL resolution for pages that don't serve useful Open Graph
+ * data to scrapers.
+ *
+ * Some large platforms return a generic banner image (X/Twitter), serve
+ * their Open Graph data only to recognized embed crawlers (Reddit), or
+ * hide post media behind a login (Facebook):
+ *
+ * - X/Twitter → FxTwitter's JSON API (api.fxtwitter.com), built for embed
+ *   previews, returns the tweet text plus direct photo/video URLs.
+ * - Reddit → the regular scrape works, but only when presented with an
+ *   embed-crawler user agent (see COMPAT_USER_AGENT in index.ts).
+ * - Facebook → same crawler-UA gating as Reddit, plus its video titles
+ *   arrive polluted with localized engagement stats that need stripping.
+ *
+ * These are pure URL matchers/mappers — the platform decides what to do
+ * with the resolution (call a JSON API vs. pick a scrape user agent).
+ */
+/**
+ * Match an X/Twitter status URL and return the FxTwitter API URL for it,
+ * or null when the URL isn't a status page (profiles, search, home feed —
+ * those go through the normal OG scrape).
+ *
+ * Handles `/<user>/status/<id>` and the `/i/web/status/<id>` share form.
+ */
+export declare function resolveTwitterStatus(url: string): string | null;
+/** Resolve supported YouTube video URLs to the official oEmbed endpoint. */
+export declare function resolveYouTubeOEmbed(url: string): string | null;
+export interface YouTubeOEmbed {
+    title: string;
+    provider_name?: string;
+    thumbnail_url: string;
+    thumbnail_width?: number;
+    thumbnail_height?: number;
+}
+/** Validate the subset of YouTube's external oEmbed payload we consume. */
+export declare function parseYouTubeOEmbed(value: unknown): YouTubeOEmbed | null;
+/** Map a validated YouTube oEmbed thumbnail to the platform image shape. */
+export declare function youtubeOEmbedImage(embed?: YouTubeOEmbed): PageObject["image"] | undefined;
+/**
+ * True for Facebook URLs (including fb.watch short links). Facebook serves
+ * an unrecognized scraper a login interstitial instead of the post, but
+ * serves recognized link-preview crawlers the post's Open Graph data —
+ * title, caption, and the video/photo thumbnail — so these URLs scrape
+ * with COMPAT_USER_AGENT (see index.ts), like Reddit.
+ */
+export declare function isFacebookUrl(url: string): boolean;
+/**
+ * Remove the engagement-stats segment Facebook prepends to video titles:
+ * "2.2M views · 21K reactions | <video title> | <author>". The stats are
+ * localized to the *scraping server's* geo-IP language, so match their
+ * shape — a leading pipe-delimited segment containing a digit and the "·"
+ * separator — rather than any particular wording.
+ */
+export declare function stripFacebookEngagement(title: string | undefined): string | undefined;
+/**
+ * True for Reddit URLs (including redd.it short links). Reddit serves its
+ * Open Graph tags — with the post's real preview image — only to
+ * recognized embed-crawler user agents; everything else gets a page with
+ * no OG data (or an outright 403 from datacenter addresses).
+ */
+export declare function isRedditUrl(url: string): boolean;
+/** Map a canonical Reddit post URL to Reddit's official media-enabled embed. */
+export declare function resolveRedditEmbed(url: string): string | null;
+/** Map a canonical Reddit post URL to old.reddit.com's public JSON endpoint. */
+export declare function resolveRedditJson(url: string): string | null;
+/**
+ * Unwrap Reddit's media-viewer URL (`reddit.com/media?url=<image>`) to the
+ * image it displays. Reddit serves neither post JSON nor oEmbed for these
+ * links (its oEmbed endpoint returns HTTP 400), but the target image is
+ * embedded in the link itself, so no network request is needed at all.
+ */
+export declare function resolveRedditMedia(url: string): string | null;
+export interface RedditPost {
+    title: string;
+    selftext?: string;
+    url?: string;
+    url_overridden_by_dest?: string;
+    secure_media?: {
+        reddit_video?: {
+            fallback_url?: string;
+            width?: number;
+            height?: number;
+            duration?: number;
+        };
+    };
+    preview?: {
+        images?: Array<{
+            source?: {
+                url?: string;
+                width?: number;
+                height?: number;
+            };
+        }>;
+    };
+}
+/** Validate and extract the post object from Reddit's listing response. */
+export declare function parseRedditPost(value: unknown): RedditPost | null;
+/** Select only a direct Reddit-hosted image belonging to the post. */
+export declare function redditPostImage(post: RedditPost): PageObject["image"];
+/** Map Reddit's hosted-video payload to the common page video shape. */
+export declare function redditPostVideo(post: RedditPost): PageObject["video"];
+/** Subset of Reddit's oEmbed response used for link previews. */
+export interface RedditOEmbed {
+    title?: string;
+    author_name?: string;
+    provider_name?: string;
+    thumbnail_url?: string;
+    thumbnail_width?: number;
+    thumbnail_height?: number;
+}
+/** Validate the subset of Reddit's untrusted oEmbed JSON that we consume. */
+export declare function parseRedditOEmbed(value: unknown): RedditOEmbed | null;
+/**
+ * Convert Reddit's optional oEmbed thumbnail into the platform image shape.
+ * A missing or malformed thumbnail deliberately means "no image": Reddit's
+ * scraped Open Graph image may be a generic site hero rather than post media.
+ */
+export declare function redditOEmbedImage(embed: RedditOEmbed): PageObject["image"] | undefined;
+/** Keep only Reddit-hosted post media, excluding branding/share-card images. */
+export declare function redditPostImages(images: PageObject["image"]): PageObject["image"] | undefined;
+/**
+ * Make scraped plain text readable without flattening intentional structure.
+ * Preserve line breaks and paragraphs, but remove HTML indentation and cap
+ * excessive vertical whitespace at one blank line.
+ */
+export declare function normalizeDescription(value: string): string;
+/** Subset of the FxTwitter status API response the platform consumes. */
+export interface FxTwitterStatus {
+    code: number;
+    message?: string;
+    tweet?: {
+        url?: string;
+        text?: string;
+        author?: {
+            name?: string;
+            screen_name?: string;
+        };
+        media?: {
+            photos?: Array<{
+                url?: string;
+                width?: number;
+                height?: number;
+            }>;
+            videos?: Array<{
+                url?: string;
+                thumbnail_url?: string;
+                width?: number;
+                height?: number;
+                duration?: number;
+            }>;
+        };
+        article?: {
+            title?: string;
+            preview_text?: string;
+            cover_media?: {
+                media_info?: {
+                    original_img_url?: string;
+                    original_img_width?: number;
+                    original_img_height?: number;
+                };
+            };
+            content?: {
+                blocks?: Array<{
+                    text?: string;
+                    type?: string;
+                }>;
+            };
+        };
+    };
+}
+/** The `object` payload shape the metadata platform responds with. */
+export interface PageObject {
+    type: "page";
+    language?: string;
+    title?: string;
+    name?: string;
+    description?: string;
+    image?: Array<{
+        url: string;
+        type?: string;
+        width?: string | number;
+        height?: string | number;
+    }>;
+    video?: {
+        url: string;
+        thumbnail?: string;
+        width?: number;
+        height?: number;
+        duration?: number;
+    };
+    url?: string;
+    favicon?: string;
+    charset?: string;
+}
+/**
+ * Map an FxTwitter API response onto the platform's page object. The
+ * preview image is the post's own media — first photo, else the video
+ * thumbnail — never the generic site banner. Returns null when the API
+ * reports anything but success (the caller falls back to the OG scrape).
+ */
+export declare function tweetToPageObject(status: FxTwitterStatus): PageObject | null;

package/dist/schema.d.ts

@@ -0,0 +1,140 @@
+export declare const PlatformMetadataSchema: {
+    name: string;
+    version: string;
+    contextUrl: string;
+    contextVersion: string;
+    schemaVersion: string;
+    messages: {
+        required: string[];
+        properties: {
+            type: {
+                type: string;
+                enum: string[];
+            };
+            object: boolean;
+        };
+    };
+    responses: {
+        type: string;
+        required: string[];
+        additionalProperties: boolean;
+        properties: {
+            "@context": {
+                type: string;
+                items: {
+                    type: string;
+                };
+            };
+            id: {
+                type: string;
+            };
+            type: {
+                enum: string[];
+            };
+            actor: {
+                type: string;
+                required: string[];
+                additionalProperties: boolean;
+                properties: {
+                    id: {
+                        type: string;
+                    };
+                    type: {
+                        enum: string[];
+                    };
+                    name: {
+                        type: string;
+                    };
+                };
+            };
+            object: {
+                $ref: string;
+            };
+        };
+        definitions: {
+            responses: {
+                page: {
+                    type: string;
+                    required: string[];
+                    additionalProperties: boolean;
+                    properties: {
+                        type: {
+                            enum: string[];
+                        };
+                        language: {
+                            type: string;
+                        };
+                        title: {
+                            type: string;
+                        };
+                        name: {
+                            type: string;
+                        };
+                        description: {
+                            type: string;
+                        };
+                        image: {
+                            $ref: string;
+                        };
+                        video: {
+                            type: string;
+                            required: string[];
+                            additionalProperties: boolean;
+                            properties: {
+                                url: {
+                                    type: string;
+                                };
+                                thumbnail: {
+                                    type: string;
+                                };
+                                width: {
+                                    type: string;
+                                };
+                                height: {
+                                    type: string;
+                                };
+                                duration: {
+                                    type: string;
+                                };
+                            };
+                        };
+                        url: {
+                            type: string;
+                        };
+                        favicon: {
+                            type: string;
+                        };
+                        charset: {
+                            type: string;
+                        };
+                    };
+                };
+                ogImage: {
+                    type: string;
+                    items: {
+                        type: string;
+                        required: string[];
+                        additionalProperties: boolean;
+                        properties: {
+                            url: {
+                                type: string;
+                            };
+                            type: {
+                                type: string;
+                            };
+                            width: {
+                                type: string[];
+                            };
+                            height: {
+                                type: string[];
+                            };
+                            alt: {
+                                type: string;
+                            };
+                        };
+                    };
+                };
+            };
+        };
+    };
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sockethub/platform-metadata",
-  "version": "1.0.1-alpha.2",
+  "version": "1.0.1-alpha.20",
   "description": "Fetch metadata from a given URL",
   "author": "Nick Jennings <nick@silverbucket.net>",
   "license": "LGPL-3.0+",
@@ -11,17 +11,16 @@
   "main": "dist/index.js",
   "exports": {
     ".": {
-      "bun": "./src/index.ts",
+      "types": "./dist/index.d.ts",
       "import": "./dist/index.js",
       "default": "./dist/index.js"
     }
   },
   "files": [
-    "src/",
     "dist/"
   ],
   "engines": {
-    "bun": ">=1.2"
+    "node": ">=20"
   },
   "keywords": [
     "sockethub",
@@ -39,16 +38,20 @@
   "homepage": "https://github.com/sockethub/sockethub/tree/master/packages/platform-metadata",
   "scripts": {
     "build": "bun build src/index.ts --outdir dist --target node --format esm --sourcemap=external",
-    "clean": "rm -rf dist"
+    "clean": "rm -rf dist",
+    "clean:deps": "rm -rf node_modules"
   },
   "dependencies": {
-    "open-graph-scraper": "^6.11.0"
+    "@sockethub/util": "^1.0.0-alpha.13",
+    "open-graph-scraper": "^6.11.0",
+    "undici": "^7.16.0"
   },
   "devDependencies": {
-    "@sockethub/schemas": "^3.0.0-alpha.7"
+    "@sockethub/schemas": "^3.0.0-alpha.25"
   },
   "peerDependencies": {
-    "@sockethub/server": "^5.0.0-alpha.6"
+    "@sockethub/server": "^5.0.0-alpha.25"
   },
-  "gitHead": "38927776c210a22bc54ceb86ccc7276c5f27b463"
+  "gitHead": "1a76cff52a7b6f405fc607f1402bf2346ec5c40f",
+  "types": "./dist/index.d.ts"
 }

package/src/index.ts

@@ -1,56 +0,0 @@
-import type {
-    ActivityStream,
-    Logger,
-    PlatformCallback,
-    PlatformConfig,
-    PlatformInterface,
-    PlatformSession,
-} from "@sockethub/schemas";
-import ogs from "open-graph-scraper";
-import { PlatformMetadataSchema } from "./schema";
-
-export default class Metadata implements PlatformInterface {
-    readonly debug: Logger;
-    config: PlatformConfig = {
-        persist: false,
-    };
-    constructor(session: PlatformSession) {
-        this.debug = session.debug;
-    }
-
-    get schema() {
-        return PlatformMetadataSchema;
-    }
-
-    fetch(job: ActivityStream, cb: PlatformCallback) {
-        this.debug(`fetching ${job.actor.id}`);
-        ogs({
-            url: job.actor.id,
-        })
-            .then((data) => {
-                const { result } = data;
-                job.actor.id = result.ogUrl || job.actor.id;
-                job.actor.name = result.ogSiteName || job.actor.name || "";
-                job.object = {
-                    type: "page",
-                    language: result.ogLocale,
-                    title: result.ogTitle,
-                    name: result.ogSiteName,
-                    description: result.ogDescription || "",
-                    image: result.ogImage,
-                    url: result.ogUrl,
-                    favicon: result.favicon,
-                    charset: result.charset,
-                };
-                cb(null, job);
-            })
-            .catch((data) => {
-                const { result } = data;
-                cb(result.error);
-            });
-    }
-
-    cleanup(cb: PlatformCallback) {
-        cb();
-    }
-}

package/src/schema.ts

@@ -1,22 +0,0 @@
-import packageJson from "../package.json" with { type: "json" };
-const version = packageJson.version;
-
-export const PlatformMetadataSchema = {
-    name: "metadata",
-    version: version,
-    messages: {
-        required: ["type"],
-        properties: {
-            type: {
-                type: "string",
-                enum: ["fetch"],
-            },
-            object: {
-                type: "object",
-                oneOf: [
-                    { $ref: "#/definitions/objectTypes/feed-parameters-url" },
-                ],
-            },
-        },
-    },
-};