@sockethub/platform-metadata 1.0.1-alpha.1 → 1.0.1-alpha.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. package/README.md +112 -11
  2. package/dist/index.d.ts +163 -0
  3. package/dist/index.js +72140 -0
  4. package/dist/index.js.map +306 -0
  5. package/dist/resolvers.d.ts +90 -0
  6. package/dist/schema.d.ts +140 -0
  7. package/package.json +24 -7
  8. package/src/index.ts +0 -56
  9. package/src/schema.ts +0 -22
package/README.md CHANGED
@@ -8,7 +8,7 @@ This platform fetches and extracts metadata from web pages, including Open Graph
8
8
  site information, images, and other structured metadata. It uses the `open-graph-scraper`
9
9
  library to analyze web pages and return structured data in ActivityStreams format.
10
10
 
11
- ## Implemented Verbs (`@type`)
11
+ ## Implemented Verbs (`type`)
12
12
 
13
13
  * **fetch** - Extract metadata from a web page URL
14
14
 
@@ -21,10 +21,15 @@ to clients through the Sockethub service.
21
21
 
22
22
  ```json
23
23
  {
24
- "@type": "fetch",
25
- "context": "metadata",
24
+ "type": "fetch",
25
+ "@context": [
26
+ "https://www.w3.org/ns/activitystreams",
27
+ "https://sockethub.org/ns/context/v1.jsonld",
28
+ "https://sockethub.org/ns/context/platform/metadata/v1.jsonld"
29
+ ],
26
30
  "actor": {
27
- "@id": "https://example.com/page-to-analyze"
31
+ "id": "https://example.com/page-to-analyze",
32
+ "type": "website"
28
33
  }
29
34
  }
30
35
  ```
@@ -35,10 +40,15 @@ The platform returns an ActivityStream object with extracted metadata:
35
40
 
36
41
  ```json
37
42
  {
38
- "@type": "fetch",
39
- "context": "metadata",
43
+ "type": "fetch",
44
+ "@context": [
45
+ "https://www.w3.org/ns/activitystreams",
46
+ "https://sockethub.org/ns/context/v1.jsonld",
47
+ "https://sockethub.org/ns/context/platform/metadata/v1.jsonld"
48
+ ],
40
49
  "actor": {
41
- "@id": "https://example.com/page-to-analyze",
50
+ "id": "https://example.com/page-to-analyze",
51
+ "type": "website",
42
52
  "name": "Example Site"
43
53
  },
44
54
  "object": {
@@ -67,19 +77,110 @@ The platform returns an ActivityStream object with extracted metadata:
67
77
  The platform extracts the following metadata when available:
68
78
 
69
79
  * **Open Graph data**: Title, description, images, site name, URL, locale
70
- * **Site information**: Favicon, character encoding
80
+ * **Site information**: Favicon, character encoding — when a page declares no
81
+ icon, `favicon` falls back to the conventional relative `/favicon.ico` so
82
+ clients can resolve it against the page URL and decorate previews
71
83
  * **Canonical URLs**: Resolved and canonical page URLs
72
- * **Media**: Images with dimensions and type information
84
+ * **Media**: Images with dimensions and type information; for X/Twitter posts,
85
+ a `video` object (direct media URL, poster `thumbnail`, dimensions, duration)
86
+
87
+ ## Site-Specific Handling
88
+
89
+ Some large platforms don't expose a post's own media to Open Graph scrapers,
90
+ so those URLs are resolved through preview services built for this purpose:
91
+
92
+ * **X / Twitter status URLs** (`x.com`, `twitter.com`, incl. `/i/web/status/…`)
93
+ are fetched from the [FxTwitter API](https://github.com/FxEmbed/FxEmbed)
94
+ instead of scraped — X serves only a generic site banner to scrapers. The
95
+ response carries the post's text, its actual photo (or video + poster
96
+ thumbnail), and the canonical `x.com` URL. If the API is unavailable or the
97
+ post can't be resolved, the platform falls back to the regular scrape
98
+ (which still yields the post text).
99
+ * **Reddit URLs** (`reddit.com` hosts and `redd.it` short links) are scraped
100
+ directly, but with a *compatibility user agent* — Reddit serves its Open
101
+ Graph data (including the post's real preview image) only to recognized
102
+ embed crawlers, and returns a page with no OG tags (or a 403) to anything
103
+ else. See `compatUserAgent` below.
104
+ * **Facebook** remains best-effort: post text usually comes through, but
105
+ media requires authentication and no public preview service exists.
106
+
107
+ ### `userAgent`
108
+
109
+ All outbound requests identify as
110
+ `Mozilla/5.0 (compatible; SockethubBot/<version>; +https://sockethub.org)`.
111
+ Sites gate scraper-facing behavior on the user agent — preview services serve
112
+ their Open Graph payload to bot-like UAs, and many hosts reject undici's
113
+ default UA. Override per deployment via `packageConfig`:
114
+
115
+ ```json
116
+ {
117
+ "packageConfig": {
118
+ "@sockethub/platform-metadata": {
119
+ "userAgent": "MyDeployment/1.0 (+https://my.example)"
120
+ }
121
+ }
122
+ }
123
+ ```
124
+
125
+ ### `compatUserAgent`
126
+
127
+ Sites that only serve Open Graph data to *recognized* embed crawlers (Reddit)
128
+ are fetched with a link-preview crawler user agent instead — the established
129
+ practice for self-hosted preview fetchers. Defaults to
130
+ `Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)`;
131
+ override via `packageConfig`:
132
+
133
+ ```json
134
+ {
135
+ "packageConfig": {
136
+ "@sockethub/platform-metadata": {
137
+ "compatUserAgent": "Mozilla/5.0 (compatible; TelegramBot/1.0)"
138
+ }
139
+ }
140
+ }
141
+ ```
73
142
 
74
143
  ## Dependencies
75
144
 
76
145
  * **open-graph-scraper**: Core library for extracting Open Graph and metadata from web
77
146
  pages
78
147
 
148
+ ## Request Hardening (SSRF Guard)
149
+
150
+ Because the server fetches whatever URL a client supplies as `actor.id`, metadata
151
+ requests are hardened against server-side request forgery (SSRF) and resource
152
+ exhaustion. The fetch runs through a guarded connection layer that:
153
+
154
+ * refuses to connect to loopback, private, link-local, carrier-grade NAT, or
155
+ cloud-metadata addresses (e.g. `169.254.169.254`), in every IPv4/IPv6 spelling,
156
+ validated at connection time so it also covers redirect hops;
157
+ * caps the response body size.
158
+
159
+ ### `allowPrivateAddresses`
160
+
161
+ Set in the Sockethub config file under this platform's `packageConfig` entry:
162
+
163
+ ```json
164
+ {
165
+ "packageConfig": {
166
+ "@sockethub/platform-metadata": {
167
+ "allowPrivateAddresses": true
168
+ }
169
+ }
170
+ }
171
+ ```
172
+
173
+ This disables **only** the private/loopback destination checks. **Do not enable
174
+ it on a server that accepts requests from untrusted clients** — it lets any
175
+ client make the server issue requests to internal services. It defaults to off;
176
+ it is an escape hatch for dev/test harnesses serving fixtures from `localhost`,
177
+ or self-hosted deployments that intentionally fetch intranet pages.
178
+
79
179
  ## Error Handling
80
180
 
81
- If metadata extraction fails (invalid URL, network issues, parsing errors), the platform
82
- will return an error through the standard Sockethub error handling mechanism.
181
+ If metadata extraction fails (invalid URL, network issues, parsing errors, or a
182
+ blocked destination), the platform returns an error through the standard
183
+ Sockethub error handling mechanism.
83
184
 
84
185
  ## Use Cases
85
186
 
package/dist/index.d.ts ADDED
@@ -0,0 +1,163 @@
1
+ import type { ActivityStream, PlatformCallback, PlatformConfig, PlatformInterface, PlatformSession } from "@sockethub/schemas";
2
+ export default class Metadata implements PlatformInterface {
3
+ private readonly log;
4
+ private dispatcher?;
5
+ config: PlatformConfig;
6
+ constructor(session: PlatformSession);
7
+ /**
8
+ * The SSRF-guarded undici dispatcher, created once per instance (its
9
+ * `allowPrivateAddresses` setting comes from packageConfig, fixed before the
10
+ * first job) and reused across fetches so connections/timers are pooled.
11
+ */
12
+ private getDispatcher;
13
+ get schema(): {
14
+ name: string;
15
+ version: string;
16
+ contextUrl: string;
17
+ contextVersion: string;
18
+ schemaVersion: string;
19
+ messages: {
20
+ required: string[];
21
+ properties: {
22
+ type: {
23
+ type: string;
24
+ enum: string[];
25
+ };
26
+ object: boolean;
27
+ };
28
+ };
29
+ responses: {
30
+ type: string;
31
+ required: string[];
32
+ additionalProperties: boolean;
33
+ properties: {
34
+ "@context": {
35
+ type: string;
36
+ items: {
37
+ type: string;
38
+ };
39
+ };
40
+ id: {
41
+ type: string;
42
+ };
43
+ type: {
44
+ enum: string[];
45
+ };
46
+ actor: {
47
+ type: string;
48
+ required: string[];
49
+ additionalProperties: boolean;
50
+ properties: {
51
+ id: {
52
+ type: string;
53
+ };
54
+ type: {
55
+ enum: string[];
56
+ };
57
+ name: {
58
+ type: string;
59
+ };
60
+ };
61
+ };
62
+ object: {
63
+ $ref: string;
64
+ };
65
+ };
66
+ definitions: {
67
+ responses: {
68
+ page: {
69
+ type: string;
70
+ required: string[];
71
+ additionalProperties: boolean;
72
+ properties: {
73
+ type: {
74
+ enum: string[];
75
+ };
76
+ language: {
77
+ type: string;
78
+ };
79
+ title: {
80
+ type: string;
81
+ };
82
+ name: {
83
+ type: string;
84
+ };
85
+ description: {
86
+ type: string;
87
+ };
88
+ image: {
89
+ $ref: string;
90
+ };
91
+ video: {
92
+ type: string;
93
+ required: string[];
94
+ additionalProperties: boolean;
95
+ properties: {
96
+ url: {
97
+ type: string;
98
+ };
99
+ thumbnail: {
100
+ type: string;
101
+ };
102
+ width: {
103
+ type: string;
104
+ };
105
+ height: {
106
+ type: string;
107
+ };
108
+ duration: {
109
+ type: string;
110
+ };
111
+ };
112
+ };
113
+ url: {
114
+ type: string;
115
+ };
116
+ favicon: {
117
+ type: string;
118
+ };
119
+ charset: {
120
+ type: string;
121
+ };
122
+ };
123
+ };
124
+ ogImage: {
125
+ type: string;
126
+ items: {
127
+ type: string;
128
+ required: string[];
129
+ additionalProperties: boolean;
130
+ properties: {
131
+ url: {
132
+ type: string;
133
+ };
134
+ type: {
135
+ type: string;
136
+ };
137
+ width: {
138
+ type: string[];
139
+ };
140
+ height: {
141
+ type: string[];
142
+ };
143
+ alt: {
144
+ type: string;
145
+ };
146
+ };
147
+ };
148
+ };
149
+ };
150
+ };
151
+ };
152
+ };
153
+ /**
154
+ * Stateless platforms are always ready to handle jobs.
155
+ */
156
+ isInitialized(): boolean;
157
+ private userAgent;
158
+ private compatUserAgent;
159
+ fetch(job: ActivityStream, cb: PlatformCallback): void;
160
+ private fetchTweet;
161
+ private scrape;
162
+ cleanup(cb: PlatformCallback): void;
163
+ }