@sockethub/platform-metadata 1.0.1-alpha.1 → 1.0.1-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -11
- package/dist/index.d.ts +163 -0
- package/dist/index.js +72140 -0
- package/dist/index.js.map +306 -0
- package/dist/resolvers.d.ts +90 -0
- package/dist/schema.d.ts +140 -0
- package/package.json +24 -7
- package/src/index.ts +0 -56
- package/src/schema.ts +0 -22
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Site-specific URL resolution for pages that don't serve useful Open Graph
|
|
3
|
+
* data to scrapers.
|
|
4
|
+
*
|
|
5
|
+
* Some large platforms return a generic banner image (X/Twitter), serve
|
|
6
|
+
* their Open Graph data only to recognized embed crawlers (Reddit), or
|
|
7
|
+
* hide post media behind a login (Facebook). Two of those have working
|
|
8
|
+
* strategies:
|
|
9
|
+
*
|
|
10
|
+
* - X/Twitter → FxTwitter's JSON API (api.fxtwitter.com), built for embed
|
|
11
|
+
* previews, returns the tweet text plus direct photo/video URLs.
|
|
12
|
+
* - Reddit → the regular scrape works, but only when presented with an
|
|
13
|
+
* embed-crawler user agent (see COMPAT_USER_AGENT in index.ts).
|
|
14
|
+
*
|
|
15
|
+
* These are pure URL matchers/mappers — the platform decides what to do
|
|
16
|
+
* with the resolution (call a JSON API vs. pick a scrape user agent).
|
|
17
|
+
*/
|
|
18
|
+
/**
|
|
19
|
+
* Match an X/Twitter status URL and return the FxTwitter API URL for it,
|
|
20
|
+
* or null when the URL isn't a status page (profiles, search, home feed —
|
|
21
|
+
* those go through the normal OG scrape).
|
|
22
|
+
*
|
|
23
|
+
* Handles `/<user>/status/<id>` and the `/i/web/status/<id>` share form.
|
|
24
|
+
*/
|
|
25
|
+
export declare function resolveTwitterStatus(url: string): string | null;
|
|
26
|
+
/**
|
|
27
|
+
* True for Reddit URLs (including redd.it short links). Reddit serves its
|
|
28
|
+
* Open Graph tags — with the post's real preview image — only to
|
|
29
|
+
* recognized embed-crawler user agents; everything else gets a page with
|
|
30
|
+
* no OG data (or an outright 403 from datacenter addresses).
|
|
31
|
+
*/
|
|
32
|
+
export declare function isRedditUrl(url: string): boolean;
|
|
33
|
+
/** Subset of the FxTwitter status API response the platform consumes. */
|
|
34
|
+
export interface FxTwitterStatus {
|
|
35
|
+
code: number;
|
|
36
|
+
message?: string;
|
|
37
|
+
tweet?: {
|
|
38
|
+
url?: string;
|
|
39
|
+
text?: string;
|
|
40
|
+
author?: {
|
|
41
|
+
name?: string;
|
|
42
|
+
screen_name?: string;
|
|
43
|
+
};
|
|
44
|
+
media?: {
|
|
45
|
+
photos?: Array<{
|
|
46
|
+
url?: string;
|
|
47
|
+
width?: number;
|
|
48
|
+
height?: number;
|
|
49
|
+
}>;
|
|
50
|
+
videos?: Array<{
|
|
51
|
+
url?: string;
|
|
52
|
+
thumbnail_url?: string;
|
|
53
|
+
width?: number;
|
|
54
|
+
height?: number;
|
|
55
|
+
duration?: number;
|
|
56
|
+
}>;
|
|
57
|
+
};
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
/** The `object` payload shape the metadata platform responds with. */
|
|
61
|
+
export interface PageObject {
|
|
62
|
+
type: "page";
|
|
63
|
+
language?: string;
|
|
64
|
+
title?: string;
|
|
65
|
+
name?: string;
|
|
66
|
+
description?: string;
|
|
67
|
+
image?: Array<{
|
|
68
|
+
url: string;
|
|
69
|
+
type?: string;
|
|
70
|
+
width?: string | number;
|
|
71
|
+
height?: string | number;
|
|
72
|
+
}>;
|
|
73
|
+
video?: {
|
|
74
|
+
url: string;
|
|
75
|
+
thumbnail?: string;
|
|
76
|
+
width?: number;
|
|
77
|
+
height?: number;
|
|
78
|
+
duration?: number;
|
|
79
|
+
};
|
|
80
|
+
url?: string;
|
|
81
|
+
favicon?: string;
|
|
82
|
+
charset?: string;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Map an FxTwitter API response onto the platform's page object. The
|
|
86
|
+
* preview image is the post's own media — first photo, else the video
|
|
87
|
+
* thumbnail — never the generic site banner. Returns null when the API
|
|
88
|
+
* reports anything but success (the caller falls back to the OG scrape).
|
|
89
|
+
*/
|
|
90
|
+
export declare function tweetToPageObject(status: FxTwitterStatus): PageObject | null;
|
package/dist/schema.d.ts
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
export declare const PlatformMetadataSchema: {
|
|
2
|
+
name: string;
|
|
3
|
+
version: string;
|
|
4
|
+
contextUrl: string;
|
|
5
|
+
contextVersion: string;
|
|
6
|
+
schemaVersion: string;
|
|
7
|
+
messages: {
|
|
8
|
+
required: string[];
|
|
9
|
+
properties: {
|
|
10
|
+
type: {
|
|
11
|
+
type: string;
|
|
12
|
+
enum: string[];
|
|
13
|
+
};
|
|
14
|
+
object: boolean;
|
|
15
|
+
};
|
|
16
|
+
};
|
|
17
|
+
responses: {
|
|
18
|
+
type: string;
|
|
19
|
+
required: string[];
|
|
20
|
+
additionalProperties: boolean;
|
|
21
|
+
properties: {
|
|
22
|
+
"@context": {
|
|
23
|
+
type: string;
|
|
24
|
+
items: {
|
|
25
|
+
type: string;
|
|
26
|
+
};
|
|
27
|
+
};
|
|
28
|
+
id: {
|
|
29
|
+
type: string;
|
|
30
|
+
};
|
|
31
|
+
type: {
|
|
32
|
+
enum: string[];
|
|
33
|
+
};
|
|
34
|
+
actor: {
|
|
35
|
+
type: string;
|
|
36
|
+
required: string[];
|
|
37
|
+
additionalProperties: boolean;
|
|
38
|
+
properties: {
|
|
39
|
+
id: {
|
|
40
|
+
type: string;
|
|
41
|
+
};
|
|
42
|
+
type: {
|
|
43
|
+
enum: string[];
|
|
44
|
+
};
|
|
45
|
+
name: {
|
|
46
|
+
type: string;
|
|
47
|
+
};
|
|
48
|
+
};
|
|
49
|
+
};
|
|
50
|
+
object: {
|
|
51
|
+
$ref: string;
|
|
52
|
+
};
|
|
53
|
+
};
|
|
54
|
+
definitions: {
|
|
55
|
+
responses: {
|
|
56
|
+
page: {
|
|
57
|
+
type: string;
|
|
58
|
+
required: string[];
|
|
59
|
+
additionalProperties: boolean;
|
|
60
|
+
properties: {
|
|
61
|
+
type: {
|
|
62
|
+
enum: string[];
|
|
63
|
+
};
|
|
64
|
+
language: {
|
|
65
|
+
type: string;
|
|
66
|
+
};
|
|
67
|
+
title: {
|
|
68
|
+
type: string;
|
|
69
|
+
};
|
|
70
|
+
name: {
|
|
71
|
+
type: string;
|
|
72
|
+
};
|
|
73
|
+
description: {
|
|
74
|
+
type: string;
|
|
75
|
+
};
|
|
76
|
+
image: {
|
|
77
|
+
$ref: string;
|
|
78
|
+
};
|
|
79
|
+
video: {
|
|
80
|
+
type: string;
|
|
81
|
+
required: string[];
|
|
82
|
+
additionalProperties: boolean;
|
|
83
|
+
properties: {
|
|
84
|
+
url: {
|
|
85
|
+
type: string;
|
|
86
|
+
};
|
|
87
|
+
thumbnail: {
|
|
88
|
+
type: string;
|
|
89
|
+
};
|
|
90
|
+
width: {
|
|
91
|
+
type: string;
|
|
92
|
+
};
|
|
93
|
+
height: {
|
|
94
|
+
type: string;
|
|
95
|
+
};
|
|
96
|
+
duration: {
|
|
97
|
+
type: string;
|
|
98
|
+
};
|
|
99
|
+
};
|
|
100
|
+
};
|
|
101
|
+
url: {
|
|
102
|
+
type: string;
|
|
103
|
+
};
|
|
104
|
+
favicon: {
|
|
105
|
+
type: string;
|
|
106
|
+
};
|
|
107
|
+
charset: {
|
|
108
|
+
type: string;
|
|
109
|
+
};
|
|
110
|
+
};
|
|
111
|
+
};
|
|
112
|
+
ogImage: {
|
|
113
|
+
type: string;
|
|
114
|
+
items: {
|
|
115
|
+
type: string;
|
|
116
|
+
required: string[];
|
|
117
|
+
additionalProperties: boolean;
|
|
118
|
+
properties: {
|
|
119
|
+
url: {
|
|
120
|
+
type: string;
|
|
121
|
+
};
|
|
122
|
+
type: {
|
|
123
|
+
type: string;
|
|
124
|
+
};
|
|
125
|
+
width: {
|
|
126
|
+
type: string[];
|
|
127
|
+
};
|
|
128
|
+
height: {
|
|
129
|
+
type: string[];
|
|
130
|
+
};
|
|
131
|
+
alt: {
|
|
132
|
+
type: string;
|
|
133
|
+
};
|
|
134
|
+
};
|
|
135
|
+
};
|
|
136
|
+
};
|
|
137
|
+
};
|
|
138
|
+
};
|
|
139
|
+
};
|
|
140
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sockethub/platform-metadata",
|
|
3
|
-
"version": "1.0.1-alpha.
|
|
3
|
+
"version": "1.0.1-alpha.10",
|
|
4
4
|
"description": "Fetch metadata from a given URL",
|
|
5
5
|
"author": "Nick Jennings <nick@silverbucket.net>",
|
|
6
6
|
"license": "LGPL-3.0+",
|
|
@@ -8,9 +8,19 @@
|
|
|
8
8
|
"access": "public"
|
|
9
9
|
},
|
|
10
10
|
"type": "module",
|
|
11
|
-
"main": "
|
|
11
|
+
"main": "dist/index.js",
|
|
12
|
+
"exports": {
|
|
13
|
+
".": {
|
|
14
|
+
"types": "./dist/index.d.ts",
|
|
15
|
+
"import": "./dist/index.js",
|
|
16
|
+
"default": "./dist/index.js"
|
|
17
|
+
}
|
|
18
|
+
},
|
|
19
|
+
"files": [
|
|
20
|
+
"dist/"
|
|
21
|
+
],
|
|
12
22
|
"engines": {
|
|
13
|
-
"
|
|
23
|
+
"node": ">=20"
|
|
14
24
|
},
|
|
15
25
|
"keywords": [
|
|
16
26
|
"sockethub",
|
|
@@ -26,14 +36,21 @@
|
|
|
26
36
|
"directory": "packages/platform-metadata"
|
|
27
37
|
},
|
|
28
38
|
"homepage": "https://github.com/sockethub/sockethub/tree/master/packages/platform-metadata",
|
|
39
|
+
"scripts": {
|
|
40
|
+
"build": "bun build src/index.ts --outdir dist --target node --format esm --sourcemap=external",
|
|
41
|
+
"clean": "rm -rf dist",
|
|
42
|
+
"clean:deps": "rm -rf node_modules"
|
|
43
|
+
},
|
|
29
44
|
"dependencies": {
|
|
30
|
-
"
|
|
45
|
+
"@sockethub/util": "^1.0.0-alpha.3",
|
|
46
|
+
"open-graph-scraper": "^6.11.0"
|
|
31
47
|
},
|
|
32
48
|
"devDependencies": {
|
|
33
|
-
"@sockethub/schemas": "3.0.0-alpha.
|
|
49
|
+
"@sockethub/schemas": "^3.0.0-alpha.15"
|
|
34
50
|
},
|
|
35
51
|
"peerDependencies": {
|
|
36
|
-
"@sockethub/server": "5.0.0-alpha.
|
|
52
|
+
"@sockethub/server": "^5.0.0-alpha.15"
|
|
37
53
|
},
|
|
38
|
-
"gitHead": "
|
|
54
|
+
"gitHead": "e7cd0a35abb39c878ff82a60884f0f7ecf4a0a87",
|
|
55
|
+
"types": "./dist/index.d.ts"
|
|
39
56
|
}
|
package/src/index.ts
DELETED
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
ActivityStream,
|
|
3
|
-
Logger,
|
|
4
|
-
PlatformCallback,
|
|
5
|
-
PlatformConfig,
|
|
6
|
-
PlatformInterface,
|
|
7
|
-
PlatformSession,
|
|
8
|
-
} from "@sockethub/schemas";
|
|
9
|
-
import ogs from "open-graph-scraper";
|
|
10
|
-
import { PlatformMetadataSchema } from "./schema";
|
|
11
|
-
|
|
12
|
-
export default class Metadata implements PlatformInterface {
|
|
13
|
-
readonly debug: Logger;
|
|
14
|
-
config: PlatformConfig = {
|
|
15
|
-
persist: false,
|
|
16
|
-
};
|
|
17
|
-
constructor(session: PlatformSession) {
|
|
18
|
-
this.debug = session.debug;
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
get schema() {
|
|
22
|
-
return PlatformMetadataSchema;
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
fetch(job: ActivityStream, cb: PlatformCallback) {
|
|
26
|
-
this.debug(`fetching ${job.actor.id}`);
|
|
27
|
-
ogs({
|
|
28
|
-
url: job.actor.id,
|
|
29
|
-
})
|
|
30
|
-
.then((data) => {
|
|
31
|
-
const { result } = data;
|
|
32
|
-
job.actor.id = result.ogUrl || job.actor.id;
|
|
33
|
-
job.actor.name = result.ogSiteName || job.actor.name || "";
|
|
34
|
-
job.object = {
|
|
35
|
-
type: "page",
|
|
36
|
-
language: result.ogLocale,
|
|
37
|
-
title: result.ogTitle,
|
|
38
|
-
name: result.ogSiteName,
|
|
39
|
-
description: result.ogDescription || "",
|
|
40
|
-
image: result.ogImage,
|
|
41
|
-
url: result.ogUrl,
|
|
42
|
-
favicon: result.favicon,
|
|
43
|
-
charset: result.charset,
|
|
44
|
-
};
|
|
45
|
-
cb(null, job);
|
|
46
|
-
})
|
|
47
|
-
.catch((data) => {
|
|
48
|
-
const { result } = data;
|
|
49
|
-
cb(result.error);
|
|
50
|
-
});
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
cleanup(cb: PlatformCallback) {
|
|
54
|
-
cb();
|
|
55
|
-
}
|
|
56
|
-
}
|
package/src/schema.ts
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import packageJson from "../package.json" with { type: "json" };
|
|
2
|
-
const version = packageJson.version;
|
|
3
|
-
|
|
4
|
-
export const PlatformMetadataSchema = {
|
|
5
|
-
name: "metadata",
|
|
6
|
-
version: version,
|
|
7
|
-
messages: {
|
|
8
|
-
required: ["type"],
|
|
9
|
-
properties: {
|
|
10
|
-
type: {
|
|
11
|
-
type: "string",
|
|
12
|
-
enum: ["fetch"],
|
|
13
|
-
},
|
|
14
|
-
object: {
|
|
15
|
-
type: "object",
|
|
16
|
-
oneOf: [
|
|
17
|
-
{ $ref: "#/definitions/objectTypes/feed-parameters-url" },
|
|
18
|
-
],
|
|
19
|
-
},
|
|
20
|
-
},
|
|
21
|
-
},
|
|
22
|
-
};
|