@postfetch/core 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -1
- package/dist/index.js +189 -11
- package/dist/instagram.d.ts +4 -1
- package/dist/internal.d.ts +114 -3
- package/dist/pinterest.d.ts +4 -1
- package/dist/reddit.d.ts +4 -1
- package/dist/soundcloud.d.ts +4 -1
- package/dist/tiktok.d.ts +5 -1
- package/dist/twitter.d.ts +5 -1
- package/dist/youtube.d.ts +4 -1
- package/package.json +1 -1
- package/src/index.ts +16 -1
- package/src/instagram.ts +50 -1
- package/src/internal.ts +155 -3
- package/src/pinterest.ts +27 -1
- package/src/reddit.ts +26 -1
- package/src/soundcloud.ts +27 -1
- package/src/tiktok.ts +40 -2
- package/src/twitter.ts +28 -2
- package/src/youtube.ts +30 -1
package/dist/index.d.ts
CHANGED
|
@@ -22,4 +22,4 @@
|
|
|
22
22
|
export { postfetch, detect, type PostfetchOptions } from "./postfetch";
|
|
23
23
|
export { download, archive, toResponse, type Archive, type DownloadOptions } from "./download";
|
|
24
24
|
export { PostfetchError } from "./internal";
|
|
25
|
-
export type { MediaItem, MediaKind, Platform, PostfetchResult } from "./internal";
|
|
25
|
+
export type { InstagramExtra, MediaItem, MediaKind, PinterestExtra, Platform, PlatformResult, PostAuthor, PostfetchResult, PostMetadata, RedditExtra, SoundcloudExtra, TiktokExtra, TwitterExtra, YoutubeExtra, } from "./internal";
|
package/dist/index.js
CHANGED
|
@@ -41,6 +41,29 @@ function string(value) {
|
|
|
41
41
|
function number(value) {
|
|
42
42
|
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
43
43
|
}
|
|
44
|
+
function bool(value) {
|
|
45
|
+
return typeof value === "boolean" ? value : undefined;
|
|
46
|
+
}
|
|
47
|
+
function count(value) {
|
|
48
|
+
if (typeof value === "number" && Number.isInteger(value) && value >= 0) {
|
|
49
|
+
return value;
|
|
50
|
+
}
|
|
51
|
+
if (typeof value === "string" && /^\d+$/.test(value.trim())) {
|
|
52
|
+
return Number(value.trim());
|
|
53
|
+
}
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
function isoFromEpochSeconds(value) {
|
|
57
|
+
const seconds = count(value);
|
|
58
|
+
return seconds === undefined ? undefined : new Date(seconds * 1000).toISOString();
|
|
59
|
+
}
|
|
60
|
+
function isoFromDateString(value) {
|
|
61
|
+
if (typeof value !== "string") {
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
const time = Date.parse(value);
|
|
65
|
+
return Number.isNaN(time) ? undefined : new Date(time).toISOString();
|
|
66
|
+
}
|
|
44
67
|
function asUrl(value) {
|
|
45
68
|
try {
|
|
46
69
|
return new URL(value);
|
|
@@ -233,7 +256,43 @@ async function resolveInstagram(input) {
|
|
|
233
256
|
if (items.length === 0) {
|
|
234
257
|
throw new Error("Instagram media url not found");
|
|
235
258
|
}
|
|
236
|
-
return { archiveFilename: filename(`instagram_${code}.zip`), id: code, items, platform: "instagram" };
|
|
259
|
+
return { archiveFilename: filename(`instagram_${code}.zip`), id: code, items, metadata: instagramMetadata(media), platform: "instagram" };
|
|
260
|
+
}
|
|
261
|
+
function instagramMetadata(media) {
|
|
262
|
+
const user = object(media.user) ? media.user : object(media.owner) ? media.owner : null;
|
|
263
|
+
const countsHidden = bool(media.like_and_view_counts_disabled);
|
|
264
|
+
return {
|
|
265
|
+
text: instagramCaption(media),
|
|
266
|
+
author: user ? {
|
|
267
|
+
handle: string(user.username) ?? undefined,
|
|
268
|
+
name: string(user.full_name) ?? undefined,
|
|
269
|
+
verified: bool(user.is_verified)
|
|
270
|
+
} : undefined,
|
|
271
|
+
createdAt: isoFromEpochSeconds(media.taken_at ?? media.taken_at_timestamp),
|
|
272
|
+
likeCount: countsHidden ? undefined : count(media.like_count) ?? edgeCount(media.edge_media_preview_like) ?? edgeCount(media.edge_liked_by),
|
|
273
|
+
commentCount: count(media.comment_count) ?? edgeCount(media.edge_media_to_comment) ?? edgeCount(media.edge_media_to_parent_comment),
|
|
274
|
+
viewCount: countsHidden ? undefined : count(media.play_count) ?? count(media.view_count) ?? count(media.video_view_count),
|
|
275
|
+
extra: {
|
|
276
|
+
productType: string(media.product_type) ?? undefined,
|
|
277
|
+
countsHidden,
|
|
278
|
+
location: object(media.location) ? string(media.location.name) ?? undefined : undefined
|
|
279
|
+
}
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
function instagramCaption(media) {
|
|
283
|
+
if (object(media.caption)) {
|
|
284
|
+
return string(media.caption.text) ?? undefined;
|
|
285
|
+
}
|
|
286
|
+
const direct = string(media.caption);
|
|
287
|
+
if (direct) {
|
|
288
|
+
return direct;
|
|
289
|
+
}
|
|
290
|
+
const edges = object(media.edge_media_to_caption) && Array.isArray(media.edge_media_to_caption.edges) ? media.edge_media_to_caption.edges : [];
|
|
291
|
+
const node = object(edges[0]) && object(edges[0].node) ? edges[0].node : null;
|
|
292
|
+
return node ? string(node.text) ?? undefined : undefined;
|
|
293
|
+
}
|
|
294
|
+
function edgeCount(value) {
|
|
295
|
+
return object(value) ? count(value.count) : undefined;
|
|
237
296
|
}
|
|
238
297
|
function shortcode(input) {
|
|
239
298
|
const path = asUrl(input).pathname.split("/").filter(Boolean);
|
|
@@ -692,7 +751,24 @@ async function resolvePinterest(input) {
|
|
|
692
751
|
if (items.length === 0) {
|
|
693
752
|
throw new Error("Pinterest media not found");
|
|
694
753
|
}
|
|
695
|
-
return { archiveFilename: filename(`pinterest_${id}.zip`), id, items, platform: "pinterest" };
|
|
754
|
+
return { archiveFilename: filename(`pinterest_${id}.zip`), id, items, metadata: pinterestMetadata(pin), platform: "pinterest" };
|
|
755
|
+
}
|
|
756
|
+
function pinterestMetadata(pin) {
|
|
757
|
+
const pinner = object(pin.pinner) ? pin.pinner : null;
|
|
758
|
+
const reactions = object(pin.reaction_counts) ? Object.values(pin.reaction_counts).reduce((sum, value) => sum + (count(value) ?? 0), 0) : undefined;
|
|
759
|
+
return {
|
|
760
|
+
title: string(pin.title) ?? string(pin.grid_title) ?? undefined,
|
|
761
|
+
text: string(pin.description) ?? undefined,
|
|
762
|
+
author: pinner ? { handle: string(pinner.username) ?? undefined, name: string(pinner.full_name) ?? undefined } : undefined,
|
|
763
|
+
createdAt: isoFromDateString(pin.created_at),
|
|
764
|
+
likeCount: reactions,
|
|
765
|
+
commentCount: count(pin.comment_count),
|
|
766
|
+
extra: {
|
|
767
|
+
saveCount: count(pin.repin_count),
|
|
768
|
+
dominantColor: string(pin.dominant_color) ?? undefined,
|
|
769
|
+
outboundLink: string(pin.link) ?? undefined
|
|
770
|
+
}
|
|
771
|
+
};
|
|
696
772
|
}
|
|
697
773
|
async function pinId(net, input) {
|
|
698
774
|
const direct = parsePin(input);
|
|
@@ -893,7 +969,25 @@ async function resolveReddit(input) {
|
|
|
893
969
|
if (items.length === 0) {
|
|
894
970
|
throw new Error("Reddit media not found");
|
|
895
971
|
}
|
|
896
|
-
return { archiveFilename: filename(`reddit_${id}.zip`), id, items, platform: "reddit" };
|
|
972
|
+
return { archiveFilename: filename(`reddit_${id}.zip`), id, items, metadata: redditMetadata(post), platform: "reddit" };
|
|
973
|
+
}
|
|
974
|
+
function redditMetadata(post) {
|
|
975
|
+
const author = string(post.author);
|
|
976
|
+
return {
|
|
977
|
+
title: string(post.title) ?? undefined,
|
|
978
|
+
text: string(post.selftext) ?? undefined,
|
|
979
|
+
author: author ? { handle: author } : undefined,
|
|
980
|
+
createdAt: isoFromEpochSeconds(post.created_utc),
|
|
981
|
+
likeCount: count(post.ups),
|
|
982
|
+
commentCount: count(post.num_comments),
|
|
983
|
+
nsfw: bool(post.over_18),
|
|
984
|
+
extra: {
|
|
985
|
+
subreddit: string(post.subreddit) ?? undefined,
|
|
986
|
+
score: number(post.score) ?? undefined,
|
|
987
|
+
upvoteRatio: number(post.upvote_ratio) ?? undefined,
|
|
988
|
+
permalink: string(post.permalink) ?? undefined
|
|
989
|
+
}
|
|
990
|
+
};
|
|
897
991
|
}
|
|
898
992
|
async function postId(net, input) {
|
|
899
993
|
const direct = parseId(input);
|
|
@@ -1133,7 +1227,26 @@ async function resolveSoundcloud(input) {
|
|
|
1133
1227
|
const track = await resolveTrack(input.net, url, clientId);
|
|
1134
1228
|
const id = trackId(track);
|
|
1135
1229
|
const item = await audioItem(input.net, track, id, clientId);
|
|
1136
|
-
return { archiveFilename: filename(`soundcloud_${id}.zip`), id, items: [item], platform: "soundcloud" };
|
|
1230
|
+
return { archiveFilename: filename(`soundcloud_${id}.zip`), id, items: [item], metadata: soundcloudMetadata(track), platform: "soundcloud" };
|
|
1231
|
+
}
|
|
1232
|
+
function soundcloudMetadata(track) {
|
|
1233
|
+
const user = object(track.user) ? track.user : null;
|
|
1234
|
+
const durationMs = count(track.duration);
|
|
1235
|
+
return {
|
|
1236
|
+
title: string(track.title) ?? undefined,
|
|
1237
|
+
text: string(track.description) ?? undefined,
|
|
1238
|
+
author: user ? { handle: string(user.username) ?? undefined, name: string(user.full_name) ?? undefined } : undefined,
|
|
1239
|
+
createdAt: isoFromDateString(track.created_at),
|
|
1240
|
+
likeCount: count(track.likes_count),
|
|
1241
|
+
commentCount: count(track.comment_count),
|
|
1242
|
+
shareCount: count(track.reposts_count),
|
|
1243
|
+
viewCount: count(track.playback_count),
|
|
1244
|
+
extra: {
|
|
1245
|
+
genre: string(track.genre) ?? undefined,
|
|
1246
|
+
license: string(track.license) ?? undefined,
|
|
1247
|
+
durationSeconds: durationMs === undefined ? undefined : Math.round(durationMs / 1000)
|
|
1248
|
+
}
|
|
1249
|
+
};
|
|
1137
1250
|
}
|
|
1138
1251
|
async function canonicalUrl2(net, input) {
|
|
1139
1252
|
const host = asUrl(input).hostname;
|
|
@@ -1255,7 +1368,33 @@ async function resolveTiktok(input) {
|
|
|
1255
1368
|
if (items.length === 0) {
|
|
1256
1369
|
throw new Error("TikTok media not found");
|
|
1257
1370
|
}
|
|
1258
|
-
return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), id, items, platform: "tiktok" };
|
|
1371
|
+
return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), id, items, metadata: tiktokMetadata(page.item), platform: "tiktok" };
|
|
1372
|
+
}
|
|
1373
|
+
function tiktokMetadata(item) {
|
|
1374
|
+
const user = object(item.author) ? item.author : null;
|
|
1375
|
+
const stats = object(item.stats) ? item.stats : null;
|
|
1376
|
+
const music = object(item.music) ? item.music : null;
|
|
1377
|
+
const video = object(item.video) ? item.video : null;
|
|
1378
|
+
return {
|
|
1379
|
+
text: string(item.desc) ?? undefined,
|
|
1380
|
+
author: user ? {
|
|
1381
|
+
handle: string(user.uniqueId) ?? undefined,
|
|
1382
|
+
name: string(user.nickname) ?? undefined,
|
|
1383
|
+
verified: bool(user.verified)
|
|
1384
|
+
} : undefined,
|
|
1385
|
+
createdAt: isoFromEpochSeconds(item.createTime),
|
|
1386
|
+
likeCount: stats ? count(stats.diggCount) : undefined,
|
|
1387
|
+
commentCount: stats ? count(stats.commentCount) : undefined,
|
|
1388
|
+
shareCount: stats ? count(stats.shareCount) : undefined,
|
|
1389
|
+
viewCount: stats ? count(stats.playCount) : undefined,
|
|
1390
|
+
extra: {
|
|
1391
|
+
saveCount: stats ? count(stats.collectCount) : undefined,
|
|
1392
|
+
musicTitle: music ? string(music.title) ?? undefined : undefined,
|
|
1393
|
+
musicAuthor: music ? string(music.authorName) ?? undefined : undefined,
|
|
1394
|
+
region: string(item.locationCreated) ?? undefined,
|
|
1395
|
+
durationSeconds: video ? count(video.duration) : undefined
|
|
1396
|
+
}
|
|
1397
|
+
};
|
|
1259
1398
|
}
|
|
1260
1399
|
async function fetchVideoPage(net, id, userAgent) {
|
|
1261
1400
|
const page = await net(`https://www.tiktok.com/@i/video/${id}`, {
|
|
@@ -1267,9 +1406,12 @@ async function fetchVideoPage(net, id, userAgent) {
|
|
|
1267
1406
|
});
|
|
1268
1407
|
return { cookie: cookieHeader(page.headers), item: itemStruct(await page.text()) };
|
|
1269
1408
|
}
|
|
1409
|
+
function isShortlinkHost(hostname) {
|
|
1410
|
+
return hostname === "vm.tiktok.com" || hostname === "vt.tiktok.com";
|
|
1411
|
+
}
|
|
1270
1412
|
async function followShortlink(net, userAgent, input) {
|
|
1271
1413
|
const url = asUrl(input);
|
|
1272
|
-
if (!url.hostname
|
|
1414
|
+
if (!isShortlinkHost(url.hostname)) {
|
|
1273
1415
|
return input;
|
|
1274
1416
|
}
|
|
1275
1417
|
const response = await net(input, { headers: { "user-agent": userAgent }, redirect: "manual" }, 1);
|
|
@@ -1395,7 +1537,25 @@ async function resolveTwitter(input) {
|
|
|
1395
1537
|
if (items.length === 0) {
|
|
1396
1538
|
throw new Error("Twitter media not found");
|
|
1397
1539
|
}
|
|
1398
|
-
return { archiveFilename: filename(`twitter_${id}.zip`), id, items, platform: "twitter" };
|
|
1540
|
+
return { archiveFilename: filename(`twitter_${id}.zip`), id, items, metadata: twitterMetadata(tweet), platform: "twitter" };
|
|
1541
|
+
}
|
|
1542
|
+
function twitterMetadata(tweet) {
|
|
1543
|
+
const user = object(tweet.user) ? tweet.user : null;
|
|
1544
|
+
return {
|
|
1545
|
+
text: string(tweet.text) ?? string(tweet.full_text) ?? undefined,
|
|
1546
|
+
author: user ? {
|
|
1547
|
+
handle: string(user.screen_name) ?? undefined,
|
|
1548
|
+
name: string(user.name) ?? undefined,
|
|
1549
|
+
verified: bool(user.verified) ?? bool(user.is_blue_verified)
|
|
1550
|
+
} : undefined,
|
|
1551
|
+
createdAt: isoFromDateString(tweet.created_at),
|
|
1552
|
+
likeCount: count(tweet.favorite_count),
|
|
1553
|
+
commentCount: count(tweet.conversation_count),
|
|
1554
|
+
shareCount: count(tweet.retweet_count),
|
|
1555
|
+
viewCount: count(tweet.view_count),
|
|
1556
|
+
nsfw: bool(tweet.possibly_sensitive),
|
|
1557
|
+
extra: { lang: string(tweet.lang) ?? undefined }
|
|
1558
|
+
};
|
|
1399
1559
|
}
|
|
1400
1560
|
async function syndication(net, id) {
|
|
1401
1561
|
const url = `https://cdn.syndication.twimg.com/tweet-result?id=${id}&lang=en&token=${syndicationToken(id)}`;
|
|
@@ -1480,7 +1640,25 @@ async function resolveYoutube(input) {
|
|
|
1480
1640
|
url: streams.video,
|
|
1481
1641
|
...streams.audio ? { audio: { headers, url: streams.audio } } : {}
|
|
1482
1642
|
};
|
|
1483
|
-
return { archiveFilename: filename(`youtube_${id}.zip`), id, items: [media], platform: "youtube" };
|
|
1643
|
+
return { archiveFilename: filename(`youtube_${id}.zip`), id, items: [media], metadata: youtubeMetadata(payload), platform: "youtube" };
|
|
1644
|
+
}
|
|
1645
|
+
function youtubeMetadata(payload) {
|
|
1646
|
+
const details = object(payload) && object(payload.videoDetails) ? payload.videoDetails : null;
|
|
1647
|
+
const renderer = object(payload) && object(payload.microformat) && object(payload.microformat.playerMicroformatRenderer) ? payload.microformat.playerMicroformatRenderer : null;
|
|
1648
|
+
const author2 = details ? string(details.author) : null;
|
|
1649
|
+
const keywords = details && Array.isArray(details.keywords) ? details.keywords.filter((keyword) => typeof keyword === "string") : [];
|
|
1650
|
+
return {
|
|
1651
|
+
title: details ? string(details.title) ?? undefined : undefined,
|
|
1652
|
+
text: details ? string(details.shortDescription) ?? undefined : undefined,
|
|
1653
|
+
author: author2 ? { name: author2 } : undefined,
|
|
1654
|
+
createdAt: renderer ? isoFromDateString(renderer.publishDate) : undefined,
|
|
1655
|
+
viewCount: details ? count(details.viewCount) : undefined,
|
|
1656
|
+
extra: {
|
|
1657
|
+
channelId: details ? string(details.channelId) ?? undefined : undefined,
|
|
1658
|
+
durationSeconds: details ? count(details.lengthSeconds) : undefined,
|
|
1659
|
+
keywords: keywords.length > 0 ? keywords : undefined
|
|
1660
|
+
}
|
|
1661
|
+
};
|
|
1484
1662
|
}
|
|
1485
1663
|
function selectStreams(payload, preferredWidth) {
|
|
1486
1664
|
const adaptive = adaptiveFormats(payload);
|
|
@@ -1893,10 +2071,10 @@ function writeCentral(view, target, offset, file, localOffset) {
|
|
|
1893
2071
|
target.set(file.name, offset + 46);
|
|
1894
2072
|
return offset + 46 + file.name.length;
|
|
1895
2073
|
}
|
|
1896
|
-
function writeEnd(view, offset,
|
|
2074
|
+
function writeEnd(view, offset, count2, centralSize, centralStart) {
|
|
1897
2075
|
view.setUint32(offset, 101010256, true);
|
|
1898
|
-
view.setUint16(offset + 8,
|
|
1899
|
-
view.setUint16(offset + 10,
|
|
2076
|
+
view.setUint16(offset + 8, count2, true);
|
|
2077
|
+
view.setUint16(offset + 10, count2, true);
|
|
1900
2078
|
view.setUint32(offset + 12, centralSize, true);
|
|
1901
2079
|
view.setUint32(offset + 16, centralStart, true);
|
|
1902
2080
|
}
|
package/dist/instagram.d.ts
CHANGED
|
@@ -1,2 +1,5 @@
|
|
|
1
|
-
import { type ResolveContext, type PostfetchResult } from "./internal";
|
|
1
|
+
import { type InstagramExtra, type PostMetadata, type ResolveContext, type Json, type PostfetchResult } from "./internal";
|
|
2
2
|
export declare function resolveInstagram(input: ResolveContext): Promise<PostfetchResult>;
|
|
3
|
+
export declare function instagramMetadata(media: Json): PostMetadata & {
|
|
4
|
+
extra?: InstagramExtra;
|
|
5
|
+
};
|
package/dist/internal.d.ts
CHANGED
|
@@ -36,8 +36,110 @@ export type MediaItem = {
|
|
|
36
36
|
/** Direct CDN URL of the media. */
|
|
37
37
|
url: string;
|
|
38
38
|
};
|
|
39
|
-
/** The
|
|
40
|
-
export type
|
|
39
|
+
/** The author of a post, normalized across platforms. */
|
|
40
|
+
export type PostAuthor = {
|
|
41
|
+
/** Account handle (the `@username`), where the platform exposes one. */
|
|
42
|
+
handle?: string;
|
|
43
|
+
/** Display name, where it differs from the handle. */
|
|
44
|
+
name?: string;
|
|
45
|
+
/** Whether the account is verified, where the platform reports it. */
|
|
46
|
+
verified?: boolean;
|
|
47
|
+
};
|
|
48
|
+
/**
|
|
49
|
+
* Post metadata normalized across platforms. Every field is optional: an absent
|
|
50
|
+
* field means the platform did not expose it in the payload already fetched to
|
|
51
|
+
* resolve the media (never a fabricated default — a missing count is `undefined`,
|
|
52
|
+
* not `0`). All counts are non-negative. Platform-specific fields live in `extra`.
|
|
53
|
+
*/
|
|
54
|
+
export type PostMetadata = {
|
|
55
|
+
/** Short headline, where the platform has one distinct from the body (Reddit, YouTube, Pinterest, SoundCloud). */
|
|
56
|
+
title?: string;
|
|
57
|
+
/** Body text: caption, tweet, description or self-text. */
|
|
58
|
+
text?: string;
|
|
59
|
+
/** Who posted it. */
|
|
60
|
+
author?: PostAuthor;
|
|
61
|
+
/** Publication time as an ISO 8601 string. */
|
|
62
|
+
createdAt?: string;
|
|
63
|
+
/** Positive reactions (likes, upvotes, favorites, diggs). */
|
|
64
|
+
likeCount?: number;
|
|
65
|
+
/** Number of comments or replies. */
|
|
66
|
+
commentCount?: number;
|
|
67
|
+
/** Genuine reshares (TikTok shares, retweets, reposts) — not saves. */
|
|
68
|
+
shareCount?: number;
|
|
69
|
+
/** Views or plays. */
|
|
70
|
+
viewCount?: number;
|
|
71
|
+
/** Whether the post is marked not-safe-for-work, where the platform reports it. */
|
|
72
|
+
nsfw?: boolean;
|
|
73
|
+
};
|
|
74
|
+
/** Reddit-specific metadata. */
|
|
75
|
+
export type RedditExtra = {
|
|
76
|
+
/** Subreddit the post belongs to, without the `r/` prefix. */
|
|
77
|
+
subreddit?: string;
|
|
78
|
+
/** Net score (upvotes minus downvotes); can be negative. */
|
|
79
|
+
score?: number;
|
|
80
|
+
/** Fraction of votes that are upvotes, from 0 to 1. */
|
|
81
|
+
upvoteRatio?: number;
|
|
82
|
+
/** Path of the post on reddit.com, starting with `/r/`. */
|
|
83
|
+
permalink?: string;
|
|
84
|
+
};
|
|
85
|
+
/** TikTok-specific metadata. */
|
|
86
|
+
export type TiktokExtra = {
|
|
87
|
+
/** Times the video was added to favorites. */
|
|
88
|
+
saveCount?: number;
|
|
89
|
+
/** Title of the sound used. */
|
|
90
|
+
musicTitle?: string;
|
|
91
|
+
/** Author of the sound used. */
|
|
92
|
+
musicAuthor?: string;
|
|
93
|
+
/** Two-letter region the video was created in. */
|
|
94
|
+
region?: string;
|
|
95
|
+
/** Video duration in seconds. */
|
|
96
|
+
durationSeconds?: number;
|
|
97
|
+
};
|
|
98
|
+
/** X (Twitter)-specific metadata. */
|
|
99
|
+
export type TwitterExtra = {
|
|
100
|
+
/** BCP 47 language tag detected for the tweet. */
|
|
101
|
+
lang?: string;
|
|
102
|
+
};
|
|
103
|
+
/** Instagram-specific metadata. */
|
|
104
|
+
export type InstagramExtra = {
|
|
105
|
+
/** Product type, e.g. `clips` for a reel or `feed` for a feed post. */
|
|
106
|
+
productType?: string;
|
|
107
|
+
/** Whether the account hid its like and view counts. */
|
|
108
|
+
countsHidden?: boolean;
|
|
109
|
+
/** Tagged location name, when present. */
|
|
110
|
+
location?: string;
|
|
111
|
+
};
|
|
112
|
+
/** YouTube-specific metadata. */
|
|
113
|
+
export type YoutubeExtra = {
|
|
114
|
+
/** Channel id (`UC…`) of the uploader. */
|
|
115
|
+
channelId?: string;
|
|
116
|
+
/** Video duration in seconds. */
|
|
117
|
+
durationSeconds?: number;
|
|
118
|
+
/** Video keywords. */
|
|
119
|
+
keywords?: string[];
|
|
120
|
+
};
|
|
121
|
+
/** SoundCloud-specific metadata. */
|
|
122
|
+
export type SoundcloudExtra = {
|
|
123
|
+
/** Track genre, when set. */
|
|
124
|
+
genre?: string;
|
|
125
|
+
/** Track license, e.g. `all-rights-reserved`. */
|
|
126
|
+
license?: string;
|
|
127
|
+
/** Track duration in seconds. */
|
|
128
|
+
durationSeconds?: number;
|
|
129
|
+
};
|
|
130
|
+
/** Pinterest-specific metadata. */
|
|
131
|
+
export type PinterestExtra = {
|
|
132
|
+
/** Times the pin was saved (repinned). */
|
|
133
|
+
saveCount?: number;
|
|
134
|
+
/** Dominant color of the pin image, as a hex string. */
|
|
135
|
+
dominantColor?: string;
|
|
136
|
+
/** Outbound link the pin points to, when set. */
|
|
137
|
+
outboundLink?: string;
|
|
138
|
+
};
|
|
139
|
+
/** The result of resolving a post: its platform, id, media items and metadata. */
|
|
140
|
+
export type PostfetchResult = PlatformResult<"facebook", never> | PlatformResult<"instagram", InstagramExtra> | PlatformResult<"pinterest", PinterestExtra> | PlatformResult<"reddit", RedditExtra> | PlatformResult<"soundcloud", SoundcloudExtra> | PlatformResult<"tiktok", TiktokExtra> | PlatformResult<"twitter", TwitterExtra> | PlatformResult<"youtube", YoutubeExtra>;
|
|
141
|
+
/** One platform's variant of a {@link PostfetchResult}, carrying that platform's `extra` metadata shape. */
|
|
142
|
+
export type PlatformResult<P extends Platform, Extra> = {
|
|
41
143
|
/** Suggested filename when zipping every item. */
|
|
42
144
|
archiveFilename: string;
|
|
43
145
|
/** Platform post/media id. */
|
|
@@ -45,7 +147,12 @@ export type PostfetchResult = {
|
|
|
45
147
|
/** The post's media, in order. */
|
|
46
148
|
items: MediaItem[];
|
|
47
149
|
/** The platform the post came from. */
|
|
48
|
-
platform:
|
|
150
|
+
platform: P;
|
|
151
|
+
/** Normalized post metadata, when any was available. */
|
|
152
|
+
metadata?: PostMetadata & {
|
|
153
|
+
/** Platform-specific metadata not covered by the normalized fields. */
|
|
154
|
+
extra?: Extra;
|
|
155
|
+
};
|
|
49
156
|
};
|
|
50
157
|
export type Net = (url: string, init?: RequestInit, attempts?: number) => Promise<Response>;
|
|
51
158
|
export type ResolveContext = {
|
|
@@ -72,5 +179,9 @@ export declare function createNet(baseFetch?: typeof fetch): Net;
|
|
|
72
179
|
export declare function object(value: unknown): value is Json;
|
|
73
180
|
export declare function string(value: unknown): string | null;
|
|
74
181
|
export declare function number(value: unknown): number | null;
|
|
182
|
+
export declare function bool(value: unknown): boolean | undefined;
|
|
183
|
+
export declare function count(value: unknown): number | undefined;
|
|
184
|
+
export declare function isoFromEpochSeconds(value: unknown): string | undefined;
|
|
185
|
+
export declare function isoFromDateString(value: unknown): string | undefined;
|
|
75
186
|
export declare function asUrl(value: string): URL;
|
|
76
187
|
export declare function filename(value: string): string;
|
package/dist/pinterest.d.ts
CHANGED
|
@@ -1,2 +1,5 @@
|
|
|
1
|
-
import { type ResolveContext, type PostfetchResult } from "./internal";
|
|
1
|
+
import { type PinterestExtra, type PostMetadata, type ResolveContext, type Json, type PostfetchResult } from "./internal";
|
|
2
2
|
export declare function resolvePinterest(input: ResolveContext): Promise<PostfetchResult>;
|
|
3
|
+
export declare function pinterestMetadata(pin: Json): PostMetadata & {
|
|
4
|
+
extra?: PinterestExtra;
|
|
5
|
+
};
|
package/dist/reddit.d.ts
CHANGED
|
@@ -1,2 +1,5 @@
|
|
|
1
|
-
import { type ResolveContext, type PostfetchResult } from "./internal";
|
|
1
|
+
import { type PostMetadata, type RedditExtra, type ResolveContext, type Json, type PostfetchResult } from "./internal";
|
|
2
2
|
export declare function resolveReddit(input: ResolveContext): Promise<PostfetchResult>;
|
|
3
|
+
export declare function redditMetadata(post: Json): PostMetadata & {
|
|
4
|
+
extra?: RedditExtra;
|
|
5
|
+
};
|
package/dist/soundcloud.d.ts
CHANGED
|
@@ -1,2 +1,5 @@
|
|
|
1
|
-
import { type ResolveContext, type PostfetchResult } from "./internal";
|
|
1
|
+
import { type PostMetadata, type SoundcloudExtra, type ResolveContext, type Json, type PostfetchResult } from "./internal";
|
|
2
2
|
export declare function resolveSoundcloud(input: ResolveContext): Promise<PostfetchResult>;
|
|
3
|
+
export declare function soundcloudMetadata(track: Json): PostMetadata & {
|
|
4
|
+
extra?: SoundcloudExtra;
|
|
5
|
+
};
|
package/dist/tiktok.d.ts
CHANGED
|
@@ -1,2 +1,6 @@
|
|
|
1
|
-
import { type ResolveContext, type PostfetchResult } from "./internal";
|
|
1
|
+
import { type PostMetadata, type TiktokExtra, type ResolveContext, type Json, type PostfetchResult } from "./internal";
|
|
2
2
|
export declare function resolveTiktok(input: ResolveContext): Promise<PostfetchResult>;
|
|
3
|
+
export declare function tiktokMetadata(item: Json): PostMetadata & {
|
|
4
|
+
extra?: TiktokExtra;
|
|
5
|
+
};
|
|
6
|
+
export declare function isShortlinkHost(hostname: string): boolean;
|
package/dist/twitter.d.ts
CHANGED
|
@@ -1,2 +1,6 @@
|
|
|
1
|
-
import { type ResolveContext, type PostfetchResult } from "./internal";
|
|
1
|
+
import { type PostMetadata, type TwitterExtra, type ResolveContext, type Json, type PostfetchResult } from "./internal";
|
|
2
2
|
export declare function resolveTwitter(input: ResolveContext): Promise<PostfetchResult>;
|
|
3
|
+
export declare function twitterMetadata(tweet: Json): PostMetadata & {
|
|
4
|
+
extra?: TwitterExtra;
|
|
5
|
+
};
|
|
6
|
+
export declare function tweetId(input: string): string | null;
|
package/dist/youtube.d.ts
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
-
import { type ResolveContext, type PostfetchResult } from "./internal";
|
|
1
|
+
import { type PostMetadata, type YoutubeExtra, type ResolveContext, type PostfetchResult } from "./internal";
|
|
2
2
|
export declare function resolveYoutube(input: ResolveContext): Promise<PostfetchResult>;
|
|
3
|
+
export declare function youtubeMetadata(payload: unknown): PostMetadata & {
|
|
4
|
+
extra?: YoutubeExtra;
|
|
5
|
+
};
|
|
3
6
|
export declare function youtubeVideoId(input: string): string | null;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@postfetch/core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "Zero-dependency typed core that turns Facebook, Instagram, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube post URLs into media files.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
package/src/index.ts
CHANGED
|
@@ -22,4 +22,19 @@
|
|
|
22
22
|
export { postfetch, detect, type PostfetchOptions } from "./postfetch";
|
|
23
23
|
export { download, archive, toResponse, type Archive, type DownloadOptions } from "./download";
|
|
24
24
|
export { PostfetchError } from "./internal";
|
|
25
|
-
export type {
|
|
25
|
+
export type {
|
|
26
|
+
InstagramExtra,
|
|
27
|
+
MediaItem,
|
|
28
|
+
MediaKind,
|
|
29
|
+
PinterestExtra,
|
|
30
|
+
Platform,
|
|
31
|
+
PlatformResult,
|
|
32
|
+
PostAuthor,
|
|
33
|
+
PostfetchResult,
|
|
34
|
+
PostMetadata,
|
|
35
|
+
RedditExtra,
|
|
36
|
+
SoundcloudExtra,
|
|
37
|
+
TiktokExtra,
|
|
38
|
+
TwitterExtra,
|
|
39
|
+
YoutubeExtra,
|
|
40
|
+
} from "./internal";
|
package/src/instagram.ts
CHANGED
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
import {
|
|
2
2
|
asUrl,
|
|
3
|
+
bool,
|
|
4
|
+
count,
|
|
3
5
|
filename,
|
|
6
|
+
isoFromEpochSeconds,
|
|
4
7
|
number,
|
|
5
8
|
object,
|
|
6
9
|
string,
|
|
10
|
+
type InstagramExtra,
|
|
11
|
+
type PostMetadata,
|
|
7
12
|
type ResolveContext,
|
|
8
13
|
type Net,
|
|
9
14
|
type Json,
|
|
@@ -63,7 +68,51 @@ export async function resolveInstagram(input: ResolveContext): Promise<Postfetch
|
|
|
63
68
|
if (items.length === 0) {
|
|
64
69
|
throw new Error("Instagram media url not found");
|
|
65
70
|
}
|
|
66
|
-
return { archiveFilename: filename(`instagram_${code}.zip`), id: code, items, platform: "instagram" };
|
|
71
|
+
return { archiveFilename: filename(`instagram_${code}.zip`), id: code, items, metadata: instagramMetadata(media), platform: "instagram" };
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function instagramMetadata(media: Json): PostMetadata & { extra?: InstagramExtra } {
|
|
75
|
+
const user = object(media.user) ? media.user : object(media.owner) ? media.owner : null;
|
|
76
|
+
const countsHidden = bool(media.like_and_view_counts_disabled);
|
|
77
|
+
return {
|
|
78
|
+
text: instagramCaption(media),
|
|
79
|
+
author: user
|
|
80
|
+
? {
|
|
81
|
+
handle: string(user.username) ?? undefined,
|
|
82
|
+
name: string(user.full_name) ?? undefined,
|
|
83
|
+
verified: bool(user.is_verified),
|
|
84
|
+
}
|
|
85
|
+
: undefined,
|
|
86
|
+
createdAt: isoFromEpochSeconds(media.taken_at ?? media.taken_at_timestamp),
|
|
87
|
+
likeCount: countsHidden ? undefined : count(media.like_count) ?? edgeCount(media.edge_media_preview_like) ?? edgeCount(media.edge_liked_by),
|
|
88
|
+
commentCount: count(media.comment_count) ?? edgeCount(media.edge_media_to_comment) ?? edgeCount(media.edge_media_to_parent_comment),
|
|
89
|
+
viewCount: countsHidden ? undefined : count(media.play_count) ?? count(media.view_count) ?? count(media.video_view_count),
|
|
90
|
+
extra: {
|
|
91
|
+
productType: string(media.product_type) ?? undefined,
|
|
92
|
+
countsHidden,
|
|
93
|
+
location: object(media.location) ? string(media.location.name) ?? undefined : undefined,
|
|
94
|
+
},
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// Caption arrives as an object ({ text }), a bare string, or the GraphQL
|
|
99
|
+
// edge_media_to_caption edge list, depending on which endpoint resolved the post.
|
|
100
|
+
function instagramCaption(media: Json): string | undefined {
|
|
101
|
+
if (object(media.caption)) {
|
|
102
|
+
return string(media.caption.text) ?? undefined;
|
|
103
|
+
}
|
|
104
|
+
const direct = string(media.caption);
|
|
105
|
+
if (direct) {
|
|
106
|
+
return direct;
|
|
107
|
+
}
|
|
108
|
+
const edges =
|
|
109
|
+
object(media.edge_media_to_caption) && Array.isArray(media.edge_media_to_caption.edges) ? media.edge_media_to_caption.edges : [];
|
|
110
|
+
const node = object(edges[0]) && object(edges[0].node) ? edges[0].node : null;
|
|
111
|
+
return node ? string(node.text) ?? undefined : undefined;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function edgeCount(value: unknown): number | undefined {
|
|
115
|
+
return object(value) ? count(value.count) : undefined;
|
|
67
116
|
}
|
|
68
117
|
|
|
69
118
|
function shortcode(input: string): string {
|
package/src/internal.ts
CHANGED
|
@@ -36,8 +36,128 @@ export type MediaItem = {
|
|
|
36
36
|
url: string;
|
|
37
37
|
};
|
|
38
38
|
|
|
39
|
-
/** The
|
|
40
|
-
export type
|
|
39
|
+
/** The author of a post, normalized across platforms. */
|
|
40
|
+
export type PostAuthor = {
|
|
41
|
+
/** Account handle (the `@username`), where the platform exposes one. */
|
|
42
|
+
handle?: string;
|
|
43
|
+
/** Display name, where it differs from the handle. */
|
|
44
|
+
name?: string;
|
|
45
|
+
/** Whether the account is verified, where the platform reports it. */
|
|
46
|
+
verified?: boolean;
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Post metadata normalized across platforms. Every field is optional: an absent
|
|
51
|
+
* field means the platform did not expose it in the payload already fetched to
|
|
52
|
+
* resolve the media (never a fabricated default — a missing count is `undefined`,
|
|
53
|
+
* not `0`). All counts are non-negative. Platform-specific fields live in `extra`.
|
|
54
|
+
*/
|
|
55
|
+
export type PostMetadata = {
|
|
56
|
+
/** Short headline, where the platform has one distinct from the body (Reddit, YouTube, Pinterest, SoundCloud). */
|
|
57
|
+
title?: string;
|
|
58
|
+
/** Body text: caption, tweet, description or self-text. */
|
|
59
|
+
text?: string;
|
|
60
|
+
/** Who posted it. */
|
|
61
|
+
author?: PostAuthor;
|
|
62
|
+
/** Publication time as an ISO 8601 string. */
|
|
63
|
+
createdAt?: string;
|
|
64
|
+
/** Positive reactions (likes, upvotes, favorites, diggs). */
|
|
65
|
+
likeCount?: number;
|
|
66
|
+
/** Number of comments or replies. */
|
|
67
|
+
commentCount?: number;
|
|
68
|
+
/** Genuine reshares (TikTok shares, retweets, reposts) — not saves. */
|
|
69
|
+
shareCount?: number;
|
|
70
|
+
/** Views or plays. */
|
|
71
|
+
viewCount?: number;
|
|
72
|
+
/** Whether the post is marked not-safe-for-work, where the platform reports it. */
|
|
73
|
+
nsfw?: boolean;
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
/** Reddit-specific metadata. */
|
|
77
|
+
export type RedditExtra = {
|
|
78
|
+
/** Subreddit the post belongs to, without the `r/` prefix. */
|
|
79
|
+
subreddit?: string;
|
|
80
|
+
/** Net score (upvotes minus downvotes); can be negative. */
|
|
81
|
+
score?: number;
|
|
82
|
+
/** Fraction of votes that are upvotes, from 0 to 1. */
|
|
83
|
+
upvoteRatio?: number;
|
|
84
|
+
/** Path of the post on reddit.com, starting with `/r/`. */
|
|
85
|
+
permalink?: string;
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
/** TikTok-specific metadata. */
|
|
89
|
+
export type TiktokExtra = {
|
|
90
|
+
/** Times the video was added to favorites. */
|
|
91
|
+
saveCount?: number;
|
|
92
|
+
/** Title of the sound used. */
|
|
93
|
+
musicTitle?: string;
|
|
94
|
+
/** Author of the sound used. */
|
|
95
|
+
musicAuthor?: string;
|
|
96
|
+
/** Two-letter region the video was created in. */
|
|
97
|
+
region?: string;
|
|
98
|
+
/** Video duration in seconds. */
|
|
99
|
+
durationSeconds?: number;
|
|
100
|
+
};
|
|
101
|
+
|
|
102
|
+
/** X (Twitter)-specific metadata. */
|
|
103
|
+
export type TwitterExtra = {
|
|
104
|
+
/** BCP 47 language tag detected for the tweet. */
|
|
105
|
+
lang?: string;
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
/** Instagram-specific metadata. */
|
|
109
|
+
export type InstagramExtra = {
|
|
110
|
+
/** Product type, e.g. `clips` for a reel or `feed` for a feed post. */
|
|
111
|
+
productType?: string;
|
|
112
|
+
/** Whether the account hid its like and view counts. */
|
|
113
|
+
countsHidden?: boolean;
|
|
114
|
+
/** Tagged location name, when present. */
|
|
115
|
+
location?: string;
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
/** YouTube-specific metadata. */
|
|
119
|
+
export type YoutubeExtra = {
|
|
120
|
+
/** Channel id (`UC…`) of the uploader. */
|
|
121
|
+
channelId?: string;
|
|
122
|
+
/** Video duration in seconds. */
|
|
123
|
+
durationSeconds?: number;
|
|
124
|
+
/** Video keywords. */
|
|
125
|
+
keywords?: string[];
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
/** SoundCloud-specific metadata. */
|
|
129
|
+
export type SoundcloudExtra = {
|
|
130
|
+
/** Track genre, when set. */
|
|
131
|
+
genre?: string;
|
|
132
|
+
/** Track license, e.g. `all-rights-reserved`. */
|
|
133
|
+
license?: string;
|
|
134
|
+
/** Track duration in seconds. */
|
|
135
|
+
durationSeconds?: number;
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
/** Pinterest-specific metadata. */
|
|
139
|
+
export type PinterestExtra = {
|
|
140
|
+
/** Times the pin was saved (repinned). */
|
|
141
|
+
saveCount?: number;
|
|
142
|
+
/** Dominant color of the pin image, as a hex string. */
|
|
143
|
+
dominantColor?: string;
|
|
144
|
+
/** Outbound link the pin points to, when set. */
|
|
145
|
+
outboundLink?: string;
|
|
146
|
+
};
|
|
147
|
+
|
|
148
|
+
/** The result of resolving a post: its platform, id, media items and metadata. */
|
|
149
|
+
export type PostfetchResult =
|
|
150
|
+
| PlatformResult<"facebook", never>
|
|
151
|
+
| PlatformResult<"instagram", InstagramExtra>
|
|
152
|
+
| PlatformResult<"pinterest", PinterestExtra>
|
|
153
|
+
| PlatformResult<"reddit", RedditExtra>
|
|
154
|
+
| PlatformResult<"soundcloud", SoundcloudExtra>
|
|
155
|
+
| PlatformResult<"tiktok", TiktokExtra>
|
|
156
|
+
| PlatformResult<"twitter", TwitterExtra>
|
|
157
|
+
| PlatformResult<"youtube", YoutubeExtra>;
|
|
158
|
+
|
|
159
|
+
/** One platform's variant of a {@link PostfetchResult}, carrying that platform's `extra` metadata shape. */
|
|
160
|
+
export type PlatformResult<P extends Platform, Extra> = {
|
|
41
161
|
/** Suggested filename when zipping every item. */
|
|
42
162
|
archiveFilename: string;
|
|
43
163
|
/** Platform post/media id. */
|
|
@@ -45,7 +165,12 @@ export type PostfetchResult = {
|
|
|
45
165
|
/** The post's media, in order. */
|
|
46
166
|
items: MediaItem[];
|
|
47
167
|
/** The platform the post came from. */
|
|
48
|
-
platform:
|
|
168
|
+
platform: P;
|
|
169
|
+
/** Normalized post metadata, when any was available. */
|
|
170
|
+
metadata?: PostMetadata & {
|
|
171
|
+
/** Platform-specific metadata not covered by the normalized fields. */
|
|
172
|
+
extra?: Extra;
|
|
173
|
+
};
|
|
49
174
|
};
|
|
50
175
|
|
|
51
176
|
export type Net = (url: string, init?: RequestInit, attempts?: number) => Promise<Response>;
|
|
@@ -114,6 +239,33 @@ export function number(value: unknown): number | null {
|
|
|
114
239
|
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
115
240
|
}
|
|
116
241
|
|
|
242
|
+
export function bool(value: unknown): boolean | undefined {
|
|
243
|
+
return typeof value === "boolean" ? value : undefined;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
export function count(value: unknown): number | undefined {
|
|
247
|
+
if (typeof value === "number" && Number.isInteger(value) && value >= 0) {
|
|
248
|
+
return value;
|
|
249
|
+
}
|
|
250
|
+
if (typeof value === "string" && /^\d+$/.test(value.trim())) {
|
|
251
|
+
return Number(value.trim());
|
|
252
|
+
}
|
|
253
|
+
return undefined;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
export function isoFromEpochSeconds(value: unknown): string | undefined {
|
|
257
|
+
const seconds = count(value);
|
|
258
|
+
return seconds === undefined ? undefined : new Date(seconds * 1000).toISOString();
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
export function isoFromDateString(value: unknown): string | undefined {
|
|
262
|
+
if (typeof value !== "string") {
|
|
263
|
+
return undefined;
|
|
264
|
+
}
|
|
265
|
+
const time = Date.parse(value);
|
|
266
|
+
return Number.isNaN(time) ? undefined : new Date(time).toISOString();
|
|
267
|
+
}
|
|
268
|
+
|
|
117
269
|
export function asUrl(value: string): URL {
|
|
118
270
|
try {
|
|
119
271
|
return new URL(value);
|
package/src/pinterest.ts
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import {
|
|
2
2
|
asUrl,
|
|
3
|
+
count,
|
|
3
4
|
filename,
|
|
5
|
+
isoFromDateString,
|
|
4
6
|
number,
|
|
5
7
|
object,
|
|
6
8
|
string,
|
|
9
|
+
type PinterestExtra,
|
|
10
|
+
type PostMetadata,
|
|
7
11
|
type ResolveContext,
|
|
8
12
|
type Json,
|
|
9
13
|
type Net,
|
|
@@ -20,7 +24,29 @@ export async function resolvePinterest(input: ResolveContext): Promise<Postfetch
|
|
|
20
24
|
if (items.length === 0) {
|
|
21
25
|
throw new Error("Pinterest media not found");
|
|
22
26
|
}
|
|
23
|
-
return { archiveFilename: filename(`pinterest_${id}.zip`), id, items, platform: "pinterest" };
|
|
27
|
+
return { archiveFilename: filename(`pinterest_${id}.zip`), id, items, metadata: pinterestMetadata(pin), platform: "pinterest" };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function pinterestMetadata(pin: Json): PostMetadata & { extra?: PinterestExtra } {
|
|
31
|
+
const pinner = object(pin.pinner) ? pin.pinner : null;
|
|
32
|
+
const reactions = object(pin.reaction_counts)
|
|
33
|
+
? Object.values(pin.reaction_counts).reduce<number>((sum, value) => sum + (count(value) ?? 0), 0)
|
|
34
|
+
: undefined;
|
|
35
|
+
return {
|
|
36
|
+
title: (string(pin.title) ?? string(pin.grid_title)) ?? undefined,
|
|
37
|
+
text: string(pin.description) ?? undefined,
|
|
38
|
+
author: pinner
|
|
39
|
+
? { handle: string(pinner.username) ?? undefined, name: string(pinner.full_name) ?? undefined }
|
|
40
|
+
: undefined,
|
|
41
|
+
createdAt: isoFromDateString(pin.created_at),
|
|
42
|
+
likeCount: reactions,
|
|
43
|
+
commentCount: count(pin.comment_count),
|
|
44
|
+
extra: {
|
|
45
|
+
saveCount: count(pin.repin_count),
|
|
46
|
+
dominantColor: string(pin.dominant_color) ?? undefined,
|
|
47
|
+
outboundLink: string(pin.link) ?? undefined,
|
|
48
|
+
},
|
|
49
|
+
};
|
|
24
50
|
}
|
|
25
51
|
|
|
26
52
|
// pin.it short links carry no pin id, so they are followed once to the canonical
|
package/src/reddit.ts
CHANGED
|
@@ -1,8 +1,14 @@
|
|
|
1
1
|
import {
|
|
2
2
|
asUrl,
|
|
3
|
+
bool,
|
|
4
|
+
count,
|
|
3
5
|
filename,
|
|
6
|
+
isoFromEpochSeconds,
|
|
7
|
+
number,
|
|
4
8
|
object,
|
|
5
9
|
string,
|
|
10
|
+
type PostMetadata,
|
|
11
|
+
type RedditExtra,
|
|
6
12
|
type ResolveContext,
|
|
7
13
|
type Json,
|
|
8
14
|
type Net,
|
|
@@ -25,7 +31,26 @@ export async function resolveReddit(input: ResolveContext): Promise<PostfetchRes
|
|
|
25
31
|
if (items.length === 0) {
|
|
26
32
|
throw new Error("Reddit media not found");
|
|
27
33
|
}
|
|
28
|
-
return { archiveFilename: filename(`reddit_${id}.zip`), id, items, platform: "reddit" };
|
|
34
|
+
return { archiveFilename: filename(`reddit_${id}.zip`), id, items, metadata: redditMetadata(post), platform: "reddit" };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function redditMetadata(post: Json): PostMetadata & { extra?: RedditExtra } {
|
|
38
|
+
const author = string(post.author);
|
|
39
|
+
return {
|
|
40
|
+
title: string(post.title) ?? undefined,
|
|
41
|
+
text: string(post.selftext) ?? undefined,
|
|
42
|
+
author: author ? { handle: author } : undefined,
|
|
43
|
+
createdAt: isoFromEpochSeconds(post.created_utc),
|
|
44
|
+
likeCount: count(post.ups),
|
|
45
|
+
commentCount: count(post.num_comments),
|
|
46
|
+
nsfw: bool(post.over_18),
|
|
47
|
+
extra: {
|
|
48
|
+
subreddit: string(post.subreddit) ?? undefined,
|
|
49
|
+
score: number(post.score) ?? undefined,
|
|
50
|
+
upvoteRatio: number(post.upvote_ratio) ?? undefined,
|
|
51
|
+
permalink: string(post.permalink) ?? undefined,
|
|
52
|
+
},
|
|
53
|
+
};
|
|
29
54
|
}
|
|
30
55
|
|
|
31
56
|
// Share and short links (redd.it/<id>, /r/<sub>/s/<share>) carry no post id, so
|
package/src/soundcloud.ts
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import {
|
|
2
2
|
asUrl,
|
|
3
|
+
count,
|
|
3
4
|
filename,
|
|
5
|
+
isoFromDateString,
|
|
4
6
|
number,
|
|
5
7
|
object,
|
|
6
8
|
string,
|
|
9
|
+
type PostMetadata,
|
|
10
|
+
type SoundcloudExtra,
|
|
7
11
|
type ResolveContext,
|
|
8
12
|
type Json,
|
|
9
13
|
type Net,
|
|
@@ -18,7 +22,29 @@ export async function resolveSoundcloud(input: ResolveContext): Promise<Postfetc
|
|
|
18
22
|
const track = await resolveTrack(input.net, url, clientId);
|
|
19
23
|
const id = trackId(track);
|
|
20
24
|
const item = await audioItem(input.net, track, id, clientId);
|
|
21
|
-
return { archiveFilename: filename(`soundcloud_${id}.zip`), id, items: [item], platform: "soundcloud" };
|
|
25
|
+
return { archiveFilename: filename(`soundcloud_${id}.zip`), id, items: [item], metadata: soundcloudMetadata(track), platform: "soundcloud" };
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function soundcloudMetadata(track: Json): PostMetadata & { extra?: SoundcloudExtra } {
|
|
29
|
+
const user = object(track.user) ? track.user : null;
|
|
30
|
+
const durationMs = count(track.duration);
|
|
31
|
+
return {
|
|
32
|
+
title: string(track.title) ?? undefined,
|
|
33
|
+
text: string(track.description) ?? undefined,
|
|
34
|
+
author: user
|
|
35
|
+
? { handle: string(user.username) ?? undefined, name: string(user.full_name) ?? undefined }
|
|
36
|
+
: undefined,
|
|
37
|
+
createdAt: isoFromDateString(track.created_at),
|
|
38
|
+
likeCount: count(track.likes_count),
|
|
39
|
+
commentCount: count(track.comment_count),
|
|
40
|
+
shareCount: count(track.reposts_count),
|
|
41
|
+
viewCount: count(track.playback_count),
|
|
42
|
+
extra: {
|
|
43
|
+
genre: string(track.genre) ?? undefined,
|
|
44
|
+
license: string(track.license) ?? undefined,
|
|
45
|
+
durationSeconds: durationMs === undefined ? undefined : Math.round(durationMs / 1000),
|
|
46
|
+
},
|
|
47
|
+
};
|
|
22
48
|
}
|
|
23
49
|
|
|
24
50
|
async function canonicalUrl(net: Net, input: string): Promise<string> {
|
package/src/tiktok.ts
CHANGED
|
@@ -1,8 +1,13 @@
|
|
|
1
1
|
import {
|
|
2
2
|
asUrl,
|
|
3
|
+
bool,
|
|
4
|
+
count,
|
|
3
5
|
filename,
|
|
6
|
+
isoFromEpochSeconds,
|
|
4
7
|
object,
|
|
5
8
|
string,
|
|
9
|
+
type PostMetadata,
|
|
10
|
+
type TiktokExtra,
|
|
6
11
|
type ResolveContext,
|
|
7
12
|
type Json,
|
|
8
13
|
type Net,
|
|
@@ -38,7 +43,36 @@ export async function resolveTiktok(input: ResolveContext): Promise<PostfetchRes
|
|
|
38
43
|
if (items.length === 0) {
|
|
39
44
|
throw new Error("TikTok media not found");
|
|
40
45
|
}
|
|
41
|
-
return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), id, items, platform: "tiktok" };
|
|
46
|
+
return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), id, items, metadata: tiktokMetadata(page.item), platform: "tiktok" };
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function tiktokMetadata(item: Json): PostMetadata & { extra?: TiktokExtra } {
|
|
50
|
+
const user = object(item.author) ? item.author : null;
|
|
51
|
+
const stats = object(item.stats) ? item.stats : null;
|
|
52
|
+
const music = object(item.music) ? item.music : null;
|
|
53
|
+
const video = object(item.video) ? item.video : null;
|
|
54
|
+
return {
|
|
55
|
+
text: string(item.desc) ?? undefined,
|
|
56
|
+
author: user
|
|
57
|
+
? {
|
|
58
|
+
handle: string(user.uniqueId) ?? undefined,
|
|
59
|
+
name: string(user.nickname) ?? undefined,
|
|
60
|
+
verified: bool(user.verified),
|
|
61
|
+
}
|
|
62
|
+
: undefined,
|
|
63
|
+
createdAt: isoFromEpochSeconds(item.createTime),
|
|
64
|
+
likeCount: stats ? count(stats.diggCount) : undefined,
|
|
65
|
+
commentCount: stats ? count(stats.commentCount) : undefined,
|
|
66
|
+
shareCount: stats ? count(stats.shareCount) : undefined,
|
|
67
|
+
viewCount: stats ? count(stats.playCount) : undefined,
|
|
68
|
+
extra: {
|
|
69
|
+
saveCount: stats ? count(stats.collectCount) : undefined,
|
|
70
|
+
musicTitle: music ? string(music.title) ?? undefined : undefined,
|
|
71
|
+
musicAuthor: music ? string(music.authorName) ?? undefined : undefined,
|
|
72
|
+
region: string(item.locationCreated) ?? undefined,
|
|
73
|
+
durationSeconds: video ? count(video.duration) : undefined,
|
|
74
|
+
},
|
|
75
|
+
};
|
|
42
76
|
}
|
|
43
77
|
|
|
44
78
|
async function fetchVideoPage(
|
|
@@ -56,9 +90,13 @@ async function fetchVideoPage(
|
|
|
56
90
|
return { cookie: cookieHeader(page.headers), item: itemStruct(await page.text()) };
|
|
57
91
|
}
|
|
58
92
|
|
|
93
|
+
export function isShortlinkHost(hostname: string): boolean {
|
|
94
|
+
return hostname === "vm.tiktok.com" || hostname === "vt.tiktok.com";
|
|
95
|
+
}
|
|
96
|
+
|
|
59
97
|
async function followShortlink(net: Net, userAgent: string, input: string): Promise<string> {
|
|
60
98
|
const url = asUrl(input);
|
|
61
|
-
if (!url.hostname
|
|
99
|
+
if (!isShortlinkHost(url.hostname)) {
|
|
62
100
|
return input;
|
|
63
101
|
}
|
|
64
102
|
const response = await net(input, { headers: { "user-agent": userAgent }, redirect: "manual" }, 1);
|
package/src/twitter.ts
CHANGED
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
import {
|
|
2
2
|
asUrl,
|
|
3
|
+
bool,
|
|
4
|
+
count,
|
|
3
5
|
filename,
|
|
6
|
+
isoFromDateString,
|
|
4
7
|
number,
|
|
5
8
|
object,
|
|
6
9
|
string,
|
|
10
|
+
type PostMetadata,
|
|
11
|
+
type TwitterExtra,
|
|
7
12
|
type ResolveContext,
|
|
8
13
|
type Json,
|
|
9
14
|
type Net,
|
|
@@ -26,7 +31,28 @@ export async function resolveTwitter(input: ResolveContext): Promise<PostfetchRe
|
|
|
26
31
|
if (items.length === 0) {
|
|
27
32
|
throw new Error("Twitter media not found");
|
|
28
33
|
}
|
|
29
|
-
return { archiveFilename: filename(`twitter_${id}.zip`), id, items, platform: "twitter" };
|
|
34
|
+
return { archiveFilename: filename(`twitter_${id}.zip`), id, items, metadata: twitterMetadata(tweet), platform: "twitter" };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterExtra } {
|
|
38
|
+
const user = object(tweet.user) ? tweet.user : null;
|
|
39
|
+
return {
|
|
40
|
+
text: (string(tweet.text) ?? string(tweet.full_text)) ?? undefined,
|
|
41
|
+
author: user
|
|
42
|
+
? {
|
|
43
|
+
handle: string(user.screen_name) ?? undefined,
|
|
44
|
+
name: string(user.name) ?? undefined,
|
|
45
|
+
verified: bool(user.verified) ?? bool(user.is_blue_verified),
|
|
46
|
+
}
|
|
47
|
+
: undefined,
|
|
48
|
+
createdAt: isoFromDateString(tweet.created_at),
|
|
49
|
+
likeCount: count(tweet.favorite_count),
|
|
50
|
+
commentCount: count(tweet.conversation_count),
|
|
51
|
+
shareCount: count(tweet.retweet_count),
|
|
52
|
+
viewCount: count(tweet.view_count),
|
|
53
|
+
nsfw: bool(tweet.possibly_sensitive),
|
|
54
|
+
extra: { lang: string(tweet.lang) ?? undefined },
|
|
55
|
+
};
|
|
30
56
|
}
|
|
31
57
|
|
|
32
58
|
// The public syndication endpoint returns tweet media (video variants + photos)
|
|
@@ -76,6 +102,6 @@ function bestVariant(entry: Json): string | null {
|
|
|
76
102
|
return best ? string(best.url) : null;
|
|
77
103
|
}
|
|
78
104
|
|
|
79
|
-
function tweetId(input: string): string | null {
|
|
105
|
+
export function tweetId(input: string): string | null {
|
|
80
106
|
return asUrl(input).pathname.match(/\/status(?:es)?\/(\d+)/)?.[1] ?? null;
|
|
81
107
|
}
|
package/src/youtube.ts
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import {
|
|
2
2
|
asUrl,
|
|
3
|
+
count,
|
|
3
4
|
filename,
|
|
5
|
+
isoFromDateString,
|
|
4
6
|
number,
|
|
5
7
|
object,
|
|
6
8
|
string,
|
|
9
|
+
type PostMetadata,
|
|
10
|
+
type YoutubeExtra,
|
|
7
11
|
type ResolveContext,
|
|
8
12
|
type Json,
|
|
9
13
|
type Net,
|
|
@@ -63,7 +67,32 @@ export async function resolveYoutube(input: ResolveContext): Promise<PostfetchRe
|
|
|
63
67
|
url: streams.video,
|
|
64
68
|
...(streams.audio ? { audio: { headers, url: streams.audio } } : {}),
|
|
65
69
|
};
|
|
66
|
-
return { archiveFilename: filename(`youtube_${id}.zip`), id, items: [media], platform: "youtube" };
|
|
70
|
+
return { archiveFilename: filename(`youtube_${id}.zip`), id, items: [media], metadata: youtubeMetadata(payload), platform: "youtube" };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function youtubeMetadata(payload: unknown): PostMetadata & { extra?: YoutubeExtra } {
|
|
74
|
+
const details = object(payload) && object(payload.videoDetails) ? payload.videoDetails : null;
|
|
75
|
+
const renderer =
|
|
76
|
+
object(payload) && object(payload.microformat) && object(payload.microformat.playerMicroformatRenderer)
|
|
77
|
+
? payload.microformat.playerMicroformatRenderer
|
|
78
|
+
: null;
|
|
79
|
+
const author = details ? string(details.author) : null;
|
|
80
|
+
const keywords =
|
|
81
|
+
details && Array.isArray(details.keywords)
|
|
82
|
+
? details.keywords.filter((keyword): keyword is string => typeof keyword === "string")
|
|
83
|
+
: [];
|
|
84
|
+
return {
|
|
85
|
+
title: details ? string(details.title) ?? undefined : undefined,
|
|
86
|
+
text: details ? string(details.shortDescription) ?? undefined : undefined,
|
|
87
|
+
author: author ? { name: author } : undefined,
|
|
88
|
+
createdAt: renderer ? isoFromDateString(renderer.publishDate) : undefined,
|
|
89
|
+
viewCount: details ? count(details.viewCount) : undefined,
|
|
90
|
+
extra: {
|
|
91
|
+
channelId: details ? string(details.channelId) ?? undefined : undefined,
|
|
92
|
+
durationSeconds: details ? count(details.lengthSeconds) : undefined,
|
|
93
|
+
keywords: keywords.length > 0 ? keywords : undefined,
|
|
94
|
+
},
|
|
95
|
+
};
|
|
67
96
|
}
|
|
68
97
|
|
|
69
98
|
// Adaptive H.264 video and AAC audio go up to 1080p with direct URLs; merging them
|