@postfetch/core 0.10.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -1
- package/dist/index.js +109 -20
- package/dist/internal.d.ts +16 -0
- package/dist/postfetch.d.ts +2 -0
- package/package.json +1 -1
- package/src/facebook.ts +2 -1
- package/src/index.ts +1 -0
- package/src/instagram.ts +1 -1
- package/src/internal.ts +17 -0
- package/src/linkedin.ts +1 -0
- package/src/pinterest.ts +1 -1
- package/src/postfetch.ts +3 -0
- package/src/reddit.ts +1 -1
- package/src/soundcloud.ts +1 -1
- package/src/tiktok.ts +1 -1
- package/src/twitter.ts +90 -13
- package/src/youtube.ts +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -23,4 +23,4 @@ export { postfetch, detect, type PostfetchOptions } from "./postfetch";
|
|
|
23
23
|
export { buildAudioSliderVideo, type AudioSliderVideoOptions } from "./audio-slider";
|
|
24
24
|
export { download, downloadBlob, archive, toResponse, type Archive, type DownloadBlobOptions, type DownloadOptions, type RemuxedVideo, } from "./download";
|
|
25
25
|
export { PostfetchError } from "./internal";
|
|
26
|
-
export type { InstagramExtra, MediaItem, MediaKind, PinterestExtra, Platform, PlatformResult, PostAuthor, PostfetchReason, PostfetchResult, PostMetadata, RedditExtra, SoundcloudExtra, TiktokExtra, TwitterExtra, TwitterQuotedTweet, YoutubeExtra, } from "./internal";
|
|
26
|
+
export type { InstagramExtra, MediaItem, MediaKind, PinterestExtra, Platform, PlatformResult, PostAuthor, PostComment, PostfetchReason, PostfetchResult, PostMetadata, RedditExtra, SoundcloudExtra, TiktokExtra, TwitterExtra, TwitterQuotedTweet, YoutubeExtra, } from "./internal";
|
package/dist/index.js
CHANGED
|
@@ -174,7 +174,7 @@ async function resolveFacebook(input) {
|
|
|
174
174
|
const url = embed ?? watch?.url ?? null;
|
|
175
175
|
if (!url) {
|
|
176
176
|
if (pageMetadata?.text) {
|
|
177
|
-
return { archiveFilename: filename(`facebook_${id}.zip`), id, items: [], metadata: pageMetadata, platform: "facebook" };
|
|
177
|
+
return { archiveFilename: filename(`facebook_${id}.zip`), comments: [], id, items: [], metadata: pageMetadata, platform: "facebook" };
|
|
178
178
|
}
|
|
179
179
|
throw new PostfetchError(404, "Facebook video not found", "notFound");
|
|
180
180
|
}
|
|
@@ -189,6 +189,7 @@ async function resolveFacebook(input) {
|
|
|
189
189
|
};
|
|
190
190
|
return {
|
|
191
191
|
archiveFilename: filename(`facebook_${id}.zip`),
|
|
192
|
+
comments: [],
|
|
192
193
|
id,
|
|
193
194
|
items: [item],
|
|
194
195
|
metadata: mergeMetadata(pageMetadata, watch?.metadata),
|
|
@@ -385,7 +386,7 @@ async function resolveInstagram(input) {
|
|
|
385
386
|
if (items.length === 0) {
|
|
386
387
|
throw new Error("Instagram media url not found");
|
|
387
388
|
}
|
|
388
|
-
return { archiveFilename: filename(`instagram_${code}.zip`), id: code, items, metadata: instagramMetadata(media), platform: "instagram" };
|
|
389
|
+
return { archiveFilename: filename(`instagram_${code}.zip`), comments: [], id: code, items, metadata: instagramMetadata(media), platform: "instagram" };
|
|
389
390
|
}
|
|
390
391
|
async function instagramUnavailable(net, code) {
|
|
391
392
|
const url = new URL("https://i.instagram.com/api/v1/oembed/");
|
|
@@ -804,6 +805,7 @@ async function resolveLinkedin(input) {
|
|
|
804
805
|
}
|
|
805
806
|
return {
|
|
806
807
|
archiveFilename: filename(`linkedin_${id}.zip`),
|
|
808
|
+
comments: [],
|
|
807
809
|
id,
|
|
808
810
|
items,
|
|
809
811
|
metadata: linkedinMetadata(post),
|
|
@@ -1323,7 +1325,7 @@ async function resolvePinterest(input) {
|
|
|
1323
1325
|
if (items.length === 0) {
|
|
1324
1326
|
throw new Error("Pinterest media not found");
|
|
1325
1327
|
}
|
|
1326
|
-
return { archiveFilename: filename(`pinterest_${id}.zip`), id, items, metadata: pinterestMetadata(pin), platform: "pinterest" };
|
|
1328
|
+
return { archiveFilename: filename(`pinterest_${id}.zip`), comments: [], id, items, metadata: pinterestMetadata(pin), platform: "pinterest" };
|
|
1327
1329
|
}
|
|
1328
1330
|
function pinterestMetadata(pin) {
|
|
1329
1331
|
const pinner = object(pin.pinner) ? pin.pinner : null;
|
|
@@ -1538,7 +1540,7 @@ async function resolveReddit(input) {
|
|
|
1538
1540
|
const token = await appToken(input.net);
|
|
1539
1541
|
const post = await fetchPost(input.net, token, id);
|
|
1540
1542
|
const items = await mediaItems3(input, post, id);
|
|
1541
|
-
return { archiveFilename: filename(`reddit_${id}.zip`), id, items, metadata: redditMetadata(post), platform: "reddit" };
|
|
1543
|
+
return { archiveFilename: filename(`reddit_${id}.zip`), comments: [], id, items, metadata: redditMetadata(post), platform: "reddit" };
|
|
1542
1544
|
}
|
|
1543
1545
|
function redditMetadata(post) {
|
|
1544
1546
|
const author = string(post.author);
|
|
@@ -1796,7 +1798,7 @@ async function resolveSoundcloud(input) {
|
|
|
1796
1798
|
const track = await resolveTrack(input.net, url, clientId);
|
|
1797
1799
|
const id = trackId(track);
|
|
1798
1800
|
const item = await audioItem(input.net, track, id, clientId);
|
|
1799
|
-
return { archiveFilename: filename(`soundcloud_${id}.zip`), id, items: [item], metadata: soundcloudMetadata(track), platform: "soundcloud" };
|
|
1801
|
+
return { archiveFilename: filename(`soundcloud_${id}.zip`), comments: [], id, items: [item], metadata: soundcloudMetadata(track), platform: "soundcloud" };
|
|
1800
1802
|
}
|
|
1801
1803
|
function soundcloudMetadata(track) {
|
|
1802
1804
|
const user = object(track.user) ? track.user : null;
|
|
@@ -1932,7 +1934,7 @@ async function resolveTiktok(input) {
|
|
|
1932
1934
|
if (items.length === 0) {
|
|
1933
1935
|
throw new Error("TikTok media not found");
|
|
1934
1936
|
}
|
|
1935
|
-
return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), id, items, metadata: tiktokMetadata(page.item), platform: "tiktok" };
|
|
1937
|
+
return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), comments: [], id, items, metadata: tiktokMetadata(page.item), platform: "tiktok" };
|
|
1936
1938
|
}
|
|
1937
1939
|
async function videoPage(net, id, photo) {
|
|
1938
1940
|
for (let attempt = 0;attempt < pageAttemptCount; attempt += 1) {
|
|
@@ -2194,7 +2196,7 @@ async function resolveTwitter(input) {
|
|
|
2194
2196
|
});
|
|
2195
2197
|
});
|
|
2196
2198
|
const items = input.tryMaxBytes === undefined ? groups.map((group) => group[0].item) : await sizeLimitedItems(input, groups);
|
|
2197
|
-
return { archiveFilename: filename(`twitter_${id}.zip`), id, items, metadata: twitterMetadata(tweet), platform: "twitter" };
|
|
2199
|
+
return { archiveFilename: filename(`twitter_${id}.zip`), comments: await twitterComments(input, id), id, items, metadata: twitterMetadata(tweet), platform: "twitter" };
|
|
2198
2200
|
}
|
|
2199
2201
|
function twitterMetadata(tweet) {
|
|
2200
2202
|
const user = object(tweet.user) ? tweet.user : null;
|
|
@@ -2208,6 +2210,11 @@ function twitterMetadata(tweet) {
|
|
|
2208
2210
|
if (quotedTweet && quotedId) {
|
|
2209
2211
|
extra.quotedTweet = { id: quotedId, metadata: twitterMetadata(quotedTweet) };
|
|
2210
2212
|
}
|
|
2213
|
+
const parent = object(tweet.parent) ? tweet.parent : null;
|
|
2214
|
+
const parentId = parent ? string(parent.id_str) : null;
|
|
2215
|
+
if (parent && parentId) {
|
|
2216
|
+
extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
|
|
2217
|
+
}
|
|
2211
2218
|
return {
|
|
2212
2219
|
text: twitterText(tweet) ?? undefined,
|
|
2213
2220
|
author: user ? {
|
|
@@ -2240,7 +2247,7 @@ function needsFullText(tweet) {
|
|
|
2240
2247
|
return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
|
|
2241
2248
|
}
|
|
2242
2249
|
async function expandTwitterText(net, tweet, rootId) {
|
|
2243
|
-
const posts = new Map(twitterTweets(tweet, rootId).map(({ id, tweet: post }) => [id, post]));
|
|
2250
|
+
const posts = new Map(twitterTweets(tweet, rootId, true).map(({ id, tweet: post }) => [id, post]));
|
|
2244
2251
|
for (const [id, post] of posts) {
|
|
2245
2252
|
if (!needsFullText(post)) {
|
|
2246
2253
|
continue;
|
|
@@ -2275,21 +2282,21 @@ async function expandTwitterText(net, tweet, rootId) {
|
|
|
2275
2282
|
} catch {}
|
|
2276
2283
|
}
|
|
2277
2284
|
}
|
|
2278
|
-
function twitterTweets(tweet, rootId) {
|
|
2285
|
+
function twitterTweets(tweet, rootId, includeParents = false) {
|
|
2279
2286
|
const tweets = [];
|
|
2280
2287
|
const seen = new Set;
|
|
2281
|
-
|
|
2282
|
-
let fallbackId = rootId;
|
|
2283
|
-
while (current) {
|
|
2288
|
+
const visit = (current, fallbackId) => {
|
|
2284
2289
|
const id = string(current.id_str) ?? fallbackId;
|
|
2285
|
-
if (!id || seen.has(id))
|
|
2286
|
-
|
|
2287
|
-
}
|
|
2288
|
-
tweets.push({ id, tweet: current });
|
|
2290
|
+
if (!id || seen.has(id))
|
|
2291
|
+
return;
|
|
2289
2292
|
seen.add(id);
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
+
tweets.push({ id, tweet: current });
|
|
2294
|
+
if (object(current.quoted_tweet))
|
|
2295
|
+
visit(current.quoted_tweet);
|
|
2296
|
+
if (includeParents && object(current.parent))
|
|
2297
|
+
visit(current.parent);
|
|
2298
|
+
};
|
|
2299
|
+
visit(tweet, rootId);
|
|
2293
2300
|
return tweets;
|
|
2294
2301
|
}
|
|
2295
2302
|
async function syndication(net, id) {
|
|
@@ -2397,6 +2404,87 @@ async function contentLength(net, item) {
|
|
|
2397
2404
|
function tweetId(input) {
|
|
2398
2405
|
return asUrl(input).pathname.match(/\/status(?:es)?\/(\d+)/)?.[1] ?? null;
|
|
2399
2406
|
}
|
|
2407
|
+
async function twitterComments(input, id) {
|
|
2408
|
+
const limit = input.comments;
|
|
2409
|
+
if (!limit || !Number.isSafeInteger(limit) || limit < 0)
|
|
2410
|
+
return [];
|
|
2411
|
+
try {
|
|
2412
|
+
const comments = [];
|
|
2413
|
+
const seen = new Set;
|
|
2414
|
+
const cursors = new Set;
|
|
2415
|
+
let cursor = null;
|
|
2416
|
+
do {
|
|
2417
|
+
const url = new URL(`https://api.fxtwitter.com/2/conversation/${id}`);
|
|
2418
|
+
if (cursor)
|
|
2419
|
+
url.searchParams.set("cursor", cursor);
|
|
2420
|
+
const response = await input.net(url.href, { headers: { accept: "application/json", "user-agent": browserUserAgent() } }, 1);
|
|
2421
|
+
if (!response.ok)
|
|
2422
|
+
throw new Error("Comment lookup failed");
|
|
2423
|
+
const payload = await response.json();
|
|
2424
|
+
if (!object(payload) || payload.code !== 200 || !Array.isArray(payload.replies))
|
|
2425
|
+
throw new Error("Invalid comments");
|
|
2426
|
+
const previousCount = comments.length;
|
|
2427
|
+
for (const reply of payload.replies) {
|
|
2428
|
+
if (!object(reply))
|
|
2429
|
+
continue;
|
|
2430
|
+
const replyId = string(reply.id);
|
|
2431
|
+
const parent = object(reply.replying_to) ? string(reply.replying_to.status) : null;
|
|
2432
|
+
if (!replyId || parent !== id || seen.has(replyId))
|
|
2433
|
+
continue;
|
|
2434
|
+
seen.add(replyId);
|
|
2435
|
+
comments.push(twitterComment(reply, replyId));
|
|
2436
|
+
if (comments.length >= limit)
|
|
2437
|
+
return comments;
|
|
2438
|
+
}
|
|
2439
|
+
cursor = object(payload.cursor) ? string(payload.cursor.bottom) : null;
|
|
2440
|
+
if (!cursor || cursors.has(cursor) || comments.length === previousCount)
|
|
2441
|
+
break;
|
|
2442
|
+
cursors.add(cursor);
|
|
2443
|
+
} while (comments.length < limit);
|
|
2444
|
+
return comments;
|
|
2445
|
+
} catch {
|
|
2446
|
+
return [];
|
|
2447
|
+
}
|
|
2448
|
+
}
|
|
2449
|
+
function twitterComment(reply, id) {
|
|
2450
|
+
const author2 = object(reply.author) ? reply.author : {};
|
|
2451
|
+
const verification = object(author2.verification) ? author2.verification : {};
|
|
2452
|
+
const media = object(reply.media) && Array.isArray(reply.media.all) ? reply.media.all.filter(object) : [];
|
|
2453
|
+
const items = media.flatMap((entry, index) => {
|
|
2454
|
+
const url = string(entry.url);
|
|
2455
|
+
const photo = entry.type === "photo";
|
|
2456
|
+
if (!url || !photo && entry.type !== "video" && entry.type !== "gif" && entry.type !== "animated_gif")
|
|
2457
|
+
return [];
|
|
2458
|
+
return [{
|
|
2459
|
+
id,
|
|
2460
|
+
platform: "twitter",
|
|
2461
|
+
filename: filename(`twitter_${id}_${index + 1}.${photo ? "jpg" : "mp4"}`),
|
|
2462
|
+
kind: photo ? "image" : "video",
|
|
2463
|
+
mime: photo ? "image/jpeg" : "video/mp4",
|
|
2464
|
+
url,
|
|
2465
|
+
headers: { "user-agent": browserUserAgent() }
|
|
2466
|
+
}];
|
|
2467
|
+
});
|
|
2468
|
+
return {
|
|
2469
|
+
id,
|
|
2470
|
+
url: `https://x.com/i/status/${id}`,
|
|
2471
|
+
items,
|
|
2472
|
+
metadata: {
|
|
2473
|
+
text: string(reply.text) ?? undefined,
|
|
2474
|
+
author: {
|
|
2475
|
+
handle: string(author2.screen_name) ?? undefined,
|
|
2476
|
+
name: string(author2.name) ?? undefined,
|
|
2477
|
+
verified: bool(verification.verified)
|
|
2478
|
+
},
|
|
2479
|
+
createdAt: isoFromDateString(reply.created_at),
|
|
2480
|
+
likeCount: count(reply.likes),
|
|
2481
|
+
commentCount: count(reply.replies),
|
|
2482
|
+
shareCount: count(reply.reposts),
|
|
2483
|
+
viewCount: count(reply.views),
|
|
2484
|
+
nsfw: bool(reply.possibly_sensitive)
|
|
2485
|
+
}
|
|
2486
|
+
};
|
|
2487
|
+
}
|
|
2400
2488
|
|
|
2401
2489
|
// src/youtube.ts
|
|
2402
2490
|
var androidVrClient = {
|
|
@@ -2470,7 +2558,7 @@ function youtubeResult(id, payload, streams, headers) {
|
|
|
2470
2558
|
...streams.hls ? { hls: true } : {},
|
|
2471
2559
|
...streams.audio ? { audio: { headers, url: streams.audio } } : {}
|
|
2472
2560
|
};
|
|
2473
|
-
return { archiveFilename: filename(`youtube_${id}.zip`), id, items: [media], metadata: youtubeMetadata(payload), platform: "youtube" };
|
|
2561
|
+
return { archiveFilename: filename(`youtube_${id}.zip`), comments: [], id, items: [media], metadata: youtubeMetadata(payload), platform: "youtube" };
|
|
2474
2562
|
}
|
|
2475
2563
|
async function probeStream(net, url, headers) {
|
|
2476
2564
|
const response = await net(url, { headers }, 1);
|
|
@@ -2702,6 +2790,7 @@ async function postfetch(url, options = {}) {
|
|
|
2702
2790
|
}
|
|
2703
2791
|
const context = {
|
|
2704
2792
|
net: createNet(options.fetch ?? globalThis.fetch),
|
|
2793
|
+
comments: options.comments,
|
|
2705
2794
|
preferredWidth: options.preferredWidth ?? 720,
|
|
2706
2795
|
tryMaxBytes: options.tryMaxBytes,
|
|
2707
2796
|
url: trimmed
|
package/dist/internal.d.ts
CHANGED
|
@@ -101,6 +101,8 @@ export type TwitterExtra = {
|
|
|
101
101
|
lang?: string;
|
|
102
102
|
/** The post quoted by this post, when X includes it in the resolved payload. */
|
|
103
103
|
quotedTweet?: TwitterQuotedTweet;
|
|
104
|
+
/** The parent this post replies to, when included by X. Media is not collected. */
|
|
105
|
+
parentTweet?: TwitterQuotedTweet;
|
|
104
106
|
};
|
|
105
107
|
/** A quoted X post and its normalized metadata. */
|
|
106
108
|
export type TwitterQuotedTweet = {
|
|
@@ -147,12 +149,25 @@ export type PinterestExtra = {
|
|
|
147
149
|
/** Outbound link the pin points to, when set. */
|
|
148
150
|
outboundLink?: string;
|
|
149
151
|
};
|
|
152
|
+
/** A reply with its own metadata and downloadable media, separate from the main post. */
|
|
153
|
+
export type PostComment = {
|
|
154
|
+
/** Reply post id. */
|
|
155
|
+
id: string;
|
|
156
|
+
/** Permalink to the reply. */
|
|
157
|
+
url: string;
|
|
158
|
+
/** Normalized reply metadata. */
|
|
159
|
+
metadata: PostMetadata;
|
|
160
|
+
/** Attachments belonging to this reply. */
|
|
161
|
+
items: MediaItem[];
|
|
162
|
+
};
|
|
150
163
|
/** The result of resolving a post: its platform, id, media items and metadata. */
|
|
151
164
|
export type PostfetchResult = PlatformResult<"facebook", never> | PlatformResult<"instagram", InstagramExtra> | PlatformResult<"linkedin", never> | PlatformResult<"pinterest", PinterestExtra> | PlatformResult<"reddit", RedditExtra> | PlatformResult<"soundcloud", SoundcloudExtra> | PlatformResult<"tiktok", TiktokExtra> | PlatformResult<"twitter", TwitterExtra> | PlatformResult<"youtube", YoutubeExtra>;
|
|
152
165
|
/** One platform's variant of a {@link PostfetchResult}, carrying that platform's `extra` metadata shape. */
|
|
153
166
|
export type PlatformResult<P extends Platform, Extra> = {
|
|
154
167
|
/** Suggested filename when zipping every item. */
|
|
155
168
|
archiveFilename: string;
|
|
169
|
+
/** Requested replies, or an empty array when not requested, unsupported or unavailable. */
|
|
170
|
+
comments: PostComment[];
|
|
156
171
|
/** Platform post/media id. */
|
|
157
172
|
id: string;
|
|
158
173
|
/** The post's media, in order. */
|
|
@@ -169,6 +184,7 @@ export type Net = (url: string, init?: RequestInit, attempts?: number) => Promis
|
|
|
169
184
|
export type ResolveContext = {
|
|
170
185
|
net: Net;
|
|
171
186
|
preferredWidth: number;
|
|
187
|
+
comments?: number;
|
|
172
188
|
/** Requested soft byte cap, for resolvers that can rank exact rendition sizes. */
|
|
173
189
|
tryMaxBytes?: number;
|
|
174
190
|
url: string;
|
package/dist/postfetch.d.ts
CHANGED
|
@@ -3,6 +3,8 @@ import { type Platform, type PostfetchResult } from "./internal";
|
|
|
3
3
|
export type PostfetchOptions = {
|
|
4
4
|
/** Custom `fetch` implementation — inject one to unit-test resolvers offline. Defaults to the global `fetch`. */
|
|
5
5
|
fetch?: typeof fetch;
|
|
6
|
+
/** Maximum replies to fetch for X. Other platforms ignore this option. Failures return an empty array. */
|
|
7
|
+
comments?: number;
|
|
6
8
|
/** Preferred media width in pixels; the closest available rendition is chosen. Defaults to `720`. */
|
|
7
9
|
preferredWidth?: number;
|
|
8
10
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@postfetch/core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.0",
|
|
4
4
|
"description": "Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube post URLs into media files.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
package/src/facebook.ts
CHANGED
|
@@ -16,7 +16,7 @@ export async function resolveFacebook(input: ResolveContext): Promise<PostfetchR
|
|
|
16
16
|
// has no downloadable media. Require post metadata from the canonical page
|
|
17
17
|
// so a missing or login-walled video does not become a false success.
|
|
18
18
|
if (pageMetadata?.text) {
|
|
19
|
-
return { archiveFilename: filename(`facebook_${id}.zip`), id, items: [], metadata: pageMetadata, platform: "facebook" };
|
|
19
|
+
return { archiveFilename: filename(`facebook_${id}.zip`), comments: [], id, items: [], metadata: pageMetadata, platform: "facebook" };
|
|
20
20
|
}
|
|
21
21
|
throw new PostfetchError(404, "Facebook video not found", "notFound");
|
|
22
22
|
}
|
|
@@ -31,6 +31,7 @@ export async function resolveFacebook(input: ResolveContext): Promise<PostfetchR
|
|
|
31
31
|
};
|
|
32
32
|
return {
|
|
33
33
|
archiveFilename: filename(`facebook_${id}.zip`),
|
|
34
|
+
comments: [],
|
|
34
35
|
id,
|
|
35
36
|
items: [item],
|
|
36
37
|
metadata: mergeMetadata(pageMetadata, watch?.metadata),
|
package/src/index.ts
CHANGED
package/src/instagram.ts
CHANGED
|
@@ -80,7 +80,7 @@ export async function resolveInstagram(input: ResolveContext): Promise<Postfetch
|
|
|
80
80
|
if (items.length === 0) {
|
|
81
81
|
throw new Error("Instagram media url not found");
|
|
82
82
|
}
|
|
83
|
-
return { archiveFilename: filename(`instagram_${code}.zip`), id: code, items, metadata: instagramMetadata(media), platform: "instagram" };
|
|
83
|
+
return { archiveFilename: filename(`instagram_${code}.zip`), comments: [], id: code, items, metadata: instagramMetadata(media), platform: "instagram" };
|
|
84
84
|
}
|
|
85
85
|
|
|
86
86
|
async function instagramUnavailable(net: Net, code: string): Promise<PostfetchError> {
|
package/src/internal.ts
CHANGED
|
@@ -105,6 +105,8 @@ export type TwitterExtra = {
|
|
|
105
105
|
lang?: string;
|
|
106
106
|
/** The post quoted by this post, when X includes it in the resolved payload. */
|
|
107
107
|
quotedTweet?: TwitterQuotedTweet;
|
|
108
|
+
/** The parent this post replies to, when included by X. Media is not collected. */
|
|
109
|
+
parentTweet?: TwitterQuotedTweet;
|
|
108
110
|
};
|
|
109
111
|
|
|
110
112
|
/** A quoted X post and its normalized metadata. */
|
|
@@ -155,6 +157,18 @@ export type PinterestExtra = {
|
|
|
155
157
|
outboundLink?: string;
|
|
156
158
|
};
|
|
157
159
|
|
|
160
|
+
/** A reply with its own metadata and downloadable media, separate from the main post. */
|
|
161
|
+
export type PostComment = {
|
|
162
|
+
/** Reply post id. */
|
|
163
|
+
id: string;
|
|
164
|
+
/** Permalink to the reply. */
|
|
165
|
+
url: string;
|
|
166
|
+
/** Normalized reply metadata. */
|
|
167
|
+
metadata: PostMetadata;
|
|
168
|
+
/** Attachments belonging to this reply. */
|
|
169
|
+
items: MediaItem[];
|
|
170
|
+
};
|
|
171
|
+
|
|
158
172
|
/** The result of resolving a post: its platform, id, media items and metadata. */
|
|
159
173
|
export type PostfetchResult =
|
|
160
174
|
| PlatformResult<"facebook", never>
|
|
@@ -171,6 +185,8 @@ export type PostfetchResult =
|
|
|
171
185
|
export type PlatformResult<P extends Platform, Extra> = {
|
|
172
186
|
/** Suggested filename when zipping every item. */
|
|
173
187
|
archiveFilename: string;
|
|
188
|
+
/** Requested replies, or an empty array when not requested, unsupported or unavailable. */
|
|
189
|
+
comments: PostComment[];
|
|
174
190
|
/** Platform post/media id. */
|
|
175
191
|
id: string;
|
|
176
192
|
/** The post's media, in order. */
|
|
@@ -189,6 +205,7 @@ export type Net = (url: string, init?: RequestInit, attempts?: number) => Promis
|
|
|
189
205
|
export type ResolveContext = {
|
|
190
206
|
net: Net;
|
|
191
207
|
preferredWidth: number;
|
|
208
|
+
comments?: number;
|
|
192
209
|
/** Requested soft byte cap, for resolvers that can rank exact rendition sizes. */
|
|
193
210
|
tryMaxBytes?: number;
|
|
194
211
|
url: string;
|
package/src/linkedin.ts
CHANGED
package/src/pinterest.ts
CHANGED
|
@@ -24,7 +24,7 @@ export async function resolvePinterest(input: ResolveContext): Promise<Postfetch
|
|
|
24
24
|
if (items.length === 0) {
|
|
25
25
|
throw new Error("Pinterest media not found");
|
|
26
26
|
}
|
|
27
|
-
return { archiveFilename: filename(`pinterest_${id}.zip`), id, items, metadata: pinterestMetadata(pin), platform: "pinterest" };
|
|
27
|
+
return { archiveFilename: filename(`pinterest_${id}.zip`), comments: [], id, items, metadata: pinterestMetadata(pin), platform: "pinterest" };
|
|
28
28
|
}
|
|
29
29
|
|
|
30
30
|
export function pinterestMetadata(pin: Json): PostMetadata & { extra?: PinterestExtra } {
|
package/src/postfetch.ts
CHANGED
|
@@ -13,6 +13,8 @@ import { resolveYoutube } from "./youtube";
|
|
|
13
13
|
export type PostfetchOptions = {
|
|
14
14
|
/** Custom `fetch` implementation — inject one to unit-test resolvers offline. Defaults to the global `fetch`. */
|
|
15
15
|
fetch?: typeof fetch;
|
|
16
|
+
/** Maximum replies to fetch for X. Other platforms ignore this option. Failures return an empty array. */
|
|
17
|
+
comments?: number;
|
|
16
18
|
/** Preferred media width in pixels; the closest available rendition is chosen. Defaults to `720`. */
|
|
17
19
|
preferredWidth?: number;
|
|
18
20
|
/**
|
|
@@ -58,6 +60,7 @@ export async function postfetch(url: string, options: PostfetchOptions = {}): Pr
|
|
|
58
60
|
}
|
|
59
61
|
const context: ResolveContext = {
|
|
60
62
|
net: createNet(options.fetch ?? globalThis.fetch),
|
|
63
|
+
comments: options.comments,
|
|
61
64
|
preferredWidth: options.preferredWidth ?? 720,
|
|
62
65
|
tryMaxBytes: options.tryMaxBytes,
|
|
63
66
|
url: trimmed,
|
package/src/reddit.ts
CHANGED
|
@@ -31,7 +31,7 @@ export async function resolveReddit(input: ResolveContext): Promise<PostfetchRes
|
|
|
31
31
|
// already thrown if the post itself is missing, so empty items here mean a
|
|
32
32
|
// self/text post, not a failure.
|
|
33
33
|
const items = await mediaItems(input, post, id);
|
|
34
|
-
return { archiveFilename: filename(`reddit_${id}.zip`), id, items, metadata: redditMetadata(post), platform: "reddit" };
|
|
34
|
+
return { archiveFilename: filename(`reddit_${id}.zip`), comments: [], id, items, metadata: redditMetadata(post), platform: "reddit" };
|
|
35
35
|
}
|
|
36
36
|
|
|
37
37
|
export function redditMetadata(post: Json): PostMetadata & { extra?: RedditExtra } {
|
package/src/soundcloud.ts
CHANGED
|
@@ -22,7 +22,7 @@ export async function resolveSoundcloud(input: ResolveContext): Promise<Postfetc
|
|
|
22
22
|
const track = await resolveTrack(input.net, url, clientId);
|
|
23
23
|
const id = trackId(track);
|
|
24
24
|
const item = await audioItem(input.net, track, id, clientId);
|
|
25
|
-
return { archiveFilename: filename(`soundcloud_${id}.zip`), id, items: [item], metadata: soundcloudMetadata(track), platform: "soundcloud" };
|
|
25
|
+
return { archiveFilename: filename(`soundcloud_${id}.zip`), comments: [], id, items: [item], metadata: soundcloudMetadata(track), platform: "soundcloud" };
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
export function soundcloudMetadata(track: Json): PostMetadata & { extra?: SoundcloudExtra } {
|
package/src/tiktok.ts
CHANGED
|
@@ -42,7 +42,7 @@ export async function resolveTiktok(input: ResolveContext): Promise<PostfetchRes
|
|
|
42
42
|
if (items.length === 0) {
|
|
43
43
|
throw new Error("TikTok media not found");
|
|
44
44
|
}
|
|
45
|
-
return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), id, items, metadata: tiktokMetadata(page.item), platform: "tiktok" };
|
|
45
|
+
return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), comments: [], id, items, metadata: tiktokMetadata(page.item), platform: "tiktok" };
|
|
46
46
|
}
|
|
47
47
|
|
|
48
48
|
async function videoPage(
|
package/src/twitter.ts
CHANGED
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
string,
|
|
10
10
|
type PostMetadata,
|
|
11
11
|
type TwitterExtra,
|
|
12
|
+
type PostComment,
|
|
12
13
|
type ResolveContext,
|
|
13
14
|
type Json,
|
|
14
15
|
type Net,
|
|
@@ -43,7 +44,7 @@ export async function resolveTwitter(input: ResolveContext): Promise<PostfetchRe
|
|
|
43
44
|
// A text-only tweet is a valid result with metadata and no media. Syndication
|
|
44
45
|
// has already failed or returned a tombstone when the tweet is unavailable,
|
|
45
46
|
// so an empty item list here does not mean the lookup failed.
|
|
46
|
-
return { archiveFilename: filename(`twitter_${id}.zip`), id, items, metadata: twitterMetadata(tweet), platform: "twitter" };
|
|
47
|
+
return { archiveFilename: filename(`twitter_${id}.zip`), comments: await twitterComments(input, id), id, items, metadata: twitterMetadata(tweet), platform: "twitter" };
|
|
47
48
|
}
|
|
48
49
|
|
|
49
50
|
export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterExtra } {
|
|
@@ -58,6 +59,11 @@ export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterEx
|
|
|
58
59
|
if (quotedTweet && quotedId) {
|
|
59
60
|
extra.quotedTweet = { id: quotedId, metadata: twitterMetadata(quotedTweet) };
|
|
60
61
|
}
|
|
62
|
+
const parent = object(tweet.parent) ? tweet.parent : null;
|
|
63
|
+
const parentId = parent ? string(parent.id_str) : null;
|
|
64
|
+
if (parent && parentId) {
|
|
65
|
+
extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
|
|
66
|
+
}
|
|
61
67
|
return {
|
|
62
68
|
text: twitterText(tweet) ?? undefined,
|
|
63
69
|
author: user
|
|
@@ -98,7 +104,7 @@ function needsFullText(tweet: Json): boolean {
|
|
|
98
104
|
}
|
|
99
105
|
|
|
100
106
|
async function expandTwitterText(net: Net, tweet: Json, rootId: string): Promise<void> {
|
|
101
|
-
const posts = new Map(twitterTweets(tweet, rootId).map(({ id, tweet: post }) => [id, post]));
|
|
107
|
+
const posts = new Map(twitterTweets(tweet, rootId, true).map(({ id, tweet: post }) => [id, post]));
|
|
102
108
|
for (const [id, post] of posts) {
|
|
103
109
|
if (!needsFullText(post)) {
|
|
104
110
|
continue;
|
|
@@ -138,21 +144,18 @@ async function expandTwitterText(net: Net, tweet: Json, rootId: string): Promise
|
|
|
138
144
|
}
|
|
139
145
|
}
|
|
140
146
|
|
|
141
|
-
function twitterTweets(tweet: Json, rootId: string): Array<{ id: string; tweet: Json }> {
|
|
147
|
+
function twitterTweets(tweet: Json, rootId: string, includeParents = false): Array<{ id: string; tweet: Json }> {
|
|
142
148
|
const tweets: Array<{ id: string; tweet: Json }> = [];
|
|
143
149
|
const seen = new Set<string>();
|
|
144
|
-
|
|
145
|
-
let fallbackId: string | null = rootId;
|
|
146
|
-
while (current) {
|
|
150
|
+
const visit = (current: Json, fallbackId?: string): void => {
|
|
147
151
|
const id = string(current.id_str) ?? fallbackId;
|
|
148
|
-
if (!id || seen.has(id))
|
|
149
|
-
break;
|
|
150
|
-
}
|
|
151
|
-
tweets.push({ id, tweet: current });
|
|
152
|
+
if (!id || seen.has(id)) return;
|
|
152
153
|
seen.add(id);
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
154
|
+
tweets.push({ id, tweet: current });
|
|
155
|
+
if (object(current.quoted_tweet)) visit(current.quoted_tweet);
|
|
156
|
+
if (includeParents && object(current.parent)) visit(current.parent);
|
|
157
|
+
};
|
|
158
|
+
visit(tweet, rootId);
|
|
156
159
|
return tweets;
|
|
157
160
|
}
|
|
158
161
|
|
|
@@ -286,3 +289,77 @@ async function contentLength(net: Net, item: MediaItem): Promise<number | null>
|
|
|
286
289
|
export function tweetId(input: string): string | null {
|
|
287
290
|
return asUrl(input).pathname.match(/\/status(?:es)?\/(\d+)/)?.[1] ?? null;
|
|
288
291
|
}
|
|
292
|
+
|
|
293
|
+
async function twitterComments(input: ResolveContext, id: string): Promise<PostComment[]> {
|
|
294
|
+
const limit = input.comments;
|
|
295
|
+
if (!limit || !Number.isSafeInteger(limit) || limit < 0) return [];
|
|
296
|
+
try {
|
|
297
|
+
const comments: PostComment[] = [];
|
|
298
|
+
const seen = new Set<string>();
|
|
299
|
+
const cursors = new Set<string>();
|
|
300
|
+
let cursor: string | null = null;
|
|
301
|
+
do {
|
|
302
|
+
const url = new URL(`https://api.fxtwitter.com/2/conversation/${id}`);
|
|
303
|
+
if (cursor) url.searchParams.set("cursor", cursor);
|
|
304
|
+
const response = await input.net(url.href, { headers: { accept: "application/json", "user-agent": browserUserAgent() } }, 1);
|
|
305
|
+
if (!response.ok) throw new Error("Comment lookup failed");
|
|
306
|
+
const payload: unknown = await response.json();
|
|
307
|
+
if (!object(payload) || payload.code !== 200 || !Array.isArray(payload.replies)) throw new Error("Invalid comments");
|
|
308
|
+
const previousCount = comments.length;
|
|
309
|
+
for (const reply of payload.replies) {
|
|
310
|
+
if (!object(reply)) continue;
|
|
311
|
+
const replyId = string(reply.id);
|
|
312
|
+
const parent = object(reply.replying_to) ? string(reply.replying_to.status) : null;
|
|
313
|
+
if (!replyId || parent !== id || seen.has(replyId)) continue;
|
|
314
|
+
seen.add(replyId);
|
|
315
|
+
comments.push(twitterComment(reply, replyId));
|
|
316
|
+
if (comments.length >= limit) return comments;
|
|
317
|
+
}
|
|
318
|
+
cursor = object(payload.cursor) ? string(payload.cursor.bottom) : null;
|
|
319
|
+
if (!cursor || cursors.has(cursor) || comments.length === previousCount) break;
|
|
320
|
+
cursors.add(cursor);
|
|
321
|
+
} while (comments.length < limit);
|
|
322
|
+
return comments;
|
|
323
|
+
} catch {
|
|
324
|
+
return [];
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
function twitterComment(reply: Json, id: string): PostComment {
|
|
329
|
+
const author = object(reply.author) ? reply.author : {};
|
|
330
|
+
const verification = object(author.verification) ? author.verification : {};
|
|
331
|
+
const media = object(reply.media) && Array.isArray(reply.media.all) ? reply.media.all.filter(object) : [];
|
|
332
|
+
const items: MediaItem[] = media.flatMap((entry, index) => {
|
|
333
|
+
const url = string(entry.url);
|
|
334
|
+
const photo = entry.type === "photo";
|
|
335
|
+
if (!url || (!photo && entry.type !== "video" && entry.type !== "gif" && entry.type !== "animated_gif")) return [];
|
|
336
|
+
return [{
|
|
337
|
+
id,
|
|
338
|
+
platform: "twitter",
|
|
339
|
+
filename: filename(`twitter_${id}_${index + 1}.${photo ? "jpg" : "mp4"}`),
|
|
340
|
+
kind: photo ? "image" : "video",
|
|
341
|
+
mime: photo ? "image/jpeg" : "video/mp4",
|
|
342
|
+
url,
|
|
343
|
+
headers: { "user-agent": browserUserAgent() },
|
|
344
|
+
}];
|
|
345
|
+
});
|
|
346
|
+
return {
|
|
347
|
+
id,
|
|
348
|
+
url: `https://x.com/i/status/${id}`,
|
|
349
|
+
items,
|
|
350
|
+
metadata: {
|
|
351
|
+
text: string(reply.text) ?? undefined,
|
|
352
|
+
author: {
|
|
353
|
+
handle: string(author.screen_name) ?? undefined,
|
|
354
|
+
name: string(author.name) ?? undefined,
|
|
355
|
+
verified: bool(verification.verified),
|
|
356
|
+
},
|
|
357
|
+
createdAt: isoFromDateString(reply.created_at),
|
|
358
|
+
likeCount: count(reply.likes),
|
|
359
|
+
commentCount: count(reply.replies),
|
|
360
|
+
shareCount: count(reply.reposts),
|
|
361
|
+
viewCount: count(reply.views),
|
|
362
|
+
nsfw: bool(reply.possibly_sensitive),
|
|
363
|
+
},
|
|
364
|
+
};
|
|
365
|
+
}
|
package/src/youtube.ts
CHANGED
|
@@ -105,7 +105,7 @@ function youtubeResult(
|
|
|
105
105
|
...(streams.hls ? { hls: true } : {}),
|
|
106
106
|
...(streams.audio ? { audio: { headers, url: streams.audio } } : {}),
|
|
107
107
|
};
|
|
108
|
-
return { archiveFilename: filename(`youtube_${id}.zip`), id, items: [media], metadata: youtubeMetadata(payload), platform: "youtube" };
|
|
108
|
+
return { archiveFilename: filename(`youtube_${id}.zip`), comments: [], id, items: [media], metadata: youtubeMetadata(payload), platform: "youtube" };
|
|
109
109
|
}
|
|
110
110
|
|
|
111
111
|
async function probeStream(net: Net, url: string, headers: HeadersInit): Promise<void> {
|