@postfetch/core 0.10.2 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -23,4 +23,4 @@ export { postfetch, detect, type PostfetchOptions } from "./postfetch";
23
23
  export { buildAudioSliderVideo, type AudioSliderVideoOptions } from "./audio-slider";
24
24
  export { download, downloadBlob, archive, toResponse, type Archive, type DownloadBlobOptions, type DownloadOptions, type RemuxedVideo, } from "./download";
25
25
  export { PostfetchError } from "./internal";
26
- export type { InstagramExtra, MediaItem, MediaKind, PinterestExtra, Platform, PlatformResult, PostAuthor, PostfetchReason, PostfetchResult, PostMetadata, RedditExtra, SoundcloudExtra, TiktokExtra, TwitterExtra, TwitterQuotedTweet, YoutubeExtra, } from "./internal";
26
+ export type { InstagramExtra, MediaItem, MediaKind, PinterestExtra, Platform, PlatformResult, PostAuthor, PostComment, PostfetchReason, PostfetchResult, PostMetadata, RedditExtra, SoundcloudExtra, TiktokExtra, TwitterExtra, TwitterQuotedTweet, YoutubeExtra, } from "./internal";
package/dist/index.js CHANGED
@@ -174,7 +174,7 @@ async function resolveFacebook(input) {
174
174
  const url = embed ?? watch?.url ?? null;
175
175
  if (!url) {
176
176
  if (pageMetadata?.text) {
177
- return { archiveFilename: filename(`facebook_${id}.zip`), id, items: [], metadata: pageMetadata, platform: "facebook" };
177
+ return { archiveFilename: filename(`facebook_${id}.zip`), comments: [], id, items: [], metadata: pageMetadata, platform: "facebook" };
178
178
  }
179
179
  throw new PostfetchError(404, "Facebook video not found", "notFound");
180
180
  }
@@ -189,6 +189,7 @@ async function resolveFacebook(input) {
189
189
  };
190
190
  return {
191
191
  archiveFilename: filename(`facebook_${id}.zip`),
192
+ comments: [],
192
193
  id,
193
194
  items: [item],
194
195
  metadata: mergeMetadata(pageMetadata, watch?.metadata),
@@ -385,7 +386,7 @@ async function resolveInstagram(input) {
385
386
  if (items.length === 0) {
386
387
  throw new Error("Instagram media url not found");
387
388
  }
388
- return { archiveFilename: filename(`instagram_${code}.zip`), id: code, items, metadata: instagramMetadata(media), platform: "instagram" };
389
+ return { archiveFilename: filename(`instagram_${code}.zip`), comments: [], id: code, items, metadata: instagramMetadata(media), platform: "instagram" };
389
390
  }
390
391
  async function instagramUnavailable(net, code) {
391
392
  const url = new URL("https://i.instagram.com/api/v1/oembed/");
@@ -804,6 +805,7 @@ async function resolveLinkedin(input) {
804
805
  }
805
806
  return {
806
807
  archiveFilename: filename(`linkedin_${id}.zip`),
808
+ comments: [],
807
809
  id,
808
810
  items,
809
811
  metadata: linkedinMetadata(post),
@@ -1323,7 +1325,7 @@ async function resolvePinterest(input) {
1323
1325
  if (items.length === 0) {
1324
1326
  throw new Error("Pinterest media not found");
1325
1327
  }
1326
- return { archiveFilename: filename(`pinterest_${id}.zip`), id, items, metadata: pinterestMetadata(pin), platform: "pinterest" };
1328
+ return { archiveFilename: filename(`pinterest_${id}.zip`), comments: [], id, items, metadata: pinterestMetadata(pin), platform: "pinterest" };
1327
1329
  }
1328
1330
  function pinterestMetadata(pin) {
1329
1331
  const pinner = object(pin.pinner) ? pin.pinner : null;
@@ -1538,7 +1540,7 @@ async function resolveReddit(input) {
1538
1540
  const token = await appToken(input.net);
1539
1541
  const post = await fetchPost(input.net, token, id);
1540
1542
  const items = await mediaItems3(input, post, id);
1541
- return { archiveFilename: filename(`reddit_${id}.zip`), id, items, metadata: redditMetadata(post), platform: "reddit" };
1543
+ return { archiveFilename: filename(`reddit_${id}.zip`), comments: [], id, items, metadata: redditMetadata(post), platform: "reddit" };
1542
1544
  }
1543
1545
  function redditMetadata(post) {
1544
1546
  const author = string(post.author);
@@ -1796,7 +1798,7 @@ async function resolveSoundcloud(input) {
1796
1798
  const track = await resolveTrack(input.net, url, clientId);
1797
1799
  const id = trackId(track);
1798
1800
  const item = await audioItem(input.net, track, id, clientId);
1799
- return { archiveFilename: filename(`soundcloud_${id}.zip`), id, items: [item], metadata: soundcloudMetadata(track), platform: "soundcloud" };
1801
+ return { archiveFilename: filename(`soundcloud_${id}.zip`), comments: [], id, items: [item], metadata: soundcloudMetadata(track), platform: "soundcloud" };
1800
1802
  }
1801
1803
  function soundcloudMetadata(track) {
1802
1804
  const user = object(track.user) ? track.user : null;
@@ -1932,7 +1934,7 @@ async function resolveTiktok(input) {
1932
1934
  if (items.length === 0) {
1933
1935
  throw new Error("TikTok media not found");
1934
1936
  }
1935
- return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), id, items, metadata: tiktokMetadata(page.item), platform: "tiktok" };
1937
+ return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), comments: [], id, items, metadata: tiktokMetadata(page.item), platform: "tiktok" };
1936
1938
  }
1937
1939
  async function videoPage(net, id, photo) {
1938
1940
  for (let attempt = 0;attempt < pageAttemptCount; attempt += 1) {
@@ -2194,7 +2196,7 @@ async function resolveTwitter(input) {
2194
2196
  });
2195
2197
  });
2196
2198
  const items = input.tryMaxBytes === undefined ? groups.map((group) => group[0].item) : await sizeLimitedItems(input, groups);
2197
- return { archiveFilename: filename(`twitter_${id}.zip`), id, items, metadata: twitterMetadata(tweet), platform: "twitter" };
2199
+ return { archiveFilename: filename(`twitter_${id}.zip`), comments: await twitterComments(input, id), id, items, metadata: twitterMetadata(tweet), platform: "twitter" };
2198
2200
  }
2199
2201
  function twitterMetadata(tweet) {
2200
2202
  const user = object(tweet.user) ? tweet.user : null;
@@ -2208,6 +2210,11 @@ function twitterMetadata(tweet) {
2208
2210
  if (quotedTweet && quotedId) {
2209
2211
  extra.quotedTweet = { id: quotedId, metadata: twitterMetadata(quotedTweet) };
2210
2212
  }
2213
+ const parent = object(tweet.parent) ? tweet.parent : null;
2214
+ const parentId = parent ? string(parent.id_str) : null;
2215
+ if (parent && parentId) {
2216
+ extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
2217
+ }
2211
2218
  return {
2212
2219
  text: twitterText(tweet) ?? undefined,
2213
2220
  author: user ? {
@@ -2240,7 +2247,7 @@ function needsFullText(tweet) {
2240
2247
  return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
2241
2248
  }
2242
2249
  async function expandTwitterText(net, tweet, rootId) {
2243
- const posts = new Map(twitterTweets(tweet, rootId).map(({ id, tweet: post }) => [id, post]));
2250
+ const posts = new Map(twitterTweets(tweet, rootId, true).map(({ id, tweet: post }) => [id, post]));
2244
2251
  for (const [id, post] of posts) {
2245
2252
  if (!needsFullText(post)) {
2246
2253
  continue;
@@ -2275,21 +2282,21 @@ async function expandTwitterText(net, tweet, rootId) {
2275
2282
  } catch {}
2276
2283
  }
2277
2284
  }
2278
- function twitterTweets(tweet, rootId) {
2285
+ function twitterTweets(tweet, rootId, includeParents = false) {
2279
2286
  const tweets = [];
2280
2287
  const seen = new Set;
2281
- let current = tweet;
2282
- let fallbackId = rootId;
2283
- while (current) {
2288
+ const visit = (current, fallbackId) => {
2284
2289
  const id = string(current.id_str) ?? fallbackId;
2285
- if (!id || seen.has(id)) {
2286
- break;
2287
- }
2288
- tweets.push({ id, tweet: current });
2290
+ if (!id || seen.has(id))
2291
+ return;
2289
2292
  seen.add(id);
2290
- current = object(current.quoted_tweet) ? current.quoted_tweet : null;
2291
- fallbackId = null;
2292
- }
2293
+ tweets.push({ id, tweet: current });
2294
+ if (object(current.quoted_tweet))
2295
+ visit(current.quoted_tweet);
2296
+ if (includeParents && object(current.parent))
2297
+ visit(current.parent);
2298
+ };
2299
+ visit(tweet, rootId);
2293
2300
  return tweets;
2294
2301
  }
2295
2302
  async function syndication(net, id) {
@@ -2397,6 +2404,87 @@ async function contentLength(net, item) {
2397
2404
  function tweetId(input) {
2398
2405
  return asUrl(input).pathname.match(/\/status(?:es)?\/(\d+)/)?.[1] ?? null;
2399
2406
  }
2407
+ async function twitterComments(input, id) {
2408
+ const limit = input.comments;
2409
+ if (!limit || !Number.isSafeInteger(limit) || limit < 0)
2410
+ return [];
2411
+ try {
2412
+ const comments = [];
2413
+ const seen = new Set;
2414
+ const cursors = new Set;
2415
+ let cursor = null;
2416
+ do {
2417
+ const url = new URL(`https://api.fxtwitter.com/2/conversation/${id}`);
2418
+ if (cursor)
2419
+ url.searchParams.set("cursor", cursor);
2420
+ const response = await input.net(url.href, { headers: { accept: "application/json", "user-agent": browserUserAgent() } }, 1);
2421
+ if (!response.ok)
2422
+ throw new Error("Comment lookup failed");
2423
+ const payload = await response.json();
2424
+ if (!object(payload) || payload.code !== 200 || !Array.isArray(payload.replies))
2425
+ throw new Error("Invalid comments");
2426
+ const previousCount = comments.length;
2427
+ for (const reply of payload.replies) {
2428
+ if (!object(reply))
2429
+ continue;
2430
+ const replyId = string(reply.id);
2431
+ const parent = object(reply.replying_to) ? string(reply.replying_to.status) : null;
2432
+ if (!replyId || parent !== id || seen.has(replyId))
2433
+ continue;
2434
+ seen.add(replyId);
2435
+ comments.push(twitterComment(reply, replyId));
2436
+ if (comments.length >= limit)
2437
+ return comments;
2438
+ }
2439
+ cursor = object(payload.cursor) ? string(payload.cursor.bottom) : null;
2440
+ if (!cursor || cursors.has(cursor) || comments.length === previousCount)
2441
+ break;
2442
+ cursors.add(cursor);
2443
+ } while (comments.length < limit);
2444
+ return comments;
2445
+ } catch {
2446
+ return [];
2447
+ }
2448
+ }
2449
+ function twitterComment(reply, id) {
2450
+ const author2 = object(reply.author) ? reply.author : {};
2451
+ const verification = object(author2.verification) ? author2.verification : {};
2452
+ const media = object(reply.media) && Array.isArray(reply.media.all) ? reply.media.all.filter(object) : [];
2453
+ const items = media.flatMap((entry, index) => {
2454
+ const url = string(entry.url);
2455
+ const photo = entry.type === "photo";
2456
+ if (!url || !photo && entry.type !== "video" && entry.type !== "gif" && entry.type !== "animated_gif")
2457
+ return [];
2458
+ return [{
2459
+ id,
2460
+ platform: "twitter",
2461
+ filename: filename(`twitter_${id}_${index + 1}.${photo ? "jpg" : "mp4"}`),
2462
+ kind: photo ? "image" : "video",
2463
+ mime: photo ? "image/jpeg" : "video/mp4",
2464
+ url,
2465
+ headers: { "user-agent": browserUserAgent() }
2466
+ }];
2467
+ });
2468
+ return {
2469
+ id,
2470
+ url: `https://x.com/i/status/${id}`,
2471
+ items,
2472
+ metadata: {
2473
+ text: string(reply.text) ?? undefined,
2474
+ author: {
2475
+ handle: string(author2.screen_name) ?? undefined,
2476
+ name: string(author2.name) ?? undefined,
2477
+ verified: bool(verification.verified)
2478
+ },
2479
+ createdAt: isoFromDateString(reply.created_at),
2480
+ likeCount: count(reply.likes),
2481
+ commentCount: count(reply.replies),
2482
+ shareCount: count(reply.reposts),
2483
+ viewCount: count(reply.views),
2484
+ nsfw: bool(reply.possibly_sensitive)
2485
+ }
2486
+ };
2487
+ }
2400
2488
 
2401
2489
  // src/youtube.ts
2402
2490
  var androidVrClient = {
@@ -2470,7 +2558,7 @@ function youtubeResult(id, payload, streams, headers) {
2470
2558
  ...streams.hls ? { hls: true } : {},
2471
2559
  ...streams.audio ? { audio: { headers, url: streams.audio } } : {}
2472
2560
  };
2473
- return { archiveFilename: filename(`youtube_${id}.zip`), id, items: [media], metadata: youtubeMetadata(payload), platform: "youtube" };
2561
+ return { archiveFilename: filename(`youtube_${id}.zip`), comments: [], id, items: [media], metadata: youtubeMetadata(payload), platform: "youtube" };
2474
2562
  }
2475
2563
  async function probeStream(net, url, headers) {
2476
2564
  const response = await net(url, { headers }, 1);
@@ -2702,6 +2790,7 @@ async function postfetch(url, options = {}) {
2702
2790
  }
2703
2791
  const context = {
2704
2792
  net: createNet(options.fetch ?? globalThis.fetch),
2793
+ comments: options.comments,
2705
2794
  preferredWidth: options.preferredWidth ?? 720,
2706
2795
  tryMaxBytes: options.tryMaxBytes,
2707
2796
  url: trimmed
@@ -101,6 +101,8 @@ export type TwitterExtra = {
101
101
  lang?: string;
102
102
  /** The post quoted by this post, when X includes it in the resolved payload. */
103
103
  quotedTweet?: TwitterQuotedTweet;
104
+ /** The parent this post replies to, when included by X. Media is not collected. */
105
+ parentTweet?: TwitterQuotedTweet;
104
106
  };
105
107
  /** A quoted X post and its normalized metadata. */
106
108
  export type TwitterQuotedTweet = {
@@ -147,12 +149,25 @@ export type PinterestExtra = {
147
149
  /** Outbound link the pin points to, when set. */
148
150
  outboundLink?: string;
149
151
  };
152
+ /** A reply with its own metadata and downloadable media, separate from the main post. */
153
+ export type PostComment = {
154
+ /** Reply post id. */
155
+ id: string;
156
+ /** Permalink to the reply. */
157
+ url: string;
158
+ /** Normalized reply metadata. */
159
+ metadata: PostMetadata;
160
+ /** Attachments belonging to this reply. */
161
+ items: MediaItem[];
162
+ };
150
163
  /** The result of resolving a post: its platform, id, media items and metadata. */
151
164
  export type PostfetchResult = PlatformResult<"facebook", never> | PlatformResult<"instagram", InstagramExtra> | PlatformResult<"linkedin", never> | PlatformResult<"pinterest", PinterestExtra> | PlatformResult<"reddit", RedditExtra> | PlatformResult<"soundcloud", SoundcloudExtra> | PlatformResult<"tiktok", TiktokExtra> | PlatformResult<"twitter", TwitterExtra> | PlatformResult<"youtube", YoutubeExtra>;
152
165
  /** One platform's variant of a {@link PostfetchResult}, carrying that platform's `extra` metadata shape. */
153
166
  export type PlatformResult<P extends Platform, Extra> = {
154
167
  /** Suggested filename when zipping every item. */
155
168
  archiveFilename: string;
169
+ /** Requested replies, or an empty array when not requested, unsupported or unavailable. */
170
+ comments: PostComment[];
156
171
  /** Platform post/media id. */
157
172
  id: string;
158
173
  /** The post's media, in order. */
@@ -169,6 +184,7 @@ export type Net = (url: string, init?: RequestInit, attempts?: number) => Promis
169
184
  export type ResolveContext = {
170
185
  net: Net;
171
186
  preferredWidth: number;
187
+ comments?: number;
172
188
  /** Requested soft byte cap, for resolvers that can rank exact rendition sizes. */
173
189
  tryMaxBytes?: number;
174
190
  url: string;
@@ -3,6 +3,8 @@ import { type Platform, type PostfetchResult } from "./internal";
3
3
  export type PostfetchOptions = {
4
4
  /** Custom `fetch` implementation — inject one to unit-test resolvers offline. Defaults to the global `fetch`. */
5
5
  fetch?: typeof fetch;
6
+ /** Maximum replies to fetch for X. Other platforms ignore this option. Failures return an empty array. */
7
+ comments?: number;
6
8
  /** Preferred media width in pixels; the closest available rendition is chosen. Defaults to `720`. */
7
9
  preferredWidth?: number;
8
10
  /**
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@postfetch/core",
3
- "version": "0.10.2",
3
+ "version": "0.11.0",
4
4
  "description": "Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube post URLs into media files.",
5
5
  "license": "MIT",
6
6
  "type": "module",
package/src/facebook.ts CHANGED
@@ -16,7 +16,7 @@ export async function resolveFacebook(input: ResolveContext): Promise<PostfetchR
16
16
  // has no downloadable media. Require post metadata from the canonical page
17
17
  // so a missing or login-walled video does not become a false success.
18
18
  if (pageMetadata?.text) {
19
- return { archiveFilename: filename(`facebook_${id}.zip`), id, items: [], metadata: pageMetadata, platform: "facebook" };
19
+ return { archiveFilename: filename(`facebook_${id}.zip`), comments: [], id, items: [], metadata: pageMetadata, platform: "facebook" };
20
20
  }
21
21
  throw new PostfetchError(404, "Facebook video not found", "notFound");
22
22
  }
@@ -31,6 +31,7 @@ export async function resolveFacebook(input: ResolveContext): Promise<PostfetchR
31
31
  };
32
32
  return {
33
33
  archiveFilename: filename(`facebook_${id}.zip`),
34
+ comments: [],
34
35
  id,
35
36
  items: [item],
36
37
  metadata: mergeMetadata(pageMetadata, watch?.metadata),
package/src/index.ts CHANGED
@@ -40,6 +40,7 @@ export type {
40
40
  Platform,
41
41
  PlatformResult,
42
42
  PostAuthor,
43
+ PostComment,
43
44
  PostfetchReason,
44
45
  PostfetchResult,
45
46
  PostMetadata,
package/src/instagram.ts CHANGED
@@ -80,7 +80,7 @@ export async function resolveInstagram(input: ResolveContext): Promise<Postfetch
80
80
  if (items.length === 0) {
81
81
  throw new Error("Instagram media url not found");
82
82
  }
83
- return { archiveFilename: filename(`instagram_${code}.zip`), id: code, items, metadata: instagramMetadata(media), platform: "instagram" };
83
+ return { archiveFilename: filename(`instagram_${code}.zip`), comments: [], id: code, items, metadata: instagramMetadata(media), platform: "instagram" };
84
84
  }
85
85
 
86
86
  async function instagramUnavailable(net: Net, code: string): Promise<PostfetchError> {
package/src/internal.ts CHANGED
@@ -105,6 +105,8 @@ export type TwitterExtra = {
105
105
  lang?: string;
106
106
  /** The post quoted by this post, when X includes it in the resolved payload. */
107
107
  quotedTweet?: TwitterQuotedTweet;
108
+ /** The parent this post replies to, when included by X. Media is not collected. */
109
+ parentTweet?: TwitterQuotedTweet;
108
110
  };
109
111
 
110
112
  /** A quoted X post and its normalized metadata. */
@@ -155,6 +157,18 @@ export type PinterestExtra = {
155
157
  outboundLink?: string;
156
158
  };
157
159
 
160
+ /** A reply with its own metadata and downloadable media, separate from the main post. */
161
+ export type PostComment = {
162
+ /** Reply post id. */
163
+ id: string;
164
+ /** Permalink to the reply. */
165
+ url: string;
166
+ /** Normalized reply metadata. */
167
+ metadata: PostMetadata;
168
+ /** Attachments belonging to this reply. */
169
+ items: MediaItem[];
170
+ };
171
+
158
172
  /** The result of resolving a post: its platform, id, media items and metadata. */
159
173
  export type PostfetchResult =
160
174
  | PlatformResult<"facebook", never>
@@ -171,6 +185,8 @@ export type PostfetchResult =
171
185
  export type PlatformResult<P extends Platform, Extra> = {
172
186
  /** Suggested filename when zipping every item. */
173
187
  archiveFilename: string;
188
+ /** Requested replies, or an empty array when not requested, unsupported or unavailable. */
189
+ comments: PostComment[];
174
190
  /** Platform post/media id. */
175
191
  id: string;
176
192
  /** The post's media, in order. */
@@ -189,6 +205,7 @@ export type Net = (url: string, init?: RequestInit, attempts?: number) => Promis
189
205
  export type ResolveContext = {
190
206
  net: Net;
191
207
  preferredWidth: number;
208
+ comments?: number;
192
209
  /** Requested soft byte cap, for resolvers that can rank exact rendition sizes. */
193
210
  tryMaxBytes?: number;
194
211
  url: string;
package/src/linkedin.ts CHANGED
@@ -51,6 +51,7 @@ export async function resolveLinkedin(input: ResolveContext): Promise<PostfetchR
51
51
  }
52
52
  return {
53
53
  archiveFilename: filename(`linkedin_${id}.zip`),
54
+ comments: [],
54
55
  id,
55
56
  items,
56
57
  metadata: linkedinMetadata(post),
package/src/pinterest.ts CHANGED
@@ -24,7 +24,7 @@ export async function resolvePinterest(input: ResolveContext): Promise<Postfetch
24
24
  if (items.length === 0) {
25
25
  throw new Error("Pinterest media not found");
26
26
  }
27
- return { archiveFilename: filename(`pinterest_${id}.zip`), id, items, metadata: pinterestMetadata(pin), platform: "pinterest" };
27
+ return { archiveFilename: filename(`pinterest_${id}.zip`), comments: [], id, items, metadata: pinterestMetadata(pin), platform: "pinterest" };
28
28
  }
29
29
 
30
30
  export function pinterestMetadata(pin: Json): PostMetadata & { extra?: PinterestExtra } {
package/src/postfetch.ts CHANGED
@@ -13,6 +13,8 @@ import { resolveYoutube } from "./youtube";
13
13
  export type PostfetchOptions = {
14
14
  /** Custom `fetch` implementation — inject one to unit-test resolvers offline. Defaults to the global `fetch`. */
15
15
  fetch?: typeof fetch;
16
+ /** Maximum replies to fetch for X. Other platforms ignore this option. Failures return an empty array. */
17
+ comments?: number;
16
18
  /** Preferred media width in pixels; the closest available rendition is chosen. Defaults to `720`. */
17
19
  preferredWidth?: number;
18
20
  /**
@@ -58,6 +60,7 @@ export async function postfetch(url: string, options: PostfetchOptions = {}): Pr
58
60
  }
59
61
  const context: ResolveContext = {
60
62
  net: createNet(options.fetch ?? globalThis.fetch),
63
+ comments: options.comments,
61
64
  preferredWidth: options.preferredWidth ?? 720,
62
65
  tryMaxBytes: options.tryMaxBytes,
63
66
  url: trimmed,
package/src/reddit.ts CHANGED
@@ -31,7 +31,7 @@ export async function resolveReddit(input: ResolveContext): Promise<PostfetchRes
31
31
  // already thrown if the post itself is missing, so empty items here mean a
32
32
  // self/text post, not a failure.
33
33
  const items = await mediaItems(input, post, id);
34
- return { archiveFilename: filename(`reddit_${id}.zip`), id, items, metadata: redditMetadata(post), platform: "reddit" };
34
+ return { archiveFilename: filename(`reddit_${id}.zip`), comments: [], id, items, metadata: redditMetadata(post), platform: "reddit" };
35
35
  }
36
36
 
37
37
  export function redditMetadata(post: Json): PostMetadata & { extra?: RedditExtra } {
package/src/soundcloud.ts CHANGED
@@ -22,7 +22,7 @@ export async function resolveSoundcloud(input: ResolveContext): Promise<Postfetc
22
22
  const track = await resolveTrack(input.net, url, clientId);
23
23
  const id = trackId(track);
24
24
  const item = await audioItem(input.net, track, id, clientId);
25
- return { archiveFilename: filename(`soundcloud_${id}.zip`), id, items: [item], metadata: soundcloudMetadata(track), platform: "soundcloud" };
25
+ return { archiveFilename: filename(`soundcloud_${id}.zip`), comments: [], id, items: [item], metadata: soundcloudMetadata(track), platform: "soundcloud" };
26
26
  }
27
27
 
28
28
  export function soundcloudMetadata(track: Json): PostMetadata & { extra?: SoundcloudExtra } {
package/src/tiktok.ts CHANGED
@@ -42,7 +42,7 @@ export async function resolveTiktok(input: ResolveContext): Promise<PostfetchRes
42
42
  if (items.length === 0) {
43
43
  throw new Error("TikTok media not found");
44
44
  }
45
- return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), id, items, metadata: tiktokMetadata(page.item), platform: "tiktok" };
45
+ return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), comments: [], id, items, metadata: tiktokMetadata(page.item), platform: "tiktok" };
46
46
  }
47
47
 
48
48
  async function videoPage(
package/src/twitter.ts CHANGED
@@ -9,6 +9,7 @@ import {
9
9
  string,
10
10
  type PostMetadata,
11
11
  type TwitterExtra,
12
+ type PostComment,
12
13
  type ResolveContext,
13
14
  type Json,
14
15
  type Net,
@@ -43,7 +44,7 @@ export async function resolveTwitter(input: ResolveContext): Promise<PostfetchRe
43
44
  // A text-only tweet is a valid result with metadata and no media. Syndication
44
45
  // has already failed or returned a tombstone when the tweet is unavailable,
45
46
  // so an empty item list here does not mean the lookup failed.
46
- return { archiveFilename: filename(`twitter_${id}.zip`), id, items, metadata: twitterMetadata(tweet), platform: "twitter" };
47
+ return { archiveFilename: filename(`twitter_${id}.zip`), comments: await twitterComments(input, id), id, items, metadata: twitterMetadata(tweet), platform: "twitter" };
47
48
  }
48
49
 
49
50
  export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterExtra } {
@@ -58,6 +59,11 @@ export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterEx
58
59
  if (quotedTweet && quotedId) {
59
60
  extra.quotedTweet = { id: quotedId, metadata: twitterMetadata(quotedTweet) };
60
61
  }
62
+ const parent = object(tweet.parent) ? tweet.parent : null;
63
+ const parentId = parent ? string(parent.id_str) : null;
64
+ if (parent && parentId) {
65
+ extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
66
+ }
61
67
  return {
62
68
  text: twitterText(tweet) ?? undefined,
63
69
  author: user
@@ -98,7 +104,7 @@ function needsFullText(tweet: Json): boolean {
98
104
  }
99
105
 
100
106
  async function expandTwitterText(net: Net, tweet: Json, rootId: string): Promise<void> {
101
- const posts = new Map(twitterTweets(tweet, rootId).map(({ id, tweet: post }) => [id, post]));
107
+ const posts = new Map(twitterTweets(tweet, rootId, true).map(({ id, tweet: post }) => [id, post]));
102
108
  for (const [id, post] of posts) {
103
109
  if (!needsFullText(post)) {
104
110
  continue;
@@ -138,21 +144,18 @@ async function expandTwitterText(net: Net, tweet: Json, rootId: string): Promise
138
144
  }
139
145
  }
140
146
 
141
- function twitterTweets(tweet: Json, rootId: string): Array<{ id: string; tweet: Json }> {
147
+ function twitterTweets(tweet: Json, rootId: string, includeParents = false): Array<{ id: string; tweet: Json }> {
142
148
  const tweets: Array<{ id: string; tweet: Json }> = [];
143
149
  const seen = new Set<string>();
144
- let current: Json | null = tweet;
145
- let fallbackId: string | null = rootId;
146
- while (current) {
150
+ const visit = (current: Json, fallbackId?: string): void => {
147
151
  const id = string(current.id_str) ?? fallbackId;
148
- if (!id || seen.has(id)) {
149
- break;
150
- }
151
- tweets.push({ id, tweet: current });
152
+ if (!id || seen.has(id)) return;
152
153
  seen.add(id);
153
- current = object(current.quoted_tweet) ? current.quoted_tweet : null;
154
- fallbackId = null;
155
- }
154
+ tweets.push({ id, tweet: current });
155
+ if (object(current.quoted_tweet)) visit(current.quoted_tweet);
156
+ if (includeParents && object(current.parent)) visit(current.parent);
157
+ };
158
+ visit(tweet, rootId);
156
159
  return tweets;
157
160
  }
158
161
 
@@ -286,3 +289,77 @@ async function contentLength(net: Net, item: MediaItem): Promise<number | null>
286
289
  export function tweetId(input: string): string | null {
287
290
  return asUrl(input).pathname.match(/\/status(?:es)?\/(\d+)/)?.[1] ?? null;
288
291
  }
292
+
293
+ async function twitterComments(input: ResolveContext, id: string): Promise<PostComment[]> {
294
+ const limit = input.comments;
295
+ if (!limit || !Number.isSafeInteger(limit) || limit < 0) return [];
296
+ try {
297
+ const comments: PostComment[] = [];
298
+ const seen = new Set<string>();
299
+ const cursors = new Set<string>();
300
+ let cursor: string | null = null;
301
+ do {
302
+ const url = new URL(`https://api.fxtwitter.com/2/conversation/${id}`);
303
+ if (cursor) url.searchParams.set("cursor", cursor);
304
+ const response = await input.net(url.href, { headers: { accept: "application/json", "user-agent": browserUserAgent() } }, 1);
305
+ if (!response.ok) throw new Error("Comment lookup failed");
306
+ const payload: unknown = await response.json();
307
+ if (!object(payload) || payload.code !== 200 || !Array.isArray(payload.replies)) throw new Error("Invalid comments");
308
+ const previousCount = comments.length;
309
+ for (const reply of payload.replies) {
310
+ if (!object(reply)) continue;
311
+ const replyId = string(reply.id);
312
+ const parent = object(reply.replying_to) ? string(reply.replying_to.status) : null;
313
+ if (!replyId || parent !== id || seen.has(replyId)) continue;
314
+ seen.add(replyId);
315
+ comments.push(twitterComment(reply, replyId));
316
+ if (comments.length >= limit) return comments;
317
+ }
318
+ cursor = object(payload.cursor) ? string(payload.cursor.bottom) : null;
319
+ if (!cursor || cursors.has(cursor) || comments.length === previousCount) break;
320
+ cursors.add(cursor);
321
+ } while (comments.length < limit);
322
+ return comments;
323
+ } catch {
324
+ return [];
325
+ }
326
+ }
327
+
328
+ function twitterComment(reply: Json, id: string): PostComment {
329
+ const author = object(reply.author) ? reply.author : {};
330
+ const verification = object(author.verification) ? author.verification : {};
331
+ const media = object(reply.media) && Array.isArray(reply.media.all) ? reply.media.all.filter(object) : [];
332
+ const items: MediaItem[] = media.flatMap((entry, index) => {
333
+ const url = string(entry.url);
334
+ const photo = entry.type === "photo";
335
+ if (!url || (!photo && entry.type !== "video" && entry.type !== "gif" && entry.type !== "animated_gif")) return [];
336
+ return [{
337
+ id,
338
+ platform: "twitter",
339
+ filename: filename(`twitter_${id}_${index + 1}.${photo ? "jpg" : "mp4"}`),
340
+ kind: photo ? "image" : "video",
341
+ mime: photo ? "image/jpeg" : "video/mp4",
342
+ url,
343
+ headers: { "user-agent": browserUserAgent() },
344
+ }];
345
+ });
346
+ return {
347
+ id,
348
+ url: `https://x.com/i/status/${id}`,
349
+ items,
350
+ metadata: {
351
+ text: string(reply.text) ?? undefined,
352
+ author: {
353
+ handle: string(author.screen_name) ?? undefined,
354
+ name: string(author.name) ?? undefined,
355
+ verified: bool(verification.verified),
356
+ },
357
+ createdAt: isoFromDateString(reply.created_at),
358
+ likeCount: count(reply.likes),
359
+ commentCount: count(reply.replies),
360
+ shareCount: count(reply.reposts),
361
+ viewCount: count(reply.views),
362
+ nsfw: bool(reply.possibly_sensitive),
363
+ },
364
+ };
365
+ }
package/src/youtube.ts CHANGED
@@ -105,7 +105,7 @@ function youtubeResult(
105
105
  ...(streams.hls ? { hls: true } : {}),
106
106
  ...(streams.audio ? { audio: { headers, url: streams.audio } } : {}),
107
107
  };
108
- return { archiveFilename: filename(`youtube_${id}.zip`), id, items: [media], metadata: youtubeMetadata(payload), platform: "youtube" };
108
+ return { archiveFilename: filename(`youtube_${id}.zip`), comments: [], id, items: [media], metadata: youtubeMetadata(payload), platform: "youtube" };
109
109
  }
110
110
 
111
111
  async function probeStream(net: Net, url: string, headers: HeadersInit): Promise<void> {