@postfetch/core 0.13.9 → 0.14.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -59,9 +59,22 @@ const blob = await buildAudioSliderVideo(result.items, { delay: 3000 });
59
59
  form.append("video", blob, "slideshow.mp4");
60
60
  ```
61
61
 
62
- - `PostfetchOptions` — `{ fetch?: typeof fetch; preferredWidth?: number; tryMaxBytes?: number }`. `tryMaxBytes` is a soft byte cap. X probes every MP4 variant and selects the highest-quality complete result that fits, falling back to its smallest variants when none do. Other platforms return a smaller available rendition when possible and otherwise keep the normal result. Inject `fetch` to unit-test resolvers offline.
62
+ - `PostfetchOptions` — `{ fetch?: typeof fetch; preferredWidth?: number; tryMaxBytes?: number; languages?: string[] }`. `tryMaxBytes` is a soft byte cap. X probes every MP4 variant and selects the highest-quality complete result that fits, falling back to its smallest variants when none do. Other platforms return a smaller available rendition when possible and otherwise keep the normal result. Inject `fetch` to unit-test resolvers offline.
63
63
  - Every request rotates a consistent browser/app fingerprint, so a fixed user-agent never gets the whole fleet blocked.
64
64
  - No runtime dependencies, no `yt-dlp`, no cookies. `ffmpeg` is only used when
65
65
  `downloadBlob(..., { remux: true })` or `buildAudioSliderVideo` is requested.
66
66
 
67
67
  See the [root README](https://github.com/chelokot/postfetch#readme) for the full API table, supported inputs and the fingerprint design.
68
+
69
+ ### X text languages
70
+
71
+ Pass `postfetch(url, { languages: ["en", "ru"] })` to keep X posts written in
72
+ any allowed language. Other posts are translated to the first language in the
73
+ list, falling back to English and then the original if translation is unavailable.
74
+ The ordered list must be non-empty; omit it to keep original text. This applies
75
+ independently to quoted posts, parent posts and requested replies. Regional
76
+ variants match a base language (`en` allows `en-US`). Other platforms ignore it.
77
+
78
+ `metadata.extra.lang` remains the detected original language. Translated posts
79
+ also expose `metadata.extra.originalText` and `metadata.extra.translatedLang`.
80
+ Translations use FxTwitter's public translation API and do not alter video audio.
package/dist/index.js CHANGED
@@ -1693,6 +1693,64 @@ async function fetchWebPost(net, id, input, initial) {
1693
1693
  }
1694
1694
  return null;
1695
1695
  }
1696
+ async function fetchEmbeddedPost(net, id) {
1697
+ const page = await fetchWebPage(net, `https://www.reddit.com/comments/${id}?embedded=true`, browserUserAgent());
1698
+ if (!page.ok)
1699
+ return null;
1700
+ for (const match of page.html.matchAll(/<div\s([^>]+)>/gi)) {
1701
+ const post = attributes(match[1]);
1702
+ if (post.id !== `thing_t3_${id}` || post["data-fullname"] !== `t3_${id}` || post["data-is-gallery"] !== "true")
1703
+ continue;
1704
+ const html = divContent(page.html, match.index + match[0].length);
1705
+ if (html === null)
1706
+ return null;
1707
+ const title = [...html.matchAll(/<a\s([^>]+)>([\s\S]*?)<\/a>/gi)].find((link) => attributes(link[1])["data-event-action"] === "title")?.[2];
1708
+ const preview = [...html.matchAll(/<div\s([^>]+)>/gi)].map((tag) => attributes(tag[1])).find((attrs) => attrs.id === `media-preview-${id}`);
1709
+ const ids = preview?.["data-media-ids"]?.split(",") ?? [];
1710
+ if (!title || !ids.length || ids.some((key) => !/^[a-z0-9]+$/i.test(key)) || new Set(ids).size !== ids.length)
1711
+ return null;
1712
+ const metadata = {};
1713
+ for (const link of html.matchAll(/<a\s([^>]+)>/gi)) {
1714
+ const attrs = attributes(link[1]);
1715
+ const url = new URL(attrs.href ?? "", "https://www.reddit.com");
1716
+ if (url.protocol !== "https:")
1717
+ continue;
1718
+ if (attrs.class?.split(/\s+/).includes("gallery-item-thumbnail-link")) {
1719
+ const index = Number(attrs["data-position"]) - 1;
1720
+ const key = ids[index];
1721
+ if (!Number.isSafeInteger(index) || !key || !/^(?:preview|i)\.redd\.it$/.test(url.hostname) || !new RegExp(`^/${key}\\.(?:png|jpe?g|webp)$`, "i").test(url.pathname))
1722
+ continue;
1723
+ metadata[key] = { s: { u: `https://i.redd.it${url.pathname}` } };
1724
+ } else if (/^(?:www\.)?reddit\.com$/.test(url.hostname)) {
1725
+ const video = url.pathname.match(/^\/link\/([a-z0-9]+)\/video\/([a-z0-9]+)\/player\/?$/i);
1726
+ if (video?.[1] === id && ids.includes(video[2])) {
1727
+ metadata[video[2]] = { e: "RedditVideo", dashUrl: `https://v.redd.it/${video[2]}/DASHPlaylist.mpd` };
1728
+ }
1729
+ }
1730
+ }
1731
+ if (ids.some((key) => !metadata[key]))
1732
+ return null;
1733
+ const body = [...html.matchAll(/<div\s([^>]+)>/gi)].find((tag) => attributes(tag[1]).class?.split(/\s+/).includes("usertext-body"));
1734
+ const text = body ? divContent(html, body.index + body[0].length) : null;
1735
+ const timestamp = finiteNumber(post["data-timestamp"]);
1736
+ return {
1737
+ id,
1738
+ title: plainText(title),
1739
+ selftext: text === null ? undefined : plainText(text.replace(/<a\s[^>]*>https:\/\/(?:www\.)?reddit\.com\/link\/[a-z0-9]+\/video\/[a-z0-9]+\/player<\/a>/gi, "")),
1740
+ author: post["data-author"],
1741
+ subreddit: post["data-subreddit"],
1742
+ permalink: post["data-permalink"],
1743
+ created_utc: timestamp === undefined ? undefined : Math.floor(timestamp / 1000),
1744
+ ups: finiteNumber(post["data-score"]),
1745
+ score: finiteNumber(post["data-score"]),
1746
+ num_comments: finiteNumber(post["data-comments-count"]),
1747
+ over_18: post["data-nsfw"] === "true" ? true : post["data-nsfw"] === "false" ? false : undefined,
1748
+ gallery_data: { items: ids.map((media_id) => ({ media_id })) },
1749
+ media_metadata: metadata
1750
+ };
1751
+ }
1752
+ return null;
1753
+ }
1696
1754
  async function fetchRssPost(net, id) {
1697
1755
  const url = `https://www.reddit.com/comments/${id}/.rss?limit=1`;
1698
1756
  const headers = { "user-agent": browserUserAgent(), accept: "application/atom+xml" };
@@ -1852,18 +1910,23 @@ function postText(html, id) {
1852
1910
  const attrs = attributes(match[1]);
1853
1911
  if (attrs.id !== `t3_${id}-post-rtjson-content`)
1854
1912
  continue;
1855
- const start = match.index + match[0].length;
1856
- const tags = /<div\b[^>]*>|<\/div\s*>/gi;
1857
- tags.lastIndex = start;
1858
- let depth = 1;
1859
- for (let tag = tags.exec(html);tag; tag = tags.exec(html)) {
1860
- depth += tag[0].startsWith("</") ? -1 : 1;
1861
- if (depth === 0)
1862
- return plainText(html.slice(start, tag.index));
1863
- }
1913
+ const content = divContent(html, match.index + match[0].length);
1914
+ if (content !== null)
1915
+ return plainText(content);
1864
1916
  }
1865
1917
  return;
1866
1918
  }
1919
+ function divContent(html, start) {
1920
+ const tags = /<div\b[^>]*>|<\/div\s*>/gi;
1921
+ tags.lastIndex = start;
1922
+ let depth = 1;
1923
+ for (let tag = tags.exec(html);tag; tag = tags.exec(html)) {
1924
+ depth += tag[0].startsWith("</") ? -1 : 1;
1925
+ if (depth === 0)
1926
+ return html.slice(start, tag.index);
1927
+ }
1928
+ return null;
1929
+ }
1867
1930
  function plainText(html) {
1868
1931
  return decodeHtml3(html.replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, "").replace(/<\/?(?:p|div|li|blockquote|pre|h[1-6])\b[^>]*>|<br\b[^>]*>/gi, `
1869
1932
  `).replace(/<[^>]*>/g, "")).replace(/[\t ]+/g, " ").replace(/ *\n */g, `
@@ -2004,6 +2067,9 @@ async function fetchPost(net, token, id, url, page) {
2004
2067
  const post2 = await fetchWebPost(net, id, url, page).catch(() => null);
2005
2068
  if (post2)
2006
2069
  return post2;
2070
+ const embedded = await fetchEmbeddedPost(net, id).catch(() => null);
2071
+ if (embedded)
2072
+ return embedded;
2007
2073
  const rss = await fetchRssPost(net, id).catch(() => null);
2008
2074
  if (rss)
2009
2075
  return rss;
@@ -2034,7 +2100,7 @@ async function mediaItems3(input, post, id) {
2034
2100
  return linkPreviewItems(post, id).concat(parent ? linkPreviewItems(parent, id) : []).slice(0, 1);
2035
2101
  }
2036
2102
  async function extract(input, post, id) {
2037
- const gallery = galleryItems(post, id);
2103
+ const gallery = await galleryMedia(input, post, id);
2038
2104
  if (gallery.length > 0) {
2039
2105
  return gallery;
2040
2106
  }
@@ -2044,7 +2110,28 @@ async function extract(input, post, id) {
2044
2110
  }
2045
2111
  return imageItems2(post, id);
2046
2112
  }
2047
- function galleryItems(post, id) {
2113
+ async function galleryMedia(input, post, id) {
2114
+ const metadata = object(post.media_metadata) ? post.media_metadata : {};
2115
+ if (!Object.values(metadata).some((meta) => object(meta) && meta.e === "RedditVideo"))
2116
+ return galleryItems(post, id);
2117
+ const data = object(post.gallery_data) ? post.gallery_data : null;
2118
+ const order = data && Array.isArray(data.items) ? data.items.filter(object) : [];
2119
+ const items = [];
2120
+ for (const [index, entry] of order.entries()) {
2121
+ const key = string(entry.media_id);
2122
+ const meta = key && object(metadata[key]) ? metadata[key] : null;
2123
+ if (meta?.e === "RedditVideo") {
2124
+ const videos = await videoItems(input, { secure_media: { reddit_video: { dash_url: meta.dashUrl } } }, id);
2125
+ if (!videos.length)
2126
+ throw new Error("Reddit gallery video unavailable");
2127
+ items.push(...videos.map((video) => ({ ...video, filename: filename(`reddit_${id}_${index + 1}.mp4`) })));
2128
+ } else {
2129
+ items.push(...galleryItems({ ...post, gallery_data: { items: [entry] } }, id, index + 1));
2130
+ }
2131
+ }
2132
+ return items;
2133
+ }
2134
+ function galleryItems(post, id, startIndex = 1) {
2048
2135
  const data = object(post.gallery_data) ? post.gallery_data : null;
2049
2136
  const order = data && Array.isArray(data.items) ? data.items.filter(object) : [];
2050
2137
  const metadata = object(post.media_metadata) ? post.media_metadata : {};
@@ -2057,10 +2144,10 @@ function galleryItems(post, id) {
2057
2144
  }
2058
2145
  const video = string(best.mp4);
2059
2146
  if (video) {
2060
- return [mediaItem(id, index + 1, "video", "video/mp4", video)];
2147
+ return [mediaItem(id, index + startIndex, "video", "video/mp4", video)];
2061
2148
  }
2062
2149
  const image = string(best.u) ?? string(best.gif);
2063
- return image ? [mediaItem(id, index + 1, "image", mimeOf(string(meta?.m), image), image)] : [];
2150
+ return image ? [mediaItem(id, index + startIndex, "image", mimeOf(string(meta?.m), image), image)] : [];
2064
2151
  });
2065
2152
  }
2066
2153
  async function videoItems(input, post, id) {
@@ -3398,6 +3485,7 @@ async function resolveTwitter(input) {
3398
3485
  throw new Error("Tweet is unavailable or age-restricted");
3399
3486
  }
3400
3487
  await expandTwitterText(input.net, tweet, id);
3488
+ await Promise.all(twitterTweets(tweet, id, true).map(({ id: id2, tweet: tweet2 }) => translateTwitterText(input, tweet2, id2)));
3401
3489
  const posts = twitterTweets(tweet, id);
3402
3490
  const groups = posts.flatMap(({ id: postId2, tweet: post }) => {
3403
3491
  const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
@@ -3421,6 +3509,11 @@ function twitterMetadata(tweet) {
3421
3509
  if (lang) {
3422
3510
  extra.lang = lang;
3423
3511
  }
3512
+ const translated = string(tweet.translated_text);
3513
+ if (translated) {
3514
+ extra.originalText = cleanTwitterText(twitterText(tweet), tweet) ?? undefined;
3515
+ extra.translatedLang = string(tweet.translated_lang) ?? undefined;
3516
+ }
3424
3517
  const quotedTweet = object(tweet.quoted_tweet) ? tweet.quoted_tweet : null;
3425
3518
  const quotedId = quotedTweet ? string(quotedTweet.id_str) : null;
3426
3519
  if (quotedTweet && quotedId) {
@@ -3435,7 +3528,7 @@ function twitterMetadata(tweet) {
3435
3528
  extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
3436
3529
  }
3437
3530
  return {
3438
- text: cleanTwitterText(twitterText(tweet), tweet) ?? undefined,
3531
+ text: cleanTwitterText(translated ?? twitterText(tweet), tweet) ?? undefined,
3439
3532
  author: user ? {
3440
3533
  handle: string(user.screen_name) ?? undefined,
3441
3534
  name: string(user.name) ?? undefined,
@@ -3482,6 +3575,43 @@ function needsFullText(tweet) {
3482
3575
  }
3483
3576
  return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
3484
3577
  }
3578
+ async function translateTwitterText(input, tweet, id) {
3579
+ const languages = input.languages;
3580
+ if (!languages?.length || !twitterText(tweet))
3581
+ return;
3582
+ const lang = string(tweet.lang)?.toLowerCase();
3583
+ const allowed = (source2) => languages.some((language) => source2 === language || source2.startsWith(`${language}-`));
3584
+ if (lang && allowed(lang))
3585
+ return;
3586
+ for (const target of new Set([languages[0], "en"])) {
3587
+ try {
3588
+ const response = await input.net(`https://api.fxtwitter.com/status/${id}/${target}`, {
3589
+ headers: { accept: "application/json", "user-agent": browserUserAgent() }
3590
+ }, 1);
3591
+ if (!response.ok)
3592
+ continue;
3593
+ const payload = await response.json();
3594
+ if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id)
3595
+ continue;
3596
+ const post = payload.tweet;
3597
+ const translation = object(post.translation) ? post.translation : null;
3598
+ const source2 = (string(post.lang) ?? string(translation?.source_lang))?.toLowerCase();
3599
+ if (source2 && source2 !== "und") {
3600
+ tweet.lang = source2;
3601
+ if (allowed(source2))
3602
+ return;
3603
+ }
3604
+ const text2 = translation ? string(translation.text) : null;
3605
+ if (!text2?.trim() || string(translation?.target_lang)?.toLowerCase() !== target)
3606
+ continue;
3607
+ if (text2.includes("You're using an older version of DeepL."))
3608
+ continue;
3609
+ tweet.translated_text = text2;
3610
+ tweet.translated_lang = target;
3611
+ return;
3612
+ } catch {}
3613
+ }
3614
+ }
3485
3615
  async function expandTwitterText(net, tweet, rootId) {
3486
3616
  const posts = new Map(twitterTweets(tweet, rootId, true).map(({ id, tweet: post }) => [id, post]));
3487
3617
  for (const [id, post] of posts) {
@@ -3702,6 +3832,7 @@ async function twitterComments(input, id) {
3702
3832
  if (!object(payload) || payload.code !== 200 || !Array.isArray(payload.replies))
3703
3833
  throw new Error("Invalid comments");
3704
3834
  const previousCount = comments.length;
3835
+ const page = [];
3705
3836
  for (const reply of payload.replies) {
3706
3837
  if (!object(reply))
3707
3838
  continue;
@@ -3710,10 +3841,16 @@ async function twitterComments(input, id) {
3710
3841
  if (!replyId || parent !== id || seen.has(replyId))
3711
3842
  continue;
3712
3843
  seen.add(replyId);
3713
- comments.push(twitterComment(reply, replyId));
3714
- if (comments.length >= limit)
3715
- return comments;
3844
+ page.push({ id: replyId, reply });
3845
+ if (comments.length + page.length >= limit)
3846
+ break;
3716
3847
  }
3848
+ comments.push(...await Promise.all(page.map(async ({ id: id2, reply }) => {
3849
+ await translateTwitterText(input, reply, id2);
3850
+ return twitterComment(reply, id2);
3851
+ })));
3852
+ if (comments.length >= limit)
3853
+ return comments;
3717
3854
  cursor = object(payload.cursor) ? string(payload.cursor.bottom) : null;
3718
3855
  if (!cursor || cursors.has(cursor) || comments.length === previousCount)
3719
3856
  break;
@@ -3748,7 +3885,7 @@ function twitterComment(reply, id) {
3748
3885
  url: `https://x.com/i/status/${id}`,
3749
3886
  items,
3750
3887
  metadata: {
3751
- text: cleanTwitterText(string(reply.text), reply) ?? undefined,
3888
+ text: cleanTwitterText(string(reply.translated_text) ?? string(reply.text), reply) ?? undefined,
3752
3889
  author: {
3753
3890
  handle: string(author2.screen_name) ?? undefined,
3754
3891
  name: string(author2.name) ?? undefined,
@@ -3775,6 +3912,7 @@ async function postfetch(url, options = {}) {
3775
3912
  }
3776
3913
  const context = {
3777
3914
  net: createNet(options.fetch ?? globalThis.fetch),
3915
+ languages: options.languages === undefined ? undefined : normalizeLanguages(options.languages),
3778
3916
  comments: options.comments,
3779
3917
  preferredWidth: options.preferredWidth ?? 720,
3780
3918
  tryMaxBytes: options.tryMaxBytes,
@@ -3783,6 +3921,12 @@ async function postfetch(url, options = {}) {
3783
3921
  const result = await resolve(context);
3784
3922
  return options.tryMaxBytes === undefined || result.platform === "twitter" ? result : trySmaller(context, result, options.tryMaxBytes);
3785
3923
  }
3924
+ function normalizeLanguages(languages) {
3925
+ if (!Array.isArray(languages) || languages.length === 0 || languages.some((lang) => typeof lang !== "string" || !/^[a-z]{2,3}(?:-[a-z0-9]{2,8})*$/i.test(lang))) {
3926
+ throw new PostfetchError(400, "languages must be a non-empty list of language codes");
3927
+ }
3928
+ return [...new Set(languages.map((lang) => lang.toLowerCase()))];
3929
+ }
3786
3930
  async function resolve(context) {
3787
3931
  switch (detect(context.url)) {
3788
3932
  case "facebook":
@@ -112,6 +112,10 @@ export type TwitterExtra = {
112
112
  replyToId?: string;
113
113
  /** BCP 47 language tag detected for the tweet. */
114
114
  lang?: string;
115
+ /** Original text retained when a requested translation is used. */
116
+ originalText?: string;
117
+ /** Language of the translated text. `lang` still describes the original. */
118
+ translatedLang?: string;
115
119
  /** The post quoted by this post, when X includes it in the resolved payload. */
116
120
  quotedTweet?: TwitterQuotedTweet;
117
121
  /** The parent this post replies to, when included by X. Media is not collected. */
@@ -208,6 +212,7 @@ export type Net = (url: string, init?: RequestInit, attempts?: number) => Promis
208
212
  export type ResolveContext = {
209
213
  net: Net;
210
214
  preferredWidth: number;
215
+ languages?: string[];
211
216
  comments?: number;
212
217
  /** Requested soft byte cap, for resolvers that can rank exact rendition sizes. */
213
218
  tryMaxBytes?: number;
@@ -5,6 +5,11 @@ export type PostfetchOptions = {
5
5
  fetch?: typeof fetch;
6
6
  /** Maximum direct comments to fetch for X and Reddit. Other platforms ignore this option. Failures return an empty array. */
7
7
  comments?: number;
8
+ /** Allowed X text languages, in preference order. Preserve allowed originals;
9
+ * otherwise translate to the first language, then English, then keep the original.
10
+ * Omit to keep original text. Applies to quotes, parents and requested replies too.
11
+ */
12
+ languages?: string[];
8
13
  /** Preferred media width in pixels; the closest available rendition is chosen. Defaults to `720`. */
9
14
  preferredWidth?: number;
10
15
  /**
@@ -9,4 +9,5 @@ export type RedditPage = {
9
9
  export declare function readWebPage(response: Response, userAgent: string): Promise<RedditPage>;
10
10
  export declare function fetchWebPage(net: Net, input: string, userAgent: string, cookies?: string[]): Promise<RedditPage>;
11
11
  export declare function fetchWebPost(net: Net, id: string, input: string, initial?: RedditPage): Promise<Json | null>;
12
+ export declare function fetchEmbeddedPost(net: Net, id: string): Promise<Json | null>;
12
13
  export declare function fetchRssPost(net: Net, id: string): Promise<Json | null>;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@postfetch/core",
3
- "version": "0.13.9",
3
+ "version": "0.14.1",
4
4
  "description": "Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, Spotify, TikTok, X and YouTube post URLs into media files.",
5
5
  "license": "MIT",
6
6
  "type": "module",
package/src/internal.ts CHANGED
@@ -113,6 +113,10 @@ export type TwitterExtra = {
113
113
  replyToId?: string;
114
114
  /** BCP 47 language tag detected for the tweet. */
115
115
  lang?: string;
116
+ /** Original text retained when a requested translation is used. */
117
+ originalText?: string;
118
+ /** Language of the translated text. `lang` still describes the original. */
119
+ translatedLang?: string;
116
120
  /** The post quoted by this post, when X includes it in the resolved payload. */
117
121
  quotedTweet?: TwitterQuotedTweet;
118
122
  /** The parent this post replies to, when included by X. Media is not collected. */
@@ -228,6 +232,7 @@ export type Net = (url: string, init?: RequestInit, attempts?: number) => Promis
228
232
  export type ResolveContext = {
229
233
  net: Net;
230
234
  preferredWidth: number;
235
+ languages?: string[];
231
236
  comments?: number;
232
237
  /** Requested soft byte cap, for resolvers that can rank exact rendition sizes. */
233
238
  tryMaxBytes?: number;
package/src/postfetch.ts CHANGED
@@ -16,6 +16,11 @@ export type PostfetchOptions = {
16
16
  fetch?: typeof fetch;
17
17
  /** Maximum direct comments to fetch for X and Reddit. Other platforms ignore this option. Failures return an empty array. */
18
18
  comments?: number;
19
+ /** Allowed X text languages, in preference order. Preserve allowed originals;
20
+ * otherwise translate to the first language, then English, then keep the original.
21
+ * Omit to keep original text. Applies to quotes, parents and requested replies too.
22
+ */
23
+ languages?: string[];
19
24
  /** Preferred media width in pixels; the closest available rendition is chosen. Defaults to `720`. */
20
25
  preferredWidth?: number;
21
26
  /**
@@ -61,6 +66,7 @@ export async function postfetch(url: string, options: PostfetchOptions = {}): Pr
61
66
  }
62
67
  const context: ResolveContext = {
63
68
  net: createNet(options.fetch ?? globalThis.fetch),
69
+ languages: options.languages === undefined ? undefined : normalizeLanguages(options.languages),
64
70
  comments: options.comments,
65
71
  preferredWidth: options.preferredWidth ?? 720,
66
72
  tryMaxBytes: options.tryMaxBytes,
@@ -72,6 +78,13 @@ export async function postfetch(url: string, options: PostfetchOptions = {}): Pr
72
78
  : trySmaller(context, result, options.tryMaxBytes);
73
79
  }
74
80
 
81
+ function normalizeLanguages(languages: string[]): string[] {
82
+ if (!Array.isArray(languages) || languages.length === 0 || languages.some((lang) => typeof lang !== "string" || !/^[a-z]{2,3}(?:-[a-z0-9]{2,8})*$/i.test(lang))) {
83
+ throw new PostfetchError(400, "languages must be a non-empty list of language codes");
84
+ }
85
+ return [...new Set(languages.map((lang) => lang.toLowerCase()))];
86
+ }
87
+
75
88
  async function resolve(context: ResolveContext): Promise<PostfetchResult> {
76
89
  switch (detect(context.url)) {
77
90
  case "facebook":
package/src/reddit-web.ts CHANGED
@@ -79,6 +79,59 @@ export async function fetchWebPost(net: Net, id: string, input: string, initial?
79
79
  return null;
80
80
  }
81
81
 
82
+ // The embedded view still serves legacy gallery markup when the main page is
83
+ // a CAPTCHA. Its media-id list includes videos that have no gallery thumbnail.
84
+ export async function fetchEmbeddedPost(net: Net, id: string): Promise<Json | null> {
85
+ const page = await fetchWebPage(net, `https://www.reddit.com/comments/${id}?embedded=true`, browserUserAgent());
86
+ if (!page.ok) return null;
87
+ for (const match of page.html.matchAll(/<div\s([^>]+)>/gi)) {
88
+ const post = attributes(match[1]);
89
+ if (post.id !== `thing_t3_${id}` || post["data-fullname"] !== `t3_${id}` || post["data-is-gallery"] !== "true") continue;
90
+ const html = divContent(page.html, match.index + match[0].length);
91
+ if (html === null) return null;
92
+ const title = [...html.matchAll(/<a\s([^>]+)>([\s\S]*?)<\/a>/gi)]
93
+ .find((link) => attributes(link[1])["data-event-action"] === "title")?.[2];
94
+ const preview = [...html.matchAll(/<div\s([^>]+)>/gi)]
95
+ .map((tag) => attributes(tag[1])).find((attrs) => attrs.id === `media-preview-${id}`);
96
+ const ids = preview?.["data-media-ids"]?.split(",") ?? [];
97
+ if (!title || !ids.length || ids.some((key) => !/^[a-z0-9]+$/i.test(key)) || new Set(ids).size !== ids.length) return null;
98
+ const metadata: Record<string, Json> = {};
99
+ for (const link of html.matchAll(/<a\s([^>]+)>/gi)) {
100
+ const attrs = attributes(link[1]);
101
+ const url = new URL(attrs.href ?? "", "https://www.reddit.com");
102
+ if (url.protocol !== "https:") continue;
103
+ if (attrs.class?.split(/\s+/).includes("gallery-item-thumbnail-link")) {
104
+ const index = Number(attrs["data-position"]) - 1;
105
+ const key = ids[index];
106
+ // Only the gallery's full-size link identifies an original. Never
107
+ // promote a tile, cover, comment image, or recommendation to an item.
108
+ if (!Number.isSafeInteger(index) || !key || !/^(?:preview|i)\.redd\.it$/.test(url.hostname)
109
+ || !new RegExp(`^/${key}\\.(?:png|jpe?g|webp)$`, "i").test(url.pathname)) continue;
110
+ metadata[key] = { s: { u: `https://i.redd.it${url.pathname}` } };
111
+ } else if (/^(?:www\.)?reddit\.com$/.test(url.hostname)) {
112
+ const video = url.pathname.match(/^\/link\/([a-z0-9]+)\/video\/([a-z0-9]+)\/player\/?$/i);
113
+ if (video?.[1] === id && ids.includes(video[2])) {
114
+ metadata[video[2]] = { e: "RedditVideo", dashUrl: `https://v.redd.it/${video[2]}/DASHPlaylist.mpd` };
115
+ }
116
+ }
117
+ }
118
+ if (ids.some((key) => !metadata[key])) return null;
119
+ const body = [...html.matchAll(/<div\s([^>]+)>/gi)]
120
+ .find((tag) => attributes(tag[1]).class?.split(/\s+/).includes("usertext-body"));
121
+ const text = body ? divContent(html, body.index + body[0].length) : null;
122
+ const timestamp = finiteNumber(post["data-timestamp"]);
123
+ return {
124
+ id, title: plainText(title), selftext: text === null ? undefined : plainText(text.replace(/<a\s[^>]*>https:\/\/(?:www\.)?reddit\.com\/link\/[a-z0-9]+\/video\/[a-z0-9]+\/player<\/a>/gi, "")),
125
+ author: post["data-author"], subreddit: post["data-subreddit"], permalink: post["data-permalink"],
126
+ created_utc: timestamp === undefined ? undefined : Math.floor(timestamp / 1000),
127
+ ups: finiteNumber(post["data-score"]), score: finiteNumber(post["data-score"]), num_comments: finiteNumber(post["data-comments-count"]),
128
+ over_18: post["data-nsfw"] === "true" ? true : post["data-nsfw"] === "false" ? false : undefined,
129
+ gallery_data: { items: ids.map((media_id) => ({ media_id })) }, media_metadata: metadata,
130
+ };
131
+ }
132
+ return null;
133
+ }
134
+
82
135
  // The public Atom feed remains usable on some routes that reject OAuth and
83
136
  // browser pages. Match the root entry explicitly: replies have their own media.
84
137
  export async function fetchRssPost(net: Net, id: string): Promise<Json | null> {
@@ -222,18 +275,23 @@ function postText(html: string, id: string): string | undefined {
222
275
  for (const match of html.matchAll(/<div\s([^>]+)>/gi)) {
223
276
  const attrs = attributes(match[1]);
224
277
  if (attrs.id !== `t3_${id}-post-rtjson-content`) continue;
225
- const start = match.index + match[0].length;
226
- const tags = /<div\b[^>]*>|<\/div\s*>/gi;
227
- tags.lastIndex = start;
228
- let depth = 1;
229
- for (let tag = tags.exec(html); tag; tag = tags.exec(html)) {
230
- depth += tag[0].startsWith("</") ? -1 : 1;
231
- if (depth === 0) return plainText(html.slice(start, tag.index));
232
- }
278
+ const content = divContent(html, match.index + match[0].length);
279
+ if (content !== null) return plainText(content);
233
280
  }
234
281
  return undefined;
235
282
  }
236
283
 
284
+ function divContent(html: string, start: number): string | null {
285
+ const tags = /<div\b[^>]*>|<\/div\s*>/gi;
286
+ tags.lastIndex = start;
287
+ let depth = 1;
288
+ for (let tag = tags.exec(html); tag; tag = tags.exec(html)) {
289
+ depth += tag[0].startsWith("</") ? -1 : 1;
290
+ if (depth === 0) return html.slice(start, tag.index);
291
+ }
292
+ return null;
293
+ }
294
+
237
295
  function plainText(html: string): string {
238
296
  return decodeHtml(html
239
297
  .replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, "")
package/src/reddit.ts CHANGED
@@ -17,7 +17,7 @@ import {
17
17
  type MediaItem,
18
18
  } from "./internal";
19
19
  import { browserUserAgent } from "./fingerprint";
20
- import { fetchWebPost, fetchRssPost, readWebPage, type RedditPage } from "./reddit-web";
20
+ import { fetchWebPost, fetchEmbeddedPost, fetchRssPost, readWebPage, type RedditPage } from "./reddit-web";
21
21
 
22
22
  // The installed-app token can still be issued while both OAuth post routes
23
23
  // reject it with Reddit's network-security page. Keep the public page as a
@@ -116,6 +116,8 @@ async function fetchPost(net: Net, token: string, id: string, url: string, page?
116
116
  // blocked. Keep the original failure if the page cannot supply the post.
117
117
  const post = await fetchWebPost(net, id, url, page).catch(() => null);
118
118
  if (post) return post;
119
+ const embedded = await fetchEmbeddedPost(net, id).catch(() => null);
120
+ if (embedded) return embedded;
119
121
  const rss = await fetchRssPost(net, id).catch(() => null);
120
122
  if (rss) return rss;
121
123
  }
@@ -147,7 +149,7 @@ async function mediaItems(input: ResolveContext, post: Json, id: string): Promis
147
149
  }
148
150
 
149
151
  async function extract(input: ResolveContext, post: Json, id: string): Promise<MediaItem[]> {
150
- const gallery = galleryItems(post, id);
152
+ const gallery = await galleryMedia(input, post, id);
151
153
  if (gallery.length > 0) {
152
154
  return gallery;
153
155
  }
@@ -158,7 +160,28 @@ async function extract(input: ResolveContext, post: Json, id: string): Promise<M
158
160
  return imageItems(post, id);
159
161
  }
160
162
 
161
- function galleryItems(post: Json, id: string): MediaItem[] {
163
+ async function galleryMedia(input: ResolveContext, post: Json, id: string): Promise<MediaItem[]> {
164
+ const metadata = object(post.media_metadata) ? post.media_metadata : {};
165
+ if (!Object.values(metadata).some((meta) => object(meta) && meta.e === "RedditVideo")) return galleryItems(post, id);
166
+ const data = object(post.gallery_data) ? post.gallery_data : null;
167
+ const order = data && Array.isArray(data.items) ? data.items.filter(object) : [];
168
+ const items: MediaItem[] = [];
169
+ for (const [index, entry] of order.entries()) {
170
+ const key = string(entry.media_id);
171
+ const meta = key && object(metadata[key]) ? metadata[key] : null;
172
+ if (meta?.e === "RedditVideo") {
173
+ const videos = await videoItems(input, { secure_media: { reddit_video: { dash_url: meta.dashUrl } } }, id);
174
+ if (!videos.length) throw new Error("Reddit gallery video unavailable");
175
+ items.push(...videos.map((video) => ({ ...video, filename: filename(`reddit_${id}_${index + 1}.mp4`) })));
176
+ } else {
177
+ // Retain the original position in filenames when videos split the images.
178
+ items.push(...galleryItems({ ...post, gallery_data: { items: [entry] } }, id, index + 1));
179
+ }
180
+ }
181
+ return items;
182
+ }
183
+
184
+ function galleryItems(post: Json, id: string, startIndex = 1): MediaItem[] {
162
185
  const data = object(post.gallery_data) ? post.gallery_data : null;
163
186
  const order = data && Array.isArray(data.items) ? data.items.filter(object) : [];
164
187
  const metadata = object(post.media_metadata) ? post.media_metadata : {};
@@ -171,10 +194,10 @@ function galleryItems(post: Json, id: string): MediaItem[] {
171
194
  }
172
195
  const video = string(best.mp4);
173
196
  if (video) {
174
- return [mediaItem(id, index + 1, "video", "video/mp4", video)];
197
+ return [mediaItem(id, index + startIndex, "video", "video/mp4", video)];
175
198
  }
176
199
  const image = string(best.u) ?? string(best.gif);
177
- return image ? [mediaItem(id, index + 1, "image", mimeOf(string(meta?.m), image), image)] : [];
200
+ return image ? [mediaItem(id, index + startIndex, "image", mimeOf(string(meta?.m), image), image)] : [];
178
201
  });
179
202
  }
180
203
 
package/src/twitter.ts CHANGED
@@ -33,6 +33,7 @@ export async function resolveTwitter(input: ResolveContext): Promise<PostfetchRe
33
33
  throw new Error("Tweet is unavailable or age-restricted");
34
34
  }
35
35
  await expandTwitterText(input.net, tweet, id);
36
+ await Promise.all(twitterTweets(tweet, id, true).map(({ id, tweet }) => translateTwitterText(input, tweet, id)));
36
37
  const posts = twitterTweets(tweet, id);
37
38
  const groups = posts.flatMap(({ id: postId, tweet: post }) => {
38
39
  const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
@@ -61,6 +62,11 @@ export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterEx
61
62
  if (lang) {
62
63
  extra.lang = lang;
63
64
  }
65
+ const translated = string(tweet.translated_text);
66
+ if (translated) {
67
+ extra.originalText = cleanTwitterText(twitterText(tweet), tweet) ?? undefined;
68
+ extra.translatedLang = string(tweet.translated_lang) ?? undefined;
69
+ }
64
70
  const quotedTweet = object(tweet.quoted_tweet) ? tweet.quoted_tweet : null;
65
71
  const quotedId = quotedTweet ? string(quotedTweet.id_str) : null;
66
72
  if (quotedTweet && quotedId) {
@@ -74,7 +80,7 @@ export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterEx
74
80
  extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
75
81
  }
76
82
  return {
77
- text: cleanTwitterText(twitterText(tweet), tweet) ?? undefined,
83
+ text: cleanTwitterText(translated ?? twitterText(tweet), tweet) ?? undefined,
78
84
  author: user
79
85
  ? {
80
86
  handle: string(user.screen_name) ?? undefined,
@@ -130,6 +136,42 @@ function needsFullText(tweet: Json): boolean {
130
136
  return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
131
137
  }
132
138
 
139
+ async function translateTwitterText(input: ResolveContext, tweet: Json, id: string): Promise<void> {
140
+ const languages = input.languages;
141
+ if (!languages?.length || !twitterText(tweet)) return;
142
+ const lang = string(tweet.lang)?.toLowerCase();
143
+ const allowed = (source: string) => languages.some((language) => source === language || source.startsWith(`${language}-`));
144
+ if (lang && allowed(lang)) return;
145
+ for (const target of new Set([languages[0], "en"])) {
146
+ try {
147
+ const response = await input.net(`https://api.fxtwitter.com/status/${id}/${target}`, {
148
+ headers: { accept: "application/json", "user-agent": browserUserAgent() },
149
+ }, 1);
150
+ if (!response.ok) continue;
151
+ const payload: unknown = await response.json();
152
+ if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id) continue;
153
+ const post = payload.tweet;
154
+ const translation = object(post.translation) ? post.translation : null;
155
+ const source = (string(post.lang) ?? string(translation?.source_lang))?.toLowerCase();
156
+ // The richer lookup may identify a language omitted by syndication.
157
+ if (source && source !== "und") {
158
+ tweet.lang = source;
159
+ if (allowed(source)) return;
160
+ }
161
+ const text = translation ? string(translation.text) : null;
162
+ if (!text?.trim() || string(translation?.target_lang)?.toLowerCase() !== target) continue;
163
+ // FxTwitter can return DeepL's client-upgrade error as a successful
164
+ // translation. It is an upstream failure, not the author's post text.
165
+ if (text.includes("You're using an older version of DeepL.")) continue;
166
+ tweet.translated_text = text;
167
+ tweet.translated_lang = target;
168
+ return;
169
+ } catch {
170
+ // Translation is optional; a failed lookup must not lose the post.
171
+ }
172
+ }
173
+ }
174
+
133
175
  async function expandTwitterText(net: Net, tweet: Json, rootId: string): Promise<void> {
134
176
  const posts = new Map(twitterTweets(tweet, rootId, true).map(({ id, tweet: post }) => [id, post]));
135
177
  for (const [id, post] of posts) {
@@ -368,15 +410,21 @@ async function twitterComments(input: ResolveContext, id: string): Promise<PostC
368
410
  const payload: unknown = await response.json();
369
411
  if (!object(payload) || payload.code !== 200 || !Array.isArray(payload.replies)) throw new Error("Invalid comments");
370
412
  const previousCount = comments.length;
413
+ const page: Array<{ id: string; reply: Json }> = [];
371
414
  for (const reply of payload.replies) {
372
415
  if (!object(reply)) continue;
373
416
  const replyId = string(reply.id);
374
417
  const parent = object(reply.replying_to) ? string(reply.replying_to.status) : null;
375
418
  if (!replyId || parent !== id || seen.has(replyId)) continue;
376
419
  seen.add(replyId);
377
- comments.push(twitterComment(reply, replyId));
378
- if (comments.length >= limit) return comments;
420
+ page.push({ id: replyId, reply });
421
+ if (comments.length + page.length >= limit) break;
379
422
  }
423
+ comments.push(...await Promise.all(page.map(async ({ id, reply }) => {
424
+ await translateTwitterText(input, reply, id);
425
+ return twitterComment(reply, id);
426
+ })));
427
+ if (comments.length >= limit) return comments;
380
428
  cursor = object(payload.cursor) ? string(payload.cursor.bottom) : null;
381
429
  if (!cursor || cursors.has(cursor) || comments.length === previousCount) break;
382
430
  cursors.add(cursor);
@@ -410,7 +458,7 @@ function twitterComment(reply: Json, id: string): PostComment {
410
458
  url: `https://x.com/i/status/${id}`,
411
459
  items,
412
460
  metadata: {
413
- text: cleanTwitterText(string(reply.text), reply) ?? undefined,
461
+ text: cleanTwitterText(string(reply.translated_text) ?? string(reply.text), reply) ?? undefined,
414
462
  author: {
415
463
  handle: string(author.screen_name) ?? undefined,
416
464
  name: string(author.name) ?? undefined,