@postfetch/core 0.13.8 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -59,9 +59,22 @@ const blob = await buildAudioSliderVideo(result.items, { delay: 3000 });
59
59
  form.append("video", blob, "slideshow.mp4");
60
60
  ```
61
61
 
62
- - `PostfetchOptions` — `{ fetch?: typeof fetch; preferredWidth?: number; tryMaxBytes?: number }`. `tryMaxBytes` is a soft byte cap. X probes every MP4 variant and selects the highest-quality complete result that fits, falling back to its smallest variants when none do. Other platforms return a smaller available rendition when possible and otherwise keep the normal result. Inject `fetch` to unit-test resolvers offline.
62
+ - `PostfetchOptions` — `{ fetch?: typeof fetch; preferredWidth?: number; tryMaxBytes?: number; languages?: string[] }`. `tryMaxBytes` is a soft byte cap. X probes every MP4 variant and selects the highest-quality complete result that fits, falling back to its smallest variants when none do. Other platforms return a smaller available rendition when possible and otherwise keep the normal result. Inject `fetch` to unit-test resolvers offline.
63
63
  - Every request rotates a consistent browser/app fingerprint, so a fixed user-agent never gets the whole fleet blocked.
64
64
  - No runtime dependencies, no `yt-dlp`, no cookies. `ffmpeg` is only used when
65
65
  `downloadBlob(..., { remux: true })` or `buildAudioSliderVideo` is requested.
66
66
 
67
67
  See the [root README](https://github.com/chelokot/postfetch#readme) for the full API table, supported inputs and the fingerprint design.
68
+
69
+ ### X text languages
70
+
71
+ Pass `postfetch(url, { languages: ["en", "ru"] })` to keep X posts written in
72
+ any allowed language. Other posts are translated to the first language in the
73
+ list, falling back to English and then the original if translation is unavailable.
74
+ The ordered list must be non-empty; omit it to keep original text. This applies
75
+ independently to quoted posts, parent posts and requested replies. Regional
76
+ variants match a base language (`en` allows `en-US`). Other platforms ignore it.
77
+
78
+ `metadata.extra.lang` remains the detected original language. Translated posts
79
+ also expose `metadata.extra.originalText` and `metadata.extra.translatedLang`.
80
+ Translations use FxTwitter's public translation API and do not alter video audio.
package/dist/index.js CHANGED
@@ -1613,12 +1613,322 @@ function imageMime(url) {
1613
1613
  return "image/jpeg";
1614
1614
  }
1615
1615
 
1616
+ // src/reddit-web.ts
1617
+ async function readWebPage(response, userAgent) {
1618
+ const cookies = response.headers.getSetCookie().map((cookie) => cookie.split(";", 1)[0]);
1619
+ if (!response.ok) {
1620
+ await response.body?.cancel();
1621
+ return { url: response.url, html: "", userAgent, cookies, ok: false };
1622
+ }
1623
+ return { url: response.url, html: await response.text(), userAgent, cookies, ok: true };
1624
+ }
1625
+ async function fetchWebPage(net, input, userAgent, cookies = []) {
1626
+ let url = new URL(input);
1627
+ let jar = new Map(cookies.map((cookie) => [cookie.split("=", 1)[0], cookie]));
1628
+ for (let redirects = 0;redirects <= 5; redirects += 1) {
1629
+ if (url.protocol !== "https:" || !/^(?:(?:www\.|old\.|m\.)?reddit\.com|redd\.it)$/.test(url.hostname))
1630
+ throw new Error("Invalid Reddit page redirect");
1631
+ const headers = { "user-agent": userAgent, ...jar.size ? { cookie: [...jar.values()].join("; ") } : {} };
1632
+ const response = await net(url.href, { headers, redirect: "manual" }, 1);
1633
+ for (const cookie of response.headers.getSetCookie()) {
1634
+ const value = cookie.split(";", 1)[0];
1635
+ const name = value.split("=", 1)[0];
1636
+ if (/;\s*max-age=0(?:;|$)/i.test(cookie))
1637
+ jar.delete(name);
1638
+ else
1639
+ jar.set(name, value);
1640
+ }
1641
+ const location = response.headers.get("location");
1642
+ if ([301, 302, 303, 307, 308].includes(response.status) && location) {
1643
+ await response.body?.cancel();
1644
+ const target = new URL(location, url);
1645
+ if (target.origin !== url.origin)
1646
+ jar = new Map;
1647
+ url = target;
1648
+ continue;
1649
+ }
1650
+ return { ...await readWebPage(response, userAgent), url: response.url || url.href, cookies: [...jar.values()] };
1651
+ }
1652
+ throw new Error("Too many Reddit page redirects");
1653
+ }
1654
+ async function fetchWebPost(net, id, input, initial) {
1655
+ const url = new URL(input);
1656
+ if (url.protocol !== "https:" || !/^(?:(?:www\.|old\.|m\.)?reddit\.com|redd\.it)$/.test(url.hostname))
1657
+ return null;
1658
+ if (url.hostname !== "redd.it")
1659
+ url.hostname = "www.reddit.com";
1660
+ const userAgent = initial?.userAgent ?? browserUserAgent();
1661
+ let page = initial ?? await fetchWebPage(net, url.href, userAgent);
1662
+ if (!page.ok)
1663
+ return null;
1664
+ const challenge = challengeUrl(page.html, page.url);
1665
+ if (challenge) {
1666
+ page = await fetchWebPage(net, challenge, userAgent, page.cookies);
1667
+ if (!page.ok)
1668
+ return null;
1669
+ }
1670
+ for (const match of page.html.matchAll(/<shreddit-post\s([^>]+)>([\s\S]*?)<\/shreddit-post>/gi)) {
1671
+ const post = attributes(match[1]);
1672
+ if (post.id !== `t3_${id}` || !post["post-title"])
1673
+ continue;
1674
+ const media = postMedia(post, match[2], id);
1675
+ if (!media)
1676
+ return null;
1677
+ const created = Date.parse(post["created-timestamp"] ?? "");
1678
+ return {
1679
+ ...media,
1680
+ id,
1681
+ title: post["post-title"],
1682
+ selftext: postText(match[2], id),
1683
+ author: post.author,
1684
+ subreddit: post["subreddit-name"],
1685
+ permalink: post.permalink,
1686
+ created_utc: Number.isFinite(created) ? Math.floor(created / 1000) : undefined,
1687
+ ups: finiteNumber(post.score),
1688
+ score: finiteNumber(post.score),
1689
+ upvote_ratio: finiteNumber(post["upvote-ratio"]),
1690
+ num_comments: finiteNumber(post["comment-count"]),
1691
+ over_18: Object.hasOwn(post, "nsfw")
1692
+ };
1693
+ }
1694
+ return null;
1695
+ }
1696
+ async function fetchRssPost(net, id) {
1697
+ const url = `https://www.reddit.com/comments/${id}/.rss?limit=1`;
1698
+ const headers = { "user-agent": browserUserAgent(), accept: "application/atom+xml" };
1699
+ let response = await net(url, { headers }, 1);
1700
+ if (response.status === 429) {
1701
+ const reset = finiteNumber(response.headers.get("x-ratelimit-reset") ?? undefined);
1702
+ await response.body?.cancel();
1703
+ if (reset === undefined || reset < 0 || reset > 15)
1704
+ return null;
1705
+ await new Promise((resolve) => setTimeout(resolve, (reset + 0.1) * 1000));
1706
+ response = await net(url, { headers }, 1);
1707
+ }
1708
+ if (!response.ok) {
1709
+ await response.body?.cancel();
1710
+ return null;
1711
+ }
1712
+ const xml = await response.text();
1713
+ if (!/<feed\b/.test(xml))
1714
+ return null;
1715
+ for (const match of xml.matchAll(/<entry\b[^>]*>([\s\S]*?)<\/entry>/g)) {
1716
+ const entry = match[1];
1717
+ if (entry.match(/<id>([^<]+)<\/id>/)?.[1] !== `t3_${id}`)
1718
+ continue;
1719
+ const title = entry.match(/<title>([\s\S]*?)<\/title>/)?.[1];
1720
+ const content = decodeHtml3(entry.match(/<content\b[^>]*>([\s\S]*?)<\/content>/)?.[1] ?? "");
1721
+ const link = [...content.matchAll(/<a\s([^>]+)>\s*\[link\]\s*<\/a>/g)][0];
1722
+ const destination = link ? attributes(link[1]).href : undefined;
1723
+ const image = hostedImage(destination);
1724
+ let media = image ? { post_hint: "image", url: image } : null;
1725
+ if (!media && destination) {
1726
+ const target = new URL(destination);
1727
+ if (target.protocol === "https:" && target.hostname === "v.redd.it" && /^\/[a-z0-9]+\/?$/i.test(target.pathname)) {
1728
+ media = { secure_media: { reddit_video: { dash_url: target.href.replace(/\/$/, "") + "/DASHPlaylist.mpd" } } };
1729
+ } else if (/^https?:$/.test(target.protocol) && !/(^|\.)(reddit\.com|redd\.it)$/.test(target.hostname)) {
1730
+ const card = content.match(/<img\s([^>]+)>/);
1731
+ const preview = card ? attributes(card[1]).src : undefined;
1732
+ const previewUrl = preview ? new URL(preview) : null;
1733
+ media = {
1734
+ post_hint: "link",
1735
+ is_self: false,
1736
+ url: target.href,
1737
+ ...previewUrl?.protocol === "https:" && previewUrl.hostname === "external-preview.redd.it" ? { preview: { images: [{ source: { url: previewUrl.href } }] } } : {}
1738
+ };
1739
+ }
1740
+ }
1741
+ const body = content.match(/<!-- SC_OFF -->([\s\S]*?)<!-- SC_ON -->/)?.[1];
1742
+ if (!media && !destination && body)
1743
+ media = { is_self: true };
1744
+ if (!title || !media)
1745
+ return null;
1746
+ const permalink = [...entry.matchAll(/<link\s([^>]+)\/?\s*>/g)].map((match2) => attributes(match2[1]).href).find((href) => href?.includes(`/comments/${id}/`));
1747
+ const created = Date.parse(entry.match(/<published>([^<]+)<\/published>/)?.[1] ?? "");
1748
+ return {
1749
+ ...media,
1750
+ id,
1751
+ title: decodeHtml3(title),
1752
+ selftext: body ? plainText(body) : undefined,
1753
+ author: decodeHtml3(entry.match(/<name>([^<]+)<\/name>/)?.[1] ?? "").replace(/^\/u\//, "") || undefined,
1754
+ subreddit: attributes(entry.match(/<category\s([^>]+)>/)?.[1] ?? "").term,
1755
+ permalink: permalink ? new URL(permalink).pathname : undefined,
1756
+ created_utc: Number.isFinite(created) ? Math.floor(created / 1000) : undefined
1757
+ };
1758
+ }
1759
+ return null;
1760
+ }
1761
+ function postMedia(post, html, id) {
1762
+ if (post["post-type"] === "text")
1763
+ return { is_self: true };
1764
+ if (post["post-type"] === "image") {
1765
+ const url = hostedImage(post["content-href"]);
1766
+ return url ? { post_hint: "image", url } : null;
1767
+ }
1768
+ if (post["post-type"] === "gallery") {
1769
+ const gallery = html.match(/<(gallery-carousel|shreddit-gallery)\b[^>]*>([\s\S]*?)<\/\1>/i);
1770
+ if (!gallery)
1771
+ return null;
1772
+ const images = [];
1773
+ for (const match of gallery[2].matchAll(/<a\s([^>]+)>/gi)) {
1774
+ const href = attributes(match[1]).href;
1775
+ if (!href)
1776
+ continue;
1777
+ const target = new URL(href, "https://www.reddit.com");
1778
+ const media = target.origin === "https://www.reddit.com" && target.pathname === "/media" ? target.searchParams.get("url") : target.hostname === "i.redd.it" ? target.href : null;
1779
+ if (!media)
1780
+ continue;
1781
+ const original = hostedImage(media);
1782
+ if (!original)
1783
+ return null;
1784
+ if (!images.includes(original))
1785
+ images.push(original);
1786
+ }
1787
+ if (images.length < 2)
1788
+ return null;
1789
+ return {
1790
+ gallery_data: { items: images.map((_, index) => ({ media_id: String(index) })) },
1791
+ media_metadata: Object.fromEntries(images.map((url, index) => [String(index), { s: { u: url } }]))
1792
+ };
1793
+ }
1794
+ if (post["post-type"] === "link") {
1795
+ const url = new URL(post["content-href"] ?? "", "https://www.reddit.com");
1796
+ if (!/^https?:$/.test(url.protocol) || /(^|\.)(reddit\.com|redd\.it)$/.test(url.hostname))
1797
+ return null;
1798
+ const preview = linkPreview(html);
1799
+ return {
1800
+ post_hint: "link",
1801
+ is_self: false,
1802
+ url: url.href,
1803
+ ...preview ? { preview: { images: [{ source: { url: preview } }] } } : {}
1804
+ };
1805
+ }
1806
+ if (post["post-type"] === "video" || post["post-type"] === "gif") {
1807
+ const destination = new URL(post["content-href"] ?? "", "https://www.reddit.com");
1808
+ if (destination.protocol !== "https:" || destination.hostname !== "v.redd.it" || !/^\/[a-z0-9]+\/?$/i.test(destination.pathname))
1809
+ return null;
1810
+ for (const match of html.matchAll(/<shreddit-player\s([^>]+)>/gi)) {
1811
+ const player = attributes(match[1]);
1812
+ if (player["post-id"] !== `t3_${id}` || player["media-placement"] === "comment")
1813
+ continue;
1814
+ const playlist = new URL(player.src ?? "", destination);
1815
+ const base = destination.href.replace(/\/$/, "") + "/";
1816
+ if (playlist.origin !== destination.origin || !playlist.href.startsWith(base) || !/\/(?:HLSPlaylist\.m3u8|DASHPlaylist\.mpd)$/.test(playlist.pathname))
1817
+ continue;
1818
+ const dash = new URL("DASHPlaylist.mpd", base);
1819
+ dash.search = playlist.search;
1820
+ return { secure_media: { reddit_video: { dash_url: dash.href } } };
1821
+ }
1822
+ }
1823
+ return null;
1824
+ }
1825
+ function linkPreview(html) {
1826
+ for (const match of html.matchAll(/<img\s([^>]+)>/gi)) {
1827
+ const image = attributes(match[1]);
1828
+ if (image.id !== "post-image" || !Object.hasOwn(image, "data-post-media-primary"))
1829
+ continue;
1830
+ const sources = (image.srcset ?? "").split(/,\s*/).flatMap((source2) => {
1831
+ const match2 = source2.trim().match(/^(\S+)\s+(\d+)w$/);
1832
+ return match2 ? [{ url: match2[1], width: Number(match2[2]) }] : [];
1833
+ });
1834
+ if (image.src)
1835
+ sources.push({ url: image.src, width: finiteNumber(image.width) ?? 0 });
1836
+ for (const source2 of sources.sort((a, b) => b.width - a.width)) {
1837
+ const url = new URL(source2.url, "https://www.reddit.com");
1838
+ if (url.protocol === "https:" && /^(?:external-preview|preview)\.redd\.it$/.test(url.hostname))
1839
+ return url.href;
1840
+ }
1841
+ }
1842
+ return null;
1843
+ }
1844
+ function hostedImage(input) {
1845
+ if (!input)
1846
+ return null;
1847
+ const url = new URL(input, "https://www.reddit.com");
1848
+ return url.protocol === "https:" && url.hostname === "i.redd.it" && /\.(?:jpe?g|png|webp|gif)$/i.test(url.pathname) ? url.href : null;
1849
+ }
1850
+ function postText(html, id) {
1851
+ for (const match of html.matchAll(/<div\s([^>]+)>/gi)) {
1852
+ const attrs = attributes(match[1]);
1853
+ if (attrs.id !== `t3_${id}-post-rtjson-content`)
1854
+ continue;
1855
+ const start = match.index + match[0].length;
1856
+ const tags = /<div\b[^>]*>|<\/div\s*>/gi;
1857
+ tags.lastIndex = start;
1858
+ let depth = 1;
1859
+ for (let tag = tags.exec(html);tag; tag = tags.exec(html)) {
1860
+ depth += tag[0].startsWith("</") ? -1 : 1;
1861
+ if (depth === 0)
1862
+ return plainText(html.slice(start, tag.index));
1863
+ }
1864
+ }
1865
+ return;
1866
+ }
1867
+ function plainText(html) {
1868
+ return decodeHtml3(html.replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, "").replace(/<\/?(?:p|div|li|blockquote|pre|h[1-6])\b[^>]*>|<br\b[^>]*>/gi, `
1869
+ `).replace(/<[^>]*>/g, "")).replace(/[\t ]+/g, " ").replace(/ *\n */g, `
1870
+ `).replace(/\n{3,}/g, `
1871
+
1872
+ `).trim();
1873
+ }
1874
+ function finiteNumber(value) {
1875
+ const number2 = value === undefined ? NaN : Number(value);
1876
+ return Number.isFinite(number2) ? number2 : undefined;
1877
+ }
1878
+ function challengeUrl(html, responseUrl) {
1879
+ const seed = html.match(/await\s*\(async\s+([a-z])\s*=>\s*\1\s*\+\s*\1\)\("([a-f0-9]{16})"\)/i)?.[2];
1880
+ const form = html.match(/<form\b([^>]*)>([\s\S]*?)<\/form>/i);
1881
+ if (!seed || !form || !responseUrl)
1882
+ return null;
1883
+ const fields = attributes(form[1]);
1884
+ if (fields.method?.toUpperCase() !== "GET" || !fields.action)
1885
+ return null;
1886
+ const base = new URL(responseUrl);
1887
+ const target = new URL(fields.action, base);
1888
+ if (base.origin !== "https://www.reddit.com" || target.origin !== base.origin || target.pathname !== base.pathname)
1889
+ return null;
1890
+ const inputs = new Map;
1891
+ for (const input of form[2].matchAll(/<input\b([^>]*)>/gi)) {
1892
+ const attrs = attributes(input[1]);
1893
+ if (attrs.name)
1894
+ inputs.set(attrs.name, attrs.value ?? "");
1895
+ }
1896
+ if (inputs.get("js_challenge") !== "1" || !inputs.get("jsc_token") || !inputs.has("solution"))
1897
+ return null;
1898
+ target.search = "";
1899
+ for (const [key, value] of inputs) {
1900
+ if (["solution", "js_challenge", "jsc_token", "jsc_orig_r"].includes(key))
1901
+ target.searchParams.append(key, key === "solution" ? seed + seed : value);
1902
+ }
1903
+ for (const [key, value] of base.searchParams) {
1904
+ if (!inputs.has(key))
1905
+ target.searchParams.append(key, value);
1906
+ }
1907
+ return target.href;
1908
+ }
1909
+ function attributes(tag) {
1910
+ const result = {};
1911
+ for (const match of tag.matchAll(/(?:^|\s)([\w-]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'))?/g)) {
1912
+ result[match[1].toLowerCase()] = decodeHtml3(match[2] ?? match[3] ?? "");
1913
+ }
1914
+ return result;
1915
+ }
1916
+ function decodeHtml3(input) {
1917
+ const named = { amp: "&", apos: "'", gt: ">", lt: "<", nbsp: " ", quot: '"' };
1918
+ return input.replace(/&(#x[\da-f]+|#\d+|amp|apos|gt|lt|nbsp|quot);/gi, (entity, code) => {
1919
+ if (!code.startsWith("#"))
1920
+ return named[code.toLowerCase()] ?? entity;
1921
+ const value = code[1]?.toLowerCase() === "x" ? Number.parseInt(code.slice(2), 16) : Number.parseInt(code.slice(1), 10);
1922
+ return Number.isSafeInteger(value) && value <= 1114111 ? String.fromCodePoint(value) : entity;
1923
+ });
1924
+ }
1925
+
1616
1926
  // src/reddit.ts
1617
1927
  var installedClientId = "ohXpoqrZYub1kg";
1618
1928
  async function resolveReddit(input) {
1619
- const id = await postId(input.net, input.url);
1929
+ const { id, url, page } = await postId(input.net, input.url);
1620
1930
  const token = await appToken(input.net);
1621
- const post = await fetchPost(input.net, token, id);
1931
+ const post = await fetchPost(input.net, token, id, url, page);
1622
1932
  const items = await mediaItems3(input, post, id);
1623
1933
  return { archiveFilename: filename(`reddit_${id}.zip`), comments: await redditComments(input, token, id), id, items, metadata: redditMetadata(post), platform: "reddit" };
1624
1934
  }
@@ -1643,14 +1953,15 @@ function redditMetadata(post) {
1643
1953
  async function postId(net, input) {
1644
1954
  const direct = parseId(input);
1645
1955
  if (direct) {
1646
- return direct;
1956
+ return { id: direct, url: input };
1647
1957
  }
1648
- const response = await net(input, { headers: { "user-agent": browserUserAgent() } }, 1);
1649
- const resolved = parseId(response.url);
1958
+ const userAgent = browserUserAgent();
1959
+ const page = await readWebPage(await net(input, { headers: { "user-agent": userAgent } }, 1), userAgent);
1960
+ const resolved = parseId(page.url);
1650
1961
  if (!resolved) {
1651
1962
  throw new Error("Reddit post id not found");
1652
1963
  }
1653
- return resolved;
1964
+ return { id: resolved, url: page.url, page };
1654
1965
  }
1655
1966
  function parseId(input) {
1656
1967
  const url = asUrl(input);
@@ -1680,7 +1991,7 @@ async function appToken(net) {
1680
1991
  }
1681
1992
  return token;
1682
1993
  }
1683
- async function fetchPost(net, token, id) {
1994
+ async function fetchPost(net, token, id, url, page) {
1684
1995
  const headers = { authorization: `Bearer ${token}`, "user-agent": browserUserAgent() };
1685
1996
  let response = await net(`https://oauth.reddit.com/comments/${id}?raw_json=1&limit=1`, { headers });
1686
1997
  if (response.status === 403) {
@@ -1688,6 +1999,15 @@ async function fetchPost(net, token, id) {
1688
1999
  response = await net(`https://oauth.reddit.com/by_id/t3_${id}?raw_json=1`, { headers });
1689
2000
  }
1690
2001
  if (!response.ok) {
2002
+ if (response.status === 403) {
2003
+ await response.body?.cancel();
2004
+ const post2 = await fetchWebPost(net, id, url, page).catch(() => null);
2005
+ if (post2)
2006
+ return post2;
2007
+ const rss = await fetchRssPost(net, id).catch(() => null);
2008
+ if (rss)
2009
+ return rss;
2010
+ }
1691
2011
  throw new Error(`Reddit post failed: ${response.status}`);
1692
2012
  }
1693
2013
  const payload = await response.json().catch(() => null);
@@ -1746,20 +2066,28 @@ function galleryItems(post, id) {
1746
2066
  async function videoItems(input, post, id) {
1747
2067
  const media = redditVideo(post);
1748
2068
  const fallback = media ? string(media.fallback_url) : null;
1749
- if (!media || !fallback) {
2069
+ if (!media) {
1750
2070
  return [];
1751
2071
  }
2072
+ const dashUrl = string(media.dash_url);
2073
+ if (!fallback && dashUrl) {
2074
+ const streams2 = await dashStreams(input.net, dashUrl, input.preferredWidth, true);
2075
+ if (!streams2)
2076
+ throw new Error("Reddit DASH manifest unavailable for muxing");
2077
+ return [streams2.audio ? muxedVideoItem(id, streams2.video, streams2.audio) : mediaItem(id, null, "video", "video/mp4", streams2.video)];
2078
+ }
2079
+ if (!fallback)
2080
+ return [];
1752
2081
  if (media.has_audio !== true) {
1753
2082
  return [mediaItem(id, null, "video", "video/mp4", fallback)];
1754
2083
  }
1755
- const dashUrl = string(media.dash_url);
1756
2084
  const streams = dashUrl ? await dashStreams(input.net, dashUrl, input.preferredWidth) : null;
1757
- if (!streams) {
2085
+ if (!streams?.audio) {
1758
2086
  throw new Error("Reddit DASH manifest unavailable for muxing");
1759
2087
  }
1760
2088
  return [muxedVideoItem(id, streams.video, streams.audio)];
1761
2089
  }
1762
- async function dashStreams(net, dashUrl, preferredWidth) {
2090
+ async function dashStreams(net, dashUrl, preferredWidth, allowSilent = false) {
1763
2091
  const response = await net(dashUrl, { headers: { "user-agent": browserUserAgent() } });
1764
2092
  if (!response.ok) {
1765
2093
  return null;
@@ -1769,10 +2097,10 @@ async function dashStreams(net, dashUrl, preferredWidth) {
1769
2097
  const query = dashUrl.includes("?") ? dashUrl.slice(dashUrl.indexOf("?")) : "";
1770
2098
  const video = videoRendition(manifest, preferredWidth);
1771
2099
  const audio = audioRendition(manifest);
1772
- if (!video || !audio) {
2100
+ if (!video || !audio && !allowSilent) {
1773
2101
  return null;
1774
2102
  }
1775
- return { video: `${base}${video}${query}`, audio: `${base}${audio}${query}` };
2103
+ return { video: `${base}${video}${query}`, ...audio ? { audio: `${base}${audio}${query}` } : {} };
1776
2104
  }
1777
2105
  function videoRendition(manifest, preferredWidth) {
1778
2106
  const renditions = renditionsOf(manifest, false);
@@ -3070,6 +3398,7 @@ async function resolveTwitter(input) {
3070
3398
  throw new Error("Tweet is unavailable or age-restricted");
3071
3399
  }
3072
3400
  await expandTwitterText(input.net, tweet, id);
3401
+ await Promise.all(twitterTweets(tweet, id, true).map(({ id: id2, tweet: tweet2 }) => translateTwitterText(input, tweet2, id2)));
3073
3402
  const posts = twitterTweets(tweet, id);
3074
3403
  const groups = posts.flatMap(({ id: postId2, tweet: post }) => {
3075
3404
  const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
@@ -3093,6 +3422,11 @@ function twitterMetadata(tweet) {
3093
3422
  if (lang) {
3094
3423
  extra.lang = lang;
3095
3424
  }
3425
+ const translated = string(tweet.translated_text);
3426
+ if (translated) {
3427
+ extra.originalText = cleanTwitterText(twitterText(tweet), tweet) ?? undefined;
3428
+ extra.translatedLang = string(tweet.translated_lang) ?? undefined;
3429
+ }
3096
3430
  const quotedTweet = object(tweet.quoted_tweet) ? tweet.quoted_tweet : null;
3097
3431
  const quotedId = quotedTweet ? string(quotedTweet.id_str) : null;
3098
3432
  if (quotedTweet && quotedId) {
@@ -3107,7 +3441,7 @@ function twitterMetadata(tweet) {
3107
3441
  extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
3108
3442
  }
3109
3443
  return {
3110
- text: cleanTwitterText(twitterText(tweet), tweet) ?? undefined,
3444
+ text: cleanTwitterText(translated ?? twitterText(tweet), tweet) ?? undefined,
3111
3445
  author: user ? {
3112
3446
  handle: string(user.screen_name) ?? undefined,
3113
3447
  name: string(user.name) ?? undefined,
@@ -3154,6 +3488,43 @@ function needsFullText(tweet) {
3154
3488
  }
3155
3489
  return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
3156
3490
  }
3491
+ async function translateTwitterText(input, tweet, id) {
3492
+ const languages = input.languages;
3493
+ if (!languages?.length || !twitterText(tweet))
3494
+ return;
3495
+ const lang = string(tweet.lang)?.toLowerCase();
3496
+ const allowed = (source2) => languages.some((language) => source2 === language || source2.startsWith(`${language}-`));
3497
+ if (lang && allowed(lang))
3498
+ return;
3499
+ for (const target of new Set([languages[0], "en"])) {
3500
+ try {
3501
+ const response = await input.net(`https://api.fxtwitter.com/status/${id}/${target}`, {
3502
+ headers: { accept: "application/json", "user-agent": browserUserAgent() }
3503
+ }, 1);
3504
+ if (!response.ok)
3505
+ continue;
3506
+ const payload = await response.json();
3507
+ if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id)
3508
+ continue;
3509
+ const post = payload.tweet;
3510
+ const translation = object(post.translation) ? post.translation : null;
3511
+ const source2 = (string(post.lang) ?? string(translation?.source_lang))?.toLowerCase();
3512
+ if (source2 && source2 !== "und") {
3513
+ tweet.lang = source2;
3514
+ if (allowed(source2))
3515
+ return;
3516
+ }
3517
+ const text2 = translation ? string(translation.text) : null;
3518
+ if (!text2?.trim() || string(translation?.target_lang)?.toLowerCase() !== target)
3519
+ continue;
3520
+ if (text2.includes("You're using an older version of DeepL."))
3521
+ continue;
3522
+ tweet.translated_text = text2;
3523
+ tweet.translated_lang = target;
3524
+ return;
3525
+ } catch {}
3526
+ }
3527
+ }
3157
3528
  async function expandTwitterText(net, tweet, rootId) {
3158
3529
  const posts = new Map(twitterTweets(tweet, rootId, true).map(({ id, tweet: post }) => [id, post]));
3159
3530
  for (const [id, post] of posts) {
@@ -3374,6 +3745,7 @@ async function twitterComments(input, id) {
3374
3745
  if (!object(payload) || payload.code !== 200 || !Array.isArray(payload.replies))
3375
3746
  throw new Error("Invalid comments");
3376
3747
  const previousCount = comments.length;
3748
+ const page = [];
3377
3749
  for (const reply of payload.replies) {
3378
3750
  if (!object(reply))
3379
3751
  continue;
@@ -3382,10 +3754,16 @@ async function twitterComments(input, id) {
3382
3754
  if (!replyId || parent !== id || seen.has(replyId))
3383
3755
  continue;
3384
3756
  seen.add(replyId);
3385
- comments.push(twitterComment(reply, replyId));
3386
- if (comments.length >= limit)
3387
- return comments;
3757
+ page.push({ id: replyId, reply });
3758
+ if (comments.length + page.length >= limit)
3759
+ break;
3388
3760
  }
3761
+ comments.push(...await Promise.all(page.map(async ({ id: id2, reply }) => {
3762
+ await translateTwitterText(input, reply, id2);
3763
+ return twitterComment(reply, id2);
3764
+ })));
3765
+ if (comments.length >= limit)
3766
+ return comments;
3389
3767
  cursor = object(payload.cursor) ? string(payload.cursor.bottom) : null;
3390
3768
  if (!cursor || cursors.has(cursor) || comments.length === previousCount)
3391
3769
  break;
@@ -3420,7 +3798,7 @@ function twitterComment(reply, id) {
3420
3798
  url: `https://x.com/i/status/${id}`,
3421
3799
  items,
3422
3800
  metadata: {
3423
- text: cleanTwitterText(string(reply.text), reply) ?? undefined,
3801
+ text: cleanTwitterText(string(reply.translated_text) ?? string(reply.text), reply) ?? undefined,
3424
3802
  author: {
3425
3803
  handle: string(author2.screen_name) ?? undefined,
3426
3804
  name: string(author2.name) ?? undefined,
@@ -3447,6 +3825,7 @@ async function postfetch(url, options = {}) {
3447
3825
  }
3448
3826
  const context = {
3449
3827
  net: createNet(options.fetch ?? globalThis.fetch),
3828
+ languages: options.languages === undefined ? undefined : normalizeLanguages(options.languages),
3450
3829
  comments: options.comments,
3451
3830
  preferredWidth: options.preferredWidth ?? 720,
3452
3831
  tryMaxBytes: options.tryMaxBytes,
@@ -3455,6 +3834,12 @@ async function postfetch(url, options = {}) {
3455
3834
  const result = await resolve(context);
3456
3835
  return options.tryMaxBytes === undefined || result.platform === "twitter" ? result : trySmaller(context, result, options.tryMaxBytes);
3457
3836
  }
3837
+ function normalizeLanguages(languages) {
3838
+ if (!Array.isArray(languages) || languages.length === 0 || languages.some((lang) => typeof lang !== "string" || !/^[a-z]{2,3}(?:-[a-z0-9]{2,8})*$/i.test(lang))) {
3839
+ throw new PostfetchError(400, "languages must be a non-empty list of language codes");
3840
+ }
3841
+ return [...new Set(languages.map((lang) => lang.toLowerCase()))];
3842
+ }
3458
3843
  async function resolve(context) {
3459
3844
  switch (detect(context.url)) {
3460
3845
  case "facebook":
@@ -112,6 +112,10 @@ export type TwitterExtra = {
112
112
  replyToId?: string;
113
113
  /** BCP 47 language tag detected for the tweet. */
114
114
  lang?: string;
115
+ /** Original text retained when a requested translation is used. */
116
+ originalText?: string;
117
+ /** Language of the translated text. `lang` still describes the original. */
118
+ translatedLang?: string;
115
119
  /** The post quoted by this post, when X includes it in the resolved payload. */
116
120
  quotedTweet?: TwitterQuotedTweet;
117
121
  /** The parent this post replies to, when included by X. Media is not collected. */
@@ -208,6 +212,7 @@ export type Net = (url: string, init?: RequestInit, attempts?: number) => Promis
208
212
  export type ResolveContext = {
209
213
  net: Net;
210
214
  preferredWidth: number;
215
+ languages?: string[];
211
216
  comments?: number;
212
217
  /** Requested soft byte cap, for resolvers that can rank exact rendition sizes. */
213
218
  tryMaxBytes?: number;
@@ -5,6 +5,11 @@ export type PostfetchOptions = {
5
5
  fetch?: typeof fetch;
6
6
  /** Maximum direct comments to fetch for X and Reddit. Other platforms ignore this option. Failures return an empty array. */
7
7
  comments?: number;
8
+ /** Allowed X text languages, in preference order. Preserve allowed originals;
9
+ * otherwise translate to the first language, then English, then keep the original.
10
+ * Omit to keep original text. Applies to quotes, parents and requested replies too.
11
+ */
12
+ languages?: string[];
8
13
  /** Preferred media width in pixels; the closest available rendition is chosen. Defaults to `720`. */
9
14
  preferredWidth?: number;
10
15
  /**
@@ -0,0 +1,12 @@
1
+ import type { Json, Net } from "./internal";
2
+ export type RedditPage = {
3
+ url: string;
4
+ html: string;
5
+ userAgent: string;
6
+ cookies: string[];
7
+ ok: boolean;
8
+ };
9
+ export declare function readWebPage(response: Response, userAgent: string): Promise<RedditPage>;
10
+ export declare function fetchWebPage(net: Net, input: string, userAgent: string, cookies?: string[]): Promise<RedditPage>;
11
+ export declare function fetchWebPost(net: Net, id: string, input: string, initial?: RedditPage): Promise<Json | null>;
12
+ export declare function fetchRssPost(net: Net, id: string): Promise<Json | null>;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@postfetch/core",
3
- "version": "0.13.8",
3
+ "version": "0.14.0",
4
4
  "description": "Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, Spotify, TikTok, X and YouTube post URLs into media files.",
5
5
  "license": "MIT",
6
6
  "type": "module",
package/src/internal.ts CHANGED
@@ -113,6 +113,10 @@ export type TwitterExtra = {
113
113
  replyToId?: string;
114
114
  /** BCP 47 language tag detected for the tweet. */
115
115
  lang?: string;
116
+ /** Original text retained when a requested translation is used. */
117
+ originalText?: string;
118
+ /** Language of the translated text. `lang` still describes the original. */
119
+ translatedLang?: string;
116
120
  /** The post quoted by this post, when X includes it in the resolved payload. */
117
121
  quotedTweet?: TwitterQuotedTweet;
118
122
  /** The parent this post replies to, when included by X. Media is not collected. */
@@ -228,6 +232,7 @@ export type Net = (url: string, init?: RequestInit, attempts?: number) => Promis
228
232
  export type ResolveContext = {
229
233
  net: Net;
230
234
  preferredWidth: number;
235
+ languages?: string[];
231
236
  comments?: number;
232
237
  /** Requested soft byte cap, for resolvers that can rank exact rendition sizes. */
233
238
  tryMaxBytes?: number;
package/src/postfetch.ts CHANGED
@@ -16,6 +16,11 @@ export type PostfetchOptions = {
16
16
  fetch?: typeof fetch;
17
17
  /** Maximum direct comments to fetch for X and Reddit. Other platforms ignore this option. Failures return an empty array. */
18
18
  comments?: number;
19
+ /** Allowed X text languages, in preference order. Preserve allowed originals;
20
+ * otherwise translate to the first language, then English, then keep the original.
21
+ * Omit to keep original text. Applies to quotes, parents and requested replies too.
22
+ */
23
+ languages?: string[];
19
24
  /** Preferred media width in pixels; the closest available rendition is chosen. Defaults to `720`. */
20
25
  preferredWidth?: number;
21
26
  /**
@@ -61,6 +66,7 @@ export async function postfetch(url: string, options: PostfetchOptions = {}): Pr
61
66
  }
62
67
  const context: ResolveContext = {
63
68
  net: createNet(options.fetch ?? globalThis.fetch),
69
+ languages: options.languages === undefined ? undefined : normalizeLanguages(options.languages),
64
70
  comments: options.comments,
65
71
  preferredWidth: options.preferredWidth ?? 720,
66
72
  tryMaxBytes: options.tryMaxBytes,
@@ -72,6 +78,13 @@ export async function postfetch(url: string, options: PostfetchOptions = {}): Pr
72
78
  : trySmaller(context, result, options.tryMaxBytes);
73
79
  }
74
80
 
81
+ function normalizeLanguages(languages: string[]): string[] {
82
+ if (!Array.isArray(languages) || languages.length === 0 || languages.some((lang) => typeof lang !== "string" || !/^[a-z]{2,3}(?:-[a-z0-9]{2,8})*$/i.test(lang))) {
83
+ throw new PostfetchError(400, "languages must be a non-empty list of language codes");
84
+ }
85
+ return [...new Set(languages.map((lang) => lang.toLowerCase()))];
86
+ }
87
+
75
88
  async function resolve(context: ResolveContext): Promise<PostfetchResult> {
76
89
  switch (detect(context.url)) {
77
90
  case "facebook":
@@ -0,0 +1,292 @@
1
+ import { browserUserAgent } from "./fingerprint";
2
+ import type { Json, Net } from "./internal";
3
+
4
+ export type RedditPage = { url: string; html: string; userAgent: string; cookies: string[]; ok: boolean };
5
+
6
+ export async function readWebPage(response: Response, userAgent: string): Promise<RedditPage> {
7
+ const cookies = response.headers.getSetCookie().map((cookie) => cookie.split(";", 1)[0]!);
8
+ if (!response.ok) {
9
+ await response.body?.cancel();
10
+ return { url: response.url, html: "", userAgent, cookies, ok: false };
11
+ }
12
+ return { url: response.url, html: await response.text(), userAgent, cookies, ok: true };
13
+ }
14
+
15
+ export async function fetchWebPage(net: Net, input: string, userAgent: string, cookies: string[] = []): Promise<RedditPage> {
16
+ let url = new URL(input);
17
+ let jar = new Map(cookies.map((cookie) => [cookie.split("=", 1)[0]!, cookie]));
18
+ for (let redirects = 0; redirects <= 5; redirects += 1) {
19
+ if (url.protocol !== "https:" || !/^(?:(?:www\.|old\.|m\.)?reddit\.com|redd\.it)$/.test(url.hostname)) throw new Error("Invalid Reddit page redirect");
20
+ const headers = { "user-agent": userAgent, ...(jar.size ? { cookie: [...jar.values()].join("; ") } : {}) };
21
+ // fetch does not keep Set-Cookie values while automatically following a
22
+ // redirect. Reddit's successful JS check redirects after setting its token.
23
+ const response = await net(url.href, { headers, redirect: "manual" }, 1);
24
+ for (const cookie of response.headers.getSetCookie()) {
25
+ const value = cookie.split(";", 1)[0]!;
26
+ const name = value.split("=", 1)[0]!;
27
+ if (/;\s*max-age=0(?:;|$)/i.test(cookie)) jar.delete(name);
28
+ else jar.set(name, value);
29
+ }
30
+ const location = response.headers.get("location");
31
+ if ([301, 302, 303, 307, 308].includes(response.status) && location) {
32
+ await response.body?.cancel();
33
+ const target = new URL(location, url);
34
+ if (target.origin !== url.origin) jar = new Map();
35
+ url = target;
36
+ continue;
37
+ }
38
+ return { ...await readWebPage(response, userAgent), url: response.url || url.href, cookies: [...jar.values()] };
39
+ }
40
+ throw new Error("Too many Reddit page redirects");
41
+ }
42
+
43
+ // Read only the requested post, never thumbnails or attachments from comments
44
+ // and recommendations. Return Reddit's normal JSON shape to reuse extraction.
45
+ export async function fetchWebPost(net: Net, id: string, input: string, initial?: RedditPage): Promise<Json | null> {
46
+ const url = new URL(input);
47
+ if (url.protocol !== "https:" || !/^(?:(?:www\.|old\.|m\.)?reddit\.com|redd\.it)$/.test(url.hostname)) return null;
48
+ if (url.hostname !== "redd.it") url.hostname = "www.reddit.com";
49
+ const userAgent = initial?.userAgent ?? browserUserAgent();
50
+ let page = initial ?? await fetchWebPage(net, url.href, userAgent);
51
+ if (!page.ok) return null;
52
+ const challenge = challengeUrl(page.html, page.url);
53
+ if (challenge) {
54
+ page = await fetchWebPage(net, challenge, userAgent, page.cookies);
55
+ if (!page.ok) return null;
56
+ }
57
+ for (const match of page.html.matchAll(/<shreddit-post\s([^>]+)>([\s\S]*?)<\/shreddit-post>/gi)) {
58
+ const post = attributes(match[1]);
59
+ if (post.id !== `t3_${id}` || !post["post-title"]) continue;
60
+ const media = postMedia(post, match[2], id);
61
+ if (!media) return null;
62
+ const created = Date.parse(post["created-timestamp"] ?? "");
63
+ return {
64
+ ...media,
65
+ id,
66
+ title: post["post-title"],
67
+ selftext: postText(match[2], id),
68
+ author: post.author,
69
+ subreddit: post["subreddit-name"],
70
+ permalink: post.permalink,
71
+ created_utc: Number.isFinite(created) ? Math.floor(created / 1000) : undefined,
72
+ ups: finiteNumber(post.score),
73
+ score: finiteNumber(post.score),
74
+ upvote_ratio: finiteNumber(post["upvote-ratio"]),
75
+ num_comments: finiteNumber(post["comment-count"]),
76
+ over_18: Object.hasOwn(post, "nsfw"),
77
+ };
78
+ }
79
+ return null;
80
+ }
81
+
82
+ // The public Atom feed remains usable on some routes that reject OAuth and
83
+ // browser pages. Match the root entry explicitly: replies have their own media.
84
+ export async function fetchRssPost(net: Net, id: string): Promise<Json | null> {
85
+ const url = `https://www.reddit.com/comments/${id}/.rss?limit=1`;
86
+ const headers = { "user-agent": browserUserAgent(), accept: "application/atom+xml" };
87
+ let response = await net(url, { headers }, 1);
88
+ if (response.status === 429) {
89
+ // Reddit currently advertises a small anonymous RSS quota and its reset
90
+ // window in this header, rather than Retry-After. Retry once within a bound.
91
+ const reset = finiteNumber(response.headers.get("x-ratelimit-reset") ?? undefined);
92
+ await response.body?.cancel();
93
+ if (reset === undefined || reset < 0 || reset > 15) return null;
94
+ await new Promise((resolve) => setTimeout(resolve, (reset + 0.1) * 1000));
95
+ response = await net(url, { headers }, 1);
96
+ }
97
+ if (!response.ok) {
98
+ await response.body?.cancel();
99
+ return null;
100
+ }
101
+ const xml = await response.text();
102
+ if (!/<feed\b/.test(xml)) return null;
103
+ for (const match of xml.matchAll(/<entry\b[^>]*>([\s\S]*?)<\/entry>/g)) {
104
+ const entry = match[1];
105
+ if (entry.match(/<id>([^<]+)<\/id>/)?.[1] !== `t3_${id}`) continue;
106
+ const title = entry.match(/<title>([\s\S]*?)<\/title>/)?.[1];
107
+ const content = decodeHtml(entry.match(/<content\b[^>]*>([\s\S]*?)<\/content>/)?.[1] ?? "");
108
+ const link = [...content.matchAll(/<a\s([^>]+)>\s*\[link\]\s*<\/a>/g)][0];
109
+ const destination = link ? attributes(link[1]).href : undefined;
110
+ const image = hostedImage(destination);
111
+ let media: Json | null = image ? { post_hint: "image", url: image } : null;
112
+ if (!media && destination) {
113
+ const target = new URL(destination);
114
+ if (target.protocol === "https:" && target.hostname === "v.redd.it" && /^\/[a-z0-9]+\/?$/i.test(target.pathname)) {
115
+ media = { secure_media: { reddit_video: { dash_url: target.href.replace(/\/$/, "") + "/DASHPlaylist.mpd" } } };
116
+ } else if (/^https?:$/.test(target.protocol) && !/(^|\.)(reddit\.com|redd\.it)$/.test(target.hostname)) {
117
+ const card = content.match(/<img\s([^>]+)>/);
118
+ const preview = card ? attributes(card[1]).src : undefined;
119
+ const previewUrl = preview ? new URL(preview) : null;
120
+ media = {
121
+ post_hint: "link", is_self: false, url: target.href,
122
+ ...(previewUrl?.protocol === "https:" && previewUrl.hostname === "external-preview.redd.it"
123
+ ? { preview: { images: [{ source: { url: previewUrl.href } }] } } : {}),
124
+ };
125
+ }
126
+ }
127
+ const body = content.match(/<!-- SC_OFF -->([\s\S]*?)<!-- SC_ON -->/)?.[1];
128
+ if (!media && !destination && body) media = { is_self: true };
129
+ if (!title || !media) return null;
130
+ const permalink = [...entry.matchAll(/<link\s([^>]+)\/?\s*>/g)]
131
+ .map((match) => attributes(match[1]).href).find((href) => href?.includes(`/comments/${id}/`));
132
+ const created = Date.parse(entry.match(/<published>([^<]+)<\/published>/)?.[1] ?? "");
133
+ return {
134
+ ...media, id, title: decodeHtml(title), selftext: body ? plainText(body) : undefined,
135
+ author: decodeHtml(entry.match(/<name>([^<]+)<\/name>/)?.[1] ?? "").replace(/^\/u\//, "") || undefined,
136
+ subreddit: attributes(entry.match(/<category\s([^>]+)>/)?.[1] ?? "").term,
137
+ permalink: permalink ? new URL(permalink).pathname : undefined,
138
+ created_utc: Number.isFinite(created) ? Math.floor(created / 1000) : undefined,
139
+ };
140
+ }
141
+ return null;
142
+ }
143
+
144
+ function postMedia(post: Record<string, string>, html: string, id: string): Json | null {
145
+ if (post["post-type"] === "text") return { is_self: true };
146
+ if (post["post-type"] === "image") {
147
+ const url = hostedImage(post["content-href"]);
148
+ return url ? { post_hint: "image", url } : null;
149
+ }
150
+ if (post["post-type"] === "gallery") {
151
+ const gallery = html.match(/<(gallery-carousel|shreddit-gallery)\b[^>]*>([\s\S]*?)<\/\1>/i);
152
+ if (!gallery) return null;
153
+ const images: string[] = [];
154
+ for (const match of gallery[2].matchAll(/<a\s([^>]+)>/gi)) {
155
+ const href = attributes(match[1]).href;
156
+ if (!href) continue;
157
+ const target = new URL(href, "https://www.reddit.com");
158
+ const media = target.origin === "https://www.reddit.com" && target.pathname === "/media"
159
+ ? target.searchParams.get("url") : target.hostname === "i.redd.it" ? target.href : null;
160
+ if (!media) continue;
161
+ const original = hostedImage(media);
162
+ if (!original) return null;
163
+ if (!images.includes(original)) images.push(original);
164
+ }
165
+ // A gallery cover alone is insufficient evidence of the complete post.
166
+ if (images.length < 2) return null;
167
+ return {
168
+ gallery_data: { items: images.map((_, index) => ({ media_id: String(index) })) },
169
+ media_metadata: Object.fromEntries(images.map((url, index) => [String(index), { s: { u: url } }])),
170
+ };
171
+ }
172
+ if (post["post-type"] === "link") {
173
+ const url = new URL(post["content-href"] ?? "", "https://www.reddit.com");
174
+ if (!/^https?:$/.test(url.protocol) || /(^|\.)(reddit\.com|redd\.it)$/.test(url.hostname)) return null;
175
+ const preview = linkPreview(html);
176
+ return {
177
+ post_hint: "link", is_self: false, url: url.href,
178
+ ...(preview ? { preview: { images: [{ source: { url: preview } }] } } : {}),
179
+ };
180
+ }
181
+ if (post["post-type"] === "video" || post["post-type"] === "gif") {
182
+ const destination = new URL(post["content-href"] ?? "", "https://www.reddit.com");
183
+ if (destination.protocol !== "https:" || destination.hostname !== "v.redd.it" || !/^\/[a-z0-9]+\/?$/i.test(destination.pathname)) return null;
184
+ for (const match of html.matchAll(/<shreddit-player\s([^>]+)>/gi)) {
185
+ const player = attributes(match[1]);
186
+ if (player["post-id"] !== `t3_${id}` || player["media-placement"] === "comment") continue;
187
+ const playlist = new URL(player.src ?? "", destination);
188
+ const base = destination.href.replace(/\/$/, "") + "/";
189
+ if (playlist.origin !== destination.origin || !playlist.href.startsWith(base) || !/\/(?:HLSPlaylist\.m3u8|DASHPlaylist\.mpd)$/.test(playlist.pathname)) continue;
190
+ const dash = new URL("DASHPlaylist.mpd", base);
191
+ dash.search = playlist.search;
192
+ return { secure_media: { reddit_video: { dash_url: dash.href } } };
193
+ }
194
+ }
195
+ return null;
196
+ }
197
+
198
+ function linkPreview(html: string): string | null {
199
+ for (const match of html.matchAll(/<img\s([^>]+)>/gi)) {
200
+ const image = attributes(match[1]);
201
+ if (image.id !== "post-image" || !Object.hasOwn(image, "data-post-media-primary")) continue;
202
+ const sources = (image.srcset ?? "").split(/,\s*/).flatMap((source) => {
203
+ const match = source.trim().match(/^(\S+)\s+(\d+)w$/);
204
+ return match ? [{ url: match[1], width: Number(match[2]) }] : [];
205
+ });
206
+ if (image.src) sources.push({ url: image.src, width: finiteNumber(image.width) ?? 0 });
207
+ for (const source of sources.sort((a, b) => b.width - a.width)) {
208
+ const url = new URL(source.url, "https://www.reddit.com");
209
+ if (url.protocol === "https:" && /^(?:external-preview|preview)\.redd\.it$/.test(url.hostname)) return url.href;
210
+ }
211
+ }
212
+ return null;
213
+ }
214
+
215
+ function hostedImage(input: string | undefined): string | null {
216
+ if (!input) return null;
217
+ const url = new URL(input, "https://www.reddit.com");
218
+ return url.protocol === "https:" && url.hostname === "i.redd.it" && /\.(?:jpe?g|png|webp|gif)$/i.test(url.pathname) ? url.href : null;
219
+ }
220
+
221
+ function postText(html: string, id: string): string | undefined {
222
+ for (const match of html.matchAll(/<div\s([^>]+)>/gi)) {
223
+ const attrs = attributes(match[1]);
224
+ if (attrs.id !== `t3_${id}-post-rtjson-content`) continue;
225
+ const start = match.index + match[0].length;
226
+ const tags = /<div\b[^>]*>|<\/div\s*>/gi;
227
+ tags.lastIndex = start;
228
+ let depth = 1;
229
+ for (let tag = tags.exec(html); tag; tag = tags.exec(html)) {
230
+ depth += tag[0].startsWith("</") ? -1 : 1;
231
+ if (depth === 0) return plainText(html.slice(start, tag.index));
232
+ }
233
+ }
234
+ return undefined;
235
+ }
236
+
237
+ function plainText(html: string): string {
238
+ return decodeHtml(html
239
+ .replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, "")
240
+ .replace(/<\/?(?:p|div|li|blockquote|pre|h[1-6])\b[^>]*>|<br\b[^>]*>/gi, "\n")
241
+ .replace(/<[^>]*>/g, ""))
242
+ .replace(/[\t ]+/g, " ").replace(/ *\n */g, "\n").replace(/\n{3,}/g, "\n\n").trim();
243
+ }
244
+
245
+ function finiteNumber(value: string | undefined): number | undefined {
246
+ const number = value === undefined ? NaN : Number(value);
247
+ return Number.isFinite(number) ? number : undefined;
248
+ }
249
+
250
+ // Reddit sometimes serves a tiny form before the public page. Recognize only
251
+ // its fixed string-doubling script; never evaluate JavaScript from the page.
252
+ function challengeUrl(html: string, responseUrl: string): string | null {
253
+ const seed = html.match(/await\s*\(async\s+([a-z])\s*=>\s*\1\s*\+\s*\1\)\("([a-f0-9]{16})"\)/i)?.[2];
254
+ const form = html.match(/<form\b([^>]*)>([\s\S]*?)<\/form>/i);
255
+ if (!seed || !form || !responseUrl) return null;
256
+ const fields = attributes(form[1]);
257
+ if (fields.method?.toUpperCase() !== "GET" || !fields.action) return null;
258
+ const base = new URL(responseUrl);
259
+ const target = new URL(fields.action, base);
260
+ if (base.origin !== "https://www.reddit.com" || target.origin !== base.origin || target.pathname !== base.pathname) return null;
261
+ const inputs = new Map<string, string>();
262
+ for (const input of form[2].matchAll(/<input\b([^>]*)>/gi)) {
263
+ const attrs = attributes(input[1]);
264
+ if (attrs.name) inputs.set(attrs.name, attrs.value ?? "");
265
+ }
266
+ if (inputs.get("js_challenge") !== "1" || !inputs.get("jsc_token") || !inputs.has("solution")) return null;
267
+ target.search = "";
268
+ for (const [key, value] of inputs) {
269
+ if (["solution", "js_challenge", "jsc_token", "jsc_orig_r"].includes(key)) target.searchParams.append(key, key === "solution" ? seed + seed : value);
270
+ }
271
+ for (const [key, value] of base.searchParams) {
272
+ if (!inputs.has(key)) target.searchParams.append(key, value);
273
+ }
274
+ return target.href;
275
+ }
276
+
277
+ function attributes(tag: string): Record<string, string> {
278
+ const result: Record<string, string> = {};
279
+ for (const match of tag.matchAll(/(?:^|\s)([\w-]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'))?/g)) {
280
+ result[match[1].toLowerCase()] = decodeHtml(match[2] ?? match[3] ?? "");
281
+ }
282
+ return result;
283
+ }
284
+
285
+ function decodeHtml(input: string): string {
286
+ const named: Record<string, string> = { amp: "&", apos: "'", gt: ">", lt: "<", nbsp: " ", quot: '"' };
287
+ return input.replace(/&(#x[\da-f]+|#\d+|amp|apos|gt|lt|nbsp|quot);/gi, (entity, code: string) => {
288
+ if (!code.startsWith("#")) return named[code.toLowerCase()] ?? entity;
289
+ const value = code[1]?.toLowerCase() === "x" ? Number.parseInt(code.slice(2), 16) : Number.parseInt(code.slice(1), 10);
290
+ return Number.isSafeInteger(value) && value <= 0x10ffff ? String.fromCodePoint(value) : entity;
291
+ });
292
+ }
package/src/reddit.ts CHANGED
@@ -17,17 +17,17 @@ import {
17
17
  type MediaItem,
18
18
  } from "./internal";
19
19
  import { browserUserAgent } from "./fingerprint";
20
+ import { fetchWebPost, fetchRssPost, readWebPage, type RedditPage } from "./reddit-web";
20
21
 
21
- // Logged-out reddit (www/old/api/.json) is fully 403-walled. The public
22
- // installed-app OAuth flow still works: a client id with no secret plus the
23
- // opt-out device id returns a bearer that reads any public post through
24
- // oauth.reddit.com.
22
+ // The installed-app token can still be issued while both OAuth post routes
23
+ // reject it with Reddit's network-security page. Keep the public page as a
24
+ // fallback rather than treating a successful token request as API availability.
25
25
  const installedClientId = "ohXpoqrZYub1kg";
26
26
 
27
27
  export async function resolveReddit(input: ResolveContext): Promise<PostfetchResult> {
28
- const id = await postId(input.net, input.url);
28
+ const { id, url, page } = await postId(input.net, input.url);
29
29
  const token = await appToken(input.net);
30
- const post = await fetchPost(input.net, token, id);
30
+ const post = await fetchPost(input.net, token, id, url, page);
31
31
  // A text post is a valid result with metadata and no media — fetchPost has
32
32
  // already thrown if the post itself is missing, so empty items here mean a
33
33
  // self/text post, not a failure.
@@ -54,19 +54,20 @@ export function redditMetadata(post: Json): PostMetadata & { extra?: RedditExtra
54
54
  };
55
55
  }
56
56
 
57
- // Share and short links (redd.it/<id>, /r/<sub>/s/<share>) carry no post id, so
58
- // they are followed once to read the canonical /comments/<id> location.
59
- async function postId(net: Net, input: string): Promise<string> {
57
+ // Share links have no post id. Retain their canonical page and client identity
58
+ // so a blocked API lookup can reuse the response instead of fetching it again.
59
+ async function postId(net: Net, input: string): Promise<{ id: string; url: string; page?: RedditPage }> {
60
60
  const direct = parseId(input);
61
61
  if (direct) {
62
- return direct;
62
+ return { id: direct, url: input };
63
63
  }
64
- const response = await net(input, { headers: { "user-agent": browserUserAgent() } }, 1);
65
- const resolved = parseId(response.url);
64
+ const userAgent = browserUserAgent();
65
+ const page = await readWebPage(await net(input, { headers: { "user-agent": userAgent } }, 1), userAgent);
66
+ const resolved = parseId(page.url);
66
67
  if (!resolved) {
67
68
  throw new Error("Reddit post id not found");
68
69
  }
69
- return resolved;
70
+ return { id: resolved, url: page.url, page };
70
71
  }
71
72
 
72
73
  function parseId(input: string): string | null {
@@ -99,7 +100,7 @@ async function appToken(net: Net): Promise<string> {
99
100
  return token;
100
101
  }
101
102
 
102
- async function fetchPost(net: Net, token: string, id: string): Promise<Json> {
103
+ async function fetchPost(net: Net, token: string, id: string, url: string, page?: RedditPage): Promise<Json> {
103
104
  const headers = { authorization: `Bearer ${token}`, "user-agent": browserUserAgent() };
104
105
  let response = await net(`https://oauth.reddit.com/comments/${id}?raw_json=1&limit=1`, { headers });
105
106
  // Reddit can block the comments route while still allowing direct post
@@ -109,6 +110,15 @@ async function fetchPost(net: Net, token: string, id: string): Promise<Json> {
109
110
  response = await net(`https://oauth.reddit.com/by_id/t3_${id}?raw_json=1`, { headers });
110
111
  }
111
112
  if (!response.ok) {
113
+ if (response.status === 403) {
114
+ await response.body?.cancel();
115
+ // Public pages can remain available when both OAuth routes are
116
+ // blocked. Keep the original failure if the page cannot supply the post.
117
+ const post = await fetchWebPost(net, id, url, page).catch(() => null);
118
+ if (post) return post;
119
+ const rss = await fetchRssPost(net, id).catch(() => null);
120
+ if (rss) return rss;
121
+ }
112
122
  throw new Error(`Reddit post failed: ${response.status}`);
113
123
  }
114
124
  const payload = await response.json().catch(() => null);
@@ -173,23 +183,34 @@ type Rendition = { width: number | null; bandwidth: number; base: string };
173
183
  async function videoItems(input: ResolveContext, post: Json, id: string): Promise<MediaItem[]> {
174
184
  const media = redditVideo(post);
175
185
  const fallback = media ? string(media.fallback_url) : null;
176
- if (!media || !fallback) {
186
+ if (!media) {
177
187
  return [];
178
188
  }
189
+ const dashUrl = string(media.dash_url);
190
+ // Public pages expose the player and manifest rather than a fallback stream
191
+ // or has_audio flag. Inspect the manifest so silent clips remain silent and
192
+ // clips with audio retain their separate track through the normal remuxer.
193
+ if (!fallback && dashUrl) {
194
+ const streams = await dashStreams(input.net, dashUrl, input.preferredWidth, true);
195
+ if (!streams) throw new Error("Reddit DASH manifest unavailable for muxing");
196
+ return [streams.audio
197
+ ? muxedVideoItem(id, streams.video, streams.audio)
198
+ : mediaItem(id, null, "video", "video/mp4", streams.video)];
199
+ }
200
+ if (!fallback) return [];
179
201
  if (media.has_audio !== true) {
180
202
  return [mediaItem(id, null, "video", "video/mp4", fallback)];
181
203
  }
182
204
  // The fallback stream is video-only; reddit keeps audio in a separate DASH
183
205
  // rendition, so a complete file is the video and audio streams merged.
184
- const dashUrl = string(media.dash_url);
185
206
  const streams = dashUrl ? await dashStreams(input.net, dashUrl, input.preferredWidth) : null;
186
- if (!streams) {
207
+ if (!streams?.audio) {
187
208
  throw new Error("Reddit DASH manifest unavailable for muxing");
188
209
  }
189
210
  return [muxedVideoItem(id, streams.video, streams.audio)];
190
211
  }
191
212
 
192
- async function dashStreams(net: Net, dashUrl: string, preferredWidth: number): Promise<{ video: string; audio: string } | null> {
213
+ async function dashStreams(net: Net, dashUrl: string, preferredWidth: number, allowSilent = false): Promise<{ video: string; audio?: string } | null> {
193
214
  const response = await net(dashUrl, { headers: { "user-agent": browserUserAgent() } });
194
215
  if (!response.ok) {
195
216
  return null;
@@ -199,10 +220,10 @@ async function dashStreams(net: Net, dashUrl: string, preferredWidth: number): P
199
220
  const query = dashUrl.includes("?") ? dashUrl.slice(dashUrl.indexOf("?")) : "";
200
221
  const video = videoRendition(manifest, preferredWidth);
201
222
  const audio = audioRendition(manifest);
202
- if (!video || !audio) {
223
+ if (!video || (!audio && !allowSilent)) {
203
224
  return null;
204
225
  }
205
- return { video: `${base}${video}${query}`, audio: `${base}${audio}${query}` };
226
+ return { video: `${base}${video}${query}`, ...(audio ? { audio: `${base}${audio}${query}` } : {}) };
206
227
  }
207
228
 
208
229
  function videoRendition(manifest: string, preferredWidth: number): string | null {
package/src/twitter.ts CHANGED
@@ -33,6 +33,7 @@ export async function resolveTwitter(input: ResolveContext): Promise<PostfetchRe
33
33
  throw new Error("Tweet is unavailable or age-restricted");
34
34
  }
35
35
  await expandTwitterText(input.net, tweet, id);
36
+ await Promise.all(twitterTweets(tweet, id, true).map(({ id, tweet }) => translateTwitterText(input, tweet, id)));
36
37
  const posts = twitterTweets(tweet, id);
37
38
  const groups = posts.flatMap(({ id: postId, tweet: post }) => {
38
39
  const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
@@ -61,6 +62,11 @@ export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterEx
61
62
  if (lang) {
62
63
  extra.lang = lang;
63
64
  }
65
+ const translated = string(tweet.translated_text);
66
+ if (translated) {
67
+ extra.originalText = cleanTwitterText(twitterText(tweet), tweet) ?? undefined;
68
+ extra.translatedLang = string(tweet.translated_lang) ?? undefined;
69
+ }
64
70
  const quotedTweet = object(tweet.quoted_tweet) ? tweet.quoted_tweet : null;
65
71
  const quotedId = quotedTweet ? string(quotedTweet.id_str) : null;
66
72
  if (quotedTweet && quotedId) {
@@ -74,7 +80,7 @@ export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterEx
74
80
  extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
75
81
  }
76
82
  return {
77
- text: cleanTwitterText(twitterText(tweet), tweet) ?? undefined,
83
+ text: cleanTwitterText(translated ?? twitterText(tweet), tweet) ?? undefined,
78
84
  author: user
79
85
  ? {
80
86
  handle: string(user.screen_name) ?? undefined,
@@ -130,6 +136,42 @@ function needsFullText(tweet: Json): boolean {
130
136
  return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
131
137
  }
132
138
 
139
+ async function translateTwitterText(input: ResolveContext, tweet: Json, id: string): Promise<void> {
140
+ const languages = input.languages;
141
+ if (!languages?.length || !twitterText(tweet)) return;
142
+ const lang = string(tweet.lang)?.toLowerCase();
143
+ const allowed = (source: string) => languages.some((language) => source === language || source.startsWith(`${language}-`));
144
+ if (lang && allowed(lang)) return;
145
+ for (const target of new Set([languages[0], "en"])) {
146
+ try {
147
+ const response = await input.net(`https://api.fxtwitter.com/status/${id}/${target}`, {
148
+ headers: { accept: "application/json", "user-agent": browserUserAgent() },
149
+ }, 1);
150
+ if (!response.ok) continue;
151
+ const payload: unknown = await response.json();
152
+ if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id) continue;
153
+ const post = payload.tweet;
154
+ const translation = object(post.translation) ? post.translation : null;
155
+ const source = (string(post.lang) ?? string(translation?.source_lang))?.toLowerCase();
156
+ // The richer lookup may identify a language omitted by syndication.
157
+ if (source && source !== "und") {
158
+ tweet.lang = source;
159
+ if (allowed(source)) return;
160
+ }
161
+ const text = translation ? string(translation.text) : null;
162
+ if (!text?.trim() || string(translation?.target_lang)?.toLowerCase() !== target) continue;
163
+ // FxTwitter can return DeepL's client-upgrade error as a successful
164
+ // translation. It is an upstream failure, not the author's post text.
165
+ if (text.includes("You're using an older version of DeepL.")) continue;
166
+ tweet.translated_text = text;
167
+ tweet.translated_lang = target;
168
+ return;
169
+ } catch {
170
+ // Translation is optional; a failed lookup must not lose the post.
171
+ }
172
+ }
173
+ }
174
+
133
175
  async function expandTwitterText(net: Net, tweet: Json, rootId: string): Promise<void> {
134
176
  const posts = new Map(twitterTweets(tweet, rootId, true).map(({ id, tweet: post }) => [id, post]));
135
177
  for (const [id, post] of posts) {
@@ -368,15 +410,21 @@ async function twitterComments(input: ResolveContext, id: string): Promise<PostC
368
410
  const payload: unknown = await response.json();
369
411
  if (!object(payload) || payload.code !== 200 || !Array.isArray(payload.replies)) throw new Error("Invalid comments");
370
412
  const previousCount = comments.length;
413
+ const page: Array<{ id: string; reply: Json }> = [];
371
414
  for (const reply of payload.replies) {
372
415
  if (!object(reply)) continue;
373
416
  const replyId = string(reply.id);
374
417
  const parent = object(reply.replying_to) ? string(reply.replying_to.status) : null;
375
418
  if (!replyId || parent !== id || seen.has(replyId)) continue;
376
419
  seen.add(replyId);
377
- comments.push(twitterComment(reply, replyId));
378
- if (comments.length >= limit) return comments;
420
+ page.push({ id: replyId, reply });
421
+ if (comments.length + page.length >= limit) break;
379
422
  }
423
+ comments.push(...await Promise.all(page.map(async ({ id, reply }) => {
424
+ await translateTwitterText(input, reply, id);
425
+ return twitterComment(reply, id);
426
+ })));
427
+ if (comments.length >= limit) return comments;
380
428
  cursor = object(payload.cursor) ? string(payload.cursor.bottom) : null;
381
429
  if (!cursor || cursors.has(cursor) || comments.length === previousCount) break;
382
430
  cursors.add(cursor);
@@ -410,7 +458,7 @@ function twitterComment(reply: Json, id: string): PostComment {
410
458
  url: `https://x.com/i/status/${id}`,
411
459
  items,
412
460
  metadata: {
413
- text: cleanTwitterText(string(reply.text), reply) ?? undefined,
461
+ text: cleanTwitterText(string(reply.translated_text) ?? string(reply.text), reply) ?? undefined,
414
462
  author: {
415
463
  handle: string(author.screen_name) ?? undefined,
416
464
  name: string(author.name) ?? undefined,