@postfetch/core 0.13.8 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -1
- package/dist/index.js +403 -18
- package/dist/internal.d.ts +5 -0
- package/dist/postfetch.d.ts +5 -0
- package/dist/reddit-web.d.ts +12 -0
- package/package.json +1 -1
- package/src/internal.ts +5 -0
- package/src/postfetch.ts +13 -0
- package/src/reddit-web.ts +292 -0
- package/src/reddit.ts +41 -20
- package/src/twitter.ts +52 -4
package/README.md
CHANGED
|
@@ -59,9 +59,22 @@ const blob = await buildAudioSliderVideo(result.items, { delay: 3000 });
|
|
|
59
59
|
form.append("video", blob, "slideshow.mp4");
|
|
60
60
|
```
|
|
61
61
|
|
|
62
|
-
- `PostfetchOptions` — `{ fetch?: typeof fetch; preferredWidth?: number; tryMaxBytes?: number }`. `tryMaxBytes` is a soft byte cap. X probes every MP4 variant and selects the highest-quality complete result that fits, falling back to its smallest variants when none do. Other platforms return a smaller available rendition when possible and otherwise keep the normal result. Inject `fetch` to unit-test resolvers offline.
|
|
62
|
+
- `PostfetchOptions` — `{ fetch?: typeof fetch; preferredWidth?: number; tryMaxBytes?: number; languages?: string[] }`. `tryMaxBytes` is a soft byte cap. X probes every MP4 variant and selects the highest-quality complete result that fits, falling back to its smallest variants when none do. Other platforms return a smaller available rendition when possible and otherwise keep the normal result. Inject `fetch` to unit-test resolvers offline.
|
|
63
63
|
- Every request rotates a consistent browser/app fingerprint, so a fixed user-agent never gets the whole fleet blocked.
|
|
64
64
|
- No runtime dependencies, no `yt-dlp`, no cookies. `ffmpeg` is only used when
|
|
65
65
|
`downloadBlob(..., { remux: true })` or `buildAudioSliderVideo` is requested.
|
|
66
66
|
|
|
67
67
|
See the [root README](https://github.com/chelokot/postfetch#readme) for the full API table, supported inputs and the fingerprint design.
|
|
68
|
+
|
|
69
|
+
### X text languages
|
|
70
|
+
|
|
71
|
+
Pass `postfetch(url, { languages: ["en", "ru"] })` to keep X posts written in
|
|
72
|
+
any allowed language. Other posts are translated to the first language in the
|
|
73
|
+
list, falling back to English and then the original if translation is unavailable.
|
|
74
|
+
The ordered list must be non-empty; omit it to keep original text. This applies
|
|
75
|
+
independently to quoted posts, parent posts and requested replies. Regional
|
|
76
|
+
variants match a base language (`en` allows `en-US`). Other platforms ignore it.
|
|
77
|
+
|
|
78
|
+
`metadata.extra.lang` remains the detected original language. Translated posts
|
|
79
|
+
also expose `metadata.extra.originalText` and `metadata.extra.translatedLang`.
|
|
80
|
+
Translations use FxTwitter's public translation API and do not alter video audio.
|
package/dist/index.js
CHANGED
|
@@ -1613,12 +1613,322 @@ function imageMime(url) {
|
|
|
1613
1613
|
return "image/jpeg";
|
|
1614
1614
|
}
|
|
1615
1615
|
|
|
1616
|
+
// src/reddit-web.ts
|
|
1617
|
+
async function readWebPage(response, userAgent) {
|
|
1618
|
+
const cookies = response.headers.getSetCookie().map((cookie) => cookie.split(";", 1)[0]);
|
|
1619
|
+
if (!response.ok) {
|
|
1620
|
+
await response.body?.cancel();
|
|
1621
|
+
return { url: response.url, html: "", userAgent, cookies, ok: false };
|
|
1622
|
+
}
|
|
1623
|
+
return { url: response.url, html: await response.text(), userAgent, cookies, ok: true };
|
|
1624
|
+
}
|
|
1625
|
+
async function fetchWebPage(net, input, userAgent, cookies = []) {
|
|
1626
|
+
let url = new URL(input);
|
|
1627
|
+
let jar = new Map(cookies.map((cookie) => [cookie.split("=", 1)[0], cookie]));
|
|
1628
|
+
for (let redirects = 0;redirects <= 5; redirects += 1) {
|
|
1629
|
+
if (url.protocol !== "https:" || !/^(?:(?:www\.|old\.|m\.)?reddit\.com|redd\.it)$/.test(url.hostname))
|
|
1630
|
+
throw new Error("Invalid Reddit page redirect");
|
|
1631
|
+
const headers = { "user-agent": userAgent, ...jar.size ? { cookie: [...jar.values()].join("; ") } : {} };
|
|
1632
|
+
const response = await net(url.href, { headers, redirect: "manual" }, 1);
|
|
1633
|
+
for (const cookie of response.headers.getSetCookie()) {
|
|
1634
|
+
const value = cookie.split(";", 1)[0];
|
|
1635
|
+
const name = value.split("=", 1)[0];
|
|
1636
|
+
if (/;\s*max-age=0(?:;|$)/i.test(cookie))
|
|
1637
|
+
jar.delete(name);
|
|
1638
|
+
else
|
|
1639
|
+
jar.set(name, value);
|
|
1640
|
+
}
|
|
1641
|
+
const location = response.headers.get("location");
|
|
1642
|
+
if ([301, 302, 303, 307, 308].includes(response.status) && location) {
|
|
1643
|
+
await response.body?.cancel();
|
|
1644
|
+
const target = new URL(location, url);
|
|
1645
|
+
if (target.origin !== url.origin)
|
|
1646
|
+
jar = new Map;
|
|
1647
|
+
url = target;
|
|
1648
|
+
continue;
|
|
1649
|
+
}
|
|
1650
|
+
return { ...await readWebPage(response, userAgent), url: response.url || url.href, cookies: [...jar.values()] };
|
|
1651
|
+
}
|
|
1652
|
+
throw new Error("Too many Reddit page redirects");
|
|
1653
|
+
}
|
|
1654
|
+
async function fetchWebPost(net, id, input, initial) {
|
|
1655
|
+
const url = new URL(input);
|
|
1656
|
+
if (url.protocol !== "https:" || !/^(?:(?:www\.|old\.|m\.)?reddit\.com|redd\.it)$/.test(url.hostname))
|
|
1657
|
+
return null;
|
|
1658
|
+
if (url.hostname !== "redd.it")
|
|
1659
|
+
url.hostname = "www.reddit.com";
|
|
1660
|
+
const userAgent = initial?.userAgent ?? browserUserAgent();
|
|
1661
|
+
let page = initial ?? await fetchWebPage(net, url.href, userAgent);
|
|
1662
|
+
if (!page.ok)
|
|
1663
|
+
return null;
|
|
1664
|
+
const challenge = challengeUrl(page.html, page.url);
|
|
1665
|
+
if (challenge) {
|
|
1666
|
+
page = await fetchWebPage(net, challenge, userAgent, page.cookies);
|
|
1667
|
+
if (!page.ok)
|
|
1668
|
+
return null;
|
|
1669
|
+
}
|
|
1670
|
+
for (const match of page.html.matchAll(/<shreddit-post\s([^>]+)>([\s\S]*?)<\/shreddit-post>/gi)) {
|
|
1671
|
+
const post = attributes(match[1]);
|
|
1672
|
+
if (post.id !== `t3_${id}` || !post["post-title"])
|
|
1673
|
+
continue;
|
|
1674
|
+
const media = postMedia(post, match[2], id);
|
|
1675
|
+
if (!media)
|
|
1676
|
+
return null;
|
|
1677
|
+
const created = Date.parse(post["created-timestamp"] ?? "");
|
|
1678
|
+
return {
|
|
1679
|
+
...media,
|
|
1680
|
+
id,
|
|
1681
|
+
title: post["post-title"],
|
|
1682
|
+
selftext: postText(match[2], id),
|
|
1683
|
+
author: post.author,
|
|
1684
|
+
subreddit: post["subreddit-name"],
|
|
1685
|
+
permalink: post.permalink,
|
|
1686
|
+
created_utc: Number.isFinite(created) ? Math.floor(created / 1000) : undefined,
|
|
1687
|
+
ups: finiteNumber(post.score),
|
|
1688
|
+
score: finiteNumber(post.score),
|
|
1689
|
+
upvote_ratio: finiteNumber(post["upvote-ratio"]),
|
|
1690
|
+
num_comments: finiteNumber(post["comment-count"]),
|
|
1691
|
+
over_18: Object.hasOwn(post, "nsfw")
|
|
1692
|
+
};
|
|
1693
|
+
}
|
|
1694
|
+
return null;
|
|
1695
|
+
}
|
|
1696
|
+
async function fetchRssPost(net, id) {
|
|
1697
|
+
const url = `https://www.reddit.com/comments/${id}/.rss?limit=1`;
|
|
1698
|
+
const headers = { "user-agent": browserUserAgent(), accept: "application/atom+xml" };
|
|
1699
|
+
let response = await net(url, { headers }, 1);
|
|
1700
|
+
if (response.status === 429) {
|
|
1701
|
+
const reset = finiteNumber(response.headers.get("x-ratelimit-reset") ?? undefined);
|
|
1702
|
+
await response.body?.cancel();
|
|
1703
|
+
if (reset === undefined || reset < 0 || reset > 15)
|
|
1704
|
+
return null;
|
|
1705
|
+
await new Promise((resolve) => setTimeout(resolve, (reset + 0.1) * 1000));
|
|
1706
|
+
response = await net(url, { headers }, 1);
|
|
1707
|
+
}
|
|
1708
|
+
if (!response.ok) {
|
|
1709
|
+
await response.body?.cancel();
|
|
1710
|
+
return null;
|
|
1711
|
+
}
|
|
1712
|
+
const xml = await response.text();
|
|
1713
|
+
if (!/<feed\b/.test(xml))
|
|
1714
|
+
return null;
|
|
1715
|
+
for (const match of xml.matchAll(/<entry\b[^>]*>([\s\S]*?)<\/entry>/g)) {
|
|
1716
|
+
const entry = match[1];
|
|
1717
|
+
if (entry.match(/<id>([^<]+)<\/id>/)?.[1] !== `t3_${id}`)
|
|
1718
|
+
continue;
|
|
1719
|
+
const title = entry.match(/<title>([\s\S]*?)<\/title>/)?.[1];
|
|
1720
|
+
const content = decodeHtml3(entry.match(/<content\b[^>]*>([\s\S]*?)<\/content>/)?.[1] ?? "");
|
|
1721
|
+
const link = [...content.matchAll(/<a\s([^>]+)>\s*\[link\]\s*<\/a>/g)][0];
|
|
1722
|
+
const destination = link ? attributes(link[1]).href : undefined;
|
|
1723
|
+
const image = hostedImage(destination);
|
|
1724
|
+
let media = image ? { post_hint: "image", url: image } : null;
|
|
1725
|
+
if (!media && destination) {
|
|
1726
|
+
const target = new URL(destination);
|
|
1727
|
+
if (target.protocol === "https:" && target.hostname === "v.redd.it" && /^\/[a-z0-9]+\/?$/i.test(target.pathname)) {
|
|
1728
|
+
media = { secure_media: { reddit_video: { dash_url: target.href.replace(/\/$/, "") + "/DASHPlaylist.mpd" } } };
|
|
1729
|
+
} else if (/^https?:$/.test(target.protocol) && !/(^|\.)(reddit\.com|redd\.it)$/.test(target.hostname)) {
|
|
1730
|
+
const card = content.match(/<img\s([^>]+)>/);
|
|
1731
|
+
const preview = card ? attributes(card[1]).src : undefined;
|
|
1732
|
+
const previewUrl = preview ? new URL(preview) : null;
|
|
1733
|
+
media = {
|
|
1734
|
+
post_hint: "link",
|
|
1735
|
+
is_self: false,
|
|
1736
|
+
url: target.href,
|
|
1737
|
+
...previewUrl?.protocol === "https:" && previewUrl.hostname === "external-preview.redd.it" ? { preview: { images: [{ source: { url: previewUrl.href } }] } } : {}
|
|
1738
|
+
};
|
|
1739
|
+
}
|
|
1740
|
+
}
|
|
1741
|
+
const body = content.match(/<!-- SC_OFF -->([\s\S]*?)<!-- SC_ON -->/)?.[1];
|
|
1742
|
+
if (!media && !destination && body)
|
|
1743
|
+
media = { is_self: true };
|
|
1744
|
+
if (!title || !media)
|
|
1745
|
+
return null;
|
|
1746
|
+
const permalink = [...entry.matchAll(/<link\s([^>]+)\/?\s*>/g)].map((match2) => attributes(match2[1]).href).find((href) => href?.includes(`/comments/${id}/`));
|
|
1747
|
+
const created = Date.parse(entry.match(/<published>([^<]+)<\/published>/)?.[1] ?? "");
|
|
1748
|
+
return {
|
|
1749
|
+
...media,
|
|
1750
|
+
id,
|
|
1751
|
+
title: decodeHtml3(title),
|
|
1752
|
+
selftext: body ? plainText(body) : undefined,
|
|
1753
|
+
author: decodeHtml3(entry.match(/<name>([^<]+)<\/name>/)?.[1] ?? "").replace(/^\/u\//, "") || undefined,
|
|
1754
|
+
subreddit: attributes(entry.match(/<category\s([^>]+)>/)?.[1] ?? "").term,
|
|
1755
|
+
permalink: permalink ? new URL(permalink).pathname : undefined,
|
|
1756
|
+
created_utc: Number.isFinite(created) ? Math.floor(created / 1000) : undefined
|
|
1757
|
+
};
|
|
1758
|
+
}
|
|
1759
|
+
return null;
|
|
1760
|
+
}
|
|
1761
|
+
function postMedia(post, html, id) {
|
|
1762
|
+
if (post["post-type"] === "text")
|
|
1763
|
+
return { is_self: true };
|
|
1764
|
+
if (post["post-type"] === "image") {
|
|
1765
|
+
const url = hostedImage(post["content-href"]);
|
|
1766
|
+
return url ? { post_hint: "image", url } : null;
|
|
1767
|
+
}
|
|
1768
|
+
if (post["post-type"] === "gallery") {
|
|
1769
|
+
const gallery = html.match(/<(gallery-carousel|shreddit-gallery)\b[^>]*>([\s\S]*?)<\/\1>/i);
|
|
1770
|
+
if (!gallery)
|
|
1771
|
+
return null;
|
|
1772
|
+
const images = [];
|
|
1773
|
+
for (const match of gallery[2].matchAll(/<a\s([^>]+)>/gi)) {
|
|
1774
|
+
const href = attributes(match[1]).href;
|
|
1775
|
+
if (!href)
|
|
1776
|
+
continue;
|
|
1777
|
+
const target = new URL(href, "https://www.reddit.com");
|
|
1778
|
+
const media = target.origin === "https://www.reddit.com" && target.pathname === "/media" ? target.searchParams.get("url") : target.hostname === "i.redd.it" ? target.href : null;
|
|
1779
|
+
if (!media)
|
|
1780
|
+
continue;
|
|
1781
|
+
const original = hostedImage(media);
|
|
1782
|
+
if (!original)
|
|
1783
|
+
return null;
|
|
1784
|
+
if (!images.includes(original))
|
|
1785
|
+
images.push(original);
|
|
1786
|
+
}
|
|
1787
|
+
if (images.length < 2)
|
|
1788
|
+
return null;
|
|
1789
|
+
return {
|
|
1790
|
+
gallery_data: { items: images.map((_, index) => ({ media_id: String(index) })) },
|
|
1791
|
+
media_metadata: Object.fromEntries(images.map((url, index) => [String(index), { s: { u: url } }]))
|
|
1792
|
+
};
|
|
1793
|
+
}
|
|
1794
|
+
if (post["post-type"] === "link") {
|
|
1795
|
+
const url = new URL(post["content-href"] ?? "", "https://www.reddit.com");
|
|
1796
|
+
if (!/^https?:$/.test(url.protocol) || /(^|\.)(reddit\.com|redd\.it)$/.test(url.hostname))
|
|
1797
|
+
return null;
|
|
1798
|
+
const preview = linkPreview(html);
|
|
1799
|
+
return {
|
|
1800
|
+
post_hint: "link",
|
|
1801
|
+
is_self: false,
|
|
1802
|
+
url: url.href,
|
|
1803
|
+
...preview ? { preview: { images: [{ source: { url: preview } }] } } : {}
|
|
1804
|
+
};
|
|
1805
|
+
}
|
|
1806
|
+
if (post["post-type"] === "video" || post["post-type"] === "gif") {
|
|
1807
|
+
const destination = new URL(post["content-href"] ?? "", "https://www.reddit.com");
|
|
1808
|
+
if (destination.protocol !== "https:" || destination.hostname !== "v.redd.it" || !/^\/[a-z0-9]+\/?$/i.test(destination.pathname))
|
|
1809
|
+
return null;
|
|
1810
|
+
for (const match of html.matchAll(/<shreddit-player\s([^>]+)>/gi)) {
|
|
1811
|
+
const player = attributes(match[1]);
|
|
1812
|
+
if (player["post-id"] !== `t3_${id}` || player["media-placement"] === "comment")
|
|
1813
|
+
continue;
|
|
1814
|
+
const playlist = new URL(player.src ?? "", destination);
|
|
1815
|
+
const base = destination.href.replace(/\/$/, "") + "/";
|
|
1816
|
+
if (playlist.origin !== destination.origin || !playlist.href.startsWith(base) || !/\/(?:HLSPlaylist\.m3u8|DASHPlaylist\.mpd)$/.test(playlist.pathname))
|
|
1817
|
+
continue;
|
|
1818
|
+
const dash = new URL("DASHPlaylist.mpd", base);
|
|
1819
|
+
dash.search = playlist.search;
|
|
1820
|
+
return { secure_media: { reddit_video: { dash_url: dash.href } } };
|
|
1821
|
+
}
|
|
1822
|
+
}
|
|
1823
|
+
return null;
|
|
1824
|
+
}
|
|
1825
|
+
function linkPreview(html) {
|
|
1826
|
+
for (const match of html.matchAll(/<img\s([^>]+)>/gi)) {
|
|
1827
|
+
const image = attributes(match[1]);
|
|
1828
|
+
if (image.id !== "post-image" || !Object.hasOwn(image, "data-post-media-primary"))
|
|
1829
|
+
continue;
|
|
1830
|
+
const sources = (image.srcset ?? "").split(/,\s*/).flatMap((source2) => {
|
|
1831
|
+
const match2 = source2.trim().match(/^(\S+)\s+(\d+)w$/);
|
|
1832
|
+
return match2 ? [{ url: match2[1], width: Number(match2[2]) }] : [];
|
|
1833
|
+
});
|
|
1834
|
+
if (image.src)
|
|
1835
|
+
sources.push({ url: image.src, width: finiteNumber(image.width) ?? 0 });
|
|
1836
|
+
for (const source2 of sources.sort((a, b) => b.width - a.width)) {
|
|
1837
|
+
const url = new URL(source2.url, "https://www.reddit.com");
|
|
1838
|
+
if (url.protocol === "https:" && /^(?:external-preview|preview)\.redd\.it$/.test(url.hostname))
|
|
1839
|
+
return url.href;
|
|
1840
|
+
}
|
|
1841
|
+
}
|
|
1842
|
+
return null;
|
|
1843
|
+
}
|
|
1844
|
+
function hostedImage(input) {
|
|
1845
|
+
if (!input)
|
|
1846
|
+
return null;
|
|
1847
|
+
const url = new URL(input, "https://www.reddit.com");
|
|
1848
|
+
return url.protocol === "https:" && url.hostname === "i.redd.it" && /\.(?:jpe?g|png|webp|gif)$/i.test(url.pathname) ? url.href : null;
|
|
1849
|
+
}
|
|
1850
|
+
function postText(html, id) {
|
|
1851
|
+
for (const match of html.matchAll(/<div\s([^>]+)>/gi)) {
|
|
1852
|
+
const attrs = attributes(match[1]);
|
|
1853
|
+
if (attrs.id !== `t3_${id}-post-rtjson-content`)
|
|
1854
|
+
continue;
|
|
1855
|
+
const start = match.index + match[0].length;
|
|
1856
|
+
const tags = /<div\b[^>]*>|<\/div\s*>/gi;
|
|
1857
|
+
tags.lastIndex = start;
|
|
1858
|
+
let depth = 1;
|
|
1859
|
+
for (let tag = tags.exec(html);tag; tag = tags.exec(html)) {
|
|
1860
|
+
depth += tag[0].startsWith("</") ? -1 : 1;
|
|
1861
|
+
if (depth === 0)
|
|
1862
|
+
return plainText(html.slice(start, tag.index));
|
|
1863
|
+
}
|
|
1864
|
+
}
|
|
1865
|
+
return;
|
|
1866
|
+
}
|
|
1867
|
+
function plainText(html) {
|
|
1868
|
+
return decodeHtml3(html.replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, "").replace(/<\/?(?:p|div|li|blockquote|pre|h[1-6])\b[^>]*>|<br\b[^>]*>/gi, `
|
|
1869
|
+
`).replace(/<[^>]*>/g, "")).replace(/[\t ]+/g, " ").replace(/ *\n */g, `
|
|
1870
|
+
`).replace(/\n{3,}/g, `
|
|
1871
|
+
|
|
1872
|
+
`).trim();
|
|
1873
|
+
}
|
|
1874
|
+
function finiteNumber(value) {
|
|
1875
|
+
const number2 = value === undefined ? NaN : Number(value);
|
|
1876
|
+
return Number.isFinite(number2) ? number2 : undefined;
|
|
1877
|
+
}
|
|
1878
|
+
function challengeUrl(html, responseUrl) {
|
|
1879
|
+
const seed = html.match(/await\s*\(async\s+([a-z])\s*=>\s*\1\s*\+\s*\1\)\("([a-f0-9]{16})"\)/i)?.[2];
|
|
1880
|
+
const form = html.match(/<form\b([^>]*)>([\s\S]*?)<\/form>/i);
|
|
1881
|
+
if (!seed || !form || !responseUrl)
|
|
1882
|
+
return null;
|
|
1883
|
+
const fields = attributes(form[1]);
|
|
1884
|
+
if (fields.method?.toUpperCase() !== "GET" || !fields.action)
|
|
1885
|
+
return null;
|
|
1886
|
+
const base = new URL(responseUrl);
|
|
1887
|
+
const target = new URL(fields.action, base);
|
|
1888
|
+
if (base.origin !== "https://www.reddit.com" || target.origin !== base.origin || target.pathname !== base.pathname)
|
|
1889
|
+
return null;
|
|
1890
|
+
const inputs = new Map;
|
|
1891
|
+
for (const input of form[2].matchAll(/<input\b([^>]*)>/gi)) {
|
|
1892
|
+
const attrs = attributes(input[1]);
|
|
1893
|
+
if (attrs.name)
|
|
1894
|
+
inputs.set(attrs.name, attrs.value ?? "");
|
|
1895
|
+
}
|
|
1896
|
+
if (inputs.get("js_challenge") !== "1" || !inputs.get("jsc_token") || !inputs.has("solution"))
|
|
1897
|
+
return null;
|
|
1898
|
+
target.search = "";
|
|
1899
|
+
for (const [key, value] of inputs) {
|
|
1900
|
+
if (["solution", "js_challenge", "jsc_token", "jsc_orig_r"].includes(key))
|
|
1901
|
+
target.searchParams.append(key, key === "solution" ? seed + seed : value);
|
|
1902
|
+
}
|
|
1903
|
+
for (const [key, value] of base.searchParams) {
|
|
1904
|
+
if (!inputs.has(key))
|
|
1905
|
+
target.searchParams.append(key, value);
|
|
1906
|
+
}
|
|
1907
|
+
return target.href;
|
|
1908
|
+
}
|
|
1909
|
+
function attributes(tag) {
|
|
1910
|
+
const result = {};
|
|
1911
|
+
for (const match of tag.matchAll(/(?:^|\s)([\w-]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'))?/g)) {
|
|
1912
|
+
result[match[1].toLowerCase()] = decodeHtml3(match[2] ?? match[3] ?? "");
|
|
1913
|
+
}
|
|
1914
|
+
return result;
|
|
1915
|
+
}
|
|
1916
|
+
function decodeHtml3(input) {
|
|
1917
|
+
const named = { amp: "&", apos: "'", gt: ">", lt: "<", nbsp: " ", quot: '"' };
|
|
1918
|
+
return input.replace(/&(#x[\da-f]+|#\d+|amp|apos|gt|lt|nbsp|quot);/gi, (entity, code) => {
|
|
1919
|
+
if (!code.startsWith("#"))
|
|
1920
|
+
return named[code.toLowerCase()] ?? entity;
|
|
1921
|
+
const value = code[1]?.toLowerCase() === "x" ? Number.parseInt(code.slice(2), 16) : Number.parseInt(code.slice(1), 10);
|
|
1922
|
+
return Number.isSafeInteger(value) && value <= 1114111 ? String.fromCodePoint(value) : entity;
|
|
1923
|
+
});
|
|
1924
|
+
}
|
|
1925
|
+
|
|
1616
1926
|
// src/reddit.ts
|
|
1617
1927
|
var installedClientId = "ohXpoqrZYub1kg";
|
|
1618
1928
|
async function resolveReddit(input) {
|
|
1619
|
-
const id = await postId(input.net, input.url);
|
|
1929
|
+
const { id, url, page } = await postId(input.net, input.url);
|
|
1620
1930
|
const token = await appToken(input.net);
|
|
1621
|
-
const post = await fetchPost(input.net, token, id);
|
|
1931
|
+
const post = await fetchPost(input.net, token, id, url, page);
|
|
1622
1932
|
const items = await mediaItems3(input, post, id);
|
|
1623
1933
|
return { archiveFilename: filename(`reddit_${id}.zip`), comments: await redditComments(input, token, id), id, items, metadata: redditMetadata(post), platform: "reddit" };
|
|
1624
1934
|
}
|
|
@@ -1643,14 +1953,15 @@ function redditMetadata(post) {
|
|
|
1643
1953
|
async function postId(net, input) {
|
|
1644
1954
|
const direct = parseId(input);
|
|
1645
1955
|
if (direct) {
|
|
1646
|
-
return direct;
|
|
1956
|
+
return { id: direct, url: input };
|
|
1647
1957
|
}
|
|
1648
|
-
const
|
|
1649
|
-
const
|
|
1958
|
+
const userAgent = browserUserAgent();
|
|
1959
|
+
const page = await readWebPage(await net(input, { headers: { "user-agent": userAgent } }, 1), userAgent);
|
|
1960
|
+
const resolved = parseId(page.url);
|
|
1650
1961
|
if (!resolved) {
|
|
1651
1962
|
throw new Error("Reddit post id not found");
|
|
1652
1963
|
}
|
|
1653
|
-
return resolved;
|
|
1964
|
+
return { id: resolved, url: page.url, page };
|
|
1654
1965
|
}
|
|
1655
1966
|
function parseId(input) {
|
|
1656
1967
|
const url = asUrl(input);
|
|
@@ -1680,7 +1991,7 @@ async function appToken(net) {
|
|
|
1680
1991
|
}
|
|
1681
1992
|
return token;
|
|
1682
1993
|
}
|
|
1683
|
-
async function fetchPost(net, token, id) {
|
|
1994
|
+
async function fetchPost(net, token, id, url, page) {
|
|
1684
1995
|
const headers = { authorization: `Bearer ${token}`, "user-agent": browserUserAgent() };
|
|
1685
1996
|
let response = await net(`https://oauth.reddit.com/comments/${id}?raw_json=1&limit=1`, { headers });
|
|
1686
1997
|
if (response.status === 403) {
|
|
@@ -1688,6 +1999,15 @@ async function fetchPost(net, token, id) {
|
|
|
1688
1999
|
response = await net(`https://oauth.reddit.com/by_id/t3_${id}?raw_json=1`, { headers });
|
|
1689
2000
|
}
|
|
1690
2001
|
if (!response.ok) {
|
|
2002
|
+
if (response.status === 403) {
|
|
2003
|
+
await response.body?.cancel();
|
|
2004
|
+
const post2 = await fetchWebPost(net, id, url, page).catch(() => null);
|
|
2005
|
+
if (post2)
|
|
2006
|
+
return post2;
|
|
2007
|
+
const rss = await fetchRssPost(net, id).catch(() => null);
|
|
2008
|
+
if (rss)
|
|
2009
|
+
return rss;
|
|
2010
|
+
}
|
|
1691
2011
|
throw new Error(`Reddit post failed: ${response.status}`);
|
|
1692
2012
|
}
|
|
1693
2013
|
const payload = await response.json().catch(() => null);
|
|
@@ -1746,20 +2066,28 @@ function galleryItems(post, id) {
|
|
|
1746
2066
|
async function videoItems(input, post, id) {
|
|
1747
2067
|
const media = redditVideo(post);
|
|
1748
2068
|
const fallback = media ? string(media.fallback_url) : null;
|
|
1749
|
-
if (!media
|
|
2069
|
+
if (!media) {
|
|
1750
2070
|
return [];
|
|
1751
2071
|
}
|
|
2072
|
+
const dashUrl = string(media.dash_url);
|
|
2073
|
+
if (!fallback && dashUrl) {
|
|
2074
|
+
const streams2 = await dashStreams(input.net, dashUrl, input.preferredWidth, true);
|
|
2075
|
+
if (!streams2)
|
|
2076
|
+
throw new Error("Reddit DASH manifest unavailable for muxing");
|
|
2077
|
+
return [streams2.audio ? muxedVideoItem(id, streams2.video, streams2.audio) : mediaItem(id, null, "video", "video/mp4", streams2.video)];
|
|
2078
|
+
}
|
|
2079
|
+
if (!fallback)
|
|
2080
|
+
return [];
|
|
1752
2081
|
if (media.has_audio !== true) {
|
|
1753
2082
|
return [mediaItem(id, null, "video", "video/mp4", fallback)];
|
|
1754
2083
|
}
|
|
1755
|
-
const dashUrl = string(media.dash_url);
|
|
1756
2084
|
const streams = dashUrl ? await dashStreams(input.net, dashUrl, input.preferredWidth) : null;
|
|
1757
|
-
if (!streams) {
|
|
2085
|
+
if (!streams?.audio) {
|
|
1758
2086
|
throw new Error("Reddit DASH manifest unavailable for muxing");
|
|
1759
2087
|
}
|
|
1760
2088
|
return [muxedVideoItem(id, streams.video, streams.audio)];
|
|
1761
2089
|
}
|
|
1762
|
-
async function dashStreams(net, dashUrl, preferredWidth) {
|
|
2090
|
+
async function dashStreams(net, dashUrl, preferredWidth, allowSilent = false) {
|
|
1763
2091
|
const response = await net(dashUrl, { headers: { "user-agent": browserUserAgent() } });
|
|
1764
2092
|
if (!response.ok) {
|
|
1765
2093
|
return null;
|
|
@@ -1769,10 +2097,10 @@ async function dashStreams(net, dashUrl, preferredWidth) {
|
|
|
1769
2097
|
const query = dashUrl.includes("?") ? dashUrl.slice(dashUrl.indexOf("?")) : "";
|
|
1770
2098
|
const video = videoRendition(manifest, preferredWidth);
|
|
1771
2099
|
const audio = audioRendition(manifest);
|
|
1772
|
-
if (!video || !audio) {
|
|
2100
|
+
if (!video || !audio && !allowSilent) {
|
|
1773
2101
|
return null;
|
|
1774
2102
|
}
|
|
1775
|
-
return { video: `${base}${video}${query}`, audio: `${base}${audio}${query}` };
|
|
2103
|
+
return { video: `${base}${video}${query}`, ...audio ? { audio: `${base}${audio}${query}` } : {} };
|
|
1776
2104
|
}
|
|
1777
2105
|
function videoRendition(manifest, preferredWidth) {
|
|
1778
2106
|
const renditions = renditionsOf(manifest, false);
|
|
@@ -3070,6 +3398,7 @@ async function resolveTwitter(input) {
|
|
|
3070
3398
|
throw new Error("Tweet is unavailable or age-restricted");
|
|
3071
3399
|
}
|
|
3072
3400
|
await expandTwitterText(input.net, tweet, id);
|
|
3401
|
+
await Promise.all(twitterTweets(tweet, id, true).map(({ id: id2, tweet: tweet2 }) => translateTwitterText(input, tweet2, id2)));
|
|
3073
3402
|
const posts = twitterTweets(tweet, id);
|
|
3074
3403
|
const groups = posts.flatMap(({ id: postId2, tweet: post }) => {
|
|
3075
3404
|
const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
|
|
@@ -3093,6 +3422,11 @@ function twitterMetadata(tweet) {
|
|
|
3093
3422
|
if (lang) {
|
|
3094
3423
|
extra.lang = lang;
|
|
3095
3424
|
}
|
|
3425
|
+
const translated = string(tweet.translated_text);
|
|
3426
|
+
if (translated) {
|
|
3427
|
+
extra.originalText = cleanTwitterText(twitterText(tweet), tweet) ?? undefined;
|
|
3428
|
+
extra.translatedLang = string(tweet.translated_lang) ?? undefined;
|
|
3429
|
+
}
|
|
3096
3430
|
const quotedTweet = object(tweet.quoted_tweet) ? tweet.quoted_tweet : null;
|
|
3097
3431
|
const quotedId = quotedTweet ? string(quotedTweet.id_str) : null;
|
|
3098
3432
|
if (quotedTweet && quotedId) {
|
|
@@ -3107,7 +3441,7 @@ function twitterMetadata(tweet) {
|
|
|
3107
3441
|
extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
|
|
3108
3442
|
}
|
|
3109
3443
|
return {
|
|
3110
|
-
text: cleanTwitterText(twitterText(tweet), tweet) ?? undefined,
|
|
3444
|
+
text: cleanTwitterText(translated ?? twitterText(tweet), tweet) ?? undefined,
|
|
3111
3445
|
author: user ? {
|
|
3112
3446
|
handle: string(user.screen_name) ?? undefined,
|
|
3113
3447
|
name: string(user.name) ?? undefined,
|
|
@@ -3154,6 +3488,43 @@ function needsFullText(tweet) {
|
|
|
3154
3488
|
}
|
|
3155
3489
|
return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
|
|
3156
3490
|
}
|
|
3491
|
+
async function translateTwitterText(input, tweet, id) {
|
|
3492
|
+
const languages = input.languages;
|
|
3493
|
+
if (!languages?.length || !twitterText(tweet))
|
|
3494
|
+
return;
|
|
3495
|
+
const lang = string(tweet.lang)?.toLowerCase();
|
|
3496
|
+
const allowed = (source2) => languages.some((language) => source2 === language || source2.startsWith(`${language}-`));
|
|
3497
|
+
if (lang && allowed(lang))
|
|
3498
|
+
return;
|
|
3499
|
+
for (const target of new Set([languages[0], "en"])) {
|
|
3500
|
+
try {
|
|
3501
|
+
const response = await input.net(`https://api.fxtwitter.com/status/${id}/${target}`, {
|
|
3502
|
+
headers: { accept: "application/json", "user-agent": browserUserAgent() }
|
|
3503
|
+
}, 1);
|
|
3504
|
+
if (!response.ok)
|
|
3505
|
+
continue;
|
|
3506
|
+
const payload = await response.json();
|
|
3507
|
+
if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id)
|
|
3508
|
+
continue;
|
|
3509
|
+
const post = payload.tweet;
|
|
3510
|
+
const translation = object(post.translation) ? post.translation : null;
|
|
3511
|
+
const source2 = (string(post.lang) ?? string(translation?.source_lang))?.toLowerCase();
|
|
3512
|
+
if (source2 && source2 !== "und") {
|
|
3513
|
+
tweet.lang = source2;
|
|
3514
|
+
if (allowed(source2))
|
|
3515
|
+
return;
|
|
3516
|
+
}
|
|
3517
|
+
const text2 = translation ? string(translation.text) : null;
|
|
3518
|
+
if (!text2?.trim() || string(translation?.target_lang)?.toLowerCase() !== target)
|
|
3519
|
+
continue;
|
|
3520
|
+
if (text2.includes("You're using an older version of DeepL."))
|
|
3521
|
+
continue;
|
|
3522
|
+
tweet.translated_text = text2;
|
|
3523
|
+
tweet.translated_lang = target;
|
|
3524
|
+
return;
|
|
3525
|
+
} catch {}
|
|
3526
|
+
}
|
|
3527
|
+
}
|
|
3157
3528
|
async function expandTwitterText(net, tweet, rootId) {
|
|
3158
3529
|
const posts = new Map(twitterTweets(tweet, rootId, true).map(({ id, tweet: post }) => [id, post]));
|
|
3159
3530
|
for (const [id, post] of posts) {
|
|
@@ -3374,6 +3745,7 @@ async function twitterComments(input, id) {
|
|
|
3374
3745
|
if (!object(payload) || payload.code !== 200 || !Array.isArray(payload.replies))
|
|
3375
3746
|
throw new Error("Invalid comments");
|
|
3376
3747
|
const previousCount = comments.length;
|
|
3748
|
+
const page = [];
|
|
3377
3749
|
for (const reply of payload.replies) {
|
|
3378
3750
|
if (!object(reply))
|
|
3379
3751
|
continue;
|
|
@@ -3382,10 +3754,16 @@ async function twitterComments(input, id) {
|
|
|
3382
3754
|
if (!replyId || parent !== id || seen.has(replyId))
|
|
3383
3755
|
continue;
|
|
3384
3756
|
seen.add(replyId);
|
|
3385
|
-
|
|
3386
|
-
if (comments.length >= limit)
|
|
3387
|
-
|
|
3757
|
+
page.push({ id: replyId, reply });
|
|
3758
|
+
if (comments.length + page.length >= limit)
|
|
3759
|
+
break;
|
|
3388
3760
|
}
|
|
3761
|
+
comments.push(...await Promise.all(page.map(async ({ id: id2, reply }) => {
|
|
3762
|
+
await translateTwitterText(input, reply, id2);
|
|
3763
|
+
return twitterComment(reply, id2);
|
|
3764
|
+
})));
|
|
3765
|
+
if (comments.length >= limit)
|
|
3766
|
+
return comments;
|
|
3389
3767
|
cursor = object(payload.cursor) ? string(payload.cursor.bottom) : null;
|
|
3390
3768
|
if (!cursor || cursors.has(cursor) || comments.length === previousCount)
|
|
3391
3769
|
break;
|
|
@@ -3420,7 +3798,7 @@ function twitterComment(reply, id) {
|
|
|
3420
3798
|
url: `https://x.com/i/status/${id}`,
|
|
3421
3799
|
items,
|
|
3422
3800
|
metadata: {
|
|
3423
|
-
text: cleanTwitterText(string(reply.text), reply) ?? undefined,
|
|
3801
|
+
text: cleanTwitterText(string(reply.translated_text) ?? string(reply.text), reply) ?? undefined,
|
|
3424
3802
|
author: {
|
|
3425
3803
|
handle: string(author2.screen_name) ?? undefined,
|
|
3426
3804
|
name: string(author2.name) ?? undefined,
|
|
@@ -3447,6 +3825,7 @@ async function postfetch(url, options = {}) {
|
|
|
3447
3825
|
}
|
|
3448
3826
|
const context = {
|
|
3449
3827
|
net: createNet(options.fetch ?? globalThis.fetch),
|
|
3828
|
+
languages: options.languages === undefined ? undefined : normalizeLanguages(options.languages),
|
|
3450
3829
|
comments: options.comments,
|
|
3451
3830
|
preferredWidth: options.preferredWidth ?? 720,
|
|
3452
3831
|
tryMaxBytes: options.tryMaxBytes,
|
|
@@ -3455,6 +3834,12 @@ async function postfetch(url, options = {}) {
|
|
|
3455
3834
|
const result = await resolve(context);
|
|
3456
3835
|
return options.tryMaxBytes === undefined || result.platform === "twitter" ? result : trySmaller(context, result, options.tryMaxBytes);
|
|
3457
3836
|
}
|
|
3837
|
+
function normalizeLanguages(languages) {
|
|
3838
|
+
if (!Array.isArray(languages) || languages.length === 0 || languages.some((lang) => typeof lang !== "string" || !/^[a-z]{2,3}(?:-[a-z0-9]{2,8})*$/i.test(lang))) {
|
|
3839
|
+
throw new PostfetchError(400, "languages must be a non-empty list of language codes");
|
|
3840
|
+
}
|
|
3841
|
+
return [...new Set(languages.map((lang) => lang.toLowerCase()))];
|
|
3842
|
+
}
|
|
3458
3843
|
async function resolve(context) {
|
|
3459
3844
|
switch (detect(context.url)) {
|
|
3460
3845
|
case "facebook":
|
package/dist/internal.d.ts
CHANGED
|
@@ -112,6 +112,10 @@ export type TwitterExtra = {
|
|
|
112
112
|
replyToId?: string;
|
|
113
113
|
/** BCP 47 language tag detected for the tweet. */
|
|
114
114
|
lang?: string;
|
|
115
|
+
/** Original text retained when a requested translation is used. */
|
|
116
|
+
originalText?: string;
|
|
117
|
+
/** Language of the translated text. `lang` still describes the original. */
|
|
118
|
+
translatedLang?: string;
|
|
115
119
|
/** The post quoted by this post, when X includes it in the resolved payload. */
|
|
116
120
|
quotedTweet?: TwitterQuotedTweet;
|
|
117
121
|
/** The parent this post replies to, when included by X. Media is not collected. */
|
|
@@ -208,6 +212,7 @@ export type Net = (url: string, init?: RequestInit, attempts?: number) => Promis
|
|
|
208
212
|
export type ResolveContext = {
|
|
209
213
|
net: Net;
|
|
210
214
|
preferredWidth: number;
|
|
215
|
+
languages?: string[];
|
|
211
216
|
comments?: number;
|
|
212
217
|
/** Requested soft byte cap, for resolvers that can rank exact rendition sizes. */
|
|
213
218
|
tryMaxBytes?: number;
|
package/dist/postfetch.d.ts
CHANGED
|
@@ -5,6 +5,11 @@ export type PostfetchOptions = {
|
|
|
5
5
|
fetch?: typeof fetch;
|
|
6
6
|
/** Maximum direct comments to fetch for X and Reddit. Other platforms ignore this option. Failures return an empty array. */
|
|
7
7
|
comments?: number;
|
|
8
|
+
/** Allowed X text languages, in preference order. Preserve allowed originals;
|
|
9
|
+
* otherwise translate to the first language, then English, then keep the original.
|
|
10
|
+
* Omit to keep original text. Applies to quotes, parents and requested replies too.
|
|
11
|
+
*/
|
|
12
|
+
languages?: string[];
|
|
8
13
|
/** Preferred media width in pixels; the closest available rendition is chosen. Defaults to `720`. */
|
|
9
14
|
preferredWidth?: number;
|
|
10
15
|
/**
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Json, Net } from "./internal";
|
|
2
|
+
export type RedditPage = {
|
|
3
|
+
url: string;
|
|
4
|
+
html: string;
|
|
5
|
+
userAgent: string;
|
|
6
|
+
cookies: string[];
|
|
7
|
+
ok: boolean;
|
|
8
|
+
};
|
|
9
|
+
export declare function readWebPage(response: Response, userAgent: string): Promise<RedditPage>;
|
|
10
|
+
export declare function fetchWebPage(net: Net, input: string, userAgent: string, cookies?: string[]): Promise<RedditPage>;
|
|
11
|
+
export declare function fetchWebPost(net: Net, id: string, input: string, initial?: RedditPage): Promise<Json | null>;
|
|
12
|
+
export declare function fetchRssPost(net: Net, id: string): Promise<Json | null>;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@postfetch/core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.14.0",
|
|
4
4
|
"description": "Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, Spotify, TikTok, X and YouTube post URLs into media files.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
package/src/internal.ts
CHANGED
|
@@ -113,6 +113,10 @@ export type TwitterExtra = {
|
|
|
113
113
|
replyToId?: string;
|
|
114
114
|
/** BCP 47 language tag detected for the tweet. */
|
|
115
115
|
lang?: string;
|
|
116
|
+
/** Original text retained when a requested translation is used. */
|
|
117
|
+
originalText?: string;
|
|
118
|
+
/** Language of the translated text. `lang` still describes the original. */
|
|
119
|
+
translatedLang?: string;
|
|
116
120
|
/** The post quoted by this post, when X includes it in the resolved payload. */
|
|
117
121
|
quotedTweet?: TwitterQuotedTweet;
|
|
118
122
|
/** The parent this post replies to, when included by X. Media is not collected. */
|
|
@@ -228,6 +232,7 @@ export type Net = (url: string, init?: RequestInit, attempts?: number) => Promis
|
|
|
228
232
|
export type ResolveContext = {
|
|
229
233
|
net: Net;
|
|
230
234
|
preferredWidth: number;
|
|
235
|
+
languages?: string[];
|
|
231
236
|
comments?: number;
|
|
232
237
|
/** Requested soft byte cap, for resolvers that can rank exact rendition sizes. */
|
|
233
238
|
tryMaxBytes?: number;
|
package/src/postfetch.ts
CHANGED
|
@@ -16,6 +16,11 @@ export type PostfetchOptions = {
|
|
|
16
16
|
fetch?: typeof fetch;
|
|
17
17
|
/** Maximum direct comments to fetch for X and Reddit. Other platforms ignore this option. Failures return an empty array. */
|
|
18
18
|
comments?: number;
|
|
19
|
+
/** Allowed X text languages, in preference order. Preserve allowed originals;
|
|
20
|
+
* otherwise translate to the first language, then English, then keep the original.
|
|
21
|
+
* Omit to keep original text. Applies to quotes, parents and requested replies too.
|
|
22
|
+
*/
|
|
23
|
+
languages?: string[];
|
|
19
24
|
/** Preferred media width in pixels; the closest available rendition is chosen. Defaults to `720`. */
|
|
20
25
|
preferredWidth?: number;
|
|
21
26
|
/**
|
|
@@ -61,6 +66,7 @@ export async function postfetch(url: string, options: PostfetchOptions = {}): Pr
|
|
|
61
66
|
}
|
|
62
67
|
const context: ResolveContext = {
|
|
63
68
|
net: createNet(options.fetch ?? globalThis.fetch),
|
|
69
|
+
languages: options.languages === undefined ? undefined : normalizeLanguages(options.languages),
|
|
64
70
|
comments: options.comments,
|
|
65
71
|
preferredWidth: options.preferredWidth ?? 720,
|
|
66
72
|
tryMaxBytes: options.tryMaxBytes,
|
|
@@ -72,6 +78,13 @@ export async function postfetch(url: string, options: PostfetchOptions = {}): Pr
|
|
|
72
78
|
: trySmaller(context, result, options.tryMaxBytes);
|
|
73
79
|
}
|
|
74
80
|
|
|
81
|
+
function normalizeLanguages(languages: string[]): string[] {
|
|
82
|
+
if (!Array.isArray(languages) || languages.length === 0 || languages.some((lang) => typeof lang !== "string" || !/^[a-z]{2,3}(?:-[a-z0-9]{2,8})*$/i.test(lang))) {
|
|
83
|
+
throw new PostfetchError(400, "languages must be a non-empty list of language codes");
|
|
84
|
+
}
|
|
85
|
+
return [...new Set(languages.map((lang) => lang.toLowerCase()))];
|
|
86
|
+
}
|
|
87
|
+
|
|
75
88
|
async function resolve(context: ResolveContext): Promise<PostfetchResult> {
|
|
76
89
|
switch (detect(context.url)) {
|
|
77
90
|
case "facebook":
|
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
import { browserUserAgent } from "./fingerprint";
|
|
2
|
+
import type { Json, Net } from "./internal";
|
|
3
|
+
|
|
4
|
+
export type RedditPage = { url: string; html: string; userAgent: string; cookies: string[]; ok: boolean };
|
|
5
|
+
|
|
6
|
+
export async function readWebPage(response: Response, userAgent: string): Promise<RedditPage> {
|
|
7
|
+
const cookies = response.headers.getSetCookie().map((cookie) => cookie.split(";", 1)[0]!);
|
|
8
|
+
if (!response.ok) {
|
|
9
|
+
await response.body?.cancel();
|
|
10
|
+
return { url: response.url, html: "", userAgent, cookies, ok: false };
|
|
11
|
+
}
|
|
12
|
+
return { url: response.url, html: await response.text(), userAgent, cookies, ok: true };
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export async function fetchWebPage(net: Net, input: string, userAgent: string, cookies: string[] = []): Promise<RedditPage> {
|
|
16
|
+
let url = new URL(input);
|
|
17
|
+
let jar = new Map(cookies.map((cookie) => [cookie.split("=", 1)[0]!, cookie]));
|
|
18
|
+
for (let redirects = 0; redirects <= 5; redirects += 1) {
|
|
19
|
+
if (url.protocol !== "https:" || !/^(?:(?:www\.|old\.|m\.)?reddit\.com|redd\.it)$/.test(url.hostname)) throw new Error("Invalid Reddit page redirect");
|
|
20
|
+
const headers = { "user-agent": userAgent, ...(jar.size ? { cookie: [...jar.values()].join("; ") } : {}) };
|
|
21
|
+
// fetch does not keep Set-Cookie values while automatically following a
|
|
22
|
+
// redirect. Reddit's successful JS check redirects after setting its token.
|
|
23
|
+
const response = await net(url.href, { headers, redirect: "manual" }, 1);
|
|
24
|
+
for (const cookie of response.headers.getSetCookie()) {
|
|
25
|
+
const value = cookie.split(";", 1)[0]!;
|
|
26
|
+
const name = value.split("=", 1)[0]!;
|
|
27
|
+
if (/;\s*max-age=0(?:;|$)/i.test(cookie)) jar.delete(name);
|
|
28
|
+
else jar.set(name, value);
|
|
29
|
+
}
|
|
30
|
+
const location = response.headers.get("location");
|
|
31
|
+
if ([301, 302, 303, 307, 308].includes(response.status) && location) {
|
|
32
|
+
await response.body?.cancel();
|
|
33
|
+
const target = new URL(location, url);
|
|
34
|
+
if (target.origin !== url.origin) jar = new Map();
|
|
35
|
+
url = target;
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
return { ...await readWebPage(response, userAgent), url: response.url || url.href, cookies: [...jar.values()] };
|
|
39
|
+
}
|
|
40
|
+
throw new Error("Too many Reddit page redirects");
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// Read only the requested post, never thumbnails or attachments from comments
|
|
44
|
+
// and recommendations. Return Reddit's normal JSON shape to reuse extraction.
|
|
45
|
+
export async function fetchWebPost(net: Net, id: string, input: string, initial?: RedditPage): Promise<Json | null> {
|
|
46
|
+
const url = new URL(input);
|
|
47
|
+
if (url.protocol !== "https:" || !/^(?:(?:www\.|old\.|m\.)?reddit\.com|redd\.it)$/.test(url.hostname)) return null;
|
|
48
|
+
if (url.hostname !== "redd.it") url.hostname = "www.reddit.com";
|
|
49
|
+
const userAgent = initial?.userAgent ?? browserUserAgent();
|
|
50
|
+
let page = initial ?? await fetchWebPage(net, url.href, userAgent);
|
|
51
|
+
if (!page.ok) return null;
|
|
52
|
+
const challenge = challengeUrl(page.html, page.url);
|
|
53
|
+
if (challenge) {
|
|
54
|
+
page = await fetchWebPage(net, challenge, userAgent, page.cookies);
|
|
55
|
+
if (!page.ok) return null;
|
|
56
|
+
}
|
|
57
|
+
for (const match of page.html.matchAll(/<shreddit-post\s([^>]+)>([\s\S]*?)<\/shreddit-post>/gi)) {
|
|
58
|
+
const post = attributes(match[1]);
|
|
59
|
+
if (post.id !== `t3_${id}` || !post["post-title"]) continue;
|
|
60
|
+
const media = postMedia(post, match[2], id);
|
|
61
|
+
if (!media) return null;
|
|
62
|
+
const created = Date.parse(post["created-timestamp"] ?? "");
|
|
63
|
+
return {
|
|
64
|
+
...media,
|
|
65
|
+
id,
|
|
66
|
+
title: post["post-title"],
|
|
67
|
+
selftext: postText(match[2], id),
|
|
68
|
+
author: post.author,
|
|
69
|
+
subreddit: post["subreddit-name"],
|
|
70
|
+
permalink: post.permalink,
|
|
71
|
+
created_utc: Number.isFinite(created) ? Math.floor(created / 1000) : undefined,
|
|
72
|
+
ups: finiteNumber(post.score),
|
|
73
|
+
score: finiteNumber(post.score),
|
|
74
|
+
upvote_ratio: finiteNumber(post["upvote-ratio"]),
|
|
75
|
+
num_comments: finiteNumber(post["comment-count"]),
|
|
76
|
+
over_18: Object.hasOwn(post, "nsfw"),
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// The public Atom feed remains usable on some routes that reject OAuth and
|
|
83
|
+
// browser pages. Match the root entry explicitly: replies have their own media.
|
|
84
|
+
export async function fetchRssPost(net: Net, id: string): Promise<Json | null> {
|
|
85
|
+
const url = `https://www.reddit.com/comments/${id}/.rss?limit=1`;
|
|
86
|
+
const headers = { "user-agent": browserUserAgent(), accept: "application/atom+xml" };
|
|
87
|
+
let response = await net(url, { headers }, 1);
|
|
88
|
+
if (response.status === 429) {
|
|
89
|
+
// Reddit currently advertises a small anonymous RSS quota and its reset
|
|
90
|
+
// window in this header, rather than Retry-After. Retry once within a bound.
|
|
91
|
+
const reset = finiteNumber(response.headers.get("x-ratelimit-reset") ?? undefined);
|
|
92
|
+
await response.body?.cancel();
|
|
93
|
+
if (reset === undefined || reset < 0 || reset > 15) return null;
|
|
94
|
+
await new Promise((resolve) => setTimeout(resolve, (reset + 0.1) * 1000));
|
|
95
|
+
response = await net(url, { headers }, 1);
|
|
96
|
+
}
|
|
97
|
+
if (!response.ok) {
|
|
98
|
+
await response.body?.cancel();
|
|
99
|
+
return null;
|
|
100
|
+
}
|
|
101
|
+
const xml = await response.text();
|
|
102
|
+
if (!/<feed\b/.test(xml)) return null;
|
|
103
|
+
for (const match of xml.matchAll(/<entry\b[^>]*>([\s\S]*?)<\/entry>/g)) {
|
|
104
|
+
const entry = match[1];
|
|
105
|
+
if (entry.match(/<id>([^<]+)<\/id>/)?.[1] !== `t3_${id}`) continue;
|
|
106
|
+
const title = entry.match(/<title>([\s\S]*?)<\/title>/)?.[1];
|
|
107
|
+
const content = decodeHtml(entry.match(/<content\b[^>]*>([\s\S]*?)<\/content>/)?.[1] ?? "");
|
|
108
|
+
const link = [...content.matchAll(/<a\s([^>]+)>\s*\[link\]\s*<\/a>/g)][0];
|
|
109
|
+
const destination = link ? attributes(link[1]).href : undefined;
|
|
110
|
+
const image = hostedImage(destination);
|
|
111
|
+
let media: Json | null = image ? { post_hint: "image", url: image } : null;
|
|
112
|
+
if (!media && destination) {
|
|
113
|
+
const target = new URL(destination);
|
|
114
|
+
if (target.protocol === "https:" && target.hostname === "v.redd.it" && /^\/[a-z0-9]+\/?$/i.test(target.pathname)) {
|
|
115
|
+
media = { secure_media: { reddit_video: { dash_url: target.href.replace(/\/$/, "") + "/DASHPlaylist.mpd" } } };
|
|
116
|
+
} else if (/^https?:$/.test(target.protocol) && !/(^|\.)(reddit\.com|redd\.it)$/.test(target.hostname)) {
|
|
117
|
+
const card = content.match(/<img\s([^>]+)>/);
|
|
118
|
+
const preview = card ? attributes(card[1]).src : undefined;
|
|
119
|
+
const previewUrl = preview ? new URL(preview) : null;
|
|
120
|
+
media = {
|
|
121
|
+
post_hint: "link", is_self: false, url: target.href,
|
|
122
|
+
...(previewUrl?.protocol === "https:" && previewUrl.hostname === "external-preview.redd.it"
|
|
123
|
+
? { preview: { images: [{ source: { url: previewUrl.href } }] } } : {}),
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
const body = content.match(/<!-- SC_OFF -->([\s\S]*?)<!-- SC_ON -->/)?.[1];
|
|
128
|
+
if (!media && !destination && body) media = { is_self: true };
|
|
129
|
+
if (!title || !media) return null;
|
|
130
|
+
const permalink = [...entry.matchAll(/<link\s([^>]+)\/?\s*>/g)]
|
|
131
|
+
.map((match) => attributes(match[1]).href).find((href) => href?.includes(`/comments/${id}/`));
|
|
132
|
+
const created = Date.parse(entry.match(/<published>([^<]+)<\/published>/)?.[1] ?? "");
|
|
133
|
+
return {
|
|
134
|
+
...media, id, title: decodeHtml(title), selftext: body ? plainText(body) : undefined,
|
|
135
|
+
author: decodeHtml(entry.match(/<name>([^<]+)<\/name>/)?.[1] ?? "").replace(/^\/u\//, "") || undefined,
|
|
136
|
+
subreddit: attributes(entry.match(/<category\s([^>]+)>/)?.[1] ?? "").term,
|
|
137
|
+
permalink: permalink ? new URL(permalink).pathname : undefined,
|
|
138
|
+
created_utc: Number.isFinite(created) ? Math.floor(created / 1000) : undefined,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
return null;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function postMedia(post: Record<string, string>, html: string, id: string): Json | null {
|
|
145
|
+
if (post["post-type"] === "text") return { is_self: true };
|
|
146
|
+
if (post["post-type"] === "image") {
|
|
147
|
+
const url = hostedImage(post["content-href"]);
|
|
148
|
+
return url ? { post_hint: "image", url } : null;
|
|
149
|
+
}
|
|
150
|
+
if (post["post-type"] === "gallery") {
|
|
151
|
+
const gallery = html.match(/<(gallery-carousel|shreddit-gallery)\b[^>]*>([\s\S]*?)<\/\1>/i);
|
|
152
|
+
if (!gallery) return null;
|
|
153
|
+
const images: string[] = [];
|
|
154
|
+
for (const match of gallery[2].matchAll(/<a\s([^>]+)>/gi)) {
|
|
155
|
+
const href = attributes(match[1]).href;
|
|
156
|
+
if (!href) continue;
|
|
157
|
+
const target = new URL(href, "https://www.reddit.com");
|
|
158
|
+
const media = target.origin === "https://www.reddit.com" && target.pathname === "/media"
|
|
159
|
+
? target.searchParams.get("url") : target.hostname === "i.redd.it" ? target.href : null;
|
|
160
|
+
if (!media) continue;
|
|
161
|
+
const original = hostedImage(media);
|
|
162
|
+
if (!original) return null;
|
|
163
|
+
if (!images.includes(original)) images.push(original);
|
|
164
|
+
}
|
|
165
|
+
// A gallery cover alone is insufficient evidence of the complete post.
|
|
166
|
+
if (images.length < 2) return null;
|
|
167
|
+
return {
|
|
168
|
+
gallery_data: { items: images.map((_, index) => ({ media_id: String(index) })) },
|
|
169
|
+
media_metadata: Object.fromEntries(images.map((url, index) => [String(index), { s: { u: url } }])),
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
if (post["post-type"] === "link") {
|
|
173
|
+
const url = new URL(post["content-href"] ?? "", "https://www.reddit.com");
|
|
174
|
+
if (!/^https?:$/.test(url.protocol) || /(^|\.)(reddit\.com|redd\.it)$/.test(url.hostname)) return null;
|
|
175
|
+
const preview = linkPreview(html);
|
|
176
|
+
return {
|
|
177
|
+
post_hint: "link", is_self: false, url: url.href,
|
|
178
|
+
...(preview ? { preview: { images: [{ source: { url: preview } }] } } : {}),
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
if (post["post-type"] === "video" || post["post-type"] === "gif") {
|
|
182
|
+
const destination = new URL(post["content-href"] ?? "", "https://www.reddit.com");
|
|
183
|
+
if (destination.protocol !== "https:" || destination.hostname !== "v.redd.it" || !/^\/[a-z0-9]+\/?$/i.test(destination.pathname)) return null;
|
|
184
|
+
for (const match of html.matchAll(/<shreddit-player\s([^>]+)>/gi)) {
|
|
185
|
+
const player = attributes(match[1]);
|
|
186
|
+
if (player["post-id"] !== `t3_${id}` || player["media-placement"] === "comment") continue;
|
|
187
|
+
const playlist = new URL(player.src ?? "", destination);
|
|
188
|
+
const base = destination.href.replace(/\/$/, "") + "/";
|
|
189
|
+
if (playlist.origin !== destination.origin || !playlist.href.startsWith(base) || !/\/(?:HLSPlaylist\.m3u8|DASHPlaylist\.mpd)$/.test(playlist.pathname)) continue;
|
|
190
|
+
const dash = new URL("DASHPlaylist.mpd", base);
|
|
191
|
+
dash.search = playlist.search;
|
|
192
|
+
return { secure_media: { reddit_video: { dash_url: dash.href } } };
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
return null;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function linkPreview(html: string): string | null {
|
|
199
|
+
for (const match of html.matchAll(/<img\s([^>]+)>/gi)) {
|
|
200
|
+
const image = attributes(match[1]);
|
|
201
|
+
if (image.id !== "post-image" || !Object.hasOwn(image, "data-post-media-primary")) continue;
|
|
202
|
+
const sources = (image.srcset ?? "").split(/,\s*/).flatMap((source) => {
|
|
203
|
+
const match = source.trim().match(/^(\S+)\s+(\d+)w$/);
|
|
204
|
+
return match ? [{ url: match[1], width: Number(match[2]) }] : [];
|
|
205
|
+
});
|
|
206
|
+
if (image.src) sources.push({ url: image.src, width: finiteNumber(image.width) ?? 0 });
|
|
207
|
+
for (const source of sources.sort((a, b) => b.width - a.width)) {
|
|
208
|
+
const url = new URL(source.url, "https://www.reddit.com");
|
|
209
|
+
if (url.protocol === "https:" && /^(?:external-preview|preview)\.redd\.it$/.test(url.hostname)) return url.href;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
return null;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
function hostedImage(input: string | undefined): string | null {
|
|
216
|
+
if (!input) return null;
|
|
217
|
+
const url = new URL(input, "https://www.reddit.com");
|
|
218
|
+
return url.protocol === "https:" && url.hostname === "i.redd.it" && /\.(?:jpe?g|png|webp|gif)$/i.test(url.pathname) ? url.href : null;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
function postText(html: string, id: string): string | undefined {
|
|
222
|
+
for (const match of html.matchAll(/<div\s([^>]+)>/gi)) {
|
|
223
|
+
const attrs = attributes(match[1]);
|
|
224
|
+
if (attrs.id !== `t3_${id}-post-rtjson-content`) continue;
|
|
225
|
+
const start = match.index + match[0].length;
|
|
226
|
+
const tags = /<div\b[^>]*>|<\/div\s*>/gi;
|
|
227
|
+
tags.lastIndex = start;
|
|
228
|
+
let depth = 1;
|
|
229
|
+
for (let tag = tags.exec(html); tag; tag = tags.exec(html)) {
|
|
230
|
+
depth += tag[0].startsWith("</") ? -1 : 1;
|
|
231
|
+
if (depth === 0) return plainText(html.slice(start, tag.index));
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
return undefined;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function plainText(html: string): string {
|
|
238
|
+
return decodeHtml(html
|
|
239
|
+
.replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, "")
|
|
240
|
+
.replace(/<\/?(?:p|div|li|blockquote|pre|h[1-6])\b[^>]*>|<br\b[^>]*>/gi, "\n")
|
|
241
|
+
.replace(/<[^>]*>/g, ""))
|
|
242
|
+
.replace(/[\t ]+/g, " ").replace(/ *\n */g, "\n").replace(/\n{3,}/g, "\n\n").trim();
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
function finiteNumber(value: string | undefined): number | undefined {
|
|
246
|
+
const number = value === undefined ? NaN : Number(value);
|
|
247
|
+
return Number.isFinite(number) ? number : undefined;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// Reddit sometimes serves a tiny form before the public page. Recognize only
|
|
251
|
+
// its fixed string-doubling script; never evaluate JavaScript from the page.
|
|
252
|
+
function challengeUrl(html: string, responseUrl: string): string | null {
|
|
253
|
+
const seed = html.match(/await\s*\(async\s+([a-z])\s*=>\s*\1\s*\+\s*\1\)\("([a-f0-9]{16})"\)/i)?.[2];
|
|
254
|
+
const form = html.match(/<form\b([^>]*)>([\s\S]*?)<\/form>/i);
|
|
255
|
+
if (!seed || !form || !responseUrl) return null;
|
|
256
|
+
const fields = attributes(form[1]);
|
|
257
|
+
if (fields.method?.toUpperCase() !== "GET" || !fields.action) return null;
|
|
258
|
+
const base = new URL(responseUrl);
|
|
259
|
+
const target = new URL(fields.action, base);
|
|
260
|
+
if (base.origin !== "https://www.reddit.com" || target.origin !== base.origin || target.pathname !== base.pathname) return null;
|
|
261
|
+
const inputs = new Map<string, string>();
|
|
262
|
+
for (const input of form[2].matchAll(/<input\b([^>]*)>/gi)) {
|
|
263
|
+
const attrs = attributes(input[1]);
|
|
264
|
+
if (attrs.name) inputs.set(attrs.name, attrs.value ?? "");
|
|
265
|
+
}
|
|
266
|
+
if (inputs.get("js_challenge") !== "1" || !inputs.get("jsc_token") || !inputs.has("solution")) return null;
|
|
267
|
+
target.search = "";
|
|
268
|
+
for (const [key, value] of inputs) {
|
|
269
|
+
if (["solution", "js_challenge", "jsc_token", "jsc_orig_r"].includes(key)) target.searchParams.append(key, key === "solution" ? seed + seed : value);
|
|
270
|
+
}
|
|
271
|
+
for (const [key, value] of base.searchParams) {
|
|
272
|
+
if (!inputs.has(key)) target.searchParams.append(key, value);
|
|
273
|
+
}
|
|
274
|
+
return target.href;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
function attributes(tag: string): Record<string, string> {
|
|
278
|
+
const result: Record<string, string> = {};
|
|
279
|
+
for (const match of tag.matchAll(/(?:^|\s)([\w-]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'))?/g)) {
|
|
280
|
+
result[match[1].toLowerCase()] = decodeHtml(match[2] ?? match[3] ?? "");
|
|
281
|
+
}
|
|
282
|
+
return result;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
function decodeHtml(input: string): string {
|
|
286
|
+
const named: Record<string, string> = { amp: "&", apos: "'", gt: ">", lt: "<", nbsp: " ", quot: '"' };
|
|
287
|
+
return input.replace(/&(#x[\da-f]+|#\d+|amp|apos|gt|lt|nbsp|quot);/gi, (entity, code: string) => {
|
|
288
|
+
if (!code.startsWith("#")) return named[code.toLowerCase()] ?? entity;
|
|
289
|
+
const value = code[1]?.toLowerCase() === "x" ? Number.parseInt(code.slice(2), 16) : Number.parseInt(code.slice(1), 10);
|
|
290
|
+
return Number.isSafeInteger(value) && value <= 0x10ffff ? String.fromCodePoint(value) : entity;
|
|
291
|
+
});
|
|
292
|
+
}
|
package/src/reddit.ts
CHANGED
|
@@ -17,17 +17,17 @@ import {
|
|
|
17
17
|
type MediaItem,
|
|
18
18
|
} from "./internal";
|
|
19
19
|
import { browserUserAgent } from "./fingerprint";
|
|
20
|
+
import { fetchWebPost, fetchRssPost, readWebPage, type RedditPage } from "./reddit-web";
|
|
20
21
|
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
//
|
|
24
|
-
// oauth.reddit.com.
|
|
22
|
+
// The installed-app token can still be issued while both OAuth post routes
|
|
23
|
+
// reject it with Reddit's network-security page. Keep the public page as a
|
|
24
|
+
// fallback rather than treating a successful token request as API availability.
|
|
25
25
|
const installedClientId = "ohXpoqrZYub1kg";
|
|
26
26
|
|
|
27
27
|
export async function resolveReddit(input: ResolveContext): Promise<PostfetchResult> {
|
|
28
|
-
const id = await postId(input.net, input.url);
|
|
28
|
+
const { id, url, page } = await postId(input.net, input.url);
|
|
29
29
|
const token = await appToken(input.net);
|
|
30
|
-
const post = await fetchPost(input.net, token, id);
|
|
30
|
+
const post = await fetchPost(input.net, token, id, url, page);
|
|
31
31
|
// A text post is a valid result with metadata and no media — fetchPost has
|
|
32
32
|
// already thrown if the post itself is missing, so empty items here mean a
|
|
33
33
|
// self/text post, not a failure.
|
|
@@ -54,19 +54,20 @@ export function redditMetadata(post: Json): PostMetadata & { extra?: RedditExtra
|
|
|
54
54
|
};
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
-
// Share
|
|
58
|
-
//
|
|
59
|
-
async function postId(net: Net, input: string): Promise<string> {
|
|
57
|
+
// Share links have no post id. Retain their canonical page and client identity
|
|
58
|
+
// so a blocked API lookup can reuse the response instead of fetching it again.
|
|
59
|
+
async function postId(net: Net, input: string): Promise<{ id: string; url: string; page?: RedditPage }> {
|
|
60
60
|
const direct = parseId(input);
|
|
61
61
|
if (direct) {
|
|
62
|
-
return direct;
|
|
62
|
+
return { id: direct, url: input };
|
|
63
63
|
}
|
|
64
|
-
const
|
|
65
|
-
const
|
|
64
|
+
const userAgent = browserUserAgent();
|
|
65
|
+
const page = await readWebPage(await net(input, { headers: { "user-agent": userAgent } }, 1), userAgent);
|
|
66
|
+
const resolved = parseId(page.url);
|
|
66
67
|
if (!resolved) {
|
|
67
68
|
throw new Error("Reddit post id not found");
|
|
68
69
|
}
|
|
69
|
-
return resolved;
|
|
70
|
+
return { id: resolved, url: page.url, page };
|
|
70
71
|
}
|
|
71
72
|
|
|
72
73
|
function parseId(input: string): string | null {
|
|
@@ -99,7 +100,7 @@ async function appToken(net: Net): Promise<string> {
|
|
|
99
100
|
return token;
|
|
100
101
|
}
|
|
101
102
|
|
|
102
|
-
async function fetchPost(net: Net, token: string, id: string): Promise<Json> {
|
|
103
|
+
async function fetchPost(net: Net, token: string, id: string, url: string, page?: RedditPage): Promise<Json> {
|
|
103
104
|
const headers = { authorization: `Bearer ${token}`, "user-agent": browserUserAgent() };
|
|
104
105
|
let response = await net(`https://oauth.reddit.com/comments/${id}?raw_json=1&limit=1`, { headers });
|
|
105
106
|
// Reddit can block the comments route while still allowing direct post
|
|
@@ -109,6 +110,15 @@ async function fetchPost(net: Net, token: string, id: string): Promise<Json> {
|
|
|
109
110
|
response = await net(`https://oauth.reddit.com/by_id/t3_${id}?raw_json=1`, { headers });
|
|
110
111
|
}
|
|
111
112
|
if (!response.ok) {
|
|
113
|
+
if (response.status === 403) {
|
|
114
|
+
await response.body?.cancel();
|
|
115
|
+
// Public pages can remain available when both OAuth routes are
|
|
116
|
+
// blocked. Keep the original failure if the page cannot supply the post.
|
|
117
|
+
const post = await fetchWebPost(net, id, url, page).catch(() => null);
|
|
118
|
+
if (post) return post;
|
|
119
|
+
const rss = await fetchRssPost(net, id).catch(() => null);
|
|
120
|
+
if (rss) return rss;
|
|
121
|
+
}
|
|
112
122
|
throw new Error(`Reddit post failed: ${response.status}`);
|
|
113
123
|
}
|
|
114
124
|
const payload = await response.json().catch(() => null);
|
|
@@ -173,23 +183,34 @@ type Rendition = { width: number | null; bandwidth: number; base: string };
|
|
|
173
183
|
async function videoItems(input: ResolveContext, post: Json, id: string): Promise<MediaItem[]> {
|
|
174
184
|
const media = redditVideo(post);
|
|
175
185
|
const fallback = media ? string(media.fallback_url) : null;
|
|
176
|
-
if (!media
|
|
186
|
+
if (!media) {
|
|
177
187
|
return [];
|
|
178
188
|
}
|
|
189
|
+
const dashUrl = string(media.dash_url);
|
|
190
|
+
// Public pages expose the player and manifest rather than a fallback stream
|
|
191
|
+
// or has_audio flag. Inspect the manifest so silent clips remain silent and
|
|
192
|
+
// clips with audio retain their separate track through the normal remuxer.
|
|
193
|
+
if (!fallback && dashUrl) {
|
|
194
|
+
const streams = await dashStreams(input.net, dashUrl, input.preferredWidth, true);
|
|
195
|
+
if (!streams) throw new Error("Reddit DASH manifest unavailable for muxing");
|
|
196
|
+
return [streams.audio
|
|
197
|
+
? muxedVideoItem(id, streams.video, streams.audio)
|
|
198
|
+
: mediaItem(id, null, "video", "video/mp4", streams.video)];
|
|
199
|
+
}
|
|
200
|
+
if (!fallback) return [];
|
|
179
201
|
if (media.has_audio !== true) {
|
|
180
202
|
return [mediaItem(id, null, "video", "video/mp4", fallback)];
|
|
181
203
|
}
|
|
182
204
|
// The fallback stream is video-only; reddit keeps audio in a separate DASH
|
|
183
205
|
// rendition, so a complete file is the video and audio streams merged.
|
|
184
|
-
const dashUrl = string(media.dash_url);
|
|
185
206
|
const streams = dashUrl ? await dashStreams(input.net, dashUrl, input.preferredWidth) : null;
|
|
186
|
-
if (!streams) {
|
|
207
|
+
if (!streams?.audio) {
|
|
187
208
|
throw new Error("Reddit DASH manifest unavailable for muxing");
|
|
188
209
|
}
|
|
189
210
|
return [muxedVideoItem(id, streams.video, streams.audio)];
|
|
190
211
|
}
|
|
191
212
|
|
|
192
|
-
async function dashStreams(net: Net, dashUrl: string, preferredWidth: number): Promise<{ video: string; audio
|
|
213
|
+
async function dashStreams(net: Net, dashUrl: string, preferredWidth: number, allowSilent = false): Promise<{ video: string; audio?: string } | null> {
|
|
193
214
|
const response = await net(dashUrl, { headers: { "user-agent": browserUserAgent() } });
|
|
194
215
|
if (!response.ok) {
|
|
195
216
|
return null;
|
|
@@ -199,10 +220,10 @@ async function dashStreams(net: Net, dashUrl: string, preferredWidth: number): P
|
|
|
199
220
|
const query = dashUrl.includes("?") ? dashUrl.slice(dashUrl.indexOf("?")) : "";
|
|
200
221
|
const video = videoRendition(manifest, preferredWidth);
|
|
201
222
|
const audio = audioRendition(manifest);
|
|
202
|
-
if (!video || !audio) {
|
|
223
|
+
if (!video || (!audio && !allowSilent)) {
|
|
203
224
|
return null;
|
|
204
225
|
}
|
|
205
|
-
return { video: `${base}${video}${query}`, audio: `${base}${audio}${query}` };
|
|
226
|
+
return { video: `${base}${video}${query}`, ...(audio ? { audio: `${base}${audio}${query}` } : {}) };
|
|
206
227
|
}
|
|
207
228
|
|
|
208
229
|
function videoRendition(manifest: string, preferredWidth: number): string | null {
|
package/src/twitter.ts
CHANGED
|
@@ -33,6 +33,7 @@ export async function resolveTwitter(input: ResolveContext): Promise<PostfetchRe
|
|
|
33
33
|
throw new Error("Tweet is unavailable or age-restricted");
|
|
34
34
|
}
|
|
35
35
|
await expandTwitterText(input.net, tweet, id);
|
|
36
|
+
await Promise.all(twitterTweets(tweet, id, true).map(({ id, tweet }) => translateTwitterText(input, tweet, id)));
|
|
36
37
|
const posts = twitterTweets(tweet, id);
|
|
37
38
|
const groups = posts.flatMap(({ id: postId, tweet: post }) => {
|
|
38
39
|
const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
|
|
@@ -61,6 +62,11 @@ export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterEx
|
|
|
61
62
|
if (lang) {
|
|
62
63
|
extra.lang = lang;
|
|
63
64
|
}
|
|
65
|
+
const translated = string(tweet.translated_text);
|
|
66
|
+
if (translated) {
|
|
67
|
+
extra.originalText = cleanTwitterText(twitterText(tweet), tweet) ?? undefined;
|
|
68
|
+
extra.translatedLang = string(tweet.translated_lang) ?? undefined;
|
|
69
|
+
}
|
|
64
70
|
const quotedTweet = object(tweet.quoted_tweet) ? tweet.quoted_tweet : null;
|
|
65
71
|
const quotedId = quotedTweet ? string(quotedTweet.id_str) : null;
|
|
66
72
|
if (quotedTweet && quotedId) {
|
|
@@ -74,7 +80,7 @@ export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterEx
|
|
|
74
80
|
extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
|
|
75
81
|
}
|
|
76
82
|
return {
|
|
77
|
-
text: cleanTwitterText(twitterText(tweet), tweet) ?? undefined,
|
|
83
|
+
text: cleanTwitterText(translated ?? twitterText(tweet), tweet) ?? undefined,
|
|
78
84
|
author: user
|
|
79
85
|
? {
|
|
80
86
|
handle: string(user.screen_name) ?? undefined,
|
|
@@ -130,6 +136,42 @@ function needsFullText(tweet: Json): boolean {
|
|
|
130
136
|
return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
|
|
131
137
|
}
|
|
132
138
|
|
|
139
|
+
async function translateTwitterText(input: ResolveContext, tweet: Json, id: string): Promise<void> {
|
|
140
|
+
const languages = input.languages;
|
|
141
|
+
if (!languages?.length || !twitterText(tweet)) return;
|
|
142
|
+
const lang = string(tweet.lang)?.toLowerCase();
|
|
143
|
+
const allowed = (source: string) => languages.some((language) => source === language || source.startsWith(`${language}-`));
|
|
144
|
+
if (lang && allowed(lang)) return;
|
|
145
|
+
for (const target of new Set([languages[0], "en"])) {
|
|
146
|
+
try {
|
|
147
|
+
const response = await input.net(`https://api.fxtwitter.com/status/${id}/${target}`, {
|
|
148
|
+
headers: { accept: "application/json", "user-agent": browserUserAgent() },
|
|
149
|
+
}, 1);
|
|
150
|
+
if (!response.ok) continue;
|
|
151
|
+
const payload: unknown = await response.json();
|
|
152
|
+
if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id) continue;
|
|
153
|
+
const post = payload.tweet;
|
|
154
|
+
const translation = object(post.translation) ? post.translation : null;
|
|
155
|
+
const source = (string(post.lang) ?? string(translation?.source_lang))?.toLowerCase();
|
|
156
|
+
// The richer lookup may identify a language omitted by syndication.
|
|
157
|
+
if (source && source !== "und") {
|
|
158
|
+
tweet.lang = source;
|
|
159
|
+
if (allowed(source)) return;
|
|
160
|
+
}
|
|
161
|
+
const text = translation ? string(translation.text) : null;
|
|
162
|
+
if (!text?.trim() || string(translation?.target_lang)?.toLowerCase() !== target) continue;
|
|
163
|
+
// FxTwitter can return DeepL's client-upgrade error as a successful
|
|
164
|
+
// translation. It is an upstream failure, not the author's post text.
|
|
165
|
+
if (text.includes("You're using an older version of DeepL.")) continue;
|
|
166
|
+
tweet.translated_text = text;
|
|
167
|
+
tweet.translated_lang = target;
|
|
168
|
+
return;
|
|
169
|
+
} catch {
|
|
170
|
+
// Translation is optional; a failed lookup must not lose the post.
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
133
175
|
async function expandTwitterText(net: Net, tweet: Json, rootId: string): Promise<void> {
|
|
134
176
|
const posts = new Map(twitterTweets(tweet, rootId, true).map(({ id, tweet: post }) => [id, post]));
|
|
135
177
|
for (const [id, post] of posts) {
|
|
@@ -368,15 +410,21 @@ async function twitterComments(input: ResolveContext, id: string): Promise<PostC
|
|
|
368
410
|
const payload: unknown = await response.json();
|
|
369
411
|
if (!object(payload) || payload.code !== 200 || !Array.isArray(payload.replies)) throw new Error("Invalid comments");
|
|
370
412
|
const previousCount = comments.length;
|
|
413
|
+
const page: Array<{ id: string; reply: Json }> = [];
|
|
371
414
|
for (const reply of payload.replies) {
|
|
372
415
|
if (!object(reply)) continue;
|
|
373
416
|
const replyId = string(reply.id);
|
|
374
417
|
const parent = object(reply.replying_to) ? string(reply.replying_to.status) : null;
|
|
375
418
|
if (!replyId || parent !== id || seen.has(replyId)) continue;
|
|
376
419
|
seen.add(replyId);
|
|
377
|
-
|
|
378
|
-
if (comments.length >= limit)
|
|
420
|
+
page.push({ id: replyId, reply });
|
|
421
|
+
if (comments.length + page.length >= limit) break;
|
|
379
422
|
}
|
|
423
|
+
comments.push(...await Promise.all(page.map(async ({ id, reply }) => {
|
|
424
|
+
await translateTwitterText(input, reply, id);
|
|
425
|
+
return twitterComment(reply, id);
|
|
426
|
+
})));
|
|
427
|
+
if (comments.length >= limit) return comments;
|
|
380
428
|
cursor = object(payload.cursor) ? string(payload.cursor.bottom) : null;
|
|
381
429
|
if (!cursor || cursors.has(cursor) || comments.length === previousCount) break;
|
|
382
430
|
cursors.add(cursor);
|
|
@@ -410,7 +458,7 @@ function twitterComment(reply: Json, id: string): PostComment {
|
|
|
410
458
|
url: `https://x.com/i/status/${id}`,
|
|
411
459
|
items,
|
|
412
460
|
metadata: {
|
|
413
|
-
text: cleanTwitterText(string(reply.text), reply) ?? undefined,
|
|
461
|
+
text: cleanTwitterText(string(reply.translated_text) ?? string(reply.text), reply) ?? undefined,
|
|
414
462
|
author: {
|
|
415
463
|
handle: string(author.screen_name) ?? undefined,
|
|
416
464
|
name: string(author.name) ?? undefined,
|