@postfetch/core 0.13.9 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -1
- package/dist/index.js +162 -18
- package/dist/internal.d.ts +5 -0
- package/dist/postfetch.d.ts +5 -0
- package/dist/reddit-web.d.ts +1 -0
- package/package.json +1 -1
- package/src/internal.ts +5 -0
- package/src/postfetch.ts +13 -0
- package/src/reddit-web.ts +66 -8
- package/src/reddit.ts +28 -5
- package/src/twitter.ts +52 -4
package/README.md
CHANGED
|
@@ -59,9 +59,22 @@ const blob = await buildAudioSliderVideo(result.items, { delay: 3000 });
|
|
|
59
59
|
form.append("video", blob, "slideshow.mp4");
|
|
60
60
|
```
|
|
61
61
|
|
|
62
|
-
- `PostfetchOptions` — `{ fetch?: typeof fetch; preferredWidth?: number; tryMaxBytes?: number }`. `tryMaxBytes` is a soft byte cap. X probes every MP4 variant and selects the highest-quality complete result that fits, falling back to its smallest variants when none do. Other platforms return a smaller available rendition when possible and otherwise keep the normal result. Inject `fetch` to unit-test resolvers offline.
|
|
62
|
+
- `PostfetchOptions` — `{ fetch?: typeof fetch; preferredWidth?: number; tryMaxBytes?: number; languages?: string[] }`. `tryMaxBytes` is a soft byte cap. X probes every MP4 variant and selects the highest-quality complete result that fits, falling back to its smallest variants when none do. Other platforms return a smaller available rendition when possible and otherwise keep the normal result. Inject `fetch` to unit-test resolvers offline.
|
|
63
63
|
- Every request rotates a consistent browser/app fingerprint, so a fixed user-agent never gets the whole fleet blocked.
|
|
64
64
|
- No runtime dependencies, no `yt-dlp`, no cookies. `ffmpeg` is only used when
|
|
65
65
|
`downloadBlob(..., { remux: true })` or `buildAudioSliderVideo` is requested.
|
|
66
66
|
|
|
67
67
|
See the [root README](https://github.com/chelokot/postfetch#readme) for the full API table, supported inputs and the fingerprint design.
|
|
68
|
+
|
|
69
|
+
### X text languages
|
|
70
|
+
|
|
71
|
+
Pass `postfetch(url, { languages: ["en", "ru"] })` to keep X posts written in
|
|
72
|
+
any allowed language. Other posts are translated to the first language in the
|
|
73
|
+
list, falling back to English and then the original if translation is unavailable.
|
|
74
|
+
The ordered list must be non-empty; omit it to keep original text. This applies
|
|
75
|
+
independently to quoted posts, parent posts and requested replies. Regional
|
|
76
|
+
variants match a base language (`en` allows `en-US`). Other platforms ignore it.
|
|
77
|
+
|
|
78
|
+
`metadata.extra.lang` remains the detected original language. Translated posts
|
|
79
|
+
also expose `metadata.extra.originalText` and `metadata.extra.translatedLang`.
|
|
80
|
+
Translations use FxTwitter's public translation API and do not alter video audio.
|
package/dist/index.js
CHANGED
|
@@ -1693,6 +1693,64 @@ async function fetchWebPost(net, id, input, initial) {
|
|
|
1693
1693
|
}
|
|
1694
1694
|
return null;
|
|
1695
1695
|
}
|
|
1696
|
+
async function fetchEmbeddedPost(net, id) {
|
|
1697
|
+
const page = await fetchWebPage(net, `https://www.reddit.com/comments/${id}?embedded=true`, browserUserAgent());
|
|
1698
|
+
if (!page.ok)
|
|
1699
|
+
return null;
|
|
1700
|
+
for (const match of page.html.matchAll(/<div\s([^>]+)>/gi)) {
|
|
1701
|
+
const post = attributes(match[1]);
|
|
1702
|
+
if (post.id !== `thing_t3_${id}` || post["data-fullname"] !== `t3_${id}` || post["data-is-gallery"] !== "true")
|
|
1703
|
+
continue;
|
|
1704
|
+
const html = divContent(page.html, match.index + match[0].length);
|
|
1705
|
+
if (html === null)
|
|
1706
|
+
return null;
|
|
1707
|
+
const title = [...html.matchAll(/<a\s([^>]+)>([\s\S]*?)<\/a>/gi)].find((link) => attributes(link[1])["data-event-action"] === "title")?.[2];
|
|
1708
|
+
const preview = [...html.matchAll(/<div\s([^>]+)>/gi)].map((tag) => attributes(tag[1])).find((attrs) => attrs.id === `media-preview-${id}`);
|
|
1709
|
+
const ids = preview?.["data-media-ids"]?.split(",") ?? [];
|
|
1710
|
+
if (!title || !ids.length || ids.some((key) => !/^[a-z0-9]+$/i.test(key)) || new Set(ids).size !== ids.length)
|
|
1711
|
+
return null;
|
|
1712
|
+
const metadata = {};
|
|
1713
|
+
for (const link of html.matchAll(/<a\s([^>]+)>/gi)) {
|
|
1714
|
+
const attrs = attributes(link[1]);
|
|
1715
|
+
const url = new URL(attrs.href ?? "", "https://www.reddit.com");
|
|
1716
|
+
if (url.protocol !== "https:")
|
|
1717
|
+
continue;
|
|
1718
|
+
if (attrs.class?.split(/\s+/).includes("gallery-item-thumbnail-link")) {
|
|
1719
|
+
const index = Number(attrs["data-position"]) - 1;
|
|
1720
|
+
const key = ids[index];
|
|
1721
|
+
if (!Number.isSafeInteger(index) || !key || !/^(?:preview|i)\.redd\.it$/.test(url.hostname) || !new RegExp(`^/${key}\\.(?:png|jpe?g|webp)$`, "i").test(url.pathname))
|
|
1722
|
+
continue;
|
|
1723
|
+
metadata[key] = { s: { u: `https://i.redd.it${url.pathname}` } };
|
|
1724
|
+
} else if (/^(?:www\.)?reddit\.com$/.test(url.hostname)) {
|
|
1725
|
+
const video = url.pathname.match(/^\/link\/([a-z0-9]+)\/video\/([a-z0-9]+)\/player\/?$/i);
|
|
1726
|
+
if (video?.[1] === id && ids.includes(video[2])) {
|
|
1727
|
+
metadata[video[2]] = { e: "RedditVideo", dashUrl: `https://v.redd.it/${video[2]}/DASHPlaylist.mpd` };
|
|
1728
|
+
}
|
|
1729
|
+
}
|
|
1730
|
+
}
|
|
1731
|
+
if (ids.some((key) => !metadata[key]))
|
|
1732
|
+
return null;
|
|
1733
|
+
const body = [...html.matchAll(/<div\s([^>]+)>/gi)].find((tag) => attributes(tag[1]).class?.split(/\s+/).includes("usertext-body"));
|
|
1734
|
+
const text = body ? divContent(html, body.index + body[0].length) : null;
|
|
1735
|
+
const timestamp = finiteNumber(post["data-timestamp"]);
|
|
1736
|
+
return {
|
|
1737
|
+
id,
|
|
1738
|
+
title: plainText(title),
|
|
1739
|
+
selftext: text === null ? undefined : plainText(text.replace(/<a\s[^>]*>https:\/\/(?:www\.)?reddit\.com\/link\/[a-z0-9]+\/video\/[a-z0-9]+\/player<\/a>/gi, "")),
|
|
1740
|
+
author: post["data-author"],
|
|
1741
|
+
subreddit: post["data-subreddit"],
|
|
1742
|
+
permalink: post["data-permalink"],
|
|
1743
|
+
created_utc: timestamp === undefined ? undefined : Math.floor(timestamp / 1000),
|
|
1744
|
+
ups: finiteNumber(post["data-score"]),
|
|
1745
|
+
score: finiteNumber(post["data-score"]),
|
|
1746
|
+
num_comments: finiteNumber(post["data-comments-count"]),
|
|
1747
|
+
over_18: post["data-nsfw"] === "true" ? true : post["data-nsfw"] === "false" ? false : undefined,
|
|
1748
|
+
gallery_data: { items: ids.map((media_id) => ({ media_id })) },
|
|
1749
|
+
media_metadata: metadata
|
|
1750
|
+
};
|
|
1751
|
+
}
|
|
1752
|
+
return null;
|
|
1753
|
+
}
|
|
1696
1754
|
async function fetchRssPost(net, id) {
|
|
1697
1755
|
const url = `https://www.reddit.com/comments/${id}/.rss?limit=1`;
|
|
1698
1756
|
const headers = { "user-agent": browserUserAgent(), accept: "application/atom+xml" };
|
|
@@ -1852,18 +1910,23 @@ function postText(html, id) {
|
|
|
1852
1910
|
const attrs = attributes(match[1]);
|
|
1853
1911
|
if (attrs.id !== `t3_${id}-post-rtjson-content`)
|
|
1854
1912
|
continue;
|
|
1855
|
-
const
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
let depth = 1;
|
|
1859
|
-
for (let tag = tags.exec(html);tag; tag = tags.exec(html)) {
|
|
1860
|
-
depth += tag[0].startsWith("</") ? -1 : 1;
|
|
1861
|
-
if (depth === 0)
|
|
1862
|
-
return plainText(html.slice(start, tag.index));
|
|
1863
|
-
}
|
|
1913
|
+
const content = divContent(html, match.index + match[0].length);
|
|
1914
|
+
if (content !== null)
|
|
1915
|
+
return plainText(content);
|
|
1864
1916
|
}
|
|
1865
1917
|
return;
|
|
1866
1918
|
}
|
|
1919
|
+
function divContent(html, start) {
|
|
1920
|
+
const tags = /<div\b[^>]*>|<\/div\s*>/gi;
|
|
1921
|
+
tags.lastIndex = start;
|
|
1922
|
+
let depth = 1;
|
|
1923
|
+
for (let tag = tags.exec(html);tag; tag = tags.exec(html)) {
|
|
1924
|
+
depth += tag[0].startsWith("</") ? -1 : 1;
|
|
1925
|
+
if (depth === 0)
|
|
1926
|
+
return html.slice(start, tag.index);
|
|
1927
|
+
}
|
|
1928
|
+
return null;
|
|
1929
|
+
}
|
|
1867
1930
|
function plainText(html) {
|
|
1868
1931
|
return decodeHtml3(html.replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, "").replace(/<\/?(?:p|div|li|blockquote|pre|h[1-6])\b[^>]*>|<br\b[^>]*>/gi, `
|
|
1869
1932
|
`).replace(/<[^>]*>/g, "")).replace(/[\t ]+/g, " ").replace(/ *\n */g, `
|
|
@@ -2004,6 +2067,9 @@ async function fetchPost(net, token, id, url, page) {
|
|
|
2004
2067
|
const post2 = await fetchWebPost(net, id, url, page).catch(() => null);
|
|
2005
2068
|
if (post2)
|
|
2006
2069
|
return post2;
|
|
2070
|
+
const embedded = await fetchEmbeddedPost(net, id).catch(() => null);
|
|
2071
|
+
if (embedded)
|
|
2072
|
+
return embedded;
|
|
2007
2073
|
const rss = await fetchRssPost(net, id).catch(() => null);
|
|
2008
2074
|
if (rss)
|
|
2009
2075
|
return rss;
|
|
@@ -2034,7 +2100,7 @@ async function mediaItems3(input, post, id) {
|
|
|
2034
2100
|
return linkPreviewItems(post, id).concat(parent ? linkPreviewItems(parent, id) : []).slice(0, 1);
|
|
2035
2101
|
}
|
|
2036
2102
|
async function extract(input, post, id) {
|
|
2037
|
-
const gallery =
|
|
2103
|
+
const gallery = await galleryMedia(input, post, id);
|
|
2038
2104
|
if (gallery.length > 0) {
|
|
2039
2105
|
return gallery;
|
|
2040
2106
|
}
|
|
@@ -2044,7 +2110,28 @@ async function extract(input, post, id) {
|
|
|
2044
2110
|
}
|
|
2045
2111
|
return imageItems2(post, id);
|
|
2046
2112
|
}
|
|
2047
|
-
function
|
|
2113
|
+
async function galleryMedia(input, post, id) {
|
|
2114
|
+
const metadata = object(post.media_metadata) ? post.media_metadata : {};
|
|
2115
|
+
if (!Object.values(metadata).some((meta) => object(meta) && meta.e === "RedditVideo"))
|
|
2116
|
+
return galleryItems(post, id);
|
|
2117
|
+
const data = object(post.gallery_data) ? post.gallery_data : null;
|
|
2118
|
+
const order = data && Array.isArray(data.items) ? data.items.filter(object) : [];
|
|
2119
|
+
const items = [];
|
|
2120
|
+
for (const [index, entry] of order.entries()) {
|
|
2121
|
+
const key = string(entry.media_id);
|
|
2122
|
+
const meta = key && object(metadata[key]) ? metadata[key] : null;
|
|
2123
|
+
if (meta?.e === "RedditVideo") {
|
|
2124
|
+
const videos = await videoItems(input, { secure_media: { reddit_video: { dash_url: meta.dashUrl } } }, id);
|
|
2125
|
+
if (!videos.length)
|
|
2126
|
+
throw new Error("Reddit gallery video unavailable");
|
|
2127
|
+
items.push(...videos.map((video) => ({ ...video, filename: filename(`reddit_${id}_${index + 1}.mp4`) })));
|
|
2128
|
+
} else {
|
|
2129
|
+
items.push(...galleryItems({ ...post, gallery_data: { items: [entry] } }, id, index + 1));
|
|
2130
|
+
}
|
|
2131
|
+
}
|
|
2132
|
+
return items;
|
|
2133
|
+
}
|
|
2134
|
+
function galleryItems(post, id, startIndex = 1) {
|
|
2048
2135
|
const data = object(post.gallery_data) ? post.gallery_data : null;
|
|
2049
2136
|
const order = data && Array.isArray(data.items) ? data.items.filter(object) : [];
|
|
2050
2137
|
const metadata = object(post.media_metadata) ? post.media_metadata : {};
|
|
@@ -2057,10 +2144,10 @@ function galleryItems(post, id) {
|
|
|
2057
2144
|
}
|
|
2058
2145
|
const video = string(best.mp4);
|
|
2059
2146
|
if (video) {
|
|
2060
|
-
return [mediaItem(id, index +
|
|
2147
|
+
return [mediaItem(id, index + startIndex, "video", "video/mp4", video)];
|
|
2061
2148
|
}
|
|
2062
2149
|
const image = string(best.u) ?? string(best.gif);
|
|
2063
|
-
return image ? [mediaItem(id, index +
|
|
2150
|
+
return image ? [mediaItem(id, index + startIndex, "image", mimeOf(string(meta?.m), image), image)] : [];
|
|
2064
2151
|
});
|
|
2065
2152
|
}
|
|
2066
2153
|
async function videoItems(input, post, id) {
|
|
@@ -3398,6 +3485,7 @@ async function resolveTwitter(input) {
|
|
|
3398
3485
|
throw new Error("Tweet is unavailable or age-restricted");
|
|
3399
3486
|
}
|
|
3400
3487
|
await expandTwitterText(input.net, tweet, id);
|
|
3488
|
+
await Promise.all(twitterTweets(tweet, id, true).map(({ id: id2, tweet: tweet2 }) => translateTwitterText(input, tweet2, id2)));
|
|
3401
3489
|
const posts = twitterTweets(tweet, id);
|
|
3402
3490
|
const groups = posts.flatMap(({ id: postId2, tweet: post }) => {
|
|
3403
3491
|
const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
|
|
@@ -3421,6 +3509,11 @@ function twitterMetadata(tweet) {
|
|
|
3421
3509
|
if (lang) {
|
|
3422
3510
|
extra.lang = lang;
|
|
3423
3511
|
}
|
|
3512
|
+
const translated = string(tweet.translated_text);
|
|
3513
|
+
if (translated) {
|
|
3514
|
+
extra.originalText = cleanTwitterText(twitterText(tweet), tweet) ?? undefined;
|
|
3515
|
+
extra.translatedLang = string(tweet.translated_lang) ?? undefined;
|
|
3516
|
+
}
|
|
3424
3517
|
const quotedTweet = object(tweet.quoted_tweet) ? tweet.quoted_tweet : null;
|
|
3425
3518
|
const quotedId = quotedTweet ? string(quotedTweet.id_str) : null;
|
|
3426
3519
|
if (quotedTweet && quotedId) {
|
|
@@ -3435,7 +3528,7 @@ function twitterMetadata(tweet) {
|
|
|
3435
3528
|
extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
|
|
3436
3529
|
}
|
|
3437
3530
|
return {
|
|
3438
|
-
text: cleanTwitterText(twitterText(tweet), tweet) ?? undefined,
|
|
3531
|
+
text: cleanTwitterText(translated ?? twitterText(tweet), tweet) ?? undefined,
|
|
3439
3532
|
author: user ? {
|
|
3440
3533
|
handle: string(user.screen_name) ?? undefined,
|
|
3441
3534
|
name: string(user.name) ?? undefined,
|
|
@@ -3482,6 +3575,43 @@ function needsFullText(tweet) {
|
|
|
3482
3575
|
}
|
|
3483
3576
|
return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
|
|
3484
3577
|
}
|
|
3578
|
+
async function translateTwitterText(input, tweet, id) {
|
|
3579
|
+
const languages = input.languages;
|
|
3580
|
+
if (!languages?.length || !twitterText(tweet))
|
|
3581
|
+
return;
|
|
3582
|
+
const lang = string(tweet.lang)?.toLowerCase();
|
|
3583
|
+
const allowed = (source2) => languages.some((language) => source2 === language || source2.startsWith(`${language}-`));
|
|
3584
|
+
if (lang && allowed(lang))
|
|
3585
|
+
return;
|
|
3586
|
+
for (const target of new Set([languages[0], "en"])) {
|
|
3587
|
+
try {
|
|
3588
|
+
const response = await input.net(`https://api.fxtwitter.com/status/${id}/${target}`, {
|
|
3589
|
+
headers: { accept: "application/json", "user-agent": browserUserAgent() }
|
|
3590
|
+
}, 1);
|
|
3591
|
+
if (!response.ok)
|
|
3592
|
+
continue;
|
|
3593
|
+
const payload = await response.json();
|
|
3594
|
+
if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id)
|
|
3595
|
+
continue;
|
|
3596
|
+
const post = payload.tweet;
|
|
3597
|
+
const translation = object(post.translation) ? post.translation : null;
|
|
3598
|
+
const source2 = (string(post.lang) ?? string(translation?.source_lang))?.toLowerCase();
|
|
3599
|
+
if (source2 && source2 !== "und") {
|
|
3600
|
+
tweet.lang = source2;
|
|
3601
|
+
if (allowed(source2))
|
|
3602
|
+
return;
|
|
3603
|
+
}
|
|
3604
|
+
const text2 = translation ? string(translation.text) : null;
|
|
3605
|
+
if (!text2?.trim() || string(translation?.target_lang)?.toLowerCase() !== target)
|
|
3606
|
+
continue;
|
|
3607
|
+
if (text2.includes("You're using an older version of DeepL."))
|
|
3608
|
+
continue;
|
|
3609
|
+
tweet.translated_text = text2;
|
|
3610
|
+
tweet.translated_lang = target;
|
|
3611
|
+
return;
|
|
3612
|
+
} catch {}
|
|
3613
|
+
}
|
|
3614
|
+
}
|
|
3485
3615
|
async function expandTwitterText(net, tweet, rootId) {
|
|
3486
3616
|
const posts = new Map(twitterTweets(tweet, rootId, true).map(({ id, tweet: post }) => [id, post]));
|
|
3487
3617
|
for (const [id, post] of posts) {
|
|
@@ -3702,6 +3832,7 @@ async function twitterComments(input, id) {
|
|
|
3702
3832
|
if (!object(payload) || payload.code !== 200 || !Array.isArray(payload.replies))
|
|
3703
3833
|
throw new Error("Invalid comments");
|
|
3704
3834
|
const previousCount = comments.length;
|
|
3835
|
+
const page = [];
|
|
3705
3836
|
for (const reply of payload.replies) {
|
|
3706
3837
|
if (!object(reply))
|
|
3707
3838
|
continue;
|
|
@@ -3710,10 +3841,16 @@ async function twitterComments(input, id) {
|
|
|
3710
3841
|
if (!replyId || parent !== id || seen.has(replyId))
|
|
3711
3842
|
continue;
|
|
3712
3843
|
seen.add(replyId);
|
|
3713
|
-
|
|
3714
|
-
if (comments.length >= limit)
|
|
3715
|
-
|
|
3844
|
+
page.push({ id: replyId, reply });
|
|
3845
|
+
if (comments.length + page.length >= limit)
|
|
3846
|
+
break;
|
|
3716
3847
|
}
|
|
3848
|
+
comments.push(...await Promise.all(page.map(async ({ id: id2, reply }) => {
|
|
3849
|
+
await translateTwitterText(input, reply, id2);
|
|
3850
|
+
return twitterComment(reply, id2);
|
|
3851
|
+
})));
|
|
3852
|
+
if (comments.length >= limit)
|
|
3853
|
+
return comments;
|
|
3717
3854
|
cursor = object(payload.cursor) ? string(payload.cursor.bottom) : null;
|
|
3718
3855
|
if (!cursor || cursors.has(cursor) || comments.length === previousCount)
|
|
3719
3856
|
break;
|
|
@@ -3748,7 +3885,7 @@ function twitterComment(reply, id) {
|
|
|
3748
3885
|
url: `https://x.com/i/status/${id}`,
|
|
3749
3886
|
items,
|
|
3750
3887
|
metadata: {
|
|
3751
|
-
text: cleanTwitterText(string(reply.text), reply) ?? undefined,
|
|
3888
|
+
text: cleanTwitterText(string(reply.translated_text) ?? string(reply.text), reply) ?? undefined,
|
|
3752
3889
|
author: {
|
|
3753
3890
|
handle: string(author2.screen_name) ?? undefined,
|
|
3754
3891
|
name: string(author2.name) ?? undefined,
|
|
@@ -3775,6 +3912,7 @@ async function postfetch(url, options = {}) {
|
|
|
3775
3912
|
}
|
|
3776
3913
|
const context = {
|
|
3777
3914
|
net: createNet(options.fetch ?? globalThis.fetch),
|
|
3915
|
+
languages: options.languages === undefined ? undefined : normalizeLanguages(options.languages),
|
|
3778
3916
|
comments: options.comments,
|
|
3779
3917
|
preferredWidth: options.preferredWidth ?? 720,
|
|
3780
3918
|
tryMaxBytes: options.tryMaxBytes,
|
|
@@ -3783,6 +3921,12 @@ async function postfetch(url, options = {}) {
|
|
|
3783
3921
|
const result = await resolve(context);
|
|
3784
3922
|
return options.tryMaxBytes === undefined || result.platform === "twitter" ? result : trySmaller(context, result, options.tryMaxBytes);
|
|
3785
3923
|
}
|
|
3924
|
+
function normalizeLanguages(languages) {
|
|
3925
|
+
if (!Array.isArray(languages) || languages.length === 0 || languages.some((lang) => typeof lang !== "string" || !/^[a-z]{2,3}(?:-[a-z0-9]{2,8})*$/i.test(lang))) {
|
|
3926
|
+
throw new PostfetchError(400, "languages must be a non-empty list of language codes");
|
|
3927
|
+
}
|
|
3928
|
+
return [...new Set(languages.map((lang) => lang.toLowerCase()))];
|
|
3929
|
+
}
|
|
3786
3930
|
async function resolve(context) {
|
|
3787
3931
|
switch (detect(context.url)) {
|
|
3788
3932
|
case "facebook":
|
package/dist/internal.d.ts
CHANGED
|
@@ -112,6 +112,10 @@ export type TwitterExtra = {
|
|
|
112
112
|
replyToId?: string;
|
|
113
113
|
/** BCP 47 language tag detected for the tweet. */
|
|
114
114
|
lang?: string;
|
|
115
|
+
/** Original text retained when a requested translation is used. */
|
|
116
|
+
originalText?: string;
|
|
117
|
+
/** Language of the translated text. `lang` still describes the original. */
|
|
118
|
+
translatedLang?: string;
|
|
115
119
|
/** The post quoted by this post, when X includes it in the resolved payload. */
|
|
116
120
|
quotedTweet?: TwitterQuotedTweet;
|
|
117
121
|
/** The parent this post replies to, when included by X. Media is not collected. */
|
|
@@ -208,6 +212,7 @@ export type Net = (url: string, init?: RequestInit, attempts?: number) => Promis
|
|
|
208
212
|
export type ResolveContext = {
|
|
209
213
|
net: Net;
|
|
210
214
|
preferredWidth: number;
|
|
215
|
+
languages?: string[];
|
|
211
216
|
comments?: number;
|
|
212
217
|
/** Requested soft byte cap, for resolvers that can rank exact rendition sizes. */
|
|
213
218
|
tryMaxBytes?: number;
|
package/dist/postfetch.d.ts
CHANGED
|
@@ -5,6 +5,11 @@ export type PostfetchOptions = {
|
|
|
5
5
|
fetch?: typeof fetch;
|
|
6
6
|
/** Maximum direct comments to fetch for X and Reddit. Other platforms ignore this option. Failures return an empty array. */
|
|
7
7
|
comments?: number;
|
|
8
|
+
/** Allowed X text languages, in preference order. Preserve allowed originals;
|
|
9
|
+
* otherwise translate to the first language, then English, then keep the original.
|
|
10
|
+
* Omit to keep original text. Applies to quotes, parents and requested replies too.
|
|
11
|
+
*/
|
|
12
|
+
languages?: string[];
|
|
8
13
|
/** Preferred media width in pixels; the closest available rendition is chosen. Defaults to `720`. */
|
|
9
14
|
preferredWidth?: number;
|
|
10
15
|
/**
|
package/dist/reddit-web.d.ts
CHANGED
|
@@ -9,4 +9,5 @@ export type RedditPage = {
|
|
|
9
9
|
export declare function readWebPage(response: Response, userAgent: string): Promise<RedditPage>;
|
|
10
10
|
export declare function fetchWebPage(net: Net, input: string, userAgent: string, cookies?: string[]): Promise<RedditPage>;
|
|
11
11
|
export declare function fetchWebPost(net: Net, id: string, input: string, initial?: RedditPage): Promise<Json | null>;
|
|
12
|
+
export declare function fetchEmbeddedPost(net: Net, id: string): Promise<Json | null>;
|
|
12
13
|
export declare function fetchRssPost(net: Net, id: string): Promise<Json | null>;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@postfetch/core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.14.1",
|
|
4
4
|
"description": "Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, Spotify, TikTok, X and YouTube post URLs into media files.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
package/src/internal.ts
CHANGED
|
@@ -113,6 +113,10 @@ export type TwitterExtra = {
|
|
|
113
113
|
replyToId?: string;
|
|
114
114
|
/** BCP 47 language tag detected for the tweet. */
|
|
115
115
|
lang?: string;
|
|
116
|
+
/** Original text retained when a requested translation is used. */
|
|
117
|
+
originalText?: string;
|
|
118
|
+
/** Language of the translated text. `lang` still describes the original. */
|
|
119
|
+
translatedLang?: string;
|
|
116
120
|
/** The post quoted by this post, when X includes it in the resolved payload. */
|
|
117
121
|
quotedTweet?: TwitterQuotedTweet;
|
|
118
122
|
/** The parent this post replies to, when included by X. Media is not collected. */
|
|
@@ -228,6 +232,7 @@ export type Net = (url: string, init?: RequestInit, attempts?: number) => Promis
|
|
|
228
232
|
export type ResolveContext = {
|
|
229
233
|
net: Net;
|
|
230
234
|
preferredWidth: number;
|
|
235
|
+
languages?: string[];
|
|
231
236
|
comments?: number;
|
|
232
237
|
/** Requested soft byte cap, for resolvers that can rank exact rendition sizes. */
|
|
233
238
|
tryMaxBytes?: number;
|
package/src/postfetch.ts
CHANGED
|
@@ -16,6 +16,11 @@ export type PostfetchOptions = {
|
|
|
16
16
|
fetch?: typeof fetch;
|
|
17
17
|
/** Maximum direct comments to fetch for X and Reddit. Other platforms ignore this option. Failures return an empty array. */
|
|
18
18
|
comments?: number;
|
|
19
|
+
/** Allowed X text languages, in preference order. Preserve allowed originals;
|
|
20
|
+
* otherwise translate to the first language, then English, then keep the original.
|
|
21
|
+
* Omit to keep original text. Applies to quotes, parents and requested replies too.
|
|
22
|
+
*/
|
|
23
|
+
languages?: string[];
|
|
19
24
|
/** Preferred media width in pixels; the closest available rendition is chosen. Defaults to `720`. */
|
|
20
25
|
preferredWidth?: number;
|
|
21
26
|
/**
|
|
@@ -61,6 +66,7 @@ export async function postfetch(url: string, options: PostfetchOptions = {}): Pr
|
|
|
61
66
|
}
|
|
62
67
|
const context: ResolveContext = {
|
|
63
68
|
net: createNet(options.fetch ?? globalThis.fetch),
|
|
69
|
+
languages: options.languages === undefined ? undefined : normalizeLanguages(options.languages),
|
|
64
70
|
comments: options.comments,
|
|
65
71
|
preferredWidth: options.preferredWidth ?? 720,
|
|
66
72
|
tryMaxBytes: options.tryMaxBytes,
|
|
@@ -72,6 +78,13 @@ export async function postfetch(url: string, options: PostfetchOptions = {}): Pr
|
|
|
72
78
|
: trySmaller(context, result, options.tryMaxBytes);
|
|
73
79
|
}
|
|
74
80
|
|
|
81
|
+
function normalizeLanguages(languages: string[]): string[] {
|
|
82
|
+
if (!Array.isArray(languages) || languages.length === 0 || languages.some((lang) => typeof lang !== "string" || !/^[a-z]{2,3}(?:-[a-z0-9]{2,8})*$/i.test(lang))) {
|
|
83
|
+
throw new PostfetchError(400, "languages must be a non-empty list of language codes");
|
|
84
|
+
}
|
|
85
|
+
return [...new Set(languages.map((lang) => lang.toLowerCase()))];
|
|
86
|
+
}
|
|
87
|
+
|
|
75
88
|
async function resolve(context: ResolveContext): Promise<PostfetchResult> {
|
|
76
89
|
switch (detect(context.url)) {
|
|
77
90
|
case "facebook":
|
package/src/reddit-web.ts
CHANGED
|
@@ -79,6 +79,59 @@ export async function fetchWebPost(net: Net, id: string, input: string, initial?
|
|
|
79
79
|
return null;
|
|
80
80
|
}
|
|
81
81
|
|
|
82
|
+
// The embedded view still serves legacy gallery markup when the main page is
|
|
83
|
+
// a CAPTCHA. Its media-id list includes videos that have no gallery thumbnail.
|
|
84
|
+
export async function fetchEmbeddedPost(net: Net, id: string): Promise<Json | null> {
|
|
85
|
+
const page = await fetchWebPage(net, `https://www.reddit.com/comments/${id}?embedded=true`, browserUserAgent());
|
|
86
|
+
if (!page.ok) return null;
|
|
87
|
+
for (const match of page.html.matchAll(/<div\s([^>]+)>/gi)) {
|
|
88
|
+
const post = attributes(match[1]);
|
|
89
|
+
if (post.id !== `thing_t3_${id}` || post["data-fullname"] !== `t3_${id}` || post["data-is-gallery"] !== "true") continue;
|
|
90
|
+
const html = divContent(page.html, match.index + match[0].length);
|
|
91
|
+
if (html === null) return null;
|
|
92
|
+
const title = [...html.matchAll(/<a\s([^>]+)>([\s\S]*?)<\/a>/gi)]
|
|
93
|
+
.find((link) => attributes(link[1])["data-event-action"] === "title")?.[2];
|
|
94
|
+
const preview = [...html.matchAll(/<div\s([^>]+)>/gi)]
|
|
95
|
+
.map((tag) => attributes(tag[1])).find((attrs) => attrs.id === `media-preview-${id}`);
|
|
96
|
+
const ids = preview?.["data-media-ids"]?.split(",") ?? [];
|
|
97
|
+
if (!title || !ids.length || ids.some((key) => !/^[a-z0-9]+$/i.test(key)) || new Set(ids).size !== ids.length) return null;
|
|
98
|
+
const metadata: Record<string, Json> = {};
|
|
99
|
+
for (const link of html.matchAll(/<a\s([^>]+)>/gi)) {
|
|
100
|
+
const attrs = attributes(link[1]);
|
|
101
|
+
const url = new URL(attrs.href ?? "", "https://www.reddit.com");
|
|
102
|
+
if (url.protocol !== "https:") continue;
|
|
103
|
+
if (attrs.class?.split(/\s+/).includes("gallery-item-thumbnail-link")) {
|
|
104
|
+
const index = Number(attrs["data-position"]) - 1;
|
|
105
|
+
const key = ids[index];
|
|
106
|
+
// Only the gallery's full-size link identifies an original. Never
|
|
107
|
+
// promote a tile, cover, comment image, or recommendation to an item.
|
|
108
|
+
if (!Number.isSafeInteger(index) || !key || !/^(?:preview|i)\.redd\.it$/.test(url.hostname)
|
|
109
|
+
|| !new RegExp(`^/${key}\\.(?:png|jpe?g|webp)$`, "i").test(url.pathname)) continue;
|
|
110
|
+
metadata[key] = { s: { u: `https://i.redd.it${url.pathname}` } };
|
|
111
|
+
} else if (/^(?:www\.)?reddit\.com$/.test(url.hostname)) {
|
|
112
|
+
const video = url.pathname.match(/^\/link\/([a-z0-9]+)\/video\/([a-z0-9]+)\/player\/?$/i);
|
|
113
|
+
if (video?.[1] === id && ids.includes(video[2])) {
|
|
114
|
+
metadata[video[2]] = { e: "RedditVideo", dashUrl: `https://v.redd.it/${video[2]}/DASHPlaylist.mpd` };
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
if (ids.some((key) => !metadata[key])) return null;
|
|
119
|
+
const body = [...html.matchAll(/<div\s([^>]+)>/gi)]
|
|
120
|
+
.find((tag) => attributes(tag[1]).class?.split(/\s+/).includes("usertext-body"));
|
|
121
|
+
const text = body ? divContent(html, body.index + body[0].length) : null;
|
|
122
|
+
const timestamp = finiteNumber(post["data-timestamp"]);
|
|
123
|
+
return {
|
|
124
|
+
id, title: plainText(title), selftext: text === null ? undefined : plainText(text.replace(/<a\s[^>]*>https:\/\/(?:www\.)?reddit\.com\/link\/[a-z0-9]+\/video\/[a-z0-9]+\/player<\/a>/gi, "")),
|
|
125
|
+
author: post["data-author"], subreddit: post["data-subreddit"], permalink: post["data-permalink"],
|
|
126
|
+
created_utc: timestamp === undefined ? undefined : Math.floor(timestamp / 1000),
|
|
127
|
+
ups: finiteNumber(post["data-score"]), score: finiteNumber(post["data-score"]), num_comments: finiteNumber(post["data-comments-count"]),
|
|
128
|
+
over_18: post["data-nsfw"] === "true" ? true : post["data-nsfw"] === "false" ? false : undefined,
|
|
129
|
+
gallery_data: { items: ids.map((media_id) => ({ media_id })) }, media_metadata: metadata,
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
return null;
|
|
133
|
+
}
|
|
134
|
+
|
|
82
135
|
// The public Atom feed remains usable on some routes that reject OAuth and
|
|
83
136
|
// browser pages. Match the root entry explicitly: replies have their own media.
|
|
84
137
|
export async function fetchRssPost(net: Net, id: string): Promise<Json | null> {
|
|
@@ -222,18 +275,23 @@ function postText(html: string, id: string): string | undefined {
|
|
|
222
275
|
for (const match of html.matchAll(/<div\s([^>]+)>/gi)) {
|
|
223
276
|
const attrs = attributes(match[1]);
|
|
224
277
|
if (attrs.id !== `t3_${id}-post-rtjson-content`) continue;
|
|
225
|
-
const
|
|
226
|
-
|
|
227
|
-
tags.lastIndex = start;
|
|
228
|
-
let depth = 1;
|
|
229
|
-
for (let tag = tags.exec(html); tag; tag = tags.exec(html)) {
|
|
230
|
-
depth += tag[0].startsWith("</") ? -1 : 1;
|
|
231
|
-
if (depth === 0) return plainText(html.slice(start, tag.index));
|
|
232
|
-
}
|
|
278
|
+
const content = divContent(html, match.index + match[0].length);
|
|
279
|
+
if (content !== null) return plainText(content);
|
|
233
280
|
}
|
|
234
281
|
return undefined;
|
|
235
282
|
}
|
|
236
283
|
|
|
284
|
+
function divContent(html: string, start: number): string | null {
|
|
285
|
+
const tags = /<div\b[^>]*>|<\/div\s*>/gi;
|
|
286
|
+
tags.lastIndex = start;
|
|
287
|
+
let depth = 1;
|
|
288
|
+
for (let tag = tags.exec(html); tag; tag = tags.exec(html)) {
|
|
289
|
+
depth += tag[0].startsWith("</") ? -1 : 1;
|
|
290
|
+
if (depth === 0) return html.slice(start, tag.index);
|
|
291
|
+
}
|
|
292
|
+
return null;
|
|
293
|
+
}
|
|
294
|
+
|
|
237
295
|
function plainText(html: string): string {
|
|
238
296
|
return decodeHtml(html
|
|
239
297
|
.replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, "")
|
package/src/reddit.ts
CHANGED
|
@@ -17,7 +17,7 @@ import {
|
|
|
17
17
|
type MediaItem,
|
|
18
18
|
} from "./internal";
|
|
19
19
|
import { browserUserAgent } from "./fingerprint";
|
|
20
|
-
import { fetchWebPost, fetchRssPost, readWebPage, type RedditPage } from "./reddit-web";
|
|
20
|
+
import { fetchWebPost, fetchEmbeddedPost, fetchRssPost, readWebPage, type RedditPage } from "./reddit-web";
|
|
21
21
|
|
|
22
22
|
// The installed-app token can still be issued while both OAuth post routes
|
|
23
23
|
// reject it with Reddit's network-security page. Keep the public page as a
|
|
@@ -116,6 +116,8 @@ async function fetchPost(net: Net, token: string, id: string, url: string, page?
|
|
|
116
116
|
// blocked. Keep the original failure if the page cannot supply the post.
|
|
117
117
|
const post = await fetchWebPost(net, id, url, page).catch(() => null);
|
|
118
118
|
if (post) return post;
|
|
119
|
+
const embedded = await fetchEmbeddedPost(net, id).catch(() => null);
|
|
120
|
+
if (embedded) return embedded;
|
|
119
121
|
const rss = await fetchRssPost(net, id).catch(() => null);
|
|
120
122
|
if (rss) return rss;
|
|
121
123
|
}
|
|
@@ -147,7 +149,7 @@ async function mediaItems(input: ResolveContext, post: Json, id: string): Promis
|
|
|
147
149
|
}
|
|
148
150
|
|
|
149
151
|
async function extract(input: ResolveContext, post: Json, id: string): Promise<MediaItem[]> {
|
|
150
|
-
const gallery =
|
|
152
|
+
const gallery = await galleryMedia(input, post, id);
|
|
151
153
|
if (gallery.length > 0) {
|
|
152
154
|
return gallery;
|
|
153
155
|
}
|
|
@@ -158,7 +160,28 @@ async function extract(input: ResolveContext, post: Json, id: string): Promise<M
|
|
|
158
160
|
return imageItems(post, id);
|
|
159
161
|
}
|
|
160
162
|
|
|
161
|
-
function
|
|
163
|
+
async function galleryMedia(input: ResolveContext, post: Json, id: string): Promise<MediaItem[]> {
|
|
164
|
+
const metadata = object(post.media_metadata) ? post.media_metadata : {};
|
|
165
|
+
if (!Object.values(metadata).some((meta) => object(meta) && meta.e === "RedditVideo")) return galleryItems(post, id);
|
|
166
|
+
const data = object(post.gallery_data) ? post.gallery_data : null;
|
|
167
|
+
const order = data && Array.isArray(data.items) ? data.items.filter(object) : [];
|
|
168
|
+
const items: MediaItem[] = [];
|
|
169
|
+
for (const [index, entry] of order.entries()) {
|
|
170
|
+
const key = string(entry.media_id);
|
|
171
|
+
const meta = key && object(metadata[key]) ? metadata[key] : null;
|
|
172
|
+
if (meta?.e === "RedditVideo") {
|
|
173
|
+
const videos = await videoItems(input, { secure_media: { reddit_video: { dash_url: meta.dashUrl } } }, id);
|
|
174
|
+
if (!videos.length) throw new Error("Reddit gallery video unavailable");
|
|
175
|
+
items.push(...videos.map((video) => ({ ...video, filename: filename(`reddit_${id}_${index + 1}.mp4`) })));
|
|
176
|
+
} else {
|
|
177
|
+
// Retain the original position in filenames when videos split the images.
|
|
178
|
+
items.push(...galleryItems({ ...post, gallery_data: { items: [entry] } }, id, index + 1));
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
return items;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function galleryItems(post: Json, id: string, startIndex = 1): MediaItem[] {
|
|
162
185
|
const data = object(post.gallery_data) ? post.gallery_data : null;
|
|
163
186
|
const order = data && Array.isArray(data.items) ? data.items.filter(object) : [];
|
|
164
187
|
const metadata = object(post.media_metadata) ? post.media_metadata : {};
|
|
@@ -171,10 +194,10 @@ function galleryItems(post: Json, id: string): MediaItem[] {
|
|
|
171
194
|
}
|
|
172
195
|
const video = string(best.mp4);
|
|
173
196
|
if (video) {
|
|
174
|
-
return [mediaItem(id, index +
|
|
197
|
+
return [mediaItem(id, index + startIndex, "video", "video/mp4", video)];
|
|
175
198
|
}
|
|
176
199
|
const image = string(best.u) ?? string(best.gif);
|
|
177
|
-
return image ? [mediaItem(id, index +
|
|
200
|
+
return image ? [mediaItem(id, index + startIndex, "image", mimeOf(string(meta?.m), image), image)] : [];
|
|
178
201
|
});
|
|
179
202
|
}
|
|
180
203
|
|
package/src/twitter.ts
CHANGED
|
@@ -33,6 +33,7 @@ export async function resolveTwitter(input: ResolveContext): Promise<PostfetchRe
|
|
|
33
33
|
throw new Error("Tweet is unavailable or age-restricted");
|
|
34
34
|
}
|
|
35
35
|
await expandTwitterText(input.net, tweet, id);
|
|
36
|
+
await Promise.all(twitterTweets(tweet, id, true).map(({ id, tweet }) => translateTwitterText(input, tweet, id)));
|
|
36
37
|
const posts = twitterTweets(tweet, id);
|
|
37
38
|
const groups = posts.flatMap(({ id: postId, tweet: post }) => {
|
|
38
39
|
const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
|
|
@@ -61,6 +62,11 @@ export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterEx
|
|
|
61
62
|
if (lang) {
|
|
62
63
|
extra.lang = lang;
|
|
63
64
|
}
|
|
65
|
+
const translated = string(tweet.translated_text);
|
|
66
|
+
if (translated) {
|
|
67
|
+
extra.originalText = cleanTwitterText(twitterText(tweet), tweet) ?? undefined;
|
|
68
|
+
extra.translatedLang = string(tweet.translated_lang) ?? undefined;
|
|
69
|
+
}
|
|
64
70
|
const quotedTweet = object(tweet.quoted_tweet) ? tweet.quoted_tweet : null;
|
|
65
71
|
const quotedId = quotedTweet ? string(quotedTweet.id_str) : null;
|
|
66
72
|
if (quotedTweet && quotedId) {
|
|
@@ -74,7 +80,7 @@ export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterEx
|
|
|
74
80
|
extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
|
|
75
81
|
}
|
|
76
82
|
return {
|
|
77
|
-
text: cleanTwitterText(twitterText(tweet), tweet) ?? undefined,
|
|
83
|
+
text: cleanTwitterText(translated ?? twitterText(tweet), tweet) ?? undefined,
|
|
78
84
|
author: user
|
|
79
85
|
? {
|
|
80
86
|
handle: string(user.screen_name) ?? undefined,
|
|
@@ -130,6 +136,42 @@ function needsFullText(tweet: Json): boolean {
|
|
|
130
136
|
return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
|
|
131
137
|
}
|
|
132
138
|
|
|
139
|
+
async function translateTwitterText(input: ResolveContext, tweet: Json, id: string): Promise<void> {
|
|
140
|
+
const languages = input.languages;
|
|
141
|
+
if (!languages?.length || !twitterText(tweet)) return;
|
|
142
|
+
const lang = string(tweet.lang)?.toLowerCase();
|
|
143
|
+
const allowed = (source: string) => languages.some((language) => source === language || source.startsWith(`${language}-`));
|
|
144
|
+
if (lang && allowed(lang)) return;
|
|
145
|
+
for (const target of new Set([languages[0], "en"])) {
|
|
146
|
+
try {
|
|
147
|
+
const response = await input.net(`https://api.fxtwitter.com/status/${id}/${target}`, {
|
|
148
|
+
headers: { accept: "application/json", "user-agent": browserUserAgent() },
|
|
149
|
+
}, 1);
|
|
150
|
+
if (!response.ok) continue;
|
|
151
|
+
const payload: unknown = await response.json();
|
|
152
|
+
if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id) continue;
|
|
153
|
+
const post = payload.tweet;
|
|
154
|
+
const translation = object(post.translation) ? post.translation : null;
|
|
155
|
+
const source = (string(post.lang) ?? string(translation?.source_lang))?.toLowerCase();
|
|
156
|
+
// The richer lookup may identify a language omitted by syndication.
|
|
157
|
+
if (source && source !== "und") {
|
|
158
|
+
tweet.lang = source;
|
|
159
|
+
if (allowed(source)) return;
|
|
160
|
+
}
|
|
161
|
+
const text = translation ? string(translation.text) : null;
|
|
162
|
+
if (!text?.trim() || string(translation?.target_lang)?.toLowerCase() !== target) continue;
|
|
163
|
+
// FxTwitter can return DeepL's client-upgrade error as a successful
|
|
164
|
+
// translation. It is an upstream failure, not the author's post text.
|
|
165
|
+
if (text.includes("You're using an older version of DeepL.")) continue;
|
|
166
|
+
tweet.translated_text = text;
|
|
167
|
+
tweet.translated_lang = target;
|
|
168
|
+
return;
|
|
169
|
+
} catch {
|
|
170
|
+
// Translation is optional; a failed lookup must not lose the post.
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
133
175
|
async function expandTwitterText(net: Net, tweet: Json, rootId: string): Promise<void> {
|
|
134
176
|
const posts = new Map(twitterTweets(tweet, rootId, true).map(({ id, tweet: post }) => [id, post]));
|
|
135
177
|
for (const [id, post] of posts) {
|
|
@@ -368,15 +410,21 @@ async function twitterComments(input: ResolveContext, id: string): Promise<PostC
|
|
|
368
410
|
const payload: unknown = await response.json();
|
|
369
411
|
if (!object(payload) || payload.code !== 200 || !Array.isArray(payload.replies)) throw new Error("Invalid comments");
|
|
370
412
|
const previousCount = comments.length;
|
|
413
|
+
const page: Array<{ id: string; reply: Json }> = [];
|
|
371
414
|
for (const reply of payload.replies) {
|
|
372
415
|
if (!object(reply)) continue;
|
|
373
416
|
const replyId = string(reply.id);
|
|
374
417
|
const parent = object(reply.replying_to) ? string(reply.replying_to.status) : null;
|
|
375
418
|
if (!replyId || parent !== id || seen.has(replyId)) continue;
|
|
376
419
|
seen.add(replyId);
|
|
377
|
-
|
|
378
|
-
if (comments.length >= limit)
|
|
420
|
+
page.push({ id: replyId, reply });
|
|
421
|
+
if (comments.length + page.length >= limit) break;
|
|
379
422
|
}
|
|
423
|
+
comments.push(...await Promise.all(page.map(async ({ id, reply }) => {
|
|
424
|
+
await translateTwitterText(input, reply, id);
|
|
425
|
+
return twitterComment(reply, id);
|
|
426
|
+
})));
|
|
427
|
+
if (comments.length >= limit) return comments;
|
|
380
428
|
cursor = object(payload.cursor) ? string(payload.cursor.bottom) : null;
|
|
381
429
|
if (!cursor || cursors.has(cursor) || comments.length === previousCount) break;
|
|
382
430
|
cursors.add(cursor);
|
|
@@ -410,7 +458,7 @@ function twitterComment(reply: Json, id: string): PostComment {
|
|
|
410
458
|
url: `https://x.com/i/status/${id}`,
|
|
411
459
|
items,
|
|
412
460
|
metadata: {
|
|
413
|
-
text: cleanTwitterText(string(reply.text), reply) ?? undefined,
|
|
461
|
+
text: cleanTwitterText(string(reply.translated_text) ?? string(reply.text), reply) ?? undefined,
|
|
414
462
|
author: {
|
|
415
463
|
handle: string(author.screen_name) ?? undefined,
|
|
416
464
|
name: string(author.name) ?? undefined,
|