@postfetch/core 0.13.9 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -59,9 +59,22 @@ const blob = await buildAudioSliderVideo(result.items, { delay: 3000 });
59
59
  form.append("video", blob, "slideshow.mp4");
60
60
  ```
61
61
 
62
- - `PostfetchOptions` — `{ fetch?: typeof fetch; preferredWidth?: number; tryMaxBytes?: number }`. `tryMaxBytes` is a soft byte cap. X probes every MP4 variant and selects the highest-quality complete result that fits, falling back to its smallest variants when none do. Other platforms return a smaller available rendition when possible and otherwise keep the normal result. Inject `fetch` to unit-test resolvers offline.
62
+ - `PostfetchOptions` — `{ fetch?: typeof fetch; preferredWidth?: number; tryMaxBytes?: number; languages?: string[] }`. `tryMaxBytes` is a soft byte cap. X probes every MP4 variant and selects the highest-quality complete result that fits, falling back to its smallest variants when none do. Other platforms return a smaller available rendition when possible and otherwise keep the normal result. Inject `fetch` to unit-test resolvers offline.
63
63
  - Every request rotates a consistent browser/app fingerprint, so a fixed user-agent never gets the whole fleet blocked.
64
64
  - No runtime dependencies, no `yt-dlp`, no cookies. `ffmpeg` is only used when
65
65
  `downloadBlob(..., { remux: true })` or `buildAudioSliderVideo` is requested.
66
66
 
67
67
  See the [root README](https://github.com/chelokot/postfetch#readme) for the full API table, supported inputs and the fingerprint design.
68
+
69
+ ### X text languages
70
+
71
+ Pass `postfetch(url, { languages: ["en", "ru"] })` to keep X posts written in
72
+ any allowed language. Other posts are translated to the first language in the
73
+ list, falling back to English and then the original if translation is unavailable.
74
+ The ordered list must be non-empty; omit it to keep original text. This applies
75
+ independently to quoted posts, parent posts and requested replies. Regional
76
+ variants match a base language (`en` allows `en-US`). Other platforms ignore it.
77
+
78
+ `metadata.extra.lang` remains the detected original language. Translated posts
79
+ also expose `metadata.extra.originalText` and `metadata.extra.translatedLang`.
80
+ Translations use FxTwitter's public translation API and do not alter video audio.
package/dist/index.js CHANGED
@@ -3398,6 +3398,7 @@ async function resolveTwitter(input) {
3398
3398
  throw new Error("Tweet is unavailable or age-restricted");
3399
3399
  }
3400
3400
  await expandTwitterText(input.net, tweet, id);
3401
+ await Promise.all(twitterTweets(tweet, id, true).map(({ id: id2, tweet: tweet2 }) => translateTwitterText(input, tweet2, id2)));
3401
3402
  const posts = twitterTweets(tweet, id);
3402
3403
  const groups = posts.flatMap(({ id: postId2, tweet: post }) => {
3403
3404
  const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
@@ -3421,6 +3422,11 @@ function twitterMetadata(tweet) {
3421
3422
  if (lang) {
3422
3423
  extra.lang = lang;
3423
3424
  }
3425
+ const translated = string(tweet.translated_text);
3426
+ if (translated) {
3427
+ extra.originalText = cleanTwitterText(twitterText(tweet), tweet) ?? undefined;
3428
+ extra.translatedLang = string(tweet.translated_lang) ?? undefined;
3429
+ }
3424
3430
  const quotedTweet = object(tweet.quoted_tweet) ? tweet.quoted_tweet : null;
3425
3431
  const quotedId = quotedTweet ? string(quotedTweet.id_str) : null;
3426
3432
  if (quotedTweet && quotedId) {
@@ -3435,7 +3441,7 @@ function twitterMetadata(tweet) {
3435
3441
  extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
3436
3442
  }
3437
3443
  return {
3438
- text: cleanTwitterText(twitterText(tweet), tweet) ?? undefined,
3444
+ text: cleanTwitterText(translated ?? twitterText(tweet), tweet) ?? undefined,
3439
3445
  author: user ? {
3440
3446
  handle: string(user.screen_name) ?? undefined,
3441
3447
  name: string(user.name) ?? undefined,
@@ -3482,6 +3488,43 @@ function needsFullText(tweet) {
3482
3488
  }
3483
3489
  return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
3484
3490
  }
3491
+ async function translateTwitterText(input, tweet, id) {
3492
+ const languages = input.languages;
3493
+ if (!languages?.length || !twitterText(tweet))
3494
+ return;
3495
+ const lang = string(tweet.lang)?.toLowerCase();
3496
+ const allowed = (source2) => languages.some((language) => source2 === language || source2.startsWith(`${language}-`));
3497
+ if (lang && allowed(lang))
3498
+ return;
3499
+ for (const target of new Set([languages[0], "en"])) {
3500
+ try {
3501
+ const response = await input.net(`https://api.fxtwitter.com/status/${id}/${target}`, {
3502
+ headers: { accept: "application/json", "user-agent": browserUserAgent() }
3503
+ }, 1);
3504
+ if (!response.ok)
3505
+ continue;
3506
+ const payload = await response.json();
3507
+ if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id)
3508
+ continue;
3509
+ const post = payload.tweet;
3510
+ const translation = object(post.translation) ? post.translation : null;
3511
+ const source2 = (string(post.lang) ?? string(translation?.source_lang))?.toLowerCase();
3512
+ if (source2 && source2 !== "und") {
3513
+ tweet.lang = source2;
3514
+ if (allowed(source2))
3515
+ return;
3516
+ }
3517
+ const text2 = translation ? string(translation.text) : null;
3518
+ if (!text2?.trim() || string(translation?.target_lang)?.toLowerCase() !== target)
3519
+ continue;
3520
+ if (text2.includes("You're using an older version of DeepL."))
3521
+ continue;
3522
+ tweet.translated_text = text2;
3523
+ tweet.translated_lang = target;
3524
+ return;
3525
+ } catch {}
3526
+ }
3527
+ }
3485
3528
  async function expandTwitterText(net, tweet, rootId) {
3486
3529
  const posts = new Map(twitterTweets(tweet, rootId, true).map(({ id, tweet: post }) => [id, post]));
3487
3530
  for (const [id, post] of posts) {
@@ -3702,6 +3745,7 @@ async function twitterComments(input, id) {
3702
3745
  if (!object(payload) || payload.code !== 200 || !Array.isArray(payload.replies))
3703
3746
  throw new Error("Invalid comments");
3704
3747
  const previousCount = comments.length;
3748
+ const page = [];
3705
3749
  for (const reply of payload.replies) {
3706
3750
  if (!object(reply))
3707
3751
  continue;
@@ -3710,10 +3754,16 @@ async function twitterComments(input, id) {
3710
3754
  if (!replyId || parent !== id || seen.has(replyId))
3711
3755
  continue;
3712
3756
  seen.add(replyId);
3713
- comments.push(twitterComment(reply, replyId));
3714
- if (comments.length >= limit)
3715
- return comments;
3757
+ page.push({ id: replyId, reply });
3758
+ if (comments.length + page.length >= limit)
3759
+ break;
3716
3760
  }
3761
+ comments.push(...await Promise.all(page.map(async ({ id: id2, reply }) => {
3762
+ await translateTwitterText(input, reply, id2);
3763
+ return twitterComment(reply, id2);
3764
+ })));
3765
+ if (comments.length >= limit)
3766
+ return comments;
3717
3767
  cursor = object(payload.cursor) ? string(payload.cursor.bottom) : null;
3718
3768
  if (!cursor || cursors.has(cursor) || comments.length === previousCount)
3719
3769
  break;
@@ -3748,7 +3798,7 @@ function twitterComment(reply, id) {
3748
3798
  url: `https://x.com/i/status/${id}`,
3749
3799
  items,
3750
3800
  metadata: {
3751
- text: cleanTwitterText(string(reply.text), reply) ?? undefined,
3801
+ text: cleanTwitterText(string(reply.translated_text) ?? string(reply.text), reply) ?? undefined,
3752
3802
  author: {
3753
3803
  handle: string(author2.screen_name) ?? undefined,
3754
3804
  name: string(author2.name) ?? undefined,
@@ -3775,6 +3825,7 @@ async function postfetch(url, options = {}) {
3775
3825
  }
3776
3826
  const context = {
3777
3827
  net: createNet(options.fetch ?? globalThis.fetch),
3828
+ languages: options.languages === undefined ? undefined : normalizeLanguages(options.languages),
3778
3829
  comments: options.comments,
3779
3830
  preferredWidth: options.preferredWidth ?? 720,
3780
3831
  tryMaxBytes: options.tryMaxBytes,
@@ -3783,6 +3834,12 @@ async function postfetch(url, options = {}) {
3783
3834
  const result = await resolve(context);
3784
3835
  return options.tryMaxBytes === undefined || result.platform === "twitter" ? result : trySmaller(context, result, options.tryMaxBytes);
3785
3836
  }
3837
+ function normalizeLanguages(languages) {
3838
+ if (!Array.isArray(languages) || languages.length === 0 || languages.some((lang) => typeof lang !== "string" || !/^[a-z]{2,3}(?:-[a-z0-9]{2,8})*$/i.test(lang))) {
3839
+ throw new PostfetchError(400, "languages must be a non-empty list of language codes");
3840
+ }
3841
+ return [...new Set(languages.map((lang) => lang.toLowerCase()))];
3842
+ }
3786
3843
  async function resolve(context) {
3787
3844
  switch (detect(context.url)) {
3788
3845
  case "facebook":
@@ -112,6 +112,10 @@ export type TwitterExtra = {
112
112
  replyToId?: string;
113
113
  /** BCP 47 language tag detected for the tweet. */
114
114
  lang?: string;
115
+ /** Original text retained when a requested translation is used. */
116
+ originalText?: string;
117
+ /** Language of the translated text. `lang` still describes the original. */
118
+ translatedLang?: string;
115
119
  /** The post quoted by this post, when X includes it in the resolved payload. */
116
120
  quotedTweet?: TwitterQuotedTweet;
117
121
  /** The parent this post replies to, when included by X. Media is not collected. */
@@ -208,6 +212,7 @@ export type Net = (url: string, init?: RequestInit, attempts?: number) => Promis
208
212
  export type ResolveContext = {
209
213
  net: Net;
210
214
  preferredWidth: number;
215
+ languages?: string[];
211
216
  comments?: number;
212
217
  /** Requested soft byte cap, for resolvers that can rank exact rendition sizes. */
213
218
  tryMaxBytes?: number;
@@ -5,6 +5,11 @@ export type PostfetchOptions = {
5
5
  fetch?: typeof fetch;
6
6
  /** Maximum direct comments to fetch for X and Reddit. Other platforms ignore this option. Failures return an empty array. */
7
7
  comments?: number;
8
+ /** Allowed X text languages, in preference order. Preserve allowed originals;
9
+ * otherwise translate to the first language, then English, then keep the original.
10
+ * Omit to keep original text. Applies to quotes, parents and requested replies too.
11
+ */
12
+ languages?: string[];
8
13
  /** Preferred media width in pixels; the closest available rendition is chosen. Defaults to `720`. */
9
14
  preferredWidth?: number;
10
15
  /**
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@postfetch/core",
3
- "version": "0.13.9",
3
+ "version": "0.14.0",
4
4
  "description": "Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, Spotify, TikTok, X and YouTube post URLs into media files.",
5
5
  "license": "MIT",
6
6
  "type": "module",
package/src/internal.ts CHANGED
@@ -113,6 +113,10 @@ export type TwitterExtra = {
113
113
  replyToId?: string;
114
114
  /** BCP 47 language tag detected for the tweet. */
115
115
  lang?: string;
116
+ /** Original text retained when a requested translation is used. */
117
+ originalText?: string;
118
+ /** Language of the translated text. `lang` still describes the original. */
119
+ translatedLang?: string;
116
120
  /** The post quoted by this post, when X includes it in the resolved payload. */
117
121
  quotedTweet?: TwitterQuotedTweet;
118
122
  /** The parent this post replies to, when included by X. Media is not collected. */
@@ -228,6 +232,7 @@ export type Net = (url: string, init?: RequestInit, attempts?: number) => Promis
228
232
  export type ResolveContext = {
229
233
  net: Net;
230
234
  preferredWidth: number;
235
+ languages?: string[];
231
236
  comments?: number;
232
237
  /** Requested soft byte cap, for resolvers that can rank exact rendition sizes. */
233
238
  tryMaxBytes?: number;
package/src/postfetch.ts CHANGED
@@ -16,6 +16,11 @@ export type PostfetchOptions = {
16
16
  fetch?: typeof fetch;
17
17
  /** Maximum direct comments to fetch for X and Reddit. Other platforms ignore this option. Failures return an empty array. */
18
18
  comments?: number;
19
+ /** Allowed X text languages, in preference order. Preserve allowed originals;
20
+ * otherwise translate to the first language, then English, then keep the original.
21
+ * Omit to keep original text. Applies to quotes, parents and requested replies too.
22
+ */
23
+ languages?: string[];
19
24
  /** Preferred media width in pixels; the closest available rendition is chosen. Defaults to `720`. */
20
25
  preferredWidth?: number;
21
26
  /**
@@ -61,6 +66,7 @@ export async function postfetch(url: string, options: PostfetchOptions = {}): Pr
61
66
  }
62
67
  const context: ResolveContext = {
63
68
  net: createNet(options.fetch ?? globalThis.fetch),
69
+ languages: options.languages === undefined ? undefined : normalizeLanguages(options.languages),
64
70
  comments: options.comments,
65
71
  preferredWidth: options.preferredWidth ?? 720,
66
72
  tryMaxBytes: options.tryMaxBytes,
@@ -72,6 +78,13 @@ export async function postfetch(url: string, options: PostfetchOptions = {}): Pr
72
78
  : trySmaller(context, result, options.tryMaxBytes);
73
79
  }
74
80
 
81
+ function normalizeLanguages(languages: string[]): string[] {
82
+ if (!Array.isArray(languages) || languages.length === 0 || languages.some((lang) => typeof lang !== "string" || !/^[a-z]{2,3}(?:-[a-z0-9]{2,8})*$/i.test(lang))) {
83
+ throw new PostfetchError(400, "languages must be a non-empty list of language codes");
84
+ }
85
+ return [...new Set(languages.map((lang) => lang.toLowerCase()))];
86
+ }
87
+
75
88
  async function resolve(context: ResolveContext): Promise<PostfetchResult> {
76
89
  switch (detect(context.url)) {
77
90
  case "facebook":
package/src/twitter.ts CHANGED
@@ -33,6 +33,7 @@ export async function resolveTwitter(input: ResolveContext): Promise<PostfetchRe
33
33
  throw new Error("Tweet is unavailable or age-restricted");
34
34
  }
35
35
  await expandTwitterText(input.net, tweet, id);
36
+ await Promise.all(twitterTweets(tweet, id, true).map(({ id, tweet }) => translateTwitterText(input, tweet, id)));
36
37
  const posts = twitterTweets(tweet, id);
37
38
  const groups = posts.flatMap(({ id: postId, tweet: post }) => {
38
39
  const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
@@ -61,6 +62,11 @@ export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterEx
61
62
  if (lang) {
62
63
  extra.lang = lang;
63
64
  }
65
+ const translated = string(tweet.translated_text);
66
+ if (translated) {
67
+ extra.originalText = cleanTwitterText(twitterText(tweet), tweet) ?? undefined;
68
+ extra.translatedLang = string(tweet.translated_lang) ?? undefined;
69
+ }
64
70
  const quotedTweet = object(tweet.quoted_tweet) ? tweet.quoted_tweet : null;
65
71
  const quotedId = quotedTweet ? string(quotedTweet.id_str) : null;
66
72
  if (quotedTweet && quotedId) {
@@ -74,7 +80,7 @@ export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterEx
74
80
  extra.parentTweet = { id: parentId, metadata: twitterMetadata(parent) };
75
81
  }
76
82
  return {
77
- text: cleanTwitterText(twitterText(tweet), tweet) ?? undefined,
83
+ text: cleanTwitterText(translated ?? twitterText(tweet), tweet) ?? undefined,
78
84
  author: user
79
85
  ? {
80
86
  handle: string(user.screen_name) ?? undefined,
@@ -130,6 +136,42 @@ function needsFullText(tweet: Json): boolean {
130
136
  return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
131
137
  }
132
138
 
139
+ async function translateTwitterText(input: ResolveContext, tweet: Json, id: string): Promise<void> {
140
+ const languages = input.languages;
141
+ if (!languages?.length || !twitterText(tweet)) return;
142
+ const lang = string(tweet.lang)?.toLowerCase();
143
+ const allowed = (source: string) => languages.some((language) => source === language || source.startsWith(`${language}-`));
144
+ if (lang && allowed(lang)) return;
145
+ for (const target of new Set([languages[0], "en"])) {
146
+ try {
147
+ const response = await input.net(`https://api.fxtwitter.com/status/${id}/${target}`, {
148
+ headers: { accept: "application/json", "user-agent": browserUserAgent() },
149
+ }, 1);
150
+ if (!response.ok) continue;
151
+ const payload: unknown = await response.json();
152
+ if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id) continue;
153
+ const post = payload.tweet;
154
+ const translation = object(post.translation) ? post.translation : null;
155
+ const source = (string(post.lang) ?? string(translation?.source_lang))?.toLowerCase();
156
+ // The richer lookup may identify a language omitted by syndication.
157
+ if (source && source !== "und") {
158
+ tweet.lang = source;
159
+ if (allowed(source)) return;
160
+ }
161
+ const text = translation ? string(translation.text) : null;
162
+ if (!text?.trim() || string(translation?.target_lang)?.toLowerCase() !== target) continue;
163
+ // FxTwitter can return DeepL's client-upgrade error as a successful
164
+ // translation. It is an upstream failure, not the author's post text.
165
+ if (text.includes("You're using an older version of DeepL.")) continue;
166
+ tweet.translated_text = text;
167
+ tweet.translated_lang = target;
168
+ return;
169
+ } catch {
170
+ // Translation is optional; a failed lookup must not lose the post.
171
+ }
172
+ }
173
+ }
174
+
133
175
  async function expandTwitterText(net: Net, tweet: Json, rootId: string): Promise<void> {
134
176
  const posts = new Map(twitterTweets(tweet, rootId, true).map(({ id, tweet: post }) => [id, post]));
135
177
  for (const [id, post] of posts) {
@@ -368,15 +410,21 @@ async function twitterComments(input: ResolveContext, id: string): Promise<PostC
368
410
  const payload: unknown = await response.json();
369
411
  if (!object(payload) || payload.code !== 200 || !Array.isArray(payload.replies)) throw new Error("Invalid comments");
370
412
  const previousCount = comments.length;
413
+ const page: Array<{ id: string; reply: Json }> = [];
371
414
  for (const reply of payload.replies) {
372
415
  if (!object(reply)) continue;
373
416
  const replyId = string(reply.id);
374
417
  const parent = object(reply.replying_to) ? string(reply.replying_to.status) : null;
375
418
  if (!replyId || parent !== id || seen.has(replyId)) continue;
376
419
  seen.add(replyId);
377
- comments.push(twitterComment(reply, replyId));
378
- if (comments.length >= limit) return comments;
420
+ page.push({ id: replyId, reply });
421
+ if (comments.length + page.length >= limit) break;
379
422
  }
423
+ comments.push(...await Promise.all(page.map(async ({ id, reply }) => {
424
+ await translateTwitterText(input, reply, id);
425
+ return twitterComment(reply, id);
426
+ })));
427
+ if (comments.length >= limit) return comments;
380
428
  cursor = object(payload.cursor) ? string(payload.cursor.bottom) : null;
381
429
  if (!cursor || cursors.has(cursor) || comments.length === previousCount) break;
382
430
  cursors.add(cursor);
@@ -410,7 +458,7 @@ function twitterComment(reply: Json, id: string): PostComment {
410
458
  url: `https://x.com/i/status/${id}`,
411
459
  items,
412
460
  metadata: {
413
- text: cleanTwitterText(string(reply.text), reply) ?? undefined,
461
+ text: cleanTwitterText(string(reply.translated_text) ?? string(reply.text), reply) ?? undefined,
414
462
  author: {
415
463
  handle: string(author.screen_name) ?? undefined,
416
464
  name: string(author.name) ?? undefined,