@postfetch/core 0.9.0 → 0.9.1

package/dist/download.d.ts

@@ -73,6 +73,7 @@ export declare function download(item: MediaItem, options?: DownloadOptions): Pr
 export declare function downloadBlob(url: string, options: DownloadBlobOptions & {
     remux: true;
 }): Promise<RemuxedVideo>;
+/** Download without remuxing and return the upstream media Blob. */
 export declare function downloadBlob(url: string, options?: DownloadBlobOptions & {
     remux?: false;
 }): Promise<Blob>;

package/dist/index.js

@@ -2009,6 +2009,7 @@ async function resolveTwitter(input) {
   if (string(tweet.__typename) === "TweetTombstone") {
     throw new Error("Tweet is unavailable or age-restricted");
   }
+  await expandTwitterText(input.net, tweet, id);
   const groups = twitterTweets(tweet, id).flatMap(({ id: postId2, tweet: post }) => {
     const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
     return media.flatMap((entry, index) => {
@@ -2032,7 +2033,7 @@ function twitterMetadata(tweet) {
     extra.quotedTweet = { id: quotedId, metadata: twitterMetadata(quotedTweet) };
   }
   return {
-    text: string(tweet.text) ?? string(tweet.full_text) ?? undefined,
+    text: twitterText(tweet) ?? undefined,
     author: user ? {
       handle: string(user.screen_name) ?? undefined,
       name: string(user.name) ?? undefined,
@@ -2047,6 +2048,57 @@ function twitterMetadata(tweet) {
     extra
   };
 }
+function noteText(tweet) {
+  const note = object(tweet.note_tweet) ? tweet.note_tweet : null;
+  const results = note && object(note.note_tweet_results) ? note.note_tweet_results : null;
+  const result = results && object(results.result) ? results.result : null;
+  return (result ? string(result.text) : null) ?? (note ? string(note.text) : null);
+}
+function twitterText(tweet) {
+  return noteText(tweet) ?? string(tweet.full_text) ?? string(tweet.text);
+}
+function needsFullText(tweet) {
+  if (noteText(tweet) || string(tweet.full_text)) {
+    return false;
+  }
+  return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
+}
+async function expandTwitterText(net, tweet, rootId) {
+  const posts = new Map(twitterTweets(tweet, rootId).map(({ id, tweet: post }) => [id, post]));
+  for (const [id, post] of posts) {
+    if (!needsFullText(post)) {
+      continue;
+    }
+    try {
+      const response = await net(`https://api.fxtwitter.com/status/${id}`, {
+        headers: { accept: "application/json", "user-agent": browserUserAgent() }
+      }, 1);
+      if (!response.ok) {
+        continue;
+      }
+      const payload = await response.json();
+      if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id) {
+        continue;
+      }
+      let current = payload.tweet;
+      const seen = new Set;
+      while (current) {
+        const currentId = string(current.id);
+        if (!currentId || seen.has(currentId)) {
+          break;
+        }
+        seen.add(currentId);
+        const target = posts.get(currentId);
+        const raw = object(current.raw_text) ? string(current.raw_text.text) : null;
+        const text = raw ?? string(current.text);
+        if (target && needsFullText(target) && text && text.length > (twitterText(target)?.length ?? 0)) {
+          target.full_text = text;
+        }
+        current = object(current.quote) ? current.quote : null;
+      }
+    } catch {}
+  }
+}
 function twitterTweets(tweet, rootId) {
   const tweets = [];
   const seen = new Set;

package/dist/internal.d.ts

@@ -47,8 +47,8 @@ export type PostAuthor = {
 };
 /**
  * Post metadata normalized across platforms. Every field is optional: an absent
- * field means the platform did not expose it in the payload already fetched to
- * resolve the media (never a fabricated default — a missing count is `undefined`,
+ * field means the sources consulted did not expose it (never a fabricated
+ * default — a missing count is `undefined`,
  * not `0`). All counts are non-negative. Platform-specific fields live in `extra`.
  */
 export type PostMetadata = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@postfetch/core",
-  "version": "0.9.0",
+  "version": "0.9.1",
   "description": "Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube post URLs into media files.",
   "license": "MIT",
   "type": "module",

package/src/download.ts

@@ -101,6 +101,7 @@ export async function downloadBlob(
   url: string,
   options: DownloadBlobOptions & { remux: true },
 ): Promise<RemuxedVideo>;
+/** Download without remuxing and return the upstream media Blob. */
 export async function downloadBlob(
   url: string,
   options?: DownloadBlobOptions & { remux?: false },

package/src/internal.ts

@@ -48,8 +48,8 @@ export type PostAuthor = {
 
 /**
  * Post metadata normalized across platforms. Every field is optional: an absent
- * field means the platform did not expose it in the payload already fetched to
- * resolve the media (never a fabricated default — a missing count is `undefined`,
+ * field means the sources consulted did not expose it (never a fabricated
+ * default — a missing count is `undefined`,
  * not `0`). All counts are non-negative. Platform-specific fields live in `extra`.
  */
 export type PostMetadata = {

package/src/twitter.ts

@@ -31,6 +31,7 @@ export async function resolveTwitter(input: ResolveContext): Promise<PostfetchRe
   if (string(tweet.__typename) === "TweetTombstone") {
     throw new Error("Tweet is unavailable or age-restricted");
   }
+  await expandTwitterText(input.net, tweet, id);
   const groups = twitterTweets(tweet, id).flatMap(({ id: postId, tweet: post }) => {
     const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
     return media.flatMap((entry, index) => {
@@ -58,7 +59,7 @@ export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterEx
     extra.quotedTweet = { id: quotedId, metadata: twitterMetadata(quotedTweet) };
   }
   return {
-    text: (string(tweet.text) ?? string(tweet.full_text)) ?? undefined,
+    text: twitterText(tweet) ?? undefined,
     author: user
       ? {
           handle: string(user.screen_name) ?? undefined,
@@ -76,6 +77,67 @@ export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterEx
   };
 }
 
+function noteText(tweet: Json): string | null {
+  const note = object(tweet.note_tweet) ? tweet.note_tweet : null;
+  const results = note && object(note.note_tweet_results) ? note.note_tweet_results : null;
+  const result = results && object(results.result) ? results.result : null;
+  return (result ? string(result.text) : null) ?? (note ? string(note.text) : null);
+}
+
+function twitterText(tweet: Json): string | null {
+  return noteText(tweet) ?? string(tweet.full_text) ?? string(tweet.text);
+}
+
+function needsFullText(tweet: Json): boolean {
+  if (noteText(tweet) || string(tweet.full_text)) {
+    return false;
+  }
+  // Embedded quotes can omit note_tweet entirely. Near-limit text is only a
+  // hint: ordinary long tweets may also incur a lookup, but are never shortened.
+  return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
+}
+
+async function expandTwitterText(net: Net, tweet: Json, rootId: string): Promise<void> {
+  const posts = new Map(twitterTweets(tweet, rootId).map(({ id, tweet: post }) => [id, post]));
+  for (const [id, post] of posts) {
+    if (!needsFullText(post)) {
+      continue;
+    }
+    try {
+      // FxTwitter's public v1 API includes full note text and embedded quotes.
+      // Keep media and other metadata from X, and don't retry an optional lookup.
+      const response = await net(`https://api.fxtwitter.com/status/${id}`, {
+        headers: { accept: "application/json", "user-agent": browserUserAgent() },
+      }, 1);
+      if (!response.ok) {
+        continue;
+      }
+      const payload: unknown = await response.json();
+      if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id) {
+        continue;
+      }
+      let current: Json | null = payload.tweet;
+      const seen = new Set<string>();
+      while (current) {
+        const currentId = string(current.id);
+        if (!currentId || seen.has(currentId)) {
+          break;
+        }
+        seen.add(currentId);
+        const target = posts.get(currentId);
+        const raw = object(current.raw_text) ? string(current.raw_text.text) : null;
+        const text = raw ?? string(current.text);
+        if (target && needsFullText(target) && text && text.length > (twitterText(target)?.length ?? 0)) {
+          target.full_text = text;
+        }
+        current = object(current.quote) ? current.quote : null;
+      }
+    } catch {
+      // A text enrichment outage must not discard a valid syndication result.
+    }
+  }
+}
+
 function twitterTweets(tweet: Json, rootId: string): Array<{ id: string; tweet: Json }> {
   const tweets: Array<{ id: string; tweet: Json }> = [];
   const seen = new Set<string>();