@postfetch/core 0.9.0 → 0.9.2

package/README.md

@@ -13,9 +13,11 @@ for (const item of result.items) {
 
 `postfetch(url, options?)` detects the platform and resolves its media into a typed `PostfetchResult` (URLs plus the headers needed to fetch them). It performs no side effects beyond the lookup — `download`, `downloadBlob`, `archive` and `toResponse` materialize the result.
 
-`downloadBlob(url, options?)` materializes an already-resolved media URL for
-clients that need an uploadable `Blob`. Pass the resolved item's headers for
-protected CDN URLs: `downloadBlob(item.url, { headers: item.headers })`. Set
+`downloadBlob(itemOrUrl, options?)` materializes resolved media for clients that
+need an uploadable `Blob`. Prefer `downloadBlob(item, { remux: true })` to
+include separate audio, assemble HLS playlists and use the item's headers.
+Passing only `item.url` fetches that URL alone and may omit separate audio;
+for direct URLs, pass `{ headers: item.headers }` when needed. Set
 `remux: true` to return `{ blob, thumbnail, width, height, duration }`: a
 normalized MP4, upload thumbnail and calculated presentation metadata. Remuxing
 defaults to off, uses `ffmpeg` and `ffprobe` from `PATH` unless their paths are

package/dist/download.d.ts

@@ -50,30 +50,32 @@ export type Archive = {
  */
 export declare function download(item: MediaItem, options?: DownloadOptions): Promise<Response>;
 /**
- * Download an already-resolved media URL as a Blob.
+ * Download a resolved media item or direct media URL as a Blob.
  *
  * This is useful for consumers that need to upload media themselves instead of
- * passing its URL to a third party. When the resolved {@link MediaItem} carries
- * required CDN headers, pass them in the options object. Set `remux: true` to
+ * passing its URL to a third party. Pass the full {@link MediaItem} to include
+ * separate audio, HLS playlists and required headers. A direct URL only fetches
+ * that URL; pass its required CDN headers in the options object. Set `remux: true` to
  * normalize an MP4 with an FFmpeg stream copy; if FFmpeg is unavailable or the
  * remux or metadata extraction fails, the operation throws.
  *
- * @param url A direct media URL returned by a resolver.
+ * @param input A resolved media item (recommended) or direct media URL.
  * @param options Headers, fetch implementation and opt-in remux behavior.
  * @returns A Blob, or with `remux: true`, the normalized video and its metadata.
  *
  * @example Upload a video with FormData
  * ```ts
  * const [media] = (await postfetch(sourceUrl)).items;
- * const video = await downloadBlob(media.url, { headers: media.headers, remux: true });
+ * const video = await downloadBlob(media, { remux: true });
  * form.append("video", video.blob, media.filename);
  * form.append("thumbnail", video.thumbnail, "thumbnail.jpg");
  * ```
  */
-export declare function downloadBlob(url: string, options: DownloadBlobOptions & {
+export declare function downloadBlob(input: string | MediaItem, options: DownloadBlobOptions & {
     remux: true;
 }): Promise<RemuxedVideo>;
-export declare function downloadBlob(url: string, options?: DownloadBlobOptions & {
+/** Download without remuxing and return the upstream media Blob. */
+export declare function downloadBlob(input: string | MediaItem, options?: DownloadBlobOptions & {
     remux?: false;
 }): Promise<Blob>;
 /** @deprecated Pass headers and fetch in a single {@link DownloadBlobOptions} object. */

package/dist/index.js

@@ -2009,6 +2009,7 @@ async function resolveTwitter(input) {
   if (string(tweet.__typename) === "TweetTombstone") {
     throw new Error("Tweet is unavailable or age-restricted");
   }
+  await expandTwitterText(input.net, tweet, id);
   const groups = twitterTweets(tweet, id).flatMap(({ id: postId2, tweet: post }) => {
     const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
     return media.flatMap((entry, index) => {
@@ -2032,7 +2033,7 @@ function twitterMetadata(tweet) {
     extra.quotedTweet = { id: quotedId, metadata: twitterMetadata(quotedTweet) };
   }
   return {
-    text: string(tweet.text) ?? string(tweet.full_text) ?? undefined,
+    text: twitterText(tweet) ?? undefined,
     author: user ? {
       handle: string(user.screen_name) ?? undefined,
       name: string(user.name) ?? undefined,
@@ -2047,6 +2048,57 @@ function twitterMetadata(tweet) {
     extra
   };
 }
+function noteText(tweet) {
+  const note = object(tweet.note_tweet) ? tweet.note_tweet : null;
+  const results = note && object(note.note_tweet_results) ? note.note_tweet_results : null;
+  const result = results && object(results.result) ? results.result : null;
+  return (result ? string(result.text) : null) ?? (note ? string(note.text) : null);
+}
+function twitterText(tweet) {
+  return noteText(tweet) ?? string(tweet.full_text) ?? string(tweet.text);
+}
+function needsFullText(tweet) {
+  if (noteText(tweet) || string(tweet.full_text)) {
+    return false;
+  }
+  return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
+}
+async function expandTwitterText(net, tweet, rootId) {
+  const posts = new Map(twitterTweets(tweet, rootId).map(({ id, tweet: post }) => [id, post]));
+  for (const [id, post] of posts) {
+    if (!needsFullText(post)) {
+      continue;
+    }
+    try {
+      const response = await net(`https://api.fxtwitter.com/status/${id}`, {
+        headers: { accept: "application/json", "user-agent": browserUserAgent() }
+      }, 1);
+      if (!response.ok) {
+        continue;
+      }
+      const payload = await response.json();
+      if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id) {
+        continue;
+      }
+      let current = payload.tweet;
+      const seen = new Set;
+      while (current) {
+        const currentId = string(current.id);
+        if (!currentId || seen.has(currentId)) {
+          break;
+        }
+        seen.add(currentId);
+        const target = posts.get(currentId);
+        const raw = object(current.raw_text) ? string(current.raw_text.text) : null;
+        const text = raw ?? string(current.text);
+        if (target && needsFullText(target) && text && text.length > (twitterText(target)?.length ?? 0)) {
+          target.full_text = text;
+        }
+        current = object(current.quote) ? current.quote : null;
+      }
+    } catch {}
+  }
+}
 function twitterTweets(tweet, rootId) {
   const tweets = [];
   const seen = new Set;
@@ -2895,12 +2947,12 @@ async function download(item, options = {}) {
   }
   return response;
 }
-async function downloadBlob(url, optionsOrHeaders = {}, legacyOptions = {}) {
+async function downloadBlob(input, optionsOrHeaders = {}, legacyOptions = {}) {
   const modern = isDownloadBlobOptions(optionsOrHeaders);
   const options = modern ? optionsOrHeaders : legacyOptions;
   const headers = modern ? optionsOrHeaders.headers ?? {} : optionsOrHeaders;
   const net = createNet(options.fetch ?? globalThis.fetch);
-  const response = await net(url, { headers });
+  const response = typeof input === "string" ? await net(input, { headers }) : await download({ ...input, headers: modern ? optionsOrHeaders.headers ?? input.headers : input.headers }, options);
   if (!response.ok || !response.body) {
     throw new PostfetchError(502, `download failed: ${response.status}`);
   }

package/dist/internal.d.ts

@@ -47,8 +47,8 @@ export type PostAuthor = {
 };
 /**
  * Post metadata normalized across platforms. Every field is optional: an absent
- * field means the platform did not expose it in the payload already fetched to
- * resolve the media (never a fabricated default — a missing count is `undefined`,
+ * field means the sources consulted did not expose it (never a fabricated
+ * default — a missing count is `undefined`,
  * not `0`). All counts are non-negative. Platform-specific fields live in `extra`.
  */
 export type PostMetadata = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@postfetch/core",
-  "version": "0.9.0",
+  "version": "0.9.2",
   "description": "Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube post URLs into media files.",
   "license": "MIT",
   "type": "module",

package/src/download.ts

@@ -77,38 +77,40 @@ export async function download(item: MediaItem, options: DownloadOptions = {}):
 }
 
 /**
- * Download an already-resolved media URL as a Blob.
+ * Download a resolved media item or direct media URL as a Blob.
  *
  * This is useful for consumers that need to upload media themselves instead of
- * passing its URL to a third party. When the resolved {@link MediaItem} carries
- * required CDN headers, pass them in the options object. Set `remux: true` to
+ * passing its URL to a third party. Pass the full {@link MediaItem} to include
+ * separate audio, HLS playlists and required headers. A direct URL only fetches
+ * that URL; pass its required CDN headers in the options object. Set `remux: true` to
  * normalize an MP4 with an FFmpeg stream copy; if FFmpeg is unavailable or the
  * remux or metadata extraction fails, the operation throws.
  *
- * @param url A direct media URL returned by a resolver.
+ * @param input A resolved media item (recommended) or direct media URL.
  * @param options Headers, fetch implementation and opt-in remux behavior.
  * @returns A Blob, or with `remux: true`, the normalized video and its metadata.
  *
  * @example Upload a video with FormData
  * ```ts
  * const [media] = (await postfetch(sourceUrl)).items;
- * const video = await downloadBlob(media.url, { headers: media.headers, remux: true });
+ * const video = await downloadBlob(media, { remux: true });
  * form.append("video", video.blob, media.filename);
  * form.append("thumbnail", video.thumbnail, "thumbnail.jpg");
  * ```
  */
 export async function downloadBlob(
-  url: string,
+  input: string | MediaItem,
   options: DownloadBlobOptions & { remux: true },
 ): Promise<RemuxedVideo>;
+/** Download without remuxing and return the upstream media Blob. */
 export async function downloadBlob(
-  url: string,
+  input: string | MediaItem,
   options?: DownloadBlobOptions & { remux?: false },
 ): Promise<Blob>;
 /** @deprecated Pass headers and fetch in a single {@link DownloadBlobOptions} object. */
 export async function downloadBlob(url: string, headers?: HeadersInit, options?: DownloadOptions): Promise<Blob>;
 export async function downloadBlob(
-  url: string,
+  input: string | MediaItem,
   optionsOrHeaders: DownloadBlobOptions | HeadersInit = {},
   legacyOptions: DownloadOptions = {},
 ): Promise<Blob | RemuxedVideo> {
@@ -116,7 +118,9 @@ export async function downloadBlob(
   const options = modern ? optionsOrHeaders : legacyOptions;
   const headers = modern ? optionsOrHeaders.headers ?? {} : optionsOrHeaders;
   const net = createNet(options.fetch ?? globalThis.fetch);
-  const response = await net(url, { headers });
+  const response = typeof input === "string"
+    ? await net(input, { headers })
+    : await download({ ...input, headers: modern ? optionsOrHeaders.headers ?? input.headers : input.headers }, options);
   if (!response.ok || !response.body) {
     throw new PostfetchError(502, `download failed: ${response.status}`);
   }

package/src/internal.ts

@@ -48,8 +48,8 @@ export type PostAuthor = {
 
 /**
  * Post metadata normalized across platforms. Every field is optional: an absent
- * field means the platform did not expose it in the payload already fetched to
- * resolve the media (never a fabricated default — a missing count is `undefined`,
+ * field means the sources consulted did not expose it (never a fabricated
+ * default — a missing count is `undefined`,
  * not `0`). All counts are non-negative. Platform-specific fields live in `extra`.
  */
 export type PostMetadata = {

package/src/twitter.ts

@@ -31,6 +31,7 @@ export async function resolveTwitter(input: ResolveContext): Promise<PostfetchRe
   if (string(tweet.__typename) === "TweetTombstone") {
     throw new Error("Tweet is unavailable or age-restricted");
   }
+  await expandTwitterText(input.net, tweet, id);
   const groups = twitterTweets(tweet, id).flatMap(({ id: postId, tweet: post }) => {
     const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
     return media.flatMap((entry, index) => {
@@ -58,7 +59,7 @@ export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterEx
     extra.quotedTweet = { id: quotedId, metadata: twitterMetadata(quotedTweet) };
   }
   return {
-    text: (string(tweet.text) ?? string(tweet.full_text)) ?? undefined,
+    text: twitterText(tweet) ?? undefined,
     author: user
       ? {
           handle: string(user.screen_name) ?? undefined,
@@ -76,6 +77,67 @@ export function twitterMetadata(tweet: Json): PostMetadata & { extra?: TwitterEx
   };
 }
 
+function noteText(tweet: Json): string | null {
+  const note = object(tweet.note_tweet) ? tweet.note_tweet : null;
+  const results = note && object(note.note_tweet_results) ? note.note_tweet_results : null;
+  const result = results && object(results.result) ? results.result : null;
+  return (result ? string(result.text) : null) ?? (note ? string(note.text) : null);
+}
+
+function twitterText(tweet: Json): string | null {
+  return noteText(tweet) ?? string(tweet.full_text) ?? string(tweet.text);
+}
+
+function needsFullText(tweet: Json): boolean {
+  if (noteText(tweet) || string(tweet.full_text)) {
+    return false;
+  }
+  // Embedded quotes can omit note_tweet entirely. Near-limit text is only a
+  // hint: ordinary long tweets may also incur a lookup, but are never shortened.
+  return object(tweet.note_tweet) || bool(tweet.truncated) === true || (string(tweet.text)?.length ?? 0) >= 270;
+}
+
+async function expandTwitterText(net: Net, tweet: Json, rootId: string): Promise<void> {
+  const posts = new Map(twitterTweets(tweet, rootId).map(({ id, tweet: post }) => [id, post]));
+  for (const [id, post] of posts) {
+    if (!needsFullText(post)) {
+      continue;
+    }
+    try {
+      // FxTwitter's public v1 API includes full note text and embedded quotes.
+      // Keep media and other metadata from X, and don't retry an optional lookup.
+      const response = await net(`https://api.fxtwitter.com/status/${id}`, {
+        headers: { accept: "application/json", "user-agent": browserUserAgent() },
+      }, 1);
+      if (!response.ok) {
+        continue;
+      }
+      const payload: unknown = await response.json();
+      if (!object(payload) || payload.code !== 200 || !object(payload.tweet) || string(payload.tweet.id) !== id) {
+        continue;
+      }
+      let current: Json | null = payload.tweet;
+      const seen = new Set<string>();
+      while (current) {
+        const currentId = string(current.id);
+        if (!currentId || seen.has(currentId)) {
+          break;
+        }
+        seen.add(currentId);
+        const target = posts.get(currentId);
+        const raw = object(current.raw_text) ? string(current.raw_text.text) : null;
+        const text = raw ?? string(current.text);
+        if (target && needsFullText(target) && text && text.length > (twitterText(target)?.length ?? 0)) {
+          target.full_text = text;
+        }
+        current = object(current.quote) ? current.quote : null;
+      }
+    } catch {
+      // A text enrichment outage must not discard a valid syndication result.
+    }
+  }
+}
+
 function twitterTweets(tweet: Json, rootId: string): Array<{ id: string; tweet: Json }> {
   const tweets: Array<{ id: string; tweet: Json }> = [];
   const seen = new Set<string>();