@postfetch/core 0.7.0 → 0.8.0

package/README.md

@@ -13,12 +13,17 @@ for (const item of result.items) {
 
 `postfetch(url, options?)` detects the platform and resolves its media into a typed `PostfetchResult` (URLs plus the headers needed to fetch them). It performs no side effects beyond the lookup — `download`, `downloadBlob`, `archive` and `toResponse` materialize the result.
 
-`downloadBlob(url, headers?, options?)` materializes an already-resolved media URL for
+`downloadBlob(url, options?)` materializes an already-resolved media URL for
 clients that need an uploadable `Blob`. Pass the resolved item's headers for
-protected CDN URLs: `downloadBlob(item.url, item.headers)`.
+protected CDN URLs: `downloadBlob(item.url, { headers: item.headers })`. Set
+`remux: true` to normalize an MP4 with an FFmpeg stream copy (fast-start,
+non-negative timestamps, no edit list). Remuxing defaults to off, uses `ffmpeg`
+from `PATH` unless `ffmpegPath` is set, and falls back to the original Blob on
+failure. The legacy `(url, headers?, options?)` overload remains supported.
 
-- `PostfetchOptions` — `{ fetch?: typeof fetch; preferredWidth?: number; tryMaxBytes?: number }`. `tryMaxBytes` is a soft byte cap: when the normal media is larger, a smaller available rendition is returned; if its size cannot be discovered or no smaller rendition is available, the normal result is kept. Inject `fetch` to unit-test resolvers offline.
+- `PostfetchOptions` — `{ fetch?: typeof fetch; preferredWidth?: number; tryMaxBytes?: number }`. `tryMaxBytes` is a soft byte cap. X probes every MP4 variant and selects the highest-quality complete result that fits, falling back to its smallest variants when none do. Other platforms return a smaller available rendition when possible and otherwise keep the normal result. Inject `fetch` to unit-test resolvers offline.
 - Every request rotates a consistent browser/app fingerprint, so a fixed user-agent never gets the whole fleet blocked.
-- No runtime dependencies, no `ffmpeg` / `yt-dlp`, no cookies.
+- No runtime dependencies, no `yt-dlp`, no cookies. `ffmpeg` is only used when
+  the opt-in `downloadBlob(..., { remux: true })` path is requested.
 
 See the [root README](https://github.com/chelokot/postfetch#readme) for the full API table, supported inputs and the fingerprint design.

package/dist/download.d.ts

@@ -4,6 +4,15 @@ export type DownloadOptions = {
     /** Custom `fetch` implementation. Defaults to the global `fetch`. */
     fetch?: typeof fetch;
 };
+/** Options for {@link downloadBlob}. */
+export type DownloadBlobOptions = DownloadOptions & {
+    /** Command used for opt-in MP4 remuxing. Defaults to `ffmpeg` from `PATH`. */
+    ffmpegPath?: string;
+    /** Headers required by the resolved media URL. */
+    headers?: HeadersInit;
+    /** Normalize an MP4 container with FFmpeg. Defaults to `false`. */
+    remux?: boolean;
+};
 /** A zip archive produced by {@link archive}. */
 export type Archive = {
     /** The zip bytes (store mode, built in-process). */
@@ -35,20 +44,23 @@ export declare function download(item: MediaItem, options?: DownloadOptions): Pr
  *
  * This is useful for consumers that need to upload media themselves instead of
  * passing its URL to a third party. When the resolved {@link MediaItem} carries
- * required CDN headers, pass them as the second argument.
+ * required CDN headers, pass them in the options object. Set `remux: true` to
+ * normalize an MP4 with an FFmpeg stream copy; if FFmpeg is unavailable or the
+ * remux fails, the original Blob is returned.
  *
  * @param url A direct media URL returned by a resolver.
- * @param headers Headers required by the resolved media URL.
- * @param options Fetch options.
+ * @param options Headers, fetch implementation and opt-in remux behavior.
  * @returns The downloaded media as a Blob.
  *
  * @example Upload a video with FormData
  * ```ts
  * const [media] = (await postfetch(sourceUrl)).items;
- * const blob = await downloadBlob(media.url, media.headers);
+ * const blob = await downloadBlob(media.url, { headers: media.headers, remux: true });
  * form.append("video", blob, media.filename);
  * ```
  */
+export declare function downloadBlob(url: string, options?: DownloadBlobOptions): Promise<Blob>;
+/** @deprecated Pass headers and fetch in a single {@link DownloadBlobOptions} object. */
 export declare function downloadBlob(url: string, headers?: HeadersInit, options?: DownloadOptions): Promise<Blob>;
 /**
  * Download every item of a result and zip them in-process (store mode).

package/dist/index.d.ts

@@ -20,6 +20,6 @@
  * @module
  */
 export { postfetch, detect, type PostfetchOptions } from "./postfetch";
-export { download, downloadBlob, archive, toResponse, type Archive, type DownloadOptions, } from "./download";
+export { download, downloadBlob, archive, toResponse, type Archive, type DownloadBlobOptions, type DownloadOptions, } from "./download";
 export { PostfetchError } from "./internal";
 export type { InstagramExtra, MediaItem, MediaKind, PinterestExtra, Platform, PlatformResult, PostAuthor, PostfetchReason, PostfetchResult, PostMetadata, RedditExtra, SoundcloudExtra, TiktokExtra, TwitterExtra, TwitterQuotedTweet, YoutubeExtra, } from "./internal";

package/dist/index.js

@@ -1,3 +1,6 @@
+import { createRequire } from "node:module";
+var __require = /* @__PURE__ */ createRequire(import.meta.url);
+
 // src/internal.ts
 class PostfetchError extends Error {
   status;
@@ -162,12 +165,17 @@ function instagramAppUserAgent() {
 
 // src/facebook.ts
 async function resolveFacebook(input) {
-  const canonical = await canonicalUrl(input.net, normalizedInputUrl(input.url));
+  const page = await canonicalPage(input.net, normalizedInputUrl(input.url));
+  const canonical = page.url;
   const id = facebookId(canonical) ?? facebookId(input.url) ?? "video";
+  const pageMetadata = facebookMetadata(page.html);
   const embed = await embedVideo(input.net, canonical, input.preferredWidth);
   const watch = !embed && /^\d+$/.test(id) ? await watchVideo(input.net, id, input.preferredWidth) : null;
   const url = embed ?? watch?.url ?? null;
   if (!url) {
+    if (pageMetadata?.text) {
+      return { archiveFilename: filename(`facebook_${id}.zip`), id, items: [], metadata: pageMetadata, platform: "facebook" };
+    }
     throw new PostfetchError(404, "Facebook video not found", "notFound");
   }
   const item = {
@@ -179,7 +187,13 @@ async function resolveFacebook(input) {
     platform: "facebook",
     url
   };
-  return { archiveFilename: filename(`facebook_${id}.zip`), id, items: [item], metadata: watch?.metadata, platform: "facebook" };
+  return {
+    archiveFilename: filename(`facebook_${id}.zip`),
+    id,
+    items: [item],
+    metadata: mergeMetadata(pageMetadata, watch?.metadata),
+    platform: "facebook"
+  };
 }
 function normalizedInputUrl(input) {
   const url = asUrl(input);
@@ -193,10 +207,13 @@ function navigationHeaders2() {
     "user-agent": browserUserAgent()
   };
 }
-async function canonicalUrl(net, input) {
+async function canonicalPage(net, input) {
   const response = await net(input, { headers: navigationHeaders2() });
-  const resolved = asUrl(response.url);
-  return `${resolved.origin}${resolved.pathname}`;
+  const resolved = asUrl(response.url || input);
+  return {
+    html: response.ok ? await response.text() : "",
+    url: `${resolved.origin}${resolved.pathname}`
+  };
 }
 async function embedVideo(net, canonical, preferredWidth) {
   const embed = `https://www.facebook.com/plugins/video.php?href=${encodeURIComponent(canonical)}`;
@@ -231,6 +248,63 @@ function watchMetadata(html) {
     viewCount: count(html.match(/"video_view_count":(\d+)/)?.[1])
   };
 }
+function facebookMetadata(html) {
+  const canonical = metaContent(html, "og:url");
+  if (!canonical || !facebookPostUrl(canonical)) {
+    return;
+  }
+  const text = metaContent(html, "og:description") ?? metaContent(html, "description");
+  const name = metaContent(html, "og:title");
+  if (!text && !name) {
+    return;
+  }
+  return {
+    text: text ?? undefined,
+    author: name ? { name } : undefined
+  };
+}
+function facebookPostUrl(input) {
+  try {
+    const url = new URL(input);
+    return /\/posts\/(?:[^/]+\/)?[A-Za-z0-9]+\/?$/.test(url.pathname) || /\/share\/p\/[A-Za-z0-9]+\/?$/.test(url.pathname) || (url.pathname === "/permalink.php" || url.pathname === "/story.php") && Boolean(url.searchParams.get("story_fbid"));
+  } catch {
+    return false;
+  }
+}
+function metaContent(html, key) {
+  for (const match of html.matchAll(/<meta\b[^>]*>/gi)) {
+    const tag = match[0];
+    if ((attribute(tag, "property") ?? attribute(tag, "name")) !== key) {
+      continue;
+    }
+    const content = attribute(tag, "content");
+    return content ? decodeHtml(content) : null;
+  }
+  return null;
+}
+function attribute(tag, name) {
+  const match = tag.match(new RegExp(`\\b${name}\\s*=\\s*(["'])(.*?)\\1`, "is"));
+  return match?.[2] ?? null;
+}
+function decodeHtml(input) {
+  const named = { amp: "&", apos: "'", gt: ">", lt: "<", nbsp: " ", quot: '"' };
+  return input.replace(/&(#x[\da-f]+|#\d+|amp|apos|gt|lt|nbsp|quot);/gi, (entity, code) => {
+    if (code[0] !== "#") {
+      return named[code.toLowerCase()] ?? entity;
+    }
+    const value = code[1]?.toLowerCase() === "x" ? Number.parseInt(code.slice(2), 16) : Number.parseInt(code.slice(1), 10);
+    return Number.isSafeInteger(value) && value <= 1114111 ? String.fromCodePoint(value) : entity;
+  });
+}
+function mergeMetadata(page, watch) {
+  if (!page) {
+    return watch;
+  }
+  if (!watch) {
+    return page;
+  }
+  return { ...page, ...watch, author: watch.author ?? page.author };
+}
 function source(html, key) {
   const match = html.match(new RegExp(`"${key}":("(?:\\\\.|[^"\\\\])*")`));
   if (!match) {
@@ -809,7 +883,7 @@ function parseJson(value) {
     return JSON.parse(value);
   } catch {
     try {
-      return JSON.parse(decodeHtml(value));
+      return JSON.parse(decodeHtml2(value));
     } catch {
       return null;
     }
@@ -844,7 +918,7 @@ function bestVideoUrl(html) {
   }
   let parsed;
   try {
-    parsed = JSON.parse(decodeHtml(encoded));
+    parsed = JSON.parse(decodeHtml2(encoded));
   } catch {
     return null;
   }
@@ -881,7 +955,7 @@ function linkedinHandle(value) {
     return null;
   }
 }
-function decodeHtml(value) {
+function decodeHtml2(value) {
   return value.replace(/&(quot|amp|lt|gt|#39|#x27|#(\d+)|#x([0-9a-f]+));/gi, (_entity, name, decimal, hex) => {
     switch (name.toLowerCase()) {
       case "quot":
@@ -1541,7 +1615,7 @@ function extensionOf(mime) {
 
 // src/soundcloud.ts
 async function resolveSoundcloud(input) {
-  const url = await canonicalUrl2(input.net, input.url);
+  const url = await canonicalUrl(input.net, input.url);
   const clientId = await appClientId(input.net);
   const track = await resolveTrack(input.net, url, clientId);
   const id = trackId(track);
@@ -1567,7 +1641,7 @@ function soundcloudMetadata(track) {
     }
   };
 }
-async function canonicalUrl2(net, input) {
+async function canonicalUrl(net, input) {
   const host = asUrl(input).hostname;
   if (host !== "on.soundcloud.com" && host !== "snd.sc") {
     return input;
@@ -1935,13 +2009,14 @@ async function resolveTwitter(input) {
   if (string(tweet.__typename) === "TweetTombstone") {
     throw new Error("Tweet is unavailable or age-restricted");
   }
-  const items = twitterTweets(tweet, id).flatMap(({ id: postId2, tweet: post }) => {
+  const groups = twitterTweets(tweet, id).flatMap(({ id: postId2, tweet: post }) => {
     const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
-    return media.flatMap((entry, index) => twitterItem(entry, postId2, index + 1));
+    return media.flatMap((entry, index) => {
+      const candidates = twitterCandidates(entry, postId2, index + 1);
+      return candidates.length > 0 ? [candidates] : [];
+    });
   });
-  if (items.length === 0) {
-    throw new Error("Twitter media not found");
-  }
+  const items = input.tryMaxBytes === undefined ? groups.map((group) => group[0].item) : await sizeLimitedItems(input, groups);
   return { archiveFilename: filename(`twitter_${id}.zip`), id, items, metadata: twitterMetadata(tweet), platform: "twitter" };
 }
 function twitterMetadata(tweet) {
@@ -2004,25 +2079,92 @@ async function syndication(net, id) {
 function syndicationToken(id) {
   return (Number(id) / 1000000000000000 * Math.PI).toString(36).replace(/(0+|\.)/g, "");
 }
-function twitterItem(entry, id, index) {
+function twitterCandidates(entry, id, index) {
   const headers = { "user-agent": browserUserAgent() };
   const type = string(entry.type);
   if (type === "video" || type === "animated_gif") {
-    const url = bestVariant(entry);
-    return url ? [{ filename: filename(`twitter_${id}_${index}.mp4`), headers, id, kind: "video", mime: "video/mp4", platform: "twitter", url }] : [];
+    return videoVariants(entry).map(({ bitrate, url }) => ({
+      bitrate,
+      item: { filename: filename(`twitter_${id}_${index}.mp4`), headers, id, kind: "video", mime: "video/mp4", platform: "twitter", url }
+    }));
   }
   const photo = string(entry.media_url_https);
-  return photo ? [{ filename: filename(`twitter_${id}_${index}.jpg`), headers, id, kind: "image", mime: "image/jpeg", platform: "twitter", url: `${photo}?name=orig` }] : [];
+  return photo ? [{ bitrate: 0, item: { filename: filename(`twitter_${id}_${index}.jpg`), headers, id, kind: "image", mime: "image/jpeg", platform: "twitter", url: `${photo}?name=orig` } }] : [];
 }
-function bestVariant(entry) {
+function videoVariants(entry) {
   const info = object(entry.video_info) ? entry.video_info : null;
   const variants = info && Array.isArray(info.variants) ? info.variants.filter(object) : [];
-  const best = variants.filter((variant) => string(variant.content_type) === "video/mp4").reduce((current, variant) => {
-    const bitrate = number(variant.bitrate) ?? 0;
-    const currentBitrate = current ? number(current.bitrate) ?? 0 : -1;
-    return bitrate > currentBitrate ? variant : current;
-  }, null);
-  return best ? string(best.url) : null;
+  const seen = new Set;
+  return variants.flatMap((variant) => {
+    const url = string(variant.url);
+    if (string(variant.content_type) !== "video/mp4" || !url || seen.has(url)) {
+      return [];
+    }
+    seen.add(url);
+    return [{ bitrate: number(variant.bitrate) ?? 0, url }];
+  }).sort((left, right) => right.bitrate - left.bitrate);
+}
+async function sizeLimitedItems(input, groups) {
+  if (groups.length === 0 || input.tryMaxBytes === undefined) {
+    return groups.map((group) => group[0].item);
+  }
+  const sizes = new Map;
+  const size = (candidate) => {
+    const existing = sizes.get(candidate.item.url);
+    if (existing) {
+      return existing;
+    }
+    const pending = contentLength(input.net, candidate.item);
+    sizes.set(candidate.item.url, pending);
+    return pending;
+  };
+  const highest = groups.map((group) => group[0]);
+  const highestSizes = await Promise.all(highest.map(size));
+  const highestTotal = knownTotal(highestSizes);
+  if (highestTotal !== null && highestTotal <= input.tryMaxBytes) {
+    return highest.map(({ item }) => item);
+  }
+  const sized = await Promise.all(groups.map(async (group) => Promise.all(group.map(async (candidate) => ({ ...candidate, bytes: await size(candidate) })))));
+  const fitting = bestFitting(sized, input.tryMaxBytes);
+  return (fitting ?? groups.map((group) => group.at(-1))).map(({ item }) => item);
+}
+function knownTotal(sizes) {
+  return sizes.some((size) => size === null) ? null : sizes.reduce((total, size) => total + (size ?? 0), 0);
+}
+function bestFitting(groups, maxBytes) {
+  let selections = [
+    { bytes: 0, candidates: [], quality: 0 }
+  ];
+  for (const group of groups) {
+    selections = selections.flatMap((selection) => group.flatMap((candidate) => {
+      if (candidate.bytes === null || selection.bytes + candidate.bytes > maxBytes) {
+        return [];
+      }
+      return [{
+        bytes: selection.bytes + candidate.bytes,
+        candidates: [...selection.candidates, candidate],
+        quality: selection.quality + candidate.bitrate
+      }];
+    }));
+    if (selections.length === 0) {
+      return null;
+    }
+  }
+  const best = selections.reduce((current, selection) => selection.quality > current.quality ? selection : current);
+  return best.candidates;
+}
+async function contentLength(net, item) {
+  try {
+    const response = await net(item.url, { headers: item.headers, method: "HEAD" }, 1);
+    const raw = response.ok ? response.headers.get("content-length") : null;
+    if (!raw || !/^\d+$/.test(raw)) {
+      return null;
+    }
+    const bytes = Number(raw);
+    return Number.isSafeInteger(bytes) ? bytes : null;
+  } catch {
+    return null;
+  }
 }
 function tweetId(input) {
   return asUrl(input).pathname.match(/\/status(?:es)?\/(\d+)/)?.[1] ?? null;
@@ -2260,10 +2402,11 @@ async function postfetch(url, options = {}) {
   const context = {
     net: createNet(options.fetch ?? globalThis.fetch),
     preferredWidth: options.preferredWidth ?? 720,
+    tryMaxBytes: options.tryMaxBytes,
     url: trimmed
   };
   const result = await resolve(context);
-  return options.tryMaxBytes === undefined ? result : trySmaller(context, result, options.tryMaxBytes);
+  return options.tryMaxBytes === undefined || result.platform === "twitter" ? result : trySmaller(context, result, options.tryMaxBytes);
 }
 async function resolve(context) {
   switch (detect(context.url)) {
@@ -2315,12 +2458,12 @@ async function itemBytes(context, item) {
     return null;
   }
   const [video, audio] = await Promise.all([
-    contentLength(context, item.url, item.headers),
-    item.audio ? contentLength(context, item.audio.url, item.audio.headers) : Promise.resolve(0)
+    contentLength2(context, item.url, item.headers),
+    item.audio ? contentLength2(context, item.audio.url, item.audio.headers) : Promise.resolve(0)
   ]);
   return video === null || audio === null ? null : video + audio;
 }
-async function contentLength(context, url, headers) {
+async function contentLength2(context, url, headers) {
   try {
     const response = await context.net(url, { headers, method: "HEAD" }, 1);
     const raw = response.ok ? response.headers.get("content-length") : null;
@@ -2370,6 +2513,83 @@ function detect(url) {
   }
   throw new PostfetchError(400, "only Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube URLs are supported");
 }
+// src/mp4-remux.ts
+async function remuxMp4(bytes, ffmpegPath = "ffmpeg", runtime) {
+  let directory;
+  let io;
+  try {
+    io = runtime ?? await nodeRuntime();
+    directory = await io.makeTempDir("postfetch-remux-");
+    const inputPath = `${directory}/input.mp4`;
+    const outputPath = `${directory}/output.mp4`;
+    await io.writeFile(inputPath, bytes);
+    const result = await io.run(ffmpegPath, [
+      "-nostdin",
+      "-hide_banner",
+      "-loglevel",
+      "error",
+      "-y",
+      "-i",
+      inputPath,
+      "-map",
+      "0:v:0",
+      "-map",
+      "0:a:0?",
+      "-c",
+      "copy",
+      "-movflags",
+      "+faststart",
+      "-avoid_negative_ts",
+      "make_zero",
+      "-use_editlist",
+      "0",
+      outputPath
+    ]);
+    if (!result.success) {
+      return null;
+    }
+    const output = await io.readFile(outputPath);
+    return output.length > 0 ? output : null;
+  } catch {
+    return null;
+  } finally {
+    if (directory && io) {
+      await io.removeDir(directory).catch(() => {
+        return;
+      });
+    }
+  }
+}
+async function nodeRuntime() {
+  const [{ spawn }, { mkdtemp, readFile, rm, writeFile }, { tmpdir }, { join }] = await Promise.all([
+    import("node:child_process"),
+    import("node:fs/promises"),
+    import("node:os"),
+    import("node:path")
+  ]);
+  return {
+    makeTempDir: (prefix) => mkdtemp(join(tmpdir(), prefix)),
+    readFile: async (path) => new Uint8Array(await readFile(path)),
+    removeDir: async (path) => {
+      await rm(path, { force: true, recursive: true });
+    },
+    run: (command, args) => new Promise((resolve2) => {
+      const child = spawn(command, args, { stdio: ["ignore", "ignore", "pipe"] });
+      let stderr = "";
+      child.stderr?.on("data", (chunk) => {
+        if (stderr.length < 4096) {
+          stderr += new TextDecoder().decode(chunk).slice(0, 4096 - stderr.length);
+        }
+      });
+      child.once("error", (error) => resolve2({ stderr: error.message, success: false }));
+      child.once("close", (code) => resolve2({ stderr, success: code === 0 }));
+    }),
+    writeFile: async (path, value) => {
+      await writeFile(path, value);
+    }
+  };
+}
+
 // src/remux.ts
 function mergeAudioVideo(video, audio) {
   const videoSource = analyze(video);
@@ -2605,13 +2825,28 @@ async function download(item, options = {}) {
   }
   return response;
 }
-async function downloadBlob(url, headers = {}, options = {}) {
+async function downloadBlob(url, optionsOrHeaders = {}, legacyOptions = {}) {
+  const modern = isDownloadBlobOptions(optionsOrHeaders);
+  const options = modern ? optionsOrHeaders : legacyOptions;
+  const headers = modern ? optionsOrHeaders.headers ?? {} : optionsOrHeaders;
   const net = createNet(options.fetch ?? globalThis.fetch);
   const response = await net(url, { headers });
   if (!response.ok || !response.body) {
     throw new PostfetchError(502, `download failed: ${response.status}`);
   }
-  return response.blob();
+  const blob = await response.blob();
+  if (!modern || optionsOrHeaders.remux !== true) {
+    return blob;
+  }
+  const remuxed = await remuxMp4(new Uint8Array(await blob.arrayBuffer()), optionsOrHeaders.ffmpegPath);
+  return remuxed ? new Blob([toArrayBuffer(remuxed)], { type: blob.type || "video/mp4" }) : blob;
+}
+function isDownloadBlobOptions(value) {
+  if (value instanceof Headers || Array.isArray(value)) {
+    return false;
+  }
+  const candidate = value;
+  return "headers" in candidate && typeof candidate.headers !== "string" || typeof candidate.remux === "boolean" || typeof candidate.ffmpegPath === "string" || typeof candidate.fetch === "function";
 }
 async function archive(result, options = {}) {
   const net = createNet(options.fetch ?? globalThis.fetch);

package/dist/internal.d.ts

@@ -169,6 +169,8 @@ export type Net = (url: string, init?: RequestInit, attempts?: number) => Promis
 export type ResolveContext = {
     net: Net;
     preferredWidth: number;
+    /** Requested soft byte cap, for resolvers that can rank exact rendition sizes. */
+    tryMaxBytes?: number;
     url: string;
 };
 /**

package/dist/mp4-remux.d.ts

@@ -0,0 +1,18 @@
+type CommandResult = {
+    stderr: string;
+    success: boolean;
+};
+export type Mp4RemuxRuntime = {
+    makeTempDir(prefix: string): Promise<string>;
+    readFile(path: string): Promise<Uint8Array>;
+    removeDir(path: string): Promise<void>;
+    run(command: string, args: string[]): Promise<CommandResult>;
+    writeFile(path: string, bytes: Uint8Array): Promise<void>;
+};
+/**
+ * Normalize an MP4 with the same stream-copy remux used by UMMR. Encoded media
+ * is left untouched; FFmpeg only rebuilds the container and timing tables.
+ * Returns `null` when FFmpeg is unavailable or the input cannot be remuxed.
+ */
+export declare function remuxMp4(bytes: Uint8Array, ffmpegPath?: string, runtime?: Mp4RemuxRuntime): Promise<Uint8Array | null>;
+export {};

package/dist/postfetch.d.ts

@@ -8,8 +8,10 @@ export type PostfetchOptions = {
     /**
      * Soft file-size cap in bytes. The normal rendition is resolved and probed
      * with HEAD; when it exceeds the cap, a smaller available rendition is
-     * returned. If size discovery or the smaller rendition is unavailable, the
-     * normal result is returned unchanged.
+     * returned. X probes every available MP4 variant and picks the highest-quality
+     * complete result under the cap, or its smallest variants when none fit. For
+     * other platforms, if size discovery or a smaller rendition is unavailable,
+     * the normal result is returned unchanged.
      */
     tryMaxBytes?: number;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@postfetch/core",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "Zero-dependency typed core that turns Facebook, Instagram, LinkedIn, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube post URLs into media files.",
   "license": "MIT",
   "type": "module",

package/src/download.ts

@@ -1,5 +1,6 @@
 import { assembleHls } from "./hls";
 import { createNet, PostfetchError, type MediaItem, type Net, type PostfetchResult } from "./internal";
+import { remuxMp4 } from "./mp4-remux";
 import { mergeAudioVideo } from "./remux";
 import { zip } from "./zip";
 
@@ -15,6 +16,16 @@ export type DownloadOptions = {
   fetch?: typeof fetch;
 };
 
+/** Options for {@link downloadBlob}. */
+export type DownloadBlobOptions = DownloadOptions & {
+  /** Command used for opt-in MP4 remuxing. Defaults to `ffmpeg` from `PATH`. */
+  ffmpegPath?: string;
+  /** Headers required by the resolved media URL. */
+  headers?: HeadersInit;
+  /** Normalize an MP4 container with FFmpeg. Defaults to `false`. */
+  remux?: boolean;
+};
+
 /** A zip archive produced by {@link archive}. */
 export type Archive = {
   /** The zip bytes (store mode, built in-process). */
@@ -59,31 +70,56 @@ export async function download(item: MediaItem, options: DownloadOptions = {}):
  *
  * This is useful for consumers that need to upload media themselves instead of
  * passing its URL to a third party. When the resolved {@link MediaItem} carries
- * required CDN headers, pass them as the second argument.
+ * required CDN headers, pass them in the options object. Set `remux: true` to
+ * normalize an MP4 with an FFmpeg stream copy; if FFmpeg is unavailable or the
+ * remux fails, the original Blob is returned.
  *
  * @param url A direct media URL returned by a resolver.
- * @param headers Headers required by the resolved media URL.
- * @param options Fetch options.
+ * @param options Headers, fetch implementation and opt-in remux behavior.
  * @returns The downloaded media as a Blob.
  *
  * @example Upload a video with FormData
  * ```ts
  * const [media] = (await postfetch(sourceUrl)).items;
- * const blob = await downloadBlob(media.url, media.headers);
+ * const blob = await downloadBlob(media.url, { headers: media.headers, remux: true });
  * form.append("video", blob, media.filename);
  * ```
  */
+export async function downloadBlob(url: string, options?: DownloadBlobOptions): Promise<Blob>;
+/** @deprecated Pass headers and fetch in a single {@link DownloadBlobOptions} object. */
+export async function downloadBlob(url: string, headers?: HeadersInit, options?: DownloadOptions): Promise<Blob>;
 export async function downloadBlob(
   url: string,
-  headers: HeadersInit = {},
-  options: DownloadOptions = {},
+  optionsOrHeaders: DownloadBlobOptions | HeadersInit = {},
+  legacyOptions: DownloadOptions = {},
 ): Promise<Blob> {
+  const modern = isDownloadBlobOptions(optionsOrHeaders);
+  const options = modern ? optionsOrHeaders : legacyOptions;
+  const headers = modern ? optionsOrHeaders.headers ?? {} : optionsOrHeaders;
   const net = createNet(options.fetch ?? globalThis.fetch);
   const response = await net(url, { headers });
   if (!response.ok || !response.body) {
     throw new PostfetchError(502, `download failed: ${response.status}`);
   }
-  return response.blob();
+  const blob = await response.blob();
+  if (!modern || optionsOrHeaders.remux !== true) {
+    return blob;
+  }
+  const remuxed = await remuxMp4(new Uint8Array(await blob.arrayBuffer()), optionsOrHeaders.ffmpegPath);
+  return remuxed ? new Blob([toArrayBuffer(remuxed)], { type: blob.type || "video/mp4" }) : blob;
+}
+
+function isDownloadBlobOptions(value: DownloadBlobOptions | HeadersInit): value is DownloadBlobOptions {
+  if (value instanceof Headers || Array.isArray(value)) {
+    return false;
+  }
+  const candidate = value as Record<string, unknown>;
+  return (
+    ("headers" in candidate && typeof candidate.headers !== "string") ||
+    typeof candidate.remux === "boolean" ||
+    typeof candidate.ffmpegPath === "string" ||
+    typeof candidate.fetch === "function"
+  );
 }
 
 /**

package/src/facebook.ts

@@ -2,14 +2,22 @@ import { asUrl, count, filename, isoFromEpochSeconds, PostfetchError, type PostM
 import { browserUserAgent } from "./fingerprint";
 
 export async function resolveFacebook(input: ResolveContext): Promise<PostfetchResult> {
-  const canonical = await canonicalUrl(input.net, normalizedInputUrl(input.url));
+  const page = await canonicalPage(input.net, normalizedInputUrl(input.url));
+  const canonical = page.url;
   const id = facebookId(canonical) ?? facebookId(input.url) ?? "video";
+  const pageMetadata = facebookMetadata(page.html);
   // The embed player only exposes /share/v/ videos; reels (and anything it
   // misses) come from the watch page, which the &_rdr flag serves logged-out.
   const embed = await embedVideo(input.net, canonical, input.preferredWidth);
   const watch = !embed && /^\d+$/.test(id) ? await watchVideo(input.net, id, input.preferredWidth) : null;
   const url = embed ?? watch?.url ?? null;
   if (!url) {
+    // As with Reddit and X, a public text post is a valid result even when it
+    // has no downloadable media. Require post metadata from the canonical page
+    // so a missing or login-walled video does not become a false success.
+    if (pageMetadata?.text) {
+      return { archiveFilename: filename(`facebook_${id}.zip`), id, items: [], metadata: pageMetadata, platform: "facebook" };
+    }
     throw new PostfetchError(404, "Facebook video not found", "notFound");
   }
   const item: MediaItem = {
@@ -21,7 +29,13 @@ export async function resolveFacebook(input: ResolveContext): Promise<PostfetchR
     platform: "facebook",
     url,
   };
-  return { archiveFilename: filename(`facebook_${id}.zip`), id, items: [item], metadata: watch?.metadata, platform: "facebook" };
+  return {
+    archiveFilename: filename(`facebook_${id}.zip`),
+    id,
+    items: [item],
+    metadata: mergeMetadata(pageMetadata, watch?.metadata),
+    platform: "facebook",
+  };
 }
 
 // Facebook share links are opaque shortlinks. Messaging apps sometimes append
@@ -43,12 +57,15 @@ function navigationHeaders(): Record<string, string> {
 }
 
 // Share and fb.watch links redirect to the canonical /reel/<id> or /<page>/videos/<id>
-// URL, which the public embed player needs. We only read the resolved location, not the
-// (login-walled) page body.
-async function canonicalUrl(net: Net, input: string): Promise<string> {
+// URL, which the public embed player needs. Public post pages also carry Open
+// Graph metadata in their logged-out HTML, so retain the body after redirecting.
+async function canonicalPage(net: Net, input: string): Promise<{ html: string; url: string }> {
   const response = await net(input, { headers: navigationHeaders() });
-  const resolved = asUrl(response.url);
-  return `${resolved.origin}${resolved.pathname}`;
+  const resolved = asUrl(response.url || input);
+  return {
+    html: response.ok ? await response.text() : "",
+    url: `${resolved.origin}${resolved.pathname}`,
+  };
 }
 
 // The logged-out watch page is fingerprint-walled, but the public embed player
@@ -100,6 +117,76 @@ function watchMetadata(html: string): PostMetadata {
   };
 }
 
+// Public post pages expose their caption and page/profile name as Open Graph
+// metadata even when there is no media player. Facebook truncates long captions
+// there, but it is still enough to identify and represent a text post.
+function facebookMetadata(html: string): PostMetadata | undefined {
+  const canonical = metaContent(html, "og:url");
+  if (!canonical || !facebookPostUrl(canonical)) {
+    return undefined;
+  }
+  const text = metaContent(html, "og:description") ?? metaContent(html, "description");
+  const name = metaContent(html, "og:title");
+  if (!text && !name) {
+    return undefined;
+  }
+  return {
+    text: text ?? undefined,
+    author: name ? { name } : undefined,
+  };
+}
+
+function facebookPostUrl(input: string): boolean {
+  try {
+    const url = new URL(input);
+    return (
+      /\/posts\/(?:[^/]+\/)?[A-Za-z0-9]+\/?$/.test(url.pathname) ||
+      /\/share\/p\/[A-Za-z0-9]+\/?$/.test(url.pathname) ||
+      ((url.pathname === "/permalink.php" || url.pathname === "/story.php") && Boolean(url.searchParams.get("story_fbid")))
+    );
+  } catch {
+    return false;
+  }
+}
+
+function metaContent(html: string, key: string): string | null {
+  for (const match of html.matchAll(/<meta\b[^>]*>/gi)) {
+    const tag = match[0];
+    if ((attribute(tag, "property") ?? attribute(tag, "name")) !== key) {
+      continue;
+    }
+    const content = attribute(tag, "content");
+    return content ? decodeHtml(content) : null;
+  }
+  return null;
+}
+
+function attribute(tag: string, name: string): string | null {
+  const match = tag.match(new RegExp(`\\b${name}\\s*=\\s*(["'])(.*?)\\1`, "is"));
+  return match?.[2] ?? null;
+}
+
+function decodeHtml(input: string): string {
+  const named: Record<string, string> = { amp: "&", apos: "'", gt: ">", lt: "<", nbsp: " ", quot: '"' };
+  return input.replace(/&(#x[\da-f]+|#\d+|amp|apos|gt|lt|nbsp|quot);/gi, (entity, code: string) => {
+    if (code[0] !== "#") {
+      return named[code.toLowerCase()] ?? entity;
+    }
+    const value = code[1]?.toLowerCase() === "x" ? Number.parseInt(code.slice(2), 16) : Number.parseInt(code.slice(1), 10);
+    return Number.isSafeInteger(value) && value <= 0x10ffff ? String.fromCodePoint(value) : entity;
+  });
+}
+
+function mergeMetadata(page: PostMetadata | undefined, watch: PostMetadata | undefined): PostMetadata | undefined {
+  if (!page) {
+    return watch;
+  }
+  if (!watch) {
+    return page;
+  }
+  return { ...page, ...watch, author: watch.author ?? page.author };
+}
+
 function source(html: string, key: string): string | null {
   const match = html.match(new RegExp(`"${key}":("(?:\\\\.|[^"\\\\])*")`));
   if (!match) {

package/src/index.ts

@@ -26,6 +26,7 @@ export {
   archive,
   toResponse,
   type Archive,
+  type DownloadBlobOptions,
   type DownloadOptions,
 } from "./download";
 export { PostfetchError } from "./internal";

package/src/internal.ts

@@ -189,6 +189,8 @@ export type Net = (url: string, init?: RequestInit, attempts?: number) => Promis
 export type ResolveContext = {
   net: Net;
   preferredWidth: number;
+  /** Requested soft byte cap, for resolvers that can rank exact rendition sizes. */
+  tryMaxBytes?: number;
   url: string;
 };
 

package/src/mp4-remux.ts

@@ -0,0 +1,100 @@
+type CommandResult = {
+  stderr: string;
+  success: boolean;
+};
+
+export type Mp4RemuxRuntime = {
+  makeTempDir(prefix: string): Promise<string>;
+  readFile(path: string): Promise<Uint8Array>;
+  removeDir(path: string): Promise<void>;
+  run(command: string, args: string[]): Promise<CommandResult>;
+  writeFile(path: string, bytes: Uint8Array): Promise<void>;
+};
+
+/**
+ * Normalize an MP4 with the same stream-copy remux used by UMMR. Encoded media
+ * is left untouched; FFmpeg only rebuilds the container and timing tables.
+ * Returns `null` when FFmpeg is unavailable or the input cannot be remuxed.
+ */
+export async function remuxMp4(
+  bytes: Uint8Array,
+  ffmpegPath = "ffmpeg",
+  runtime?: Mp4RemuxRuntime,
+): Promise<Uint8Array | null> {
+  let directory: string | undefined;
+  let io: Mp4RemuxRuntime | undefined;
+  try {
+    io = runtime ?? (await nodeRuntime());
+    directory = await io.makeTempDir("postfetch-remux-");
+    const inputPath = `${directory}/input.mp4`;
+    const outputPath = `${directory}/output.mp4`;
+    await io.writeFile(inputPath, bytes);
+    const result = await io.run(ffmpegPath, [
+      "-nostdin",
+      "-hide_banner",
+      "-loglevel",
+      "error",
+      "-y",
+      "-i",
+      inputPath,
+      "-map",
+      "0:v:0",
+      "-map",
+      "0:a:0?",
+      "-c",
+      "copy",
+      "-movflags",
+      "+faststart",
+      "-avoid_negative_ts",
+      "make_zero",
+      "-use_editlist",
+      "0",
+      outputPath,
+    ]);
+    if (!result.success) {
+      return null;
+    }
+    const output = await io.readFile(outputPath);
+    return output.length > 0 ? output : null;
+  } catch {
+    return null;
+  } finally {
+    if (directory && io) {
+      await io.removeDir(directory).catch(() => undefined);
+    }
+  }
+}
+
+// Deno, Bun and Node all expose the Node compatibility modules used here. They
+// stay behind dynamic imports so browser/edge consumers that leave remux off do
+// not load or execute any process/filesystem code.
+async function nodeRuntime(): Promise<Mp4RemuxRuntime> {
+  const [{ spawn }, { mkdtemp, readFile, rm, writeFile }, { tmpdir }, { join }] = await Promise.all([
+    import("node:child_process"),
+    import("node:fs/promises"),
+    import("node:os"),
+    import("node:path"),
+  ]);
+  return {
+    makeTempDir: (prefix) => mkdtemp(join(tmpdir(), prefix)),
+    readFile: async (path) => new Uint8Array(await readFile(path)),
+    removeDir: async (path) => {
+      await rm(path, { force: true, recursive: true });
+    },
+    run: (command, args) =>
+      new Promise((resolve) => {
+        const child = spawn(command, args, { stdio: ["ignore", "ignore", "pipe"] });
+        let stderr = "";
+        child.stderr?.on("data", (chunk: Uint8Array) => {
+          if (stderr.length < 4096) {
+            stderr += new TextDecoder().decode(chunk).slice(0, 4096 - stderr.length);
+          }
+        });
+        child.once("error", (error) => resolve({ stderr: error.message, success: false }));
+        child.once("close", (code) => resolve({ stderr, success: code === 0 }));
+      }),
+    writeFile: async (path, value) => {
+      await writeFile(path, value);
+    },
+  };
+}

package/src/postfetch.ts

@@ -18,8 +18,10 @@ export type PostfetchOptions = {
   /**
    * Soft file-size cap in bytes. The normal rendition is resolved and probed
    * with HEAD; when it exceeds the cap, a smaller available rendition is
-   * returned. If size discovery or the smaller rendition is unavailable, the
-   * normal result is returned unchanged.
+   * returned. X probes every available MP4 variant and picks the highest-quality
+   * complete result under the cap, or its smallest variants when none fit. For
+   * other platforms, if size discovery or a smaller rendition is unavailable,
+   * the normal result is returned unchanged.
    */
   tryMaxBytes?: number;
 };
@@ -57,10 +59,13 @@ export async function postfetch(url: string, options: PostfetchOptions = {}): Pr
   const context: ResolveContext = {
     net: createNet(options.fetch ?? globalThis.fetch),
     preferredWidth: options.preferredWidth ?? 720,
+    tryMaxBytes: options.tryMaxBytes,
     url: trimmed,
   };
   const result = await resolve(context);
-  return options.tryMaxBytes === undefined ? result : trySmaller(context, result, options.tryMaxBytes);
+  return options.tryMaxBytes === undefined || result.platform === "twitter"
+    ? result
+    : trySmaller(context, result, options.tryMaxBytes);
 }
 
 async function resolve(context: ResolveContext): Promise<PostfetchResult> {

package/src/twitter.ts

@@ -17,6 +17,11 @@ import {
 } from "./internal";
 import { browserUserAgent } from "./fingerprint";
 
+type TwitterCandidate = {
+  bitrate: number;
+  item: MediaItem;
+};
+
 export async function resolveTwitter(input: ResolveContext): Promise<PostfetchResult> {
   const id = tweetId(input.url);
   if (!id) {
@@ -26,13 +31,17 @@ export async function resolveTwitter(input: ResolveContext): Promise<PostfetchRe
   if (string(tweet.__typename) === "TweetTombstone") {
     throw new Error("Tweet is unavailable or age-restricted");
   }
-  const items = twitterTweets(tweet, id).flatMap(({ id: postId, tweet: post }) => {
+  const groups = twitterTweets(tweet, id).flatMap(({ id: postId, tweet: post }) => {
     const media = Array.isArray(post.mediaDetails) ? post.mediaDetails.filter(object) : [];
-    return media.flatMap((entry, index) => twitterItem(entry, postId, index + 1));
+    return media.flatMap((entry, index) => {
+      const candidates = twitterCandidates(entry, postId, index + 1);
+      return candidates.length > 0 ? [candidates] : [];
+    });
   });
-  if (items.length === 0) {
-    throw new Error("Twitter media not found");
-  }
+  const items = input.tryMaxBytes === undefined ? groups.map((group) => group[0].item) : await sizeLimitedItems(input, groups);
+  // A text-only tweet is a valid result with metadata and no media. Syndication
+  // has already failed or returned a tombstone when the tweet is unavailable,
+  // so an empty item list here does not mean the lookup failed.
   return { archiveFilename: filename(`twitter_${id}.zip`), id, items, metadata: twitterMetadata(tweet), platform: "twitter" };
 }
 
@@ -104,32 +113,112 @@ function syndicationToken(id: string): string {
   return ((Number(id) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, "");
 }
 
-function twitterItem(entry: Json, id: string, index: number): MediaItem[] {
+function twitterCandidates(entry: Json, id: string, index: number): TwitterCandidate[] {
   const headers = { "user-agent": browserUserAgent() };
   const type = string(entry.type);
   if (type === "video" || type === "animated_gif") {
-    const url = bestVariant(entry);
-    return url
-      ? [{ filename: filename(`twitter_${id}_${index}.mp4`), headers, id, kind: "video", mime: "video/mp4", platform: "twitter", url }]
-      : [];
+    return videoVariants(entry).map(({ bitrate, url }) => ({
+      bitrate,
+      item: { filename: filename(`twitter_${id}_${index}.mp4`), headers, id, kind: "video", mime: "video/mp4", platform: "twitter", url },
+    }));
   }
   const photo = string(entry.media_url_https);
   return photo
-    ? [{ filename: filename(`twitter_${id}_${index}.jpg`), headers, id, kind: "image", mime: "image/jpeg", platform: "twitter", url: `${photo}?name=orig` }]
+    ? [{ bitrate: 0, item: { filename: filename(`twitter_${id}_${index}.jpg`), headers, id, kind: "image", mime: "image/jpeg", platform: "twitter", url: `${photo}?name=orig` } }]
     : [];
 }
 
-function bestVariant(entry: Json): string | null {
+function videoVariants(entry: Json): Array<{ bitrate: number; url: string }> {
   const info = object(entry.video_info) ? entry.video_info : null;
   const variants = info && Array.isArray(info.variants) ? info.variants.filter(object) : [];
-  const best = variants
-    .filter((variant) => string(variant.content_type) === "video/mp4")
-    .reduce<Json | null>((current, variant) => {
-      const bitrate = number(variant.bitrate) ?? 0;
-      const currentBitrate = current ? number(current.bitrate) ?? 0 : -1;
-      return bitrate > currentBitrate ? variant : current;
-    }, null);
-  return best ? string(best.url) : null;
+  const seen = new Set<string>();
+  return variants
+    .flatMap((variant) => {
+      const url = string(variant.url);
+      if (string(variant.content_type) !== "video/mp4" || !url || seen.has(url)) {
+        return [];
+      }
+      seen.add(url);
+      return [{ bitrate: number(variant.bitrate) ?? 0, url }];
+    })
+    .sort((left, right) => right.bitrate - left.bitrate);
+}
+
+async function sizeLimitedItems(input: ResolveContext, groups: TwitterCandidate[][]): Promise<MediaItem[]> {
+  if (groups.length === 0 || input.tryMaxBytes === undefined) {
+    return groups.map((group) => group[0].item);
+  }
+  const sizes = new Map<string, Promise<number | null>>();
+  const size = (candidate: TwitterCandidate): Promise<number | null> => {
+    const existing = sizes.get(candidate.item.url);
+    if (existing) {
+      return existing;
+    }
+    const pending = contentLength(input.net, candidate.item);
+    sizes.set(candidate.item.url, pending);
+    return pending;
+  };
+
+  const highest = groups.map((group) => group[0]);
+  const highestSizes = await Promise.all(highest.map(size));
+  const highestTotal = knownTotal(highestSizes);
+  if (highestTotal !== null && highestTotal <= input.tryMaxBytes) {
+    return highest.map(({ item }) => item);
+  }
+
+  const sized = await Promise.all(
+    groups.map(async (group) =>
+      Promise.all(group.map(async (candidate) => ({ ...candidate, bytes: await size(candidate) }))),
+    ),
+  );
+  const fitting = bestFitting(sized, input.tryMaxBytes);
+  return (fitting ?? groups.map((group) => group.at(-1) as TwitterCandidate)).map(({ item }) => item);
+}
+
+function knownTotal(sizes: Array<number | null>): number | null {
+  return sizes.some((size) => size === null) ? null : sizes.reduce<number>((total, size) => total + (size ?? 0), 0);
+}
+
+function bestFitting(
+  groups: Array<Array<TwitterCandidate & { bytes: number | null }>>,
+  maxBytes: number,
+): TwitterCandidate[] | null {
+  let selections: Array<{ bytes: number; candidates: TwitterCandidate[]; quality: number }> = [
+    { bytes: 0, candidates: [], quality: 0 },
+  ];
+  for (const group of groups) {
+    selections = selections.flatMap((selection) =>
+      group.flatMap((candidate) => {
+        if (candidate.bytes === null || selection.bytes + candidate.bytes > maxBytes) {
+          return [];
+        }
+        return [{
+          bytes: selection.bytes + candidate.bytes,
+          candidates: [...selection.candidates, candidate],
+          quality: selection.quality + candidate.bitrate,
+        }];
+      }),
+    );
+    if (selections.length === 0) {
+      return null;
+    }
+  }
+  const best = selections.reduce((current, selection) => (selection.quality > current.quality ? selection : current));
+  return best.candidates;
+}
+
+async function contentLength(net: Net, item: MediaItem): Promise<number | null> {
+  try {
+    const response = await net(item.url, { headers: item.headers, method: "HEAD" }, 1);
+    const raw = response.ok ? response.headers.get("content-length") : null;
+    if (!raw || !/^\d+$/.test(raw)) {
+      return null;
+    }
+    const bytes = Number(raw);
+    return Number.isSafeInteger(bytes) ? bytes : null;
+  } catch {
+    return null;
+  }
 }
 
 export function tweetId(input: string): string | null {