@postfetch/core 0.6.0 → 0.8.0

package/src/download.ts

@@ -1,5 +1,6 @@
 import { assembleHls } from "./hls";
 import { createNet, PostfetchError, type MediaItem, type Net, type PostfetchResult } from "./internal";
+import { remuxMp4 } from "./mp4-remux";
 import { mergeAudioVideo } from "./remux";
 import { zip } from "./zip";
 
@@ -15,6 +16,16 @@ export type DownloadOptions = {
   fetch?: typeof fetch;
 };
 
+/** Options for {@link downloadBlob}. */
+export type DownloadBlobOptions = DownloadOptions & {
+  /** Command used for opt-in MP4 remuxing. Defaults to `ffmpeg` from `PATH`. */
+  ffmpegPath?: string;
+  /** Headers required by the resolved media URL. */
+  headers?: HeadersInit;
+  /** Normalize an MP4 container with FFmpeg. Defaults to `false`. */
+  remux?: boolean;
+};
+
 /** A zip archive produced by {@link archive}. */
 export type Archive = {
   /** The zip bytes (store mode, built in-process). */
@@ -54,6 +65,63 @@ export async function download(item: MediaItem, options: DownloadOptions = {}):
   return response;
 }
 
+/**
+ * Download an already-resolved media URL as a Blob.
+ *
+ * This is useful for consumers that need to upload media themselves instead of
+ * passing its URL to a third party. When the resolved {@link MediaItem} carries
+ * required CDN headers, pass them in the options object. Set `remux: true` to
+ * normalize an MP4 with an FFmpeg stream copy; if FFmpeg is unavailable or the
+ * remux fails, the original Blob is returned.
+ *
+ * @param url A direct media URL returned by a resolver.
+ * @param options Headers, fetch implementation and opt-in remux behavior.
+ * @returns The downloaded media as a Blob.
+ *
+ * @example Upload a video with FormData
+ * ```ts
+ * const [media] = (await postfetch(sourceUrl)).items;
+ * const blob = await downloadBlob(media.url, { headers: media.headers, remux: true });
+ * form.append("video", blob, media.filename);
+ * ```
+ */
+export async function downloadBlob(url: string, options?: DownloadBlobOptions): Promise<Blob>;
+/** @deprecated Pass headers and fetch in a single {@link DownloadBlobOptions} object. */
+export async function downloadBlob(url: string, headers?: HeadersInit, options?: DownloadOptions): Promise<Blob>;
+export async function downloadBlob(
+  url: string,
+  optionsOrHeaders: DownloadBlobOptions | HeadersInit = {},
+  legacyOptions: DownloadOptions = {},
+): Promise<Blob> {
+  const modern = isDownloadBlobOptions(optionsOrHeaders);
+  const options = modern ? optionsOrHeaders : legacyOptions;
+  const headers = modern ? optionsOrHeaders.headers ?? {} : optionsOrHeaders;
+  const net = createNet(options.fetch ?? globalThis.fetch);
+  const response = await net(url, { headers });
+  if (!response.ok || !response.body) {
+    throw new PostfetchError(502, `download failed: ${response.status}`);
+  }
+  const blob = await response.blob();
+  if (!modern || optionsOrHeaders.remux !== true) {
+    return blob;
+  }
+  const remuxed = await remuxMp4(new Uint8Array(await blob.arrayBuffer()), optionsOrHeaders.ffmpegPath);
+  return remuxed ? new Blob([toArrayBuffer(remuxed)], { type: blob.type || "video/mp4" }) : blob;
+}
+
+function isDownloadBlobOptions(value: DownloadBlobOptions | HeadersInit): value is DownloadBlobOptions {
+  if (value instanceof Headers || Array.isArray(value)) {
+    return false;
+  }
+  const candidate = value as Record<string, unknown>;
+  return (
+    ("headers" in candidate && typeof candidate.headers !== "string") ||
+    typeof candidate.remux === "boolean" ||
+    typeof candidate.ffmpegPath === "string" ||
+    typeof candidate.fetch === "function"
+  );
+}
+
 /**
  * Download every item of a result and zip them in-process (store mode).
  *

package/src/facebook.ts

@@ -1,16 +1,24 @@
-import { asUrl, count, filename, isoFromEpochSeconds, type PostMetadata, type ResolveContext, type Json, type Net, type PostfetchResult, type MediaItem } from "./internal";
+import { asUrl, count, filename, isoFromEpochSeconds, PostfetchError, type PostMetadata, type ResolveContext, type Json, type Net, type PostfetchResult, type MediaItem } from "./internal";
 import { browserUserAgent } from "./fingerprint";
 
 export async function resolveFacebook(input: ResolveContext): Promise<PostfetchResult> {
-  const canonical = await canonicalUrl(input.net, input.url);
+  const page = await canonicalPage(input.net, normalizedInputUrl(input.url));
+  const canonical = page.url;
   const id = facebookId(canonical) ?? facebookId(input.url) ?? "video";
+  const pageMetadata = facebookMetadata(page.html);
   // The embed player only exposes /share/v/ videos; reels (and anything it
   // misses) come from the watch page, which the &_rdr flag serves logged-out.
-  const embed = await embedVideo(input.net, canonical);
-  const watch = !embed && /^\d+$/.test(id) ? await watchVideo(input.net, id) : null;
+  const embed = await embedVideo(input.net, canonical, input.preferredWidth);
+  const watch = !embed && /^\d+$/.test(id) ? await watchVideo(input.net, id, input.preferredWidth) : null;
   const url = embed ?? watch?.url ?? null;
   if (!url) {
-    throw new Error("Facebook video not found");
+    // As with Reddit and X, a public text post is a valid result even when it
+    // has no downloadable media. Require post metadata from the canonical page
+    // so a missing or login-walled video does not become a false success.
+    if (pageMetadata?.text) {
+      return { archiveFilename: filename(`facebook_${id}.zip`), id, items: [], metadata: pageMetadata, platform: "facebook" };
+    }
+    throw new PostfetchError(404, "Facebook video not found", "notFound");
   }
   const item: MediaItem = {
     filename: filename(`facebook_${id}.mp4`),
@@ -21,7 +29,23 @@ export async function resolveFacebook(input: ResolveContext): Promise<PostfetchR
     platform: "facebook",
     url,
   };
-  return { archiveFilename: filename(`facebook_${id}.zip`), id, items: [item], metadata: watch?.metadata, platform: "facebook" };
+  return {
+    archiveFilename: filename(`facebook_${id}.zip`),
+    id,
+    items: [item],
+    metadata: mergeMetadata(pageMetadata, watch?.metadata),
+    platform: "facebook",
+  };
+}
+
+// Facebook share links are opaque shortlinks. Messaging apps sometimes append
+// selected text to the copied URL as another path segment; Facebook returns 404
+// for that form even though the share token is still intact. Strip everything
+// after the token before following the redirect.
+function normalizedInputUrl(input: string): string {
+  const url = asUrl(input);
+  const share = url.pathname.match(/^\/share\/([a-z])\/([A-Za-z0-9]+)/i);
+  return share ? `${url.origin}/share/${share[1]}/${share[2]}/` : input;
 }
 
 function navigationHeaders(): Record<string, string> {
@@ -33,29 +57,32 @@ function navigationHeaders(): Record<string, string> {
 }
 
 // Share and fb.watch links redirect to the canonical /reel/<id> or /<page>/videos/<id>
-// URL, which the public embed player needs. We only read the resolved location, not the
-// (login-walled) page body.
-async function canonicalUrl(net: Net, input: string): Promise<string> {
+// URL, which the public embed player needs. Public post pages also carry Open
+// Graph metadata in their logged-out HTML, so retain the body after redirecting.
+async function canonicalPage(net: Net, input: string): Promise<{ html: string; url: string }> {
   const response = await net(input, { headers: navigationHeaders() });
-  const resolved = asUrl(response.url);
-  return `${resolved.origin}${resolved.pathname}`;
+  const resolved = asUrl(response.url || input);
+  return {
+    html: response.ok ? await response.text() : "",
+    url: `${resolved.origin}${resolved.pathname}`,
+  };
 }
 
 // The logged-out watch page is fingerprint-walled, but the public embed player
 // (plugins/video.php) still exposes the progressive hd_src/sd_src URLs.
-async function embedVideo(net: Net, canonical: string): Promise<string | null> {
+async function embedVideo(net: Net, canonical: string, preferredWidth: number): Promise<string | null> {
   const embed = `https://www.facebook.com/plugins/video.php?href=${encodeURIComponent(canonical)}`;
   const response = await net(embed, { headers: navigationHeaders() });
   if (!response.ok) {
     return null;
   }
   const html = await response.text();
-  return source(html, "hd_src") ?? source(html, "sd_src");
+  return preferredSource(preferredWidth, source(html, "hd_src"), source(html, "sd_src"));
 }
 
 // The logged-out watch page (with &_rdr) embeds the progressive video URLs in
 // its ScheduledServerJS blocks; these are the same fields yt-dlp reads.
-async function watchVideo(net: Net, id: string): Promise<{ url: string; metadata: PostMetadata } | null> {
+async function watchVideo(net: Net, id: string, preferredWidth: number): Promise<{ url: string; metadata: PostMetadata } | null> {
   // Without the document Accept header Facebook serves a JS shell without the
   // media JSON, so ask for HTML explicitly here.
   const headers = { ...navigationHeaders(), accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" };
@@ -64,14 +91,18 @@ async function watchVideo(net: Net, id: string): Promise<{ url: string; metadata
     return null;
   }
   const html = await response.text();
-  const url =
-    source(html, "playable_url_quality_hd") ??
-    source(html, "browser_native_hd_url") ??
-    source(html, "playable_url") ??
-    source(html, "browser_native_sd_url");
+  const hd = source(html, "playable_url_quality_hd") ?? source(html, "browser_native_hd_url");
+  const sd = source(html, "playable_url") ?? source(html, "browser_native_sd_url");
+  const url = preferredSource(preferredWidth, hd, sd);
   return url ? { url, metadata: watchMetadata(html) } : null;
 }
 
+// Facebook labels its two progressive renditions as 720p HD and 360p SD. Pick
+// the one closest to the requested width, preferring HD on the 540px midpoint.
+function preferredSource(preferredWidth: number, hd: string | null, sd: string | null): string | null {
+  return preferredWidth >= 540 ? hd ?? sd : sd ?? hd;
+}
+
 // Only the fields that can be read unambiguously from the flat page: the
 // caption (creation_story) and the reaction/comment/share counts appear in
 // shapes shared with related videos and comment threads, so they are skipped
@@ -86,6 +117,76 @@ function watchMetadata(html: string): PostMetadata {
   };
 }
 
+// Public post pages expose their caption and page/profile name as Open Graph
+// metadata even when there is no media player. Facebook truncates long captions
+// there, but it is still enough to identify and represent a text post.
+function facebookMetadata(html: string): PostMetadata | undefined {
+  const canonical = metaContent(html, "og:url");
+  if (!canonical || !facebookPostUrl(canonical)) {
+    return undefined;
+  }
+  const text = metaContent(html, "og:description") ?? metaContent(html, "description");
+  const name = metaContent(html, "og:title");
+  if (!text && !name) {
+    return undefined;
+  }
+  return {
+    text: text ?? undefined,
+    author: name ? { name } : undefined,
+  };
+}
+
+function facebookPostUrl(input: string): boolean {
+  try {
+    const url = new URL(input);
+    return (
+      /\/posts\/(?:[^/]+\/)?[A-Za-z0-9]+\/?$/.test(url.pathname) ||
+      /\/share\/p\/[A-Za-z0-9]+\/?$/.test(url.pathname) ||
+      ((url.pathname === "/permalink.php" || url.pathname === "/story.php") && Boolean(url.searchParams.get("story_fbid")))
+    );
+  } catch {
+    return false;
+  }
+}
+
+function metaContent(html: string, key: string): string | null {
+  for (const match of html.matchAll(/<meta\b[^>]*>/gi)) {
+    const tag = match[0];
+    if ((attribute(tag, "property") ?? attribute(tag, "name")) !== key) {
+      continue;
+    }
+    const content = attribute(tag, "content");
+    return content ? decodeHtml(content) : null;
+  }
+  return null;
+}
+
+function attribute(tag: string, name: string): string | null {
+  const match = tag.match(new RegExp(`\\b${name}\\s*=\\s*(["'])(.*?)\\1`, "is"));
+  return match?.[2] ?? null;
+}
+
+function decodeHtml(input: string): string {
+  const named: Record<string, string> = { amp: "&", apos: "'", gt: ">", lt: "<", nbsp: " ", quot: '"' };
+  return input.replace(/&(#x[\da-f]+|#\d+|amp|apos|gt|lt|nbsp|quot);/gi, (entity, code: string) => {
+    if (code[0] !== "#") {
+      return named[code.toLowerCase()] ?? entity;
+    }
+    const value = code[1]?.toLowerCase() === "x" ? Number.parseInt(code.slice(2), 16) : Number.parseInt(code.slice(1), 10);
+    return Number.isSafeInteger(value) && value <= 0x10ffff ? String.fromCodePoint(value) : entity;
+  });
+}
+
+function mergeMetadata(page: PostMetadata | undefined, watch: PostMetadata | undefined): PostMetadata | undefined {
+  if (!page) {
+    return watch;
+  }
+  if (!watch) {
+    return page;
+  }
+  return { ...page, ...watch, author: watch.author ?? page.author };
+}
+
 function source(html: string, key: string): string | null {
   const match = html.match(new RegExp(`"${key}":("(?:\\\\.|[^"\\\\])*")`));
   if (!match) {

package/src/index.ts

@@ -5,8 +5,8 @@
  * URLs to typed {@link MediaItem}s. Zero dependencies, only Web-standard APIs
  * (`fetch`, `Response`, `URL`, `crypto`) — so it runs on Deno, Node, Bun,
  * Cloudflare Workers and browsers alike. It only resolves the media; the
- * {@link download}, {@link archive} and {@link toResponse} helpers turn a result
- * into bytes or a `Response`.
+ * {@link download}, {@link downloadBlob}, {@link archive} and {@link toResponse}
+ * helpers turn a result into bytes, a `Blob`, or a `Response`.
  *
  * @example Resolve a reel and save the video
  * ```ts
@@ -20,7 +20,15 @@
  * @module
  */
 export { postfetch, detect, type PostfetchOptions } from "./postfetch";
-export { download, archive, toResponse, type Archive, type DownloadOptions } from "./download";
+export {
+  download,
+  downloadBlob,
+  archive,
+  toResponse,
+  type Archive,
+  type DownloadBlobOptions,
+  type DownloadOptions,
+} from "./download";
 export { PostfetchError } from "./internal";
 export type {
   InstagramExtra,
@@ -37,5 +45,6 @@ export type {
   SoundcloudExtra,
   TiktokExtra,
   TwitterExtra,
+  TwitterQuotedTweet,
   YoutubeExtra,
 } from "./internal";

package/src/internal.ts

@@ -103,6 +103,16 @@ export type TiktokExtra = {
 export type TwitterExtra = {
   /** BCP 47 language tag detected for the tweet. */
   lang?: string;
+  /** The post quoted by this post, when X includes it in the resolved payload. */
+  quotedTweet?: TwitterQuotedTweet;
+};
+
+/** A quoted X post and its normalized metadata. */
+export type TwitterQuotedTweet = {
+  /** Status id of the quoted post. */
+  id: string;
+  /** Metadata belonging to the quoted post rather than the outer post. */
+  metadata: PostMetadata & { extra?: TwitterExtra };
 };
 
 /** Instagram-specific metadata. */
@@ -179,6 +189,8 @@ export type Net = (url: string, init?: RequestInit, attempts?: number) => Promis
 export type ResolveContext = {
   net: Net;
   preferredWidth: number;
+  /** Requested soft byte cap, for resolvers that can rank exact rendition sizes. */
+  tryMaxBytes?: number;
   url: string;
 };
 

package/src/mp4-remux.ts

@@ -0,0 +1,100 @@
+type CommandResult = {
+  stderr: string;
+  success: boolean;
+};
+
+export type Mp4RemuxRuntime = {
+  makeTempDir(prefix: string): Promise<string>;
+  readFile(path: string): Promise<Uint8Array>;
+  removeDir(path: string): Promise<void>;
+  run(command: string, args: string[]): Promise<CommandResult>;
+  writeFile(path: string, bytes: Uint8Array): Promise<void>;
+};
+
+/**
+ * Normalize an MP4 with the same stream-copy remux used by UMMR. Encoded media
+ * is left untouched; FFmpeg only rebuilds the container and timing tables.
+ * Returns `null` when FFmpeg is unavailable or the input cannot be remuxed.
+ */
+export async function remuxMp4(
+  bytes: Uint8Array,
+  ffmpegPath = "ffmpeg",
+  runtime?: Mp4RemuxRuntime,
+): Promise<Uint8Array | null> {
+  let directory: string | undefined;
+  let io: Mp4RemuxRuntime | undefined;
+  try {
+    io = runtime ?? (await nodeRuntime());
+    directory = await io.makeTempDir("postfetch-remux-");
+    const inputPath = `${directory}/input.mp4`;
+    const outputPath = `${directory}/output.mp4`;
+    await io.writeFile(inputPath, bytes);
+    const result = await io.run(ffmpegPath, [
+      "-nostdin",
+      "-hide_banner",
+      "-loglevel",
+      "error",
+      "-y",
+      "-i",
+      inputPath,
+      "-map",
+      "0:v:0",
+      "-map",
+      "0:a:0?",
+      "-c",
+      "copy",
+      "-movflags",
+      "+faststart",
+      "-avoid_negative_ts",
+      "make_zero",
+      "-use_editlist",
+      "0",
+      outputPath,
+    ]);
+    if (!result.success) {
+      return null;
+    }
+    const output = await io.readFile(outputPath);
+    return output.length > 0 ? output : null;
+  } catch {
+    return null;
+  } finally {
+    if (directory && io) {
+      await io.removeDir(directory).catch(() => undefined);
+    }
+  }
+}
+
+// Deno, Bun and Node all expose the Node compatibility modules used here. They
+// stay behind dynamic imports so browser/edge consumers that leave remux off do
+// not load or execute any process/filesystem code.
+async function nodeRuntime(): Promise<Mp4RemuxRuntime> {
+  const [{ spawn }, { mkdtemp, readFile, rm, writeFile }, { tmpdir }, { join }] = await Promise.all([
+    import("node:child_process"),
+    import("node:fs/promises"),
+    import("node:os"),
+    import("node:path"),
+  ]);
+  return {
+    makeTempDir: (prefix) => mkdtemp(join(tmpdir(), prefix)),
+    readFile: async (path) => new Uint8Array(await readFile(path)),
+    removeDir: async (path) => {
+      await rm(path, { force: true, recursive: true });
+    },
+    run: (command, args) =>
+      new Promise((resolve) => {
+        const child = spawn(command, args, { stdio: ["ignore", "ignore", "pipe"] });
+        let stderr = "";
+        child.stderr?.on("data", (chunk: Uint8Array) => {
+          if (stderr.length < 4096) {
+            stderr += new TextDecoder().decode(chunk).slice(0, 4096 - stderr.length);
+          }
+        });
+        child.once("error", (error) => resolve({ stderr: error.message, success: false }));
+        child.once("close", (code) => resolve({ stderr, success: code === 0 }));
+      }),
+    writeFile: async (path, value) => {
+      await writeFile(path, value);
+    },
+  };
+}

package/src/postfetch.ts

@@ -1,4 +1,4 @@
-import { asUrl, createNet, PostfetchError, type Platform, type PostfetchResult } from "./internal";
+import { asUrl, createNet, PostfetchError, type MediaItem, type Platform, type PostfetchResult, type ResolveContext } from "./internal";
 import { resolveFacebook } from "./facebook";
 import { resolveInstagram } from "./instagram";
 import { resolveLinkedin } from "./linkedin";
@@ -15,6 +15,15 @@ export type PostfetchOptions = {
   fetch?: typeof fetch;
   /** Preferred media width in pixels; the closest available rendition is chosen. Defaults to `720`. */
   preferredWidth?: number;
+  /**
+   * Soft file-size cap in bytes. The normal rendition is resolved and probed
+   * with HEAD; when it exceeds the cap, a smaller available rendition is
+   * returned. X probes every available MP4 variant and picks the highest-quality
+   * complete result under the cap, or its smallest variants when none fit. For
+   * other platforms, if size discovery or a smaller rendition is unavailable,
+   * the normal result is returned unchanged.
+   */
+  tryMaxBytes?: number;
 };
 
 /**
@@ -22,8 +31,9 @@ export type PostfetchOptions = {
  *
  * Detects the platform from the URL host, then returns the post's media as typed
  * {@link MediaItem}s — direct CDN URLs plus the headers needed to fetch them. It
- * performs no side effects beyond the lookup; use {@link download}, {@link archive}
- * or {@link toResponse} to turn the result into bytes or a `Response`.
+ * performs no side effects beyond the lookup; use {@link download},
+ * {@link downloadBlob}, {@link archive} or {@link toResponse} to turn the result
+ * into bytes, a `Blob` or a `Response`.
  *
  * @param url A post URL from a supported platform.
  * @param options See {@link PostfetchOptions}.
@@ -43,12 +53,23 @@ export async function postfetch(url: string, options: PostfetchOptions = {}): Pr
   if (trimmed.length === 0) {
     throw new PostfetchError(400, "url is required");
   }
-  const context = {
+  if (options.tryMaxBytes !== undefined && (!Number.isSafeInteger(options.tryMaxBytes) || options.tryMaxBytes <= 0)) {
+    throw new PostfetchError(400, "tryMaxBytes must be a positive integer");
+  }
+  const context: ResolveContext = {
     net: createNet(options.fetch ?? globalThis.fetch),
     preferredWidth: options.preferredWidth ?? 720,
+    tryMaxBytes: options.tryMaxBytes,
     url: trimmed,
   };
-  switch (detect(trimmed)) {
+  const result = await resolve(context);
+  return options.tryMaxBytes === undefined || result.platform === "twitter"
+    ? result
+    : trySmaller(context, result, options.tryMaxBytes);
+}
+
+async function resolve(context: ResolveContext): Promise<PostfetchResult> {
+  switch (detect(context.url)) {
     case "facebook":
       return resolveFacebook(context);
     case "instagram":
@@ -70,6 +91,65 @@ export async function postfetch(url: string, options: PostfetchOptions = {}): Pr
   }
 }
 
+async function trySmaller(context: ResolveContext, result: PostfetchResult, maxBytes: number): Promise<PostfetchResult> {
+  const originalBytes = await resultBytes(context, result);
+  if (originalBytes === null || originalBytes <= maxBytes) {
+    return result;
+  }
+  try {
+    // Width-aware resolvers interpret 1px as "pick the smallest available".
+    // Resolvers without a smaller rendition return the same media unchanged.
+    const smaller = await resolve({ ...context, preferredWidth: 1 });
+    if (sameMedia(result, smaller)) {
+      return result;
+    }
+    const smallerBytes = await resultBytes(context, smaller);
+    return smallerBytes !== null && smallerBytes < originalBytes ? smaller : result;
+  } catch {
+    return result;
+  }
+}
+
+async function resultBytes(context: ResolveContext, result: PostfetchResult): Promise<number | null> {
+  const sizes = await Promise.all(result.items.map((item) => itemBytes(context, item)));
+  if (sizes.some((size) => size === null)) {
+    return null;
+  }
+  return sizes.reduce<number>((total, size) => total + (size ?? 0), 0);
+}
+
+async function itemBytes(context: ResolveContext, item: MediaItem): Promise<number | null> {
+  if (item.hls) {
+    return null;
+  }
+  const [video, audio] = await Promise.all([
+    contentLength(context, item.url, item.headers),
+    item.audio ? contentLength(context, item.audio.url, item.audio.headers) : Promise.resolve(0),
+  ]);
+  return video === null || audio === null ? null : video + audio;
+}
+
+async function contentLength(context: ResolveContext, url: string, headers: HeadersInit): Promise<number | null> {
+  try {
+    const response = await context.net(url, { headers, method: "HEAD" }, 1);
+    const raw = response.ok ? response.headers.get("content-length") : null;
+    if (!raw || !/^\d+$/.test(raw)) {
+      return null;
+    }
+    const bytes = Number(raw);
+    return Number.isSafeInteger(bytes) ? bytes : null;
+  } catch {
+    return null;
+  }
+}
+
+function sameMedia(left: PostfetchResult, right: PostfetchResult): boolean {
+  return left.items.length === right.items.length && left.items.every((item, index) => {
+    const other = right.items[index];
+    return item.url === other?.url && item.audio?.url === other.audio?.url;
+  });
+}
+
 /**
  * Detect which platform a URL belongs to, from its host.
  *