@postfetch/core 0.2.4 → 0.3.1

package/dist/internal.d.ts

@@ -1,13 +1,30 @@
 /** A platform that {@link postfetch} can resolve. */
-export type Platform = "facebook" | "instagram" | "tiktok" | "twitter" | "youtube";
+export type Platform = "facebook" | "instagram" | "pinterest" | "reddit" | "soundcloud" | "tiktok" | "twitter" | "youtube";
 /** The kind of a {@link MediaItem}. */
 export type MediaKind = "audio" | "image" | "video";
 /** A single resolved media file: a direct URL plus everything needed to fetch and name it. */
 export type MediaItem = {
+    /**
+     * When present, this is a video whose audio is a separate stream: {@link MediaItem.url}
+     * carries the video track and `audio.url` the audio track (DASH/CMAF hands them out
+     * apart). {@link download}, {@link archive} and {@link toResponse} merge the two with
+     * the bundled remuxer; callers that read `url` directly get the silent video.
+     */
+    audio?: {
+        headers: HeadersInit;
+        url: string;
+    };
     /** Suggested download filename. */
     filename: string;
     /** Headers required to fetch {@link MediaItem.url} from the CDN. */
     headers: HeadersInit;
+    /**
+     * When `true`, {@link MediaItem.url} (and `audio.url`, if set) are HLS media
+     * playlists rather than direct files: {@link download} assembles their segments
+     * into a fragmented MP4 before serving — and merges video with audio when both
+     * are present. Callers that read `url` directly get the playlist, not the media.
+     */
+    hls?: boolean;
     /** Platform media id. */
     id: string;
     /** Whether this item is a video, image or audio file. */

package/dist/isobmff.d.ts

@@ -0,0 +1,20 @@
+export type Box = {
+    type: string;
+    /** Offset of the box header (its size field). */
+    start: number;
+    /** Offset just past the box. */
+    end: number;
+    /** Offset of the box payload (after the size/type, and any 64-bit largesize). */
+    dataStart: number;
+};
+export declare function parseBoxes(data: Uint8Array, start?: number, end?: number): Box[];
+export declare function findBox(boxes: Box[], type: string): Box | undefined;
+export declare function childBoxes(data: Uint8Array, box: Box): Box[];
+/** A standalone copy of a box's bytes, header included. */
+export declare function sliceBox(data: Uint8Array, box: Box): Uint8Array;
+export declare function buildBox(type: string, payload: Uint8Array): Uint8Array;
+export declare function concat(parts: Uint8Array[]): Uint8Array;
+export declare function readU32(data: Uint8Array, offset: number): number;
+export declare function writeU32(data: Uint8Array, offset: number, value: number): void;
+/** The version byte of a full box (the first byte of its payload). */
+export declare function boxVersion(data: Uint8Array, box: Box): number;

package/dist/pinterest.d.ts

@@ -0,0 +1,2 @@
+import { type ResolveContext, type PostfetchResult } from "./internal";
+export declare function resolvePinterest(input: ResolveContext): Promise<PostfetchResult>;

package/dist/reddit.d.ts

@@ -0,0 +1,2 @@
+import { type ResolveContext, type PostfetchResult } from "./internal";
+export declare function resolveReddit(input: ResolveContext): Promise<PostfetchResult>;

package/dist/remux.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Merge a video-only and an audio-only fragmented MP4 into one file with both
+ * tracks. The DASH/CMAF streams reddit and YouTube hand out keep video and audio
+ * apart; this recombines them without re-encoding — every `moof`/`mdat` pair is
+ * copied verbatim (so each `trun` data offset stays valid) and only the track id
+ * and fragment sequence number are patched.
+ */
+export declare function mergeAudioVideo(video: Uint8Array, audio: Uint8Array): Uint8Array;

package/dist/soundcloud.d.ts

@@ -0,0 +1,2 @@
+import { type ResolveContext, type PostfetchResult } from "./internal";
+export declare function resolveSoundcloud(input: ResolveContext): Promise<PostfetchResult>;

package/dist/tiktok.d.ts

@@ -1,2 +1,3 @@
 import { type ResolveContext, type PostfetchResult } from "./internal";
 export declare function resolveTiktok(input: ResolveContext): Promise<PostfetchResult>;
+export declare function isShortlinkHost(hostname: string): boolean;

package/dist/twitter.d.ts

@@ -1,2 +1,3 @@
 import { type ResolveContext, type PostfetchResult } from "./internal";
 export declare function resolveTwitter(input: ResolveContext): Promise<PostfetchResult>;
+export declare function tweetId(input: string): string | null;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@postfetch/core",
-  "version": "0.2.4",
-  "description": "Zero-dependency typed core that turns Facebook, Instagram, TikTok, X and YouTube post URLs into media files.",
+  "version": "0.3.1",
+  "description": "Zero-dependency typed core that turns Facebook, Instagram, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube post URLs into media files.",
   "license": "MIT",
   "type": "module",
   "exports": {
@@ -31,6 +31,9 @@
     "facebook",
     "instagram",
     "media",
+    "pinterest",
+    "reddit",
+    "soundcloud",
     "tiktok",
     "twitter",
     "youtube",

package/src/download.ts

@@ -1,6 +1,14 @@
-import { createNet, PostfetchError, type MediaItem, type PostfetchResult } from "./internal";
+import { assembleHls } from "./hls";
+import { createNet, PostfetchError, type MediaItem, type Net, type PostfetchResult } from "./internal";
+import { mergeAudioVideo } from "./remux";
 import { zip } from "./zip";
 
+// A muxed item carries a separate audio stream; an HLS item carries playlists.
+// Either way the bytes must be assembled in-process rather than streamed.
+function buffered(item: MediaItem): boolean {
+  return Boolean(item.audio) || item.hls === true;
+}
+
 /** Options for {@link download}, {@link archive} and {@link toResponse}. */
 export type DownloadOptions = {
   /** Custom `fetch` implementation. Defaults to the global `fetch`. */
@@ -18,11 +26,13 @@ export type Archive = {
 };
 
 /**
- * Fetch a single media item from its CDN with the headers it requires.
+ * Fetch a single media item, ready to stream or buffer. A plain item streams
+ * straight from its CDN; a muxed item (one with {@link MediaItem.audio}) is
+ * downloaded as two streams and merged in-process, so its body is buffered.
  *
  * @param item A {@link MediaItem} from a {@link PostfetchResult}.
  * @param options See {@link DownloadOptions}.
- * @returns The upstream `Response`, ready to stream or buffer.
+ * @returns The upstream `Response`, or a buffered `Response` of the merged file.
  * @throws {PostfetchError} If the CDN responds with a non-OK status.
  *
  * @example
@@ -33,6 +43,10 @@ export type Archive = {
  */
 export async function download(item: MediaItem, options: DownloadOptions = {}): Promise<Response> {
   const net = createNet(options.fetch ?? globalThis.fetch);
+  if (buffered(item)) {
+    const bytes = await itemBytes(net, item);
+    return new Response(toArrayBuffer(bytes), { headers: { "content-length": String(bytes.length), "content-type": item.mime } });
+  }
   const response = await net(item.url, { headers: item.headers });
   if (!response.ok || !response.body) {
     throw new PostfetchError(502, `download failed: ${response.status}`);
@@ -50,21 +64,16 @@ export async function download(item: MediaItem, options: DownloadOptions = {}):
 export async function archive(result: PostfetchResult, options: DownloadOptions = {}): Promise<Archive> {
   const net = createNet(options.fetch ?? globalThis.fetch);
   const files = await Promise.all(
-    result.items.map(async (item) => {
-      const response = await net(item.url, { headers: item.headers });
-      if (!response.ok) {
-        throw new PostfetchError(502, `download failed: ${response.status}`);
-      }
-      return { data: new Uint8Array(await response.arrayBuffer()), name: item.filename };
-    }),
+    result.items.map(async (item) => ({ data: await itemBytes(net, item), name: item.filename })),
   );
   return { bytes: zip(files), filename: result.archiveFilename, mime: "application/zip" };
 }
 
 /**
  * Turn a result into a ready-to-serve `Response`: a single item is streamed as
- * its file, multiple items become a zip. Sets `content-disposition` and the
- * `x-media-*` headers. This is what the showcase server and templates return.
+ * its file (or, when muxed, merged and buffered), multiple items become a zip.
+ * Sets `content-disposition` and the `x-media-*` headers. This is what the
+ * showcase server and templates return.
  *
  * @param result A {@link PostfetchResult}.
  * @param options See {@link DownloadOptions}.
@@ -78,6 +87,21 @@ export async function toResponse(result: PostfetchResult, options: DownloadOptio
 }
 
 async function singleResponse(item: MediaItem, options: DownloadOptions): Promise<Response> {
+  if (buffered(item)) {
+    const net = createNet(options.fetch ?? globalThis.fetch);
+    const bytes = await itemBytes(net, item);
+    return new Response(toArrayBuffer(bytes), {
+      headers: {
+        "content-disposition": `attachment; filename="${item.filename}"`,
+        "content-length": String(bytes.length),
+        "content-type": item.mime,
+        "x-media-count": "1",
+        "x-media-id": item.id,
+        "x-media-kind": item.kind,
+        "x-media-platform": item.platform,
+      },
+    });
+  }
   const media = await download(item, options);
   const headers = new Headers({
     "content-disposition": `attachment; filename="${item.filename}"`,
@@ -96,8 +120,7 @@ async function singleResponse(item: MediaItem, options: DownloadOptions): Promis
 
 async function archiveResponse(result: PostfetchResult, options: DownloadOptions): Promise<Response> {
   const { bytes, filename } = await archive(result, options);
-  const body = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer;
-  return new Response(body, {
+  return new Response(toArrayBuffer(bytes), {
     headers: {
       "content-disposition": `attachment; filename="${filename}"`,
       "content-length": String(bytes.length),
@@ -108,3 +131,38 @@ async function archiveResponse(result: PostfetchResult, options: DownloadOptions
     },
   });
 }
+
+async function itemBytes(net: Net, item: MediaItem): Promise<Uint8Array> {
+  if (item.hls) {
+    const video = await assembleHls(net, item.url, item.headers);
+    if (item.audio) {
+      return mergeAudioVideo(video, await assembleHls(net, item.audio.url, item.audio.headers));
+    }
+    return video;
+  }
+  if (item.audio) {
+    return mergedBytes(net, item, item.audio);
+  }
+  return fetchBytes(net, item.url, item.headers);
+}
+
+// A muxed item's two streams are fetched in parallel and merged in a single pass.
+async function mergedBytes(net: Net, item: MediaItem, audio: NonNullable<MediaItem["audio"]>): Promise<Uint8Array> {
+  const [video, audioBytes] = await Promise.all([
+    fetchBytes(net, item.url, item.headers),
+    fetchBytes(net, audio.url, audio.headers),
+  ]);
+  return mergeAudioVideo(video, audioBytes);
+}
+
+async function fetchBytes(net: Net, url: string, headers: HeadersInit): Promise<Uint8Array> {
+  const response = await net(url, { headers });
+  if (!response.ok) {
+    throw new PostfetchError(502, `download failed: ${response.status}`);
+  }
+  return new Uint8Array(await response.arrayBuffer());
+}
+
+function toArrayBuffer(bytes: Uint8Array): ArrayBuffer {
+  return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer;
+}

package/src/hls.ts

@@ -0,0 +1,120 @@
+import { concat } from "./isobmff";
+import type { Net } from "./internal";
+
+// HLS support for the fMP4/CMAF playlists reddit-style muxing already covers:
+// a media playlist is an EXT-X-MAP init plus media segments (often byte ranges of
+// one CMAF file), so assembling it is just fetching those byte ranges in order and
+// concatenating them into a fragmented MP4. No MPEG-TS demuxing is involved.
+
+type Segment = { url: string; range: { length: number; offset: number } | null };
+
+export type HlsVariant = { width: number; height: number; bandwidth: number; url: string; audioGroup: string | null };
+export type HlsMaster = { variants: HlsVariant[]; audio: Record<string, string> };
+
+export function isMasterPlaylist(text: string): boolean {
+  return text.includes("#EXT-X-STREAM-INF");
+}
+
+export function parseMaster(text: string, baseUrl: string): HlsMaster {
+  const lines = text.split("\n").map((line) => line.trim());
+  const audio: Record<string, string> = {};
+  const variants: HlsVariant[] = [];
+  for (let index = 0; index < lines.length; index += 1) {
+    const line = lines[index];
+    if (line.startsWith("#EXT-X-MEDIA:") && /TYPE=AUDIO/.test(line)) {
+      const group = line.match(/GROUP-ID="([^"]+)"/)?.[1];
+      const uri = line.match(/URI="([^"]+)"/)?.[1];
+      if (group && uri) {
+        audio[group] = new URL(uri, baseUrl).href;
+      }
+    } else if (line.startsWith("#EXT-X-STREAM-INF:")) {
+      const target = lines[index + 1];
+      if (target && !target.startsWith("#")) {
+        const resolution = line.match(/RESOLUTION=(\d+)x(\d+)/);
+        variants.push({
+          width: resolution ? Number(resolution[1]) : 0,
+          height: resolution ? Number(resolution[2]) : 0,
+          bandwidth: Number(line.match(/BANDWIDTH=(\d+)/)?.[1] ?? 0),
+          url: new URL(target, baseUrl).href,
+          audioGroup: line.match(/AUDIO="([^"]+)"/)?.[1] ?? null,
+        });
+      }
+    }
+  }
+  return { variants, audio };
+}
+
+export async function assembleHls(net: Net, playlistUrl: string, headers: HeadersInit): Promise<Uint8Array> {
+  const response = await net(playlistUrl, { headers });
+  if (!response.ok) {
+    throw new Error(`HLS playlist failed: ${response.status}`);
+  }
+  const segments = mediaSegments(await response.text(), playlistUrl);
+  if (segments.length === 0) {
+    throw new Error("HLS playlist has no segments");
+  }
+  return concat(await fetchSegments(net, segments, headers));
+}
+
+function mediaSegments(text: string, baseUrl: string): Segment[] {
+  const segments: Segment[] = [];
+  let pendingRange: { length: number; offset: number } | null = null;
+  let nextOffset = 0;
+  for (const raw of text.split("\n")) {
+    const line = raw.trim();
+    if (line.startsWith("#EXT-X-MAP:")) {
+      const uri = line.match(/URI="([^"]+)"/)?.[1];
+      if (uri) {
+        const range = byteRange(line.match(/BYTERANGE="([^"]+)"/)?.[1], nextOffset);
+        segments.push({ url: new URL(uri, baseUrl).href, range });
+        nextOffset = range ? range.offset + range.length : nextOffset;
+      }
+    } else if (line.startsWith("#EXT-X-BYTERANGE:")) {
+      pendingRange = byteRange(line.slice("#EXT-X-BYTERANGE:".length), nextOffset);
+      nextOffset = pendingRange ? pendingRange.offset + pendingRange.length : nextOffset;
+    } else if (line.length > 0 && !line.startsWith("#")) {
+      segments.push({ url: new URL(line, baseUrl).href, range: pendingRange });
+      pendingRange = null;
+    }
+  }
+  return segments;
+}
+
+// EXT-X-BYTERANGE is `length[@offset]`; a missing offset continues from the end of
+// the previous sub-range of the same resource.
+function byteRange(spec: string | undefined, runningOffset: number): { length: number; offset: number } | null {
+  const match = spec?.trim().match(/^(\d+)(?:@(\d+))?$/);
+  if (!match) {
+    return null;
+  }
+  return { length: Number(match[1]), offset: match[2] === undefined ? runningOffset : Number(match[2]) };
+}
+
+// Segments are fetched with a small concurrency window: parallel enough to be fast,
+// bounded so a long playlist does not open hundreds of sockets at once.
+async function fetchSegments(net: Net, segments: Segment[], headers: HeadersInit): Promise<Uint8Array[]> {
+  const results = new Array<Uint8Array>(segments.length);
+  let cursor = 0;
+  async function worker(): Promise<void> {
+    while (cursor < segments.length) {
+      const index = cursor;
+      cursor += 1;
+      results[index] = await fetchSegment(net, segments[index], headers);
+    }
+  }
+  const workers = Array.from({ length: Math.min(8, segments.length) }, worker);
+  await Promise.all(workers);
+  return results;
+}
+
+async function fetchSegment(net: Net, segment: Segment, headers: HeadersInit): Promise<Uint8Array> {
+  const requestHeaders = new Headers(headers);
+  if (segment.range) {
+    requestHeaders.set("range", `bytes=${segment.range.offset}-${segment.range.offset + segment.range.length - 1}`);
+  }
+  const response = await net(segment.url, { headers: requestHeaders });
+  if (!response.ok) {
+    throw new Error(`HLS segment failed: ${response.status}`);
+  }
+  return new Uint8Array(await response.arrayBuffer());
+}

package/src/internal.ts

@@ -1,15 +1,29 @@
 /** A platform that {@link postfetch} can resolve. */
-export type Platform = "facebook" | "instagram" | "tiktok" | "twitter" | "youtube";
+export type Platform = "facebook" | "instagram" | "pinterest" | "reddit" | "soundcloud" | "tiktok" | "twitter" | "youtube";
 
 /** The kind of a {@link MediaItem}. */
 export type MediaKind = "audio" | "image" | "video";
 
 /** A single resolved media file: a direct URL plus everything needed to fetch and name it. */
 export type MediaItem = {
+  /**
+   * When present, this is a video whose audio is a separate stream: {@link MediaItem.url}
+   * carries the video track and `audio.url` the audio track (DASH/CMAF hands them out
+   * apart). {@link download}, {@link archive} and {@link toResponse} merge the two with
+   * the bundled remuxer; callers that read `url` directly get the silent video.
+   */
+  audio?: { headers: HeadersInit; url: string };
   /** Suggested download filename. */
   filename: string;
   /** Headers required to fetch {@link MediaItem.url} from the CDN. */
   headers: HeadersInit;
+  /**
+   * When `true`, {@link MediaItem.url} (and `audio.url`, if set) are HLS media
+   * playlists rather than direct files: {@link download} assembles their segments
+   * into a fragmented MP4 before serving — and merges video with audio when both
+   * are present. Callers that read `url` directly get the playlist, not the media.
+   */
+  hls?: boolean;
   /** Platform media id. */
   id: string;
   /** Whether this item is a video, image or audio file. */

package/src/isobmff.ts

@@ -0,0 +1,85 @@
+// A minimal ISO Base Media File Format reader/writer — just enough to remux
+// fragmented MP4 (CMAF/DASH) by recombining whole boxes. It never decodes media,
+// only walks the box tree and patches a handful of integer fields in place.
+
+export type Box = {
+  type: string;
+  /** Offset of the box header (its size field). */
+  start: number;
+  /** Offset just past the box. */
+  end: number;
+  /** Offset of the box payload (after the size/type, and any 64-bit largesize). */
+  dataStart: number;
+};
+
+export function parseBoxes(data: Uint8Array, start = 0, end = data.length): Box[] {
+  const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
+  const boxes: Box[] = [];
+  let offset = start;
+  while (offset + 8 <= end) {
+    let size = view.getUint32(offset);
+    const type = String.fromCharCode(data[offset + 4], data[offset + 5], data[offset + 6], data[offset + 7]);
+    let dataStart = offset + 8;
+    if (size === 1) {
+      size = Number(view.getBigUint64(offset + 8));
+      dataStart = offset + 16;
+    } else if (size === 0) {
+      size = end - offset;
+    }
+    if (size < 8 || offset + size > end) {
+      break;
+    }
+    boxes.push({ type, start: offset, end: offset + size, dataStart });
+    offset += size;
+  }
+  return boxes;
+}
+
+export function findBox(boxes: Box[], type: string): Box | undefined {
+  return boxes.find((box) => box.type === type);
+}
+
+export function childBoxes(data: Uint8Array, box: Box): Box[] {
+  return parseBoxes(data, box.dataStart, box.end);
+}
+
+/** A standalone copy of a box's bytes, header included. */
+export function sliceBox(data: Uint8Array, box: Box): Uint8Array {
+  return data.slice(box.start, box.end);
+}
+
+export function buildBox(type: string, payload: Uint8Array): Uint8Array {
+  const out = new Uint8Array(8 + payload.length);
+  const view = new DataView(out.buffer);
+  view.setUint32(0, out.length);
+  out[4] = type.charCodeAt(0);
+  out[5] = type.charCodeAt(1);
+  out[6] = type.charCodeAt(2);
+  out[7] = type.charCodeAt(3);
+  out.set(payload, 8);
+  return out;
+}
+
+export function concat(parts: Uint8Array[]): Uint8Array {
+  const total = parts.reduce((sum, part) => sum + part.length, 0);
+  const out = new Uint8Array(total);
+  let offset = 0;
+  for (const part of parts) {
+    out.set(part, offset);
+    offset += part.length;
+  }
+  return out;
+}
+
+export function readU32(data: Uint8Array, offset: number): number {
+  return new DataView(data.buffer, data.byteOffset, data.byteLength).getUint32(offset);
+}
+
+export function writeU32(data: Uint8Array, offset: number, value: number): void {
+  new DataView(data.buffer, data.byteOffset, data.byteLength).setUint32(offset, value);
+}
+
+/** The version byte of a full box (the first byte of its payload). */
+export function boxVersion(data: Uint8Array, box: Box): number {
+  return data[box.dataStart];
+}