@postfetch/core 0.2.3 → 0.3.0

package/src/pinterest.ts

@@ -0,0 +1,237 @@
+import {
+  asUrl,
+  filename,
+  number,
+  object,
+  string,
+  type ResolveContext,
+  type Json,
+  type Net,
+  type PostfetchResult,
+  type MediaItem,
+} from "./internal";
+import { browserUserAgent } from "./fingerprint";
+import { isMasterPlaylist, parseMaster, type HlsVariant } from "./hls";
+
+export async function resolvePinterest(input: ResolveContext): Promise<PostfetchResult> {
+  const id = await pinId(input.net, input.url);
+  const pin = await pinResource(input.net, id);
+  const items = await mediaItems(input, pin, id);
+  if (items.length === 0) {
+    throw new Error("Pinterest media not found");
+  }
+  return { archiveFilename: filename(`pinterest_${id}.zip`), id, items, platform: "pinterest" };
+}
+
+// pin.it short links carry no pin id, so they are followed once to the canonical
+// /pin/<id>/ location.
+async function pinId(net: Net, input: string): Promise<string> {
+  const direct = parsePin(input);
+  if (direct) {
+    return direct;
+  }
+  const response = await net(input, { headers: { "user-agent": browserUserAgent() } }, 1);
+  const resolved = parsePin(response.url);
+  if (!resolved) {
+    throw new Error("Pinterest pin id not found");
+  }
+  return resolved;
+}
+
+function parsePin(input: string): string | null {
+  return asUrl(input).pathname.match(/\/pin\/(\d+)/)?.[1] ?? null;
+}
+
+// Logged out, the pin page no longer carries the pin data; the public PinResource
+// endpoint returns it as JSON for any handler-tagged request, no cookie needed.
+async function pinResource(net: Net, id: string): Promise<Json> {
+  const data = JSON.stringify({
+    options: { id, field_set_key: "unauth_react_main_pin", fetch_visual_search_objects: false },
+    context: {},
+  });
+  const url = new URL("https://www.pinterest.com/resource/PinResource/get/");
+  url.searchParams.set("source_url", `/pin/${id}/`);
+  url.searchParams.set("data", data);
+  const response = await net(url.href, {
+    headers: {
+      accept: "application/json",
+      "user-agent": browserUserAgent(),
+      "x-pinterest-pws-handler": "www/pin/[id].js",
+    },
+  });
+  if (!response.ok) {
+    throw new Error(`Pinterest resource failed: ${response.status}`);
+  }
+  const payload = await response.json().catch(() => null);
+  const wrapper = object(payload) && object(payload.resource_response) ? payload.resource_response : null;
+  const pin = wrapper && object(wrapper.data) ? wrapper.data : null;
+  if (!pin) {
+    throw new Error("Pinterest pin not found");
+  }
+  return pin;
+}
+
+async function mediaItems(input: ResolveContext, pin: Json, id: string): Promise<MediaItem[]> {
+  const video = await standardVideo(input, pin, id);
+  if (video.length > 0) {
+    return video;
+  }
+  const story = await storyItems(input, pin, id);
+  if (story.length > 0) {
+    return story;
+  }
+  return imageItems(pin, id);
+}
+
+async function standardVideo(input: ResolveContext, pin: Json, id: string): Promise<MediaItem[]> {
+  const item = await videoFromList(input, id, null, videoList(object(pin.videos) ? pin.videos : null));
+  return item ? [item] : [];
+}
+
+// Idea pins carry their media as story blocks, usually as HLS rather than a
+// progressive file. Each video block is resolved to its best rendition; only a
+// block with neither a progressive nor an HLS stream is treated as undownloadable.
+async function storyItems(input: ResolveContext, pin: Json, id: string): Promise<MediaItem[]> {
+  const story = object(pin.story_pin_data) ? pin.story_pin_data : null;
+  if (!story) {
+    return [];
+  }
+  const pages = Array.isArray(story.pages) ? story.pages.filter(object) : [];
+  const blocks = pages.flatMap((page) => (Array.isArray(page.blocks) ? page.blocks.filter(object) : []));
+  const videoBlocks = blocks.filter((block) => object(block.video));
+  const videos: MediaItem[] = [];
+  for (const [index, block] of videoBlocks.entries()) {
+    const item = await videoFromList(input, id, index + 1, videoList(object(block.video) ? block.video : null));
+    if (item) {
+      videos.push(item);
+    }
+  }
+  if (videos.length > 0) {
+    return videos;
+  }
+  if (videoBlocks.length > 0) {
+    throw new Error("Pinterest idea pin video has no downloadable rendition");
+  }
+  return imageItems(pin, id);
+}
+
+// Prefer a progressive mp4; fall back to the HLS master, picking the variant
+// nearest the preferred width and pairing it with its audio group.
+async function videoFromList(input: ResolveContext, id: string, index: number | null, renditions: Json[]): Promise<MediaItem | null> {
+  const progressive = progressiveMp4(renditions);
+  if (progressive) {
+    return videoItem(id, index, progressive);
+  }
+  const master = hlsUrl(renditions);
+  return master ? hlsVideoItem(input.net, id, index, master, input.preferredWidth) : null;
+}
+
+async function hlsVideoItem(net: Net, id: string, index: number | null, masterUrl: string, preferredWidth: number): Promise<MediaItem> {
+  const headers = { "user-agent": browserUserAgent() };
+  const suffix = index === null ? "" : `_${index}`;
+  const base: MediaItem = {
+    filename: filename(`pinterest_${id}${suffix}.mp4`),
+    headers,
+    hls: true,
+    id,
+    kind: "video",
+    mime: "video/mp4",
+    platform: "pinterest",
+    url: masterUrl,
+  };
+  const response = await net(masterUrl, { headers });
+  if (!response.ok) {
+    throw new Error(`Pinterest HLS failed: ${response.status}`);
+  }
+  const playlist = await response.text();
+  if (!isMasterPlaylist(playlist)) {
+    return base;
+  }
+  const master = parseMaster(playlist, masterUrl);
+  const variant = pickVariant(master.variants, preferredWidth);
+  if (!variant) {
+    throw new Error("Pinterest HLS variant not found");
+  }
+  const audioUrl = variant.audioGroup ? master.audio[variant.audioGroup] : undefined;
+  return { ...base, url: variant.url, ...(audioUrl ? { audio: { headers, url: audioUrl } } : {}) };
+}
+
+function pickVariant(variants: HlsVariant[], preferredWidth: number): HlsVariant | null {
+  return variants.reduce<HlsVariant | null>((current, variant) => {
+    if (!current) {
+      return variant;
+    }
+    return Math.abs(variant.width - preferredWidth) < Math.abs(current.width - preferredWidth) ? variant : current;
+  }, null);
+}
+
+function hlsUrl(renditions: Json[]): string | null {
+  return renditions.map((rendition) => string(rendition.url)).find((url): url is string => url !== null && url.toLowerCase().includes(".m3u8")) ?? null;
+}
+
+function imageItems(pin: Json, id: string): MediaItem[] {
+  const images = object(pin.images) ? pin.images : null;
+  const orig = images && object(images.orig) ? images.orig : null;
+  const url = orig ? string(orig.url) : null;
+  if (!url) {
+    return [];
+  }
+  return [
+    {
+      filename: filename(`pinterest_${id}.${imageExtension(url)}`),
+      headers: { "user-agent": browserUserAgent() },
+      id,
+      kind: "image",
+      mime: imageMime(url),
+      platform: "pinterest",
+      url,
+    },
+  ];
+}
+
+function videoList(videos: Json | null): Json[] {
+  const list = videos && object(videos.video_list) ? videos.video_list : null;
+  return list ? Object.values(list).filter(object) : [];
+}
+
+function progressiveMp4(renditions: Json[]): string | null {
+  const mp4s = renditions.filter((rendition) => string(rendition.url)?.toLowerCase().endsWith(".mp4"));
+  const best = mp4s.reduce<Json | null>((current, rendition) => {
+    const width = number(rendition.width) ?? 0;
+    const currentWidth = current ? number(current.width) ?? 0 : -1;
+    return width > currentWidth ? rendition : current;
+  }, null);
+  return best ? string(best.url) : null;
+}
+
+function videoItem(id: string, index: number | null, url: string): MediaItem {
+  const suffix = index === null ? "" : `_${index}`;
+  return {
+    filename: filename(`pinterest_${id}${suffix}.mp4`),
+    headers: { "user-agent": browserUserAgent() },
+    id,
+    kind: "video",
+    mime: "video/mp4",
+    platform: "pinterest",
+    url,
+  };
+}
+
+function imageExtension(url: string): string {
+  const extension = asUrl(url).pathname.match(/\.([a-z0-9]+)$/i)?.[1]?.toLowerCase();
+  return extension === "png" || extension === "webp" || extension === "gif" ? extension : "jpg";
+}
+
+function imageMime(url: string): string {
+  const extension = imageExtension(url);
+  if (extension === "png") {
+    return "image/png";
+  }
+  if (extension === "webp") {
+    return "image/webp";
+  }
+  if (extension === "gif") {
+    return "image/gif";
+  }
+  return "image/jpeg";
+}

package/src/postfetch.ts

@@ -1,15 +1,42 @@
 import { asUrl, createNet, PostfetchError, type Platform, type PostfetchResult } from "./internal";
 import { resolveFacebook } from "./facebook";
 import { resolveInstagram } from "./instagram";
+import { resolvePinterest } from "./pinterest";
+import { resolveReddit } from "./reddit";
+import { resolveSoundcloud } from "./soundcloud";
 import { resolveTiktok } from "./tiktok";
 import { resolveTwitter } from "./twitter";
 import { resolveYoutube } from "./youtube";
 
+/** Options for {@link postfetch}. */
 export type PostfetchOptions = {
+  /** Custom `fetch` implementation — inject one to unit-test resolvers offline. Defaults to the global `fetch`. */
   fetch?: typeof fetch;
+  /** Preferred media width in pixels; the closest available rendition is chosen. Defaults to `720`. */
   preferredWidth?: number;
 };
 
+/**
+ * Resolve a public social post URL to its media.
+ *
+ * Detects the platform from the URL host, then returns the post's media as typed
+ * {@link MediaItem}s — direct CDN URLs plus the headers needed to fetch them. It
+ * performs no side effects beyond the lookup; use {@link download}, {@link archive}
+ * or {@link toResponse} to turn the result into bytes or a `Response`.
+ *
+ * @param url A post URL from Instagram, TikTok, YouTube, Facebook or X.
+ * @param options See {@link PostfetchOptions}.
+ * @returns The resolved post and its media items.
+ * @throws {PostfetchError} If the URL is empty or its host is unsupported.
+ *
+ * @example
+ * ```ts
+ * import { postfetch } from "@postfetch/core";
+ *
+ * const result = await postfetch("https://vt.tiktok.com/ZSxpHvCUM/");
+ * for (const item of result.items) console.log(item.kind, item.url);
+ * ```
+ */
 export async function postfetch(url: string, options: PostfetchOptions = {}): Promise<PostfetchResult> {
   const trimmed = url.trim();
   if (trimmed.length === 0) {
@@ -25,6 +52,12 @@ export async function postfetch(url: string, options: PostfetchOptions = {}): Pr
       return resolveFacebook(context);
     case "instagram":
       return resolveInstagram(context);
+    case "pinterest":
+      return resolvePinterest(context);
+    case "reddit":
+      return resolveReddit(context);
+    case "soundcloud":
+      return resolveSoundcloud(context);
     case "tiktok":
       return resolveTiktok(context);
     case "twitter":
@@ -34,6 +67,13 @@ export async function postfetch(url: string, options: PostfetchOptions = {}): Pr
   }
 }
 
+/**
+ * Detect which platform a URL belongs to, from its host.
+ *
+ * @param url Any URL.
+ * @returns The matched {@link Platform}.
+ * @throws {PostfetchError} If the host is not a supported platform.
+ */
 export function detect(url: string): Platform {
   const host = asUrl(url).hostname;
   if (host.includes("tiktok.com")) {
@@ -42,6 +82,15 @@ export function detect(url: string): Platform {
   if (host.includes("instagram.com")) {
     return "instagram";
   }
+  if (host.includes("pinterest.") || host === "pin.it") {
+    return "pinterest";
+  }
+  if (host === "reddit.com" || host.endsWith(".reddit.com") || host === "redd.it") {
+    return "reddit";
+  }
+  if (host.includes("soundcloud.com") || host === "snd.sc") {
+    return "soundcloud";
+  }
   if (host.includes("youtube.com") || host === "youtu.be") {
     return "youtube";
   }
@@ -51,5 +100,5 @@ export function detect(url: string): Platform {
   if (host === "x.com" || host.endsWith(".x.com") || host.includes("twitter.com")) {
     return "twitter";
   }
-  throw new PostfetchError(400, "only Facebook, Instagram, TikTok, X and YouTube URLs are supported");
+  throw new PostfetchError(400, "only Facebook, Instagram, Pinterest, Reddit, SoundCloud, TikTok, X and YouTube URLs are supported");
 }

package/src/reddit.ts

@@ -0,0 +1,293 @@
+import {
+  asUrl,
+  filename,
+  object,
+  string,
+  type ResolveContext,
+  type Json,
+  type Net,
+  type PostfetchResult,
+  type MediaItem,
+} from "./internal";
+import { browserUserAgent } from "./fingerprint";
+
+// Logged-out reddit (www/old/api/.json) is fully 403-walled. The public
+// installed-app OAuth flow still works: a client id with no secret plus the
+// opt-out device id returns a bearer that reads any public post through
+// oauth.reddit.com.
+const installedClientId = "ohXpoqrZYub1kg";
+
+export async function resolveReddit(input: ResolveContext): Promise<PostfetchResult> {
+  const id = await postId(input.net, input.url);
+  const token = await appToken(input.net);
+  const post = await fetchPost(input.net, token, id);
+  const items = await mediaItems(input, post, id);
+  if (items.length === 0) {
+    throw new Error("Reddit media not found");
+  }
+  return { archiveFilename: filename(`reddit_${id}.zip`), id, items, platform: "reddit" };
+}
+
+// Share and short links (redd.it/<id>, /r/<sub>/s/<share>) carry no post id, so
+// they are followed once to read the canonical /comments/<id> location.
+async function postId(net: Net, input: string): Promise<string> {
+  const direct = parseId(input);
+  if (direct) {
+    return direct;
+  }
+  const response = await net(input, { headers: { "user-agent": browserUserAgent() } }, 1);
+  const resolved = parseId(response.url);
+  if (!resolved) {
+    throw new Error("Reddit post id not found");
+  }
+  return resolved;
+}
+
+function parseId(input: string): string | null {
+  const url = asUrl(input);
+  if (url.hostname === "redd.it") {
+    const slug = url.pathname.split("/").filter(Boolean)[0];
+    return slug && /^[a-z0-9]+$/i.test(slug) ? slug : null;
+  }
+  return url.pathname.match(/\/comments\/([a-z0-9]+)/i)?.[1] ?? null;
+}
+
+async function appToken(net: Net): Promise<string> {
+  const response = await net("https://www.reddit.com/api/v1/access_token", {
+    body: "grant_type=https://oauth.reddit.com/grants/installed_client&device_id=DO_NOT_TRACK_THIS_DEVICE",
+    headers: {
+      authorization: `Basic ${btoa(`${installedClientId}:`)}`,
+      "content-type": "application/x-www-form-urlencoded",
+      "user-agent": browserUserAgent(),
+    },
+    method: "POST",
+  });
+  if (!response.ok) {
+    throw new Error(`Reddit auth failed: ${response.status}`);
+  }
+  const payload = await response.json().catch(() => null);
+  const token = object(payload) ? string(payload.access_token) : null;
+  if (!token) {
+    throw new Error("Reddit token not found");
+  }
+  return token;
+}
+
+async function fetchPost(net: Net, token: string, id: string): Promise<Json> {
+  const response = await net(`https://oauth.reddit.com/comments/${id}?raw_json=1&limit=1`, {
+    headers: { authorization: `Bearer ${token}`, "user-agent": browserUserAgent() },
+  });
+  if (!response.ok) {
+    throw new Error(`Reddit post failed: ${response.status}`);
+  }
+  const payload = await response.json().catch(() => null);
+  const listing = Array.isArray(payload) ? payload[0] : null;
+  const children = object(listing) && object(listing.data) && Array.isArray(listing.data.children) ? listing.data.children : [];
+  const first = children[0];
+  const post = object(first) && object(first.data) ? first.data : null;
+  if (!post) {
+    throw new Error("Reddit post not found");
+  }
+  return post;
+}
+
+async function mediaItems(input: ResolveContext, post: Json, id: string): Promise<MediaItem[]> {
+  const own = await extract(input, post, id);
+  if (own.length > 0) {
+    return own;
+  }
+  const parents = Array.isArray(post.crosspost_parent_list) ? post.crosspost_parent_list.filter(object) : [];
+  return parents[0] ? extract(input, parents[0], id) : [];
+}
+
+async function extract(input: ResolveContext, post: Json, id: string): Promise<MediaItem[]> {
+  const gallery = galleryItems(post, id);
+  if (gallery.length > 0) {
+    return gallery;
+  }
+  const video = await videoItems(input, post, id);
+  if (video.length > 0) {
+    return video;
+  }
+  return imageItems(post, id);
+}
+
+function galleryItems(post: Json, id: string): MediaItem[] {
+  const data = object(post.gallery_data) ? post.gallery_data : null;
+  const order = data && Array.isArray(data.items) ? data.items.filter(object) : [];
+  const metadata = object(post.media_metadata) ? post.media_metadata : {};
+  return order.flatMap((entry, index) => {
+    const mediaId = string(entry.media_id);
+    const meta = mediaId && object(metadata[mediaId]) ? metadata[mediaId] : null;
+    const best = meta && object(meta.s) ? meta.s : null;
+    if (!best) {
+      return [];
+    }
+    const video = string(best.mp4);
+    if (video) {
+      return [mediaItem(id, index + 1, "video", "video/mp4", video)];
+    }
+    const image = string(best.u) ?? string(best.gif);
+    return image ? [mediaItem(id, index + 1, "image", mimeOf(string(meta?.m), image), image)] : [];
+  });
+}
+
+type Rendition = { width: number | null; bandwidth: number; base: string };
+
+async function videoItems(input: ResolveContext, post: Json, id: string): Promise<MediaItem[]> {
+  const media = redditVideo(post);
+  const fallback = media ? string(media.fallback_url) : null;
+  if (!media || !fallback) {
+    return [];
+  }
+  if (media.has_audio !== true) {
+    return [mediaItem(id, null, "video", "video/mp4", fallback)];
+  }
+  // The fallback stream is video-only; reddit keeps audio in a separate DASH
+  // rendition, so a complete file is the video and audio streams merged.
+  const dashUrl = string(media.dash_url);
+  const streams = dashUrl ? await dashStreams(input.net, dashUrl, input.preferredWidth) : null;
+  if (!streams) {
+    throw new Error("Reddit DASH manifest unavailable for muxing");
+  }
+  return [muxedVideoItem(id, streams.video, streams.audio)];
+}
+
+async function dashStreams(net: Net, dashUrl: string, preferredWidth: number): Promise<{ video: string; audio: string } | null> {
+  const response = await net(dashUrl, { headers: { "user-agent": browserUserAgent() } });
+  if (!response.ok) {
+    return null;
+  }
+  const manifest = await response.text();
+  const base = dashUrl.slice(0, dashUrl.lastIndexOf("/") + 1);
+  const query = dashUrl.includes("?") ? dashUrl.slice(dashUrl.indexOf("?")) : "";
+  const video = videoRendition(manifest, preferredWidth);
+  const audio = audioRendition(manifest);
+  if (!video || !audio) {
+    return null;
+  }
+  return { video: `${base}${video}${query}`, audio: `${base}${audio}${query}` };
+}
+
+function videoRendition(manifest: string, preferredWidth: number): string | null {
+  const renditions = renditionsOf(manifest, false);
+  const sized = renditions.filter((rendition) => rendition.width !== null);
+  const pool = sized.length > 0 ? sized : renditions;
+  const best = pool.reduce<Rendition | null>((current, rendition) => {
+    if (!current) {
+      return rendition;
+    }
+    if (rendition.width !== null && current.width !== null) {
+      return Math.abs(rendition.width - preferredWidth) < Math.abs(current.width - preferredWidth) ? rendition : current;
+    }
+    return rendition.bandwidth > current.bandwidth ? rendition : current;
+  }, null);
+  return best ? best.base : null;
+}
+
+function audioRendition(manifest: string): string | null {
+  const best = renditionsOf(manifest, true).reduce<Rendition | null>(
+    (current, rendition) => (!current || rendition.bandwidth > current.bandwidth ? rendition : current),
+    null,
+  );
+  return best ? best.base : null;
+}
+
+function renditionsOf(manifest: string, audio: boolean): Rendition[] {
+  return manifest
+    .split("<AdaptationSet")
+    .slice(1)
+    .filter((chunk) => isAudioSet(chunk) === audio)
+    .flatMap((chunk) =>
+      [...chunk.matchAll(/<Representation\b([^>]*)>([\s\S]*?)<\/Representation>/g)].flatMap((match) => {
+        const segment = match[2].match(/<BaseURL>([^<]+)<\/BaseURL>/)?.[1];
+        if (!segment) {
+          return [];
+        }
+        const width = match[1].match(/(?:^|[^a-zA-Z])width="(\d+)"/)?.[1];
+        return [{ width: width ? Number(width) : null, bandwidth: Number(match[1].match(/bandwidth="(\d+)"/)?.[1] ?? 0), base: segment }];
+      }),
+    );
+}
+
+function isAudioSet(chunk: string): boolean {
+  const head = chunk.slice(0, chunk.indexOf(">"));
+  return /contentType="audio"/.test(head) || /mimeType="audio/.test(head);
+}
+
+function muxedVideoItem(id: string, videoUrl: string, audioUrl: string): MediaItem {
+  return {
+    audio: { headers: { "user-agent": browserUserAgent() }, url: audioUrl },
+    filename: filename(`reddit_${id}.mp4`),
+    headers: { "user-agent": browserUserAgent() },
+    id,
+    kind: "video",
+    mime: "video/mp4",
+    platform: "reddit",
+    url: videoUrl,
+  };
+}
+
+function redditVideo(post: Json): Json | null {
+  const secure = object(post.secure_media) ? post.secure_media : null;
+  const media = object(post.media) ? post.media : null;
+  const fromSecure = secure && object(secure.reddit_video) ? secure.reddit_video : null;
+  const fromMedia = media && object(media.reddit_video) ? media.reddit_video : null;
+  return fromSecure ?? fromMedia;
+}
+
+function imageItems(post: Json, id: string): MediaItem[] {
+  const url = string(post.url_overridden_by_dest) ?? string(post.url);
+  if (!url) {
+    return [];
+  }
+  const path = asUrl(url).pathname;
+  const isImage = string(post.post_hint) === "image" || asUrl(url).hostname === "i.redd.it" || /\.(jpe?g|png|webp|gif)$/i.test(path);
+  return isImage ? [mediaItem(id, null, "image", mimeOf(null, url), url)] : [];
+}
+
+function mediaItem(id: string, index: number | null, kind: MediaItem["kind"], mime: string, url: string): MediaItem {
+  const suffix = index === null ? "" : `_${index}`;
+  return {
+    filename: filename(`reddit_${id}${suffix}.${extensionOf(mime)}`),
+    headers: { "user-agent": browserUserAgent() },
+    id,
+    kind,
+    mime,
+    platform: "reddit",
+    url,
+  };
+}
+
+function mimeOf(declared: string | null, url: string): string {
+  if (declared && declared.includes("/")) {
+    return declared.replace("image/jpg", "image/jpeg");
+  }
+  const extension = asUrl(url).pathname.match(/\.([a-z0-9]+)$/i)?.[1]?.toLowerCase();
+  if (extension === "png") {
+    return "image/png";
+  }
+  if (extension === "webp") {
+    return "image/webp";
+  }
+  if (extension === "gif") {
+    return "image/gif";
+  }
+  return "image/jpeg";
+}
+
+function extensionOf(mime: string): string {
+  if (mime === "video/mp4") {
+    return "mp4";
+  }
+  if (mime === "image/png") {
+    return "png";
+  }
+  if (mime === "image/webp") {
+    return "webp";
+  }
+  if (mime === "image/gif") {
+    return "gif";
+  }
+  return "jpg";
+}