@postfetch/core 0.2.0

package/src/tiktok.ts

@@ -0,0 +1,187 @@
+import {
+  asUrl,
+  filename,
+  object,
+  string,
+  type ResolveContext,
+  type Json,
+  type Net,
+  type PostfetchResult,
+  type MediaItem,
+} from "./internal";
+import { browserUserAgent, firefoxUserAgent } from "./fingerprint";
+
+const marker = '<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">';
+
+export async function resolveTiktok(input: ResolveContext): Promise<PostfetchResult> {
+  const userAgent = browserUserAgent();
+  const pageUrl = await followShortlink(input.net, userAgent, input.url);
+  const id = videoId(pageUrl);
+  if (!id) {
+    throw new Error("TikTok video id not found");
+  }
+  const page = await fetchVideoPage(input.net, id, userAgent).catch((error: unknown) => {
+    if (!recoverablePageError(error)) {
+      throw error;
+    }
+    return fetchVideoPage(input.net, id, firefoxUserAgent());
+  });
+  const user = author(page.item) ?? username(pageUrl) ?? "i";
+  const headers: Record<string, string> = {
+    referer: `https://www.tiktok.com/@${encodeURIComponent(user)}/video/${encodeURIComponent(id)}`,
+    "user-agent": userAgent,
+  };
+  if (page.cookie) {
+    headers.cookie = page.cookie;
+  }
+  const items = mediaItems(page.item, user, id, headers);
+  if (items.length === 0) {
+    throw new Error("TikTok media not found");
+  }
+  return { archiveFilename: filename(`tiktok_${user}_${id}.zip`), id, items, platform: "tiktok" };
+}
+
+async function fetchVideoPage(
+  net: Net,
+  id: string,
+  userAgent: string,
+): Promise<{ item: Json; cookie: string | null }> {
+  const page = await net(`https://www.tiktok.com/@i/video/${id}`, {
+    headers: {
+      accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+      referer: "https://www.tiktok.com/",
+      "user-agent": userAgent,
+    },
+  });
+  return { cookie: cookieHeader(page.headers), item: itemStruct(await page.text()) };
+}
+
+async function followShortlink(net: Net, userAgent: string, input: string): Promise<string> {
+  const url = asUrl(input);
+  if (!url.hostname.includes("vt.tiktok.com")) {
+    return input;
+  }
+  const response = await net(input, { headers: { "user-agent": userAgent }, redirect: "manual" }, 1);
+  const location = response.headers.get("location");
+  if (location) {
+    return new URL(location, input).href;
+  }
+  const html = await response.text();
+  const href = html.match(/<a href="(https:\/\/[^"]+)"/)?.[1];
+  return href ? new URL(href.split("?")[0]).href : input;
+}
+
+function videoId(input: string): string | null {
+  return asUrl(input).pathname.match(/video\/(\d+)/)?.[1] ?? null;
+}
+
+function itemStruct(html: string): Json {
+  if (html.includes("SlardarWAF") || html.includes("_wafchallengeid")) {
+    throw new Error("TikTok WAF challenge");
+  }
+  const start = html.indexOf(marker);
+  const end = start === -1 ? -1 : html.indexOf("</script>", start + marker.length);
+  if (start === -1 || end === -1) {
+    throw new Error("TikTok hydration not found");
+  }
+  const parsed: unknown = JSON.parse(html.slice(start + marker.length, end));
+  const scope = object(parsed) && object(parsed.__DEFAULT_SCOPE__) ? parsed.__DEFAULT_SCOPE__ : null;
+  const detail = scope && object(scope["webapp.video-detail"]) ? scope["webapp.video-detail"] : null;
+  const info = detail && object(detail.itemInfo) ? detail.itemInfo : null;
+  const item = info && object(info.itemStruct) ? info.itemStruct : null;
+  if (!item) {
+    throw new Error("TikTok itemStruct not found");
+  }
+  return item;
+}
+
+function recoverablePageError(error: unknown): boolean {
+  const message = error instanceof Error ? error.message : "";
+  return message === "TikTok WAF challenge" || message === "TikTok hydration not found";
+}
+
+function downloadUrl(item: Json): string | null {
+  const video = object(item.video) ? item.video : null;
+  return video ? string(video.playAddr) ?? string(video.downloadAddr) : null;
+}
+
+function mediaItems(item: Json, user: string, id: string, headers: HeadersInit): MediaItem[] {
+  const images = imageItems(item, user, id, headers);
+  if (images.length > 0) {
+    const audio = audioItem(item, user, id, headers);
+    return audio ? [...images, audio] : images;
+  }
+  const url = downloadUrl(item);
+  return url
+    ? [{
+      filename: filename(`tiktok_${user}_${id}.mp4`),
+      headers,
+      id,
+      kind: "video",
+      mime: "video/mp4",
+      platform: "tiktok",
+      url,
+    }]
+    : [];
+}
+
+function imageItems(item: Json, user: string, id: string, headers: HeadersInit): MediaItem[] {
+  const imagePost = object(item.imagePost) ? item.imagePost : null;
+  const images = imagePost && Array.isArray(imagePost.images) ? imagePost.images.filter(object) : [];
+  return images.flatMap((image, index) => {
+    const imageUrl = object(image.imageURL) ? image.imageURL : null;
+    const list = imageUrl && Array.isArray(imageUrl.urlList) ? imageUrl.urlList : [];
+    const url = list.map(string).find((candidate): candidate is string => Boolean(candidate)) ?? null;
+    return url
+      ? [{
+        filename: filename(`tiktok_${user}_${id}_${index + 1}.jpg`),
+        headers,
+        id,
+        kind: "image" as const,
+        mime: "image/jpeg",
+        platform: "tiktok" as const,
+        url,
+      }]
+      : [];
+  });
+}
+
+function audioItem(item: Json, user: string, id: string, headers: HeadersInit): MediaItem | null {
+  const video = object(item.video) ? item.video : null;
+  const music = object(item.music) ? item.music : null;
+  const url = video ? string(video.playAddr) : null;
+  const fallback = music ? string(music.playUrl) : null;
+  const selected = url ?? fallback;
+  if (!selected) {
+    return null;
+  }
+  const extension = selected.includes("mime_type=audio_mpeg") ? "mp3" : "m4a";
+  return {
+    filename: filename(`tiktok_${user}_${id}_audio.${extension}`),
+    headers,
+    id,
+    kind: "audio",
+    mime: extension === "mp3" ? "audio/mpeg" : "audio/mp4",
+    platform: "tiktok",
+    url: selected,
+  };
+}
+
+function author(item: Json): string | null {
+  const user = object(item.author) ? item.author : null;
+  return user ? string(user.uniqueId) : null;
+}
+
+function username(input: string): string | null {
+  return asUrl(input).pathname.match(/@([^/]+)/)?.[1] ?? null;
+}
+
+function cookieHeader(headers: Headers): string | null {
+  const getter = (headers as Headers & { getSetCookie?: () => string[] }).getSetCookie;
+  const setCookie = typeof getter === "function" ? getter.call(headers).join(",") : headers.get("set-cookie");
+  const cookies = setCookie
+    ?.split(/,(?=[^;]+?=)/)
+    .map((part) => part.split(";")[0]?.trim())
+    .filter((part): part is string => Boolean(part));
+  return cookies && cookies.length > 0 ? cookies.join("; ") : null;
+}

package/src/twitter.ts

@@ -0,0 +1,81 @@
+import {
+  asUrl,
+  filename,
+  number,
+  object,
+  string,
+  type ResolveContext,
+  type Json,
+  type Net,
+  type PostfetchResult,
+  type MediaItem,
+} from "./internal";
+import { browserUserAgent } from "./fingerprint";
+
+export async function resolveTwitter(input: ResolveContext): Promise<PostfetchResult> {
+  const id = tweetId(input.url);
+  if (!id) {
+    throw new Error("Tweet id not found");
+  }
+  const tweet = await syndication(input.net, id);
+  if (string(tweet.__typename) === "TweetTombstone") {
+    throw new Error("Tweet is unavailable or age-restricted");
+  }
+  const media = Array.isArray(tweet.mediaDetails) ? tweet.mediaDetails.filter(object) : [];
+  const items = media.flatMap((entry, index) => twitterItem(entry, id, index + 1));
+  if (items.length === 0) {
+    throw new Error("Twitter media not found");
+  }
+  return { archiveFilename: filename(`twitter_${id}.zip`), id, items, platform: "twitter" };
+}
+
+// The public syndication endpoint returns tweet media (video variants + photos)
+// without a guest token or bearer; the token is a non-validated cache key.
+async function syndication(net: Net, id: string): Promise<Json> {
+  const url = `https://cdn.syndication.twimg.com/tweet-result?id=${id}&lang=en&token=${syndicationToken(id)}`;
+  const response = await net(url, { headers: { accept: "application/json", "user-agent": browserUserAgent() } });
+  if (!response.ok) {
+    throw new Error(`Twitter syndication failed: ${response.status}`);
+  }
+  const payload: unknown = await response.json();
+  if (!object(payload)) {
+    throw new Error("Twitter response invalid");
+  }
+  return payload;
+}
+
+function syndicationToken(id: string): string {
+  return ((Number(id) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, "");
+}
+
+function twitterItem(entry: Json, id: string, index: number): MediaItem[] {
+  const headers = { "user-agent": browserUserAgent() };
+  const type = string(entry.type);
+  if (type === "video" || type === "animated_gif") {
+    const url = bestVariant(entry);
+    return url
+      ? [{ filename: filename(`twitter_${id}_${index}.mp4`), headers, id, kind: "video", mime: "video/mp4", platform: "twitter", url }]
+      : [];
+  }
+  const photo = string(entry.media_url_https);
+  return photo
+    ? [{ filename: filename(`twitter_${id}_${index}.jpg`), headers, id, kind: "image", mime: "image/jpeg", platform: "twitter", url: `${photo}?name=orig` }]
+    : [];
+}
+
+function bestVariant(entry: Json): string | null {
+  const info = object(entry.video_info) ? entry.video_info : null;
+  const variants = info && Array.isArray(info.variants) ? info.variants.filter(object) : [];
+  const best = variants
+    .filter((variant) => string(variant.content_type) === "video/mp4")
+    .reduce<Json | null>((current, variant) => {
+      const bitrate = number(variant.bitrate) ?? 0;
+      const currentBitrate = current ? number(current.bitrate) ?? 0 : -1;
+      return bitrate > currentBitrate ? variant : current;
+    }, null);
+  return best ? string(best.url) : null;
+}
+
+function tweetId(input: string): string | null {
+  return asUrl(input).pathname.match(/\/status(?:es)?\/(\d+)/)?.[1] ?? null;
+}

package/src/youtube.ts

@@ -0,0 +1,194 @@
+import {
+  asUrl,
+  filename,
+  number,
+  object,
+  string,
+  type ResolveContext,
+  type Json,
+  type Net,
+  type PostfetchResult,
+  type MediaItem,
+} from "./internal";
+import { browserUserAgent } from "./fingerprint";
+
+type YoutubeSession = {
+  cookie: string;
+  signatureTimestamp: number;
+  visitorData: string;
+};
+
+const androidVrClient = {
+  name: "ANDROID_VR",
+  number: "28",
+  userAgent: "com.google.android.apps.youtube.vr.oculus/1.65.10 (Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip",
+  version: "1.65.10",
+};
+
+const browserCookie = "PREF=hl=en&tz=UTC; SOCS=CAI";
+
+export async function resolveYoutube(input: ResolveContext): Promise<PostfetchResult> {
+  const id = youtubeVideoId(input.url);
+  if (!id) {
+    throw new Error("YouTube video id not found");
+  }
+  const session = await youtubeSession(input.net, id);
+  const response = await input.net("https://www.youtube.com/youtubei/v1/player?prettyPrint=false", {
+    body: JSON.stringify(playerBody(id, session)),
+    headers: playerHeaders(session),
+    method: "POST",
+  });
+  if (!response.ok) {
+    throw new Error(`YouTube player failed: ${response.status}`);
+  }
+  const payload = await response.json();
+  const status = object(payload) && object(payload.playabilityStatus) ? string(payload.playabilityStatus.status) : null;
+  if (status !== "OK") {
+    const reason = object(payload) && object(payload.playabilityStatus) ? string(payload.playabilityStatus.reason) : null;
+    throw new Error(reason ?? "YouTube video unavailable");
+  }
+  const format = selectFormat(payload);
+  if (!format) {
+    throw new Error("YouTube progressive mp4 not found");
+  }
+  const title = object(payload) && object(payload.videoDetails) ? string(payload.videoDetails.title) : null;
+  const media: MediaItem = {
+    filename: filename(`youtube_${title ?? id}_${id}.mp4`),
+    headers: { "user-agent": androidVrClient.userAgent },
+    id,
+    kind: "video",
+    mime: "video/mp4",
+    platform: "youtube",
+    url: format,
+  };
+  return { archiveFilename: filename(`youtube_${id}.zip`), id, items: [media], platform: "youtube" };
+}
+
+function playerBody(id: string, session: YoutubeSession): Json {
+  return {
+    contentCheckOk: true,
+    context: {
+      client: {
+        androidSdkVersion: 32,
+        clientName: androidVrClient.name,
+        clientVersion: androidVrClient.version,
+        deviceMake: "Oculus",
+        deviceModel: "Quest 3",
+        gl: "US",
+        hl: "en",
+        osName: "Android",
+        osVersion: "12L",
+        timeZone: "UTC",
+        userAgent: androidVrClient.userAgent,
+        utcOffsetMinutes: 0,
+        visitorData: session.visitorData,
+      },
+    },
+    playbackContext: {
+      contentPlaybackContext: {
+        html5Preference: "HTML5_PREF_WANTS",
+        signatureTimestamp: session.signatureTimestamp,
+      },
+    },
+    racyCheckOk: true,
+    videoId: id,
+  };
+}
+
+async function youtubeSession(net: Net, id: string): Promise<YoutubeSession> {
+  const response = await net(`https://www.youtube.com/watch?v=${id}&bpctr=9999999999&has_verified=1`, {
+    headers: {
+      accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+      "accept-language": "en-us,en;q=0.5",
+      cookie: browserCookie,
+      "sec-fetch-mode": "navigate",
+      "user-agent": browserUserAgent(),
+    },
+  });
+  if (!response.ok) {
+    throw new Error(`YouTube page failed: ${response.status}`);
+  }
+  const page = await response.text();
+  const visitorData = string(page.match(/"visitorData":"([^"]+)"/)?.[1]);
+  if (!visitorData) {
+    throw new Error("YouTube visitor data not found");
+  }
+  const playerPath = string(page.match(/"jsUrl":"([^"]+)"/)?.[1]) ?? string(page.match(/"PLAYER_JS_URL":"([^"]+)"/)?.[1]);
+  if (!playerPath) {
+    throw new Error("YouTube player url not found");
+  }
+  return {
+    cookie: youtubeCookie(response.headers),
+    signatureTimestamp: await signatureTimestamp(net, playerPath),
+    visitorData,
+  };
+}
+
+function playerHeaders(session: YoutubeSession): Headers {
+  return new Headers({
+    "content-type": "application/json",
+    cookie: `${browserCookie}; ${session.cookie}`,
+    origin: "https://www.youtube.com",
+    "user-agent": androidVrClient.userAgent,
+    "x-goog-visitor-id": session.visitorData,
+    "x-youtube-client-name": androidVrClient.number,
+    "x-youtube-client-version": androidVrClient.version,
+  });
+}
+
+function youtubeCookie(headers: Headers): string {
+  const readable = headers as Headers & { getSetCookie?: () => string[] };
+  const fallback = headers.get("set-cookie");
+  const cookieHeaders = readable.getSetCookie ? readable.getSetCookie() : fallback ? [fallback] : [];
+  return cookieHeaders
+    .map((header) => string(header.split(";", 1)[0]))
+    .filter(string)
+    .join("; ");
+}
+
+async function signatureTimestamp(net: Net, playerPath: string): Promise<number> {
+  const response = await net(new URL(playerPath, "https://www.youtube.com").toString(), {
+    headers: { "user-agent": browserUserAgent() },
+  });
+  if (!response.ok) {
+    throw new Error(`YouTube player failed: ${response.status}`);
+  }
+  const player = await response.text();
+  const timestamp = number(Number(player.match(/signatureTimestamp[:=](\d+)/)?.[1] ?? player.match(/sts[:=](\d+)/)?.[1]));
+  if (!timestamp) {
+    throw new Error("YouTube signature timestamp not found");
+  }
+  return timestamp;
+}
+
+export function youtubeVideoId(input: string): string | null {
+  const url = asUrl(input);
+  if (url.hostname === "youtu.be") {
+    return cleanId(url.pathname.split("/").filter(Boolean)[0]);
+  }
+  const fromQuery = cleanId(url.searchParams.get("v"));
+  if (fromQuery) {
+    return fromQuery;
+  }
+  const parts = url.pathname.split("/").filter(Boolean);
+  const index = parts.findIndex((part) => part === "shorts" || part === "live" || part === "embed");
+  return index >= 0 ? cleanId(parts[index + 1]) : null;
+}
+
+function cleanId(value: string | null | undefined): string | null {
+  return typeof value === "string" && /^[A-Za-z0-9_-]{11}$/.test(value) ? value : null;
+}
+
+function selectFormat(payload: unknown): string | null {
+  const root = object(payload) ? payload : null;
+  const streaming = root && object(root.streamingData) ? root.streamingData : null;
+  const formats = streaming && Array.isArray(streaming.formats) ? streaming.formats.filter(object) : [];
+  const mp4 = formats
+    .filter((format) => string(format.url) && string(format.mimeType)?.startsWith("video/mp4"))
+    .sort((left, right) => height(right) - height(left));
+  return mp4[0] ? string(mp4[0].url) : null;
+}
+
+function height(format: Json): number {
+  return number(format.height) ?? Number(string(format.qualityLabel)?.match(/(\d+)p/)?.[1] ?? 0);
+}

package/src/zip.ts

@@ -0,0 +1,121 @@
+export type ZipEntry = {
+  data: Uint8Array;
+  name: string;
+};
+
+const encoder = new TextEncoder();
+const table = new Uint32Array(256);
+type PackedEntry = {
+  crc: number;
+  data: Uint8Array;
+  name: Uint8Array;
+};
+
+for (let value = 0; value < table.length; value += 1) {
+  let crc = value;
+  for (let bit = 0; bit < 8; bit += 1) {
+    crc = crc & 1 ? 0xedb88320 ^ (crc >>> 1) : crc >>> 1;
+  }
+  table[value] = crc >>> 0;
+}
+
+export function zip(entries: ZipEntry[]): Uint8Array {
+  const files = entries.map((entry) => {
+    const name = encoder.encode(entry.name);
+    const crc = crc32(entry.data);
+    return { ...entry, crc, name };
+  });
+  const locals = files.map((file) => 30 + file.name.length + file.data.length);
+  const centrals = files.map((file) => 46 + file.name.length);
+  const localSize = locals.reduce((sum, size) => sum + size, 0);
+  const centralSize = centrals.reduce((sum, size) => sum + size, 0);
+  const result = new Uint8Array(localSize + centralSize + 22);
+  const view = new DataView(result.buffer);
+  let offset = 0;
+  const centralOffsets: number[] = [];
+
+  for (const file of files) {
+    centralOffsets.push(offset);
+    offset = writeLocal(view, result, offset, file);
+  }
+
+  const centralStart = offset;
+  for (let index = 0; index < files.length; index += 1) {
+    offset = writeCentral(view, result, offset, files[index], centralOffsets[index]);
+  }
+
+  writeEnd(view, offset, files.length, centralSize, centralStart);
+  return result;
+}
+
+function writeLocal(
+  view: DataView,
+  target: Uint8Array,
+  offset: number,
+  file: PackedEntry,
+): number {
+  view.setUint32(offset, 0x04034b50, true);
+  view.setUint16(offset + 4, 20, true);
+  view.setUint16(offset + 6, 0x0800, true);
+  view.setUint16(offset + 8, 0, true);
+  writeDate(view, offset + 10);
+  view.setUint32(offset + 14, file.crc, true);
+  view.setUint32(offset + 18, file.data.length, true);
+  view.setUint32(offset + 22, file.data.length, true);
+  view.setUint16(offset + 26, file.name.length, true);
+  view.setUint16(offset + 28, 0, true);
+  target.set(file.name, offset + 30);
+  target.set(file.data, offset + 30 + file.name.length);
+  return offset + 30 + file.name.length + file.data.length;
+}
+
+function writeCentral(
+  view: DataView,
+  target: Uint8Array,
+  offset: number,
+  file: PackedEntry,
+  localOffset: number,
+): number {
+  view.setUint32(offset, 0x02014b50, true);
+  view.setUint16(offset + 4, 20, true);
+  view.setUint16(offset + 6, 20, true);
+  view.setUint16(offset + 8, 0x0800, true);
+  view.setUint16(offset + 10, 0, true);
+  writeDate(view, offset + 12);
+  view.setUint32(offset + 16, file.crc, true);
+  view.setUint32(offset + 20, file.data.length, true);
+  view.setUint32(offset + 24, file.data.length, true);
+  view.setUint16(offset + 28, file.name.length, true);
+  view.setUint16(offset + 30, 0, true);
+  view.setUint16(offset + 32, 0, true);
+  view.setUint16(offset + 34, 0, true);
+  view.setUint16(offset + 36, 0, true);
+  view.setUint32(offset + 38, 0, true);
+  view.setUint32(offset + 42, localOffset, true);
+  target.set(file.name, offset + 46);
+  return offset + 46 + file.name.length;
+}
+
+function writeEnd(view: DataView, offset: number, count: number, centralSize: number, centralStart: number): void {
+  view.setUint32(offset, 0x06054b50, true);
+  view.setUint16(offset + 8, count, true);
+  view.setUint16(offset + 10, count, true);
+  view.setUint32(offset + 12, centralSize, true);
+  view.setUint32(offset + 16, centralStart, true);
+}
+
+function writeDate(view: DataView, offset: number): void {
+  const date = new Date();
+  const time = (date.getHours() << 11) | (date.getMinutes() << 5) | Math.floor(date.getSeconds() / 2);
+  const day = ((date.getFullYear() - 1980) << 9) | ((date.getMonth() + 1) << 5) | date.getDate();
+  view.setUint16(offset, time, true);
+  view.setUint16(offset + 2, day, true);
+}
+
+function crc32(data: Uint8Array): number {
+  let crc = 0xffffffff;
+  for (const byte of data) {
+    crc = table[(crc ^ byte) & 0xff] ^ (crc >>> 8);
+  }
+  return (crc ^ 0xffffffff) >>> 0;
+}