@postfetch/core 0.2.4 → 0.3.0

package/src/remux.ts

@@ -0,0 +1,172 @@
+import {
+  boxVersion,
+  buildBox,
+  childBoxes,
+  concat,
+  findBox,
+  parseBoxes,
+  readU32,
+  sliceBox,
+  writeU32,
+  type Box,
+} from "./isobmff";
+
+// A single-track fragmented MP4 (CMAF init segment + moof/mdat fragments), broken
+// into the parts a merge needs.
+type Source = {
+  data: Uint8Array;
+  ftyp: Box;
+  moov: Box;
+  trak: Box;
+  mvex: Box;
+  trackId: number;
+  fragments: { moof: Box; mdat: Box }[];
+};
+
+/**
+ * Merge a video-only and an audio-only fragmented MP4 into one file with both
+ * tracks. The DASH/CMAF streams reddit and YouTube hand out keep video and audio
+ * apart; this recombines them without re-encoding — every `moof`/`mdat` pair is
+ * copied verbatim (so each `trun` data offset stays valid) and only the track id
+ * and fragment sequence number are patched.
+ */
+export function mergeAudioVideo(video: Uint8Array, audio: Uint8Array): Uint8Array {
+  const videoSource = analyze(video);
+  const audioSource = analyze(audio);
+  const audioTrackId = videoSource.trackId === 1 ? 2 : 1;
+
+  const moov = buildMoov(videoSource, audioSource, audioTrackId);
+  const parts: Uint8Array[] = [sliceBox(video, videoSource.ftyp), moov];
+
+  let sequence = 1;
+  const fragmentCount = Math.max(videoSource.fragments.length, audioSource.fragments.length);
+  for (let index = 0; index < fragmentCount; index += 1) {
+    const videoFragment = videoSource.fragments[index];
+    if (videoFragment) {
+      // The patched moof is a small copy; the (large) mdat is appended as a view,
+      // so the trailing concat is the only time sample bytes are copied.
+      parts.push(rewriteMoof(video, videoFragment.moof, null, sequence));
+      parts.push(video.subarray(videoFragment.mdat.start, videoFragment.mdat.end));
+      sequence += 1;
+    }
+    const audioFragment = audioSource.fragments[index];
+    if (audioFragment) {
+      parts.push(rewriteMoof(audio, audioFragment.moof, audioTrackId, sequence));
+      parts.push(audio.subarray(audioFragment.mdat.start, audioFragment.mdat.end));
+      sequence += 1;
+    }
+  }
+  return concat(parts);
+}
+
+function analyze(data: Uint8Array): Source {
+  const top = parseBoxes(data);
+  const ftyp = findBox(top, "ftyp");
+  const moov = findBox(top, "moov");
+  if (!ftyp || !moov) {
+    throw new Error("not a fragmented MP4");
+  }
+  const moovChildren = childBoxes(data, moov);
+  const trak = findBox(moovChildren, "trak");
+  const mvex = findBox(moovChildren, "mvex");
+  if (!trak || !mvex) {
+    throw new Error("MP4 is not fragmented (no trak/mvex)");
+  }
+  const tkhd = findBox(childBoxes(data, trak), "tkhd");
+  if (!tkhd) {
+    throw new Error("track header not found");
+  }
+  const fragments: { moof: Box; mdat: Box }[] = [];
+  for (let index = 0; index < top.length; index += 1) {
+    if (top[index].type === "moof") {
+      const mdat = top[index + 1];
+      if (!mdat || mdat.type !== "mdat") {
+        throw new Error("fragment without mdat");
+      }
+      fragments.push({ moof: top[index], mdat });
+    }
+  }
+  if (fragments.length === 0) {
+    throw new Error("MP4 has no fragments");
+  }
+  return { data, ftyp, moov, trak, mvex, trackId: trackHeaderId(data, tkhd), fragments };
+}
+
+// next_track_ID is the last uint32 of mvhd; track_ID sits after the version-sized
+// creation/modification timestamps in tkhd.
+function trackHeaderId(data: Uint8Array, tkhd: Box): number {
+  return readU32(data, tkhd.dataStart + (boxVersion(data, tkhd) === 1 ? 20 : 12));
+}
+
+function buildMoov(video: Source, audio: Source, audioTrackId: number): Uint8Array {
+  const moovChildren = childBoxes(video.data, video.moov);
+  const mvhd = patchedMvhd(video, audioTrackId);
+  const meta = findBox(moovChildren, "meta");
+  const videoTrak = sliceBox(video.data, video.trak);
+  const audioTrak = retrackTrak(sliceBox(audio.data, audio.trak), audioTrackId);
+  const mvex = buildMvex(video, audio, audioTrackId);
+
+  const children = [mvhd];
+  if (meta) {
+    children.push(sliceBox(video.data, meta));
+  }
+  children.push(videoTrak, audioTrak, mvex);
+  return buildBox("moov", concat(children));
+}
+
+function patchedMvhd(video: Source, audioTrackId: number): Uint8Array {
+  const mvhd = findBox(childBoxes(video.data, video.moov), "mvhd");
+  if (!mvhd) {
+    throw new Error("movie header not found");
+  }
+  const copy = sliceBox(video.data, mvhd);
+  writeU32(copy, copy.length - 4, Math.max(video.trackId, audioTrackId) + 1);
+  return copy;
+}
+
+function buildMvex(video: Source, audio: Source, audioTrackId: number): Uint8Array {
+  const videoMvex = childBoxes(video.data, video.mvex);
+  const mehd = findBox(videoMvex, "mehd");
+  const videoTrex = findBox(videoMvex, "trex");
+  const audioTrex = findBox(childBoxes(audio.data, audio.mvex), "trex");
+  if (!videoTrex || !audioTrex) {
+    throw new Error("track extends box not found");
+  }
+  const audioTrexCopy = sliceBox(audio.data, audioTrex);
+  writeU32(audioTrexCopy, audioTrex.dataStart - audioTrex.start + 4, audioTrackId);
+
+  const children: Uint8Array[] = [];
+  if (mehd) {
+    children.push(sliceBox(video.data, mehd));
+  }
+  children.push(sliceBox(video.data, videoTrex), audioTrexCopy);
+  return buildBox("mvex", concat(children));
+}
+
+function retrackTrak(trak: Uint8Array, trackId: number): Uint8Array {
+  const tkhd = findBox(parseBoxes(trak, 8, trak.length), "tkhd");
+  if (!tkhd) {
+    throw new Error("track header not found");
+  }
+  writeU32(trak, tkhd.dataStart + (boxVersion(trak, tkhd) === 1 ? 20 : 12), trackId);
+  return trak;
+}
+
+function rewriteMoof(data: Uint8Array, moofBox: Box, trackId: number | null, sequence: number): Uint8Array {
+  const moof = sliceBox(data, moofBox);
+  const moofChildren = parseBoxes(moof, 8, moof.length);
+  const mfhd = findBox(moofChildren, "mfhd");
+  if (!mfhd) {
+    throw new Error("movie fragment header not found");
+  }
+  writeU32(moof, mfhd.dataStart + 4, sequence);
+  if (trackId !== null) {
+    const traf = findBox(moofChildren, "traf");
+    const tfhd = traf ? findBox(parseBoxes(moof, traf.dataStart, traf.end), "tfhd") : undefined;
+    if (!tfhd) {
+      throw new Error("track fragment header not found");
+    }
+    writeU32(moof, tfhd.dataStart + 4, trackId);
+  }
+  return moof;
+}

package/src/soundcloud.ts

@@ -0,0 +1,127 @@
+import {
+  asUrl,
+  filename,
+  number,
+  object,
+  string,
+  type ResolveContext,
+  type Json,
+  type Net,
+  type PostfetchResult,
+  type MediaItem,
+} from "./internal";
+import { browserUserAgent } from "./fingerprint";
+
+export async function resolveSoundcloud(input: ResolveContext): Promise<PostfetchResult> {
+  const url = await canonicalUrl(input.net, input.url);
+  const clientId = await appClientId(input.net);
+  const track = await resolveTrack(input.net, url, clientId);
+  const id = trackId(track);
+  const item = await audioItem(input.net, track, id, clientId);
+  return { archiveFilename: filename(`soundcloud_${id}.zip`), id, items: [item], platform: "soundcloud" };
+}
+
+async function canonicalUrl(net: Net, input: string): Promise<string> {
+  const host = asUrl(input).hostname;
+  if (host !== "on.soundcloud.com" && host !== "snd.sc") {
+    return input;
+  }
+  const response = await net(input, { headers: { "user-agent": browserUserAgent() } }, 1);
+  return response.url;
+}
+
+// SoundCloud's public api-v2 needs a client_id, which the web app ships inside one
+// of its asset bundles. The bundle that carries it changes, so the script srcs are
+// scanned (newest first) until one yields the token.
+async function appClientId(net: Net): Promise<string> {
+  const home = await net("https://soundcloud.com/", { headers: { "user-agent": browserUserAgent() } });
+  if (!home.ok) {
+    throw new Error(`SoundCloud home failed: ${home.status}`);
+  }
+  const html = await home.text();
+  const scripts = [...html.matchAll(/<script[^>]+src="(https:\/\/a-v2\.sndcdn\.com\/assets\/[^"]+\.js)"/g)].map((match) => match[1]);
+  for (const src of scripts.reverse()) {
+    const response = await net(src, { headers: { "user-agent": browserUserAgent() } }, 1);
+    if (!response.ok) {
+      continue;
+    }
+    const code = await response.text();
+    const id = code.match(/client_id\s*[:=]\s*"([A-Za-z0-9]{20,})"/)?.[1];
+    if (id) {
+      return id;
+    }
+  }
+  throw new Error("SoundCloud client id not found");
+}
+
+async function resolveTrack(net: Net, url: string, clientId: string): Promise<Json> {
+  const api = new URL("https://api-v2.soundcloud.com/resolve");
+  api.searchParams.set("url", url);
+  api.searchParams.set("client_id", clientId);
+  const response = await net(api.href, { headers: { accept: "application/json", "user-agent": browserUserAgent() } });
+  if (!response.ok) {
+    throw new Error(`SoundCloud resolve failed: ${response.status}`);
+  }
+  const payload = await response.json().catch(() => null);
+  if (!object(payload) || string(payload.kind) !== "track") {
+    throw new Error("SoundCloud track not found");
+  }
+  return payload;
+}
+
+async function audioItem(net: Net, track: Json, id: string, clientId: string): Promise<MediaItem> {
+  const media = object(track.media) ? track.media : null;
+  const transcodings = media && Array.isArray(media.transcodings) ? media.transcodings.filter(object) : [];
+  // Prefer the direct progressive mp3; otherwise assemble the AAC HLS playlist
+  // (its CMAF segments concatenate into a fragmented mp4) at download time.
+  const progressive = transcodings.find((transcoding) => protocol(transcoding) === "progressive");
+  const hls = transcodings.find((transcoding) => protocol(transcoding) === "hls" && mimeType(transcoding)?.includes("mp4") === true);
+  const chosen = progressive ?? hls;
+  if (!chosen) {
+    throw new Error("SoundCloud track has no downloadable stream");
+  }
+  const stream = await streamUrl(net, chosen, clientId);
+  const title = string(track.title) ?? id;
+  const headers = { "user-agent": browserUserAgent() };
+  if (chosen === progressive) {
+    return { filename: filename(`soundcloud_${title}.mp3`), headers, id, kind: "audio", mime: "audio/mpeg", platform: "soundcloud", url: stream };
+  }
+  return { filename: filename(`soundcloud_${title}.m4a`), headers, hls: true, id, kind: "audio", mime: "audio/mp4", platform: "soundcloud", url: stream };
+}
+
+async function streamUrl(net: Net, transcoding: Json, clientId: string): Promise<string> {
+  const endpoint = string(transcoding.url);
+  if (!endpoint) {
+    throw new Error("SoundCloud stream endpoint not found");
+  }
+  const url = new URL(endpoint);
+  url.searchParams.set("client_id", clientId);
+  const response = await net(url.href, { headers: { accept: "application/json", "user-agent": browserUserAgent() } });
+  if (!response.ok) {
+    throw new Error(`SoundCloud stream failed: ${response.status}`);
+  }
+  const payload = await response.json().catch(() => null);
+  const stream = object(payload) ? string(payload.url) : null;
+  if (!stream) {
+    throw new Error("SoundCloud stream url not found");
+  }
+  return stream;
+}
+
+function protocol(transcoding: Json): string | null {
+  const format = object(transcoding.format) ? transcoding.format : null;
+  return format ? string(format.protocol) : null;
+}
+
+function mimeType(transcoding: Json): string | null {
+  const format = object(transcoding.format) ? transcoding.format : null;
+  return format ? string(format.mime_type) : null;
+}
+
+function trackId(track: Json): string {
+  const numeric = number(track.id);
+  if (numeric !== null) {
+    return String(numeric);
+  }
+  return string(track.permalink) ?? "track";
+}

package/src/youtube.ts

@@ -47,23 +47,72 @@ export async function resolveYoutube(input: ResolveContext): Promise<PostfetchRe
     const reason = object(payload) && object(payload.playabilityStatus) ? string(payload.playabilityStatus.reason) : null;
     throw new Error(reason ?? "YouTube video unavailable");
   }
-  const format = selectFormat(payload);
-  if (!format) {
-    throw new Error("YouTube progressive mp4 not found");
+  const streams = selectStreams(payload, input.preferredWidth);
+  if (!streams) {
+    throw new Error("YouTube mp4 stream not found");
   }
   const title = object(payload) && object(payload.videoDetails) ? string(payload.videoDetails.title) : null;
+  const headers = { "user-agent": androidVrClient.userAgent };
   const media: MediaItem = {
     filename: filename(`youtube_${title ?? id}_${id}.mp4`),
-    headers: { "user-agent": androidVrClient.userAgent },
+    headers,
     id,
     kind: "video",
     mime: "video/mp4",
     platform: "youtube",
-    url: format,
+    url: streams.video,
+    ...(streams.audio ? { audio: { headers, url: streams.audio } } : {}),
   };
   return { archiveFilename: filename(`youtube_${id}.zip`), id, items: [media], platform: "youtube" };
 }
 
+// Adaptive H.264 video and AAC audio go up to 1080p with direct URLs; merging them
+// beats the progressive formats (capped at 360-720p). The progressive single-file
+// stream is the fallback when a separate-stream pair is unavailable.
+function selectStreams(payload: unknown, preferredWidth: number): { video: string; audio: string | null } | null {
+  const adaptive = adaptiveFormats(payload);
+  const video = bestVideo(adaptive, preferredWidth);
+  const audio = bestAudio(adaptive);
+  if (video && audio) {
+    return { video, audio };
+  }
+  const progressive = selectFormat(payload);
+  return progressive ? { video: progressive, audio: null } : null;
+}
+
+function adaptiveFormats(payload: unknown): Json[] {
+  const root = object(payload) ? payload : null;
+  const streaming = root && object(root.streamingData) ? root.streamingData : null;
+  return streaming && Array.isArray(streaming.adaptiveFormats) ? streaming.adaptiveFormats.filter(object) : [];
+}
+
+function bestVideo(formats: Json[], preferredWidth: number): string | null {
+  const candidates = formats.filter((format) => {
+    const mimeType = string(format.mimeType);
+    return Boolean(string(format.url)) && mimeType?.startsWith("video/mp4") === true && mimeType.includes("avc1");
+  });
+  const best = candidates.reduce<Json | null>((current, format) => {
+    if (!current) {
+      return format;
+    }
+    const width = number(format.width) ?? 0;
+    const currentWidth = number(current.width) ?? 0;
+    return Math.abs(width - preferredWidth) < Math.abs(currentWidth - preferredWidth) ? format : current;
+  }, null);
+  return best ? string(best.url) : null;
+}
+
+function bestAudio(formats: Json[]): string | null {
+  const candidates = formats.filter((format) => Boolean(string(format.url)) && string(format.mimeType)?.startsWith("audio/mp4") === true);
+  const best = candidates.reduce<Json | null>((current, format) => {
+    if (!current) {
+      return format;
+    }
+    return (number(format.bitrate) ?? 0) > (number(current.bitrate) ?? 0) ? format : current;
+  }, null);
+  return best ? string(best.url) : null;
+}
+
 function playerBody(id: string, session: YoutubeSession): Json {
   return {
     contentCheckOk: true,