getraw 0.2.2 → 0.3.0

package/bun.lock

@@ -5,7 +5,9 @@
     "": {
       "name": "dlpx",
       "dependencies": {
+        "googlevideo": "^4.0.4",
         "hls-parser": "^0.13.6",
+        "meriyah": "^6.0.7",
         "mpd-parser": "^1.3.0",
         "youtubei.js": "^17.0.1",
       },
@@ -36,6 +38,8 @@
 
     "global": ["global@4.4.0", "", { "dependencies": { "min-document": "^2.19.0", "process": "^0.11.10" } }, "sha512-wv/LAoHdRE3BeTGz53FAamhGlPLhlssK45usmGFThIi4XqnBmjKQ16u+RNbP7WvigRZDxUsM0J3gcQ5yicaL0w=="],
 
+    "googlevideo": ["googlevideo@4.0.4", "", { "dependencies": { "@bufbuild/protobuf": "^2.0.0" } }, "sha512-S/rfuoPBI+qXCEUPJeVhXsHoISMgVhOz8hHSpGWa0OztfHhh+g9EKaEcqAb/+ttO7meoNQNqIy9dfIpz7HPc4g=="],
+
     "hls-parser": ["hls-parser@0.13.6", "", {}, "sha512-I40sl22E2muqeSTpG8kMN2dAegAhubkXPXtnsUXFwdKwZK47d1Q+XwuX32VMZ++AZU5oeQIZqAnGNHxSG1sWaw=="],
 
     "meriyah": ["meriyah@6.1.4", "", {}, "sha512-Sz8FzjzI0kN13GK/6MVEsVzMZEPvOhnmmI1lU5+/1cGOiK3QUahntrNNtdVeihrO7t9JpoH75iMNXg6R6uWflQ=="],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "getraw",
-  "version": "0.2.2",
+  "version": "0.3.0",
   "description": "Fast media downloader CLI built natively in Bun/TypeScript",
   "type": "module",
   "bin": {
@@ -10,11 +10,12 @@
     "dev": "bun run src/cli/index.ts",
     "test": "bun test",
     "build": "bun build src/cli/index.ts --compile --outfile=getraw",
-    "dashboard": "bun run tools/dashboard.ts",
-    "postinstall": "bun run scripts/patch-youtubei.js"
+    "dashboard": "bun run tools/dashboard.ts"
   },
   "dependencies": {
+    "googlevideo": "^4.0.4",
     "hls-parser": "^0.13.6",
+    "meriyah": "^6.0.7",
     "mpd-parser": "^1.3.0",
     "youtubei.js": "^17.0.1"
   },

package/src/extractors/youtube/index.ts

@@ -1,28 +1,50 @@
 import { BaseExtractor, ExtractorError } from "../../core/types";
 import type { InfoDict, Format, Thumbnail } from "../../core/types";
 import { parseCaptionTracks } from "./captions";
+import { InnerTubeClient } from "./innertube";
+import type { RawFormat, PlayerResponse, StreamingData } from "./innertube";
+import { decipherStreamUrl, setPageHtmlForPlayerExtraction } from "./player";
 
 const VALID_URL = /^https?:\/\/(?:(?:www|m|music)\.)?(?:youtube\.com\/(?:watch\?.*v=|shorts\/|live\/|embed\/|v\/)|youtu\.be\/)([a-zA-Z0-9_-]{11})/;
 const PLAYLIST_URL = /^https?:\/\/(?:(?:www|m|music)\.)?youtube\.com\/playlist\?.*list=([a-zA-Z0-9_-]+)/;
 const CHANNEL_URL = /^https?:\/\/(?:(?:www|m|music)\.)?youtube\.com\/(?:channel\/|@)([a-zA-Z0-9_-]+)/;
 
+const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36";
+
 function generateCpn(): string {
   const chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_";
   return Array.from({ length: 16 }, () => chars[Math.floor(Math.random() * 64)]).join("");
 }
 
-let _innertube: Awaited<ReturnType<typeof createInnertube>> | null = null;
-
-async function createInnertube() {
-  const { Innertube } = await import("youtubei.js");
-  return Innertube.create({ generate_session_locally: true });
+interface PageData {
+  playerResponse: PlayerResponse;
+  html: string;
 }
 
-async function getInnertube() {
-  if (!_innertube) {
-    _innertube = await createInnertube();
+async function fetchPageData(videoId: string): Promise<PageData> {
+  const resp = await fetch(`https://www.youtube.com/watch?v=${videoId}`, {
+    headers: {
+      "User-Agent": USER_AGENT,
+      "Accept-Language": "en-US,en;q=0.9",
+    },
+  });
+
+  if (!resp.ok) {
+    throw new ExtractorError(`Failed to fetch YouTube page: ${resp.status}`);
   }
-  return _innertube;
+
+  const html = await resp.text();
+  setPageHtmlForPlayerExtraction(html);
+
+  const prMatch = html.match(/var\s+ytInitialPlayerResponse\s*=\s*(\{.+?\});/s);
+  if (!prMatch) {
+    throw new ExtractorError("Could not extract player response from page");
+  }
+
+  return {
+    playerResponse: JSON.parse(prMatch[1]) as PlayerResponse,
+    html,
+  };
 }
 
 export class YouTubeExtractor extends BaseExtractor {
@@ -41,16 +63,22 @@ export class YouTubeExtractor extends BaseExtractor {
   }
 
   private async extractVideo(videoId: string): Promise<InfoDict> {
-    const yt = await getInnertube();
-    const info = await yt.getInfo(videoId);
+    const pageData = await fetchPageData(videoId);
+    const playerResponse = pageData.playerResponse;
+
+    const status = playerResponse.playabilityStatus;
+    if (status?.status !== "OK") {
+      throw new ExtractorError(status?.reason ?? "Video unavailable");
+    }
 
-    if (!info.basic_info.title) {
+    const details = playerResponse.videoDetails;
+    if (!details?.title) {
       throw new ExtractorError("Could not extract video info");
     }
 
-    const formats = await this.extractFormats(info, yt);
+    const formats = await this.extractFormats(playerResponse.streamingData, pageData.html, videoId);
 
-    const thumbnails: Thumbnail[] = (info.basic_info.thumbnail ?? []).map((t: { url: string; width: number; height: number }) => ({
+    const thumbnails: Thumbnail[] = (details.thumbnail?.thumbnails ?? []).map((t) => ({
       url: t.url,
       width: t.width,
       height: t.height,
@@ -58,104 +86,146 @@ export class YouTubeExtractor extends BaseExtractor {
 
     const result: InfoDict = {
       id: videoId,
-      title: info.basic_info.title,
+      title: details.title,
       formats,
       thumbnails,
-      description: info.basic_info.short_description,
-      channel: info.basic_info.author,
-      channel_id: info.basic_info.channel_id,
-      duration: info.basic_info.duration,
-      view_count: info.basic_info.view_count,
+      description: details.shortDescription,
+      channel: details.author,
+      channel_id: details.channelId,
+      duration: parseInt(details.lengthSeconds, 10) || undefined,
+      view_count: parseInt(details.viewCount, 10) || undefined,
       webpage_url: `https://www.youtube.com/watch?v=${videoId}`,
-      live_status: info.basic_info.is_live ? "is_live" : "not_live",
+      live_status: details.isLive ? "is_live" : "not_live",
     };
 
-    // Extract captions from page response
-    const pageResponse = await this.fetchPagePlayerResponse(videoId);
-    if (pageResponse) {
-      const captionTracks = pageResponse.captions?.playerCaptionsTracklistRenderer?.captionTracks;
-      if (captionTracks?.length) {
-        const { subtitles, automatic_captions } = parseCaptionTracks(captionTracks);
-        result.subtitles = subtitles;
-        result.automatic_captions = automatic_captions;
-      }
+    const captionTracks = playerResponse.captions?.playerCaptionsTracklistRenderer?.captionTracks;
+    if (captionTracks?.length) {
+      const { subtitles, automatic_captions } = parseCaptionTracks(captionTracks);
+      result.subtitles = subtitles;
+      result.automatic_captions = automatic_captions;
     }
 
     return result;
   }
 
-  private async extractFormats(info: { streaming_data?: { formats?: unknown[]; adaptive_formats?: unknown[] }; chooseFormat: (opts: { type: string; quality: string }) => unknown }, yt: { session: { player: unknown } }): Promise<Format[]> {
-    const formats: Format[] = [];
-    const player = yt.session.player;
+  private async extractFormats(streamingData: StreamingData | undefined, pageHtml: string, videoId?: string): Promise<Format[]> {
     const cpn = generateCpn();
+    const formats: Format[] = [];
 
-    const allFormats = [
-      ...(info.streaming_data?.formats ?? []),
-      ...(info.streaming_data?.adaptive_formats ?? []),
-    ];
+    // First: get formats from page response (muxed formats with signatureCipher)
+    if (streamingData) {
+      const pageFormats: RawFormat[] = [
+        ...(streamingData.formats ?? []),
+        ...(streamingData.adaptiveFormats ?? []),
+      ];
+
+      for (const raw of pageFormats) {
+        if (!raw.url && !raw.signatureCipher) continue;
+        try {
+          const url = await decipherStreamUrl(raw.url, raw.signatureCipher, pageHtml);
+          if (!url) continue;
+          const parsed = new URL(url);
+          parsed.searchParams.set("cpn", cpn);
+          formats.push(this.buildFormat(raw, parsed.toString()));
+        } catch {
+          continue;
+        }
+      }
+    }
 
-    for (const raw of allFormats) {
-      const f = raw as Record<string, unknown>;
+    if (videoId) {
       try {
-        let url: string | undefined;
-
-        if (typeof (f as { decipher?: unknown }).decipher === "function") {
-          const deciphered = await (f as { decipher: (p: unknown) => Promise<unknown> }).decipher(player);
-          if (typeof deciphered === "string") {
-            const parsed = new URL(deciphered);
-            parsed.searchParams.set("cpn", cpn);
-            url = parsed.toString();
+        const iosFormats = await this.fetchIosFormats(videoId, pageHtml, cpn);
+        const existingItags = new Set(formats.map((f) => f.format_id));
+        for (const f of iosFormats) {
+          if (!existingItags.has(f.format_id)) {
+            formats.push(f);
           }
         }
-
-        if (!url) continue;
-
-        const mime = String(f.mime_type ?? "");
-        const mimeMatch = mime.match(/^(video|audio)\/(\w+);\s*codecs="([^"]+)"/);
-        const ext = mimeMatch?.[2] ?? "mp4";
-        const codecs = mimeMatch?.[3] ?? "";
-        const isVideo = mime.startsWith("video");
-        const isAudio = mime.startsWith("audio");
-
-        formats.push({
-          format_id: String(f.itag ?? ""),
-          url,
-          ext,
-          vcodec: isVideo ? codecs.split(",")[0]?.trim() : "none",
-          acodec: isAudio ? codecs : (isVideo && codecs.includes(",") ? codecs.split(",")[1]?.trim() : undefined),
-          width: (f.width as number) ?? undefined,
-          height: (f.height as number) ?? undefined,
-          fps: (f.fps as number) ?? undefined,
-          tbr: f.bitrate ? Math.round((f.bitrate as number) / 1000) : undefined,
-          filesize: f.content_length ? parseInt(String(f.content_length), 10) : undefined,
-          format_note: String(f.quality_label ?? f.quality ?? ""),
-          audio_channels: (f.audio_channels as number) ?? undefined,
-          http_headers: {
-            "Origin": "https://www.youtube.com",
-            "Referer": "https://www.youtube.com/",
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36",
-          },
-        });
       } catch {
-        continue;
+        // IOS client failed, continue with page formats
       }
     }
 
     return formats;
   }
 
-  private async fetchPagePlayerResponse(videoId: string): Promise<Record<string, unknown> | null> {
+  private async fetchIosFormats(videoId: string, pageHtml: string, cpn: string): Promise<Format[]> {
+    const iosClient = InnerTubeClient.withClient("IOS");
+    let response: PlayerResponse;
     try {
-      const resp = await fetch(`https://www.youtube.com/watch?v=${videoId}`, {
-        headers: {
-          "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36",
-        },
-      });
-      const html = await resp.text();
-      const match = html.match(/var\s+ytInitialPlayerResponse\s*=\s*(\{.+?\});/s);
-      return match ? JSON.parse(match[1]) : null;
+      response = await iosClient.getPlayerResponse(videoId);
     } catch {
-      return null;
+      const androidClient = InnerTubeClient.withClient("ANDROID");
+      response = await androidClient.getPlayerResponse(videoId);
+    }
+
+    if (response.playabilityStatus?.status !== "OK") return [];
+
+    const allRaw: RawFormat[] = [
+      ...(response.streamingData?.formats ?? []),
+      ...(response.streamingData?.adaptiveFormats ?? []),
+    ];
+
+    const formats: Format[] = [];
+    const IOS_UA = "com.google.ios.youtube/19.45.4 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X;)";
+
+    for (const raw of allRaw) {
+      if (!raw.url) continue;
+      let finalUrl: string;
+      try {
+        const deciphered = await decipherStreamUrl(raw.url, undefined, pageHtml);
+        finalUrl = deciphered ?? raw.url;
+      } catch {
+        finalUrl = raw.url;
+      }
+
+      const parsed = new URL(finalUrl);
+      parsed.searchParams.set("cpn", cpn);
+      const format = this.buildFormat(raw, parsed.toString());
+      format.http_headers = {
+        "User-Agent": IOS_UA,
+        "Origin": "https://www.youtube.com",
+        "Referer": "https://www.youtube.com/",
+      };
+      formats.push(format);
     }
+
+    return formats;
+  }
+
+  private buildFormat(raw: RawFormat, url: string): Format {
+    const mime = raw.mimeType;
+    const mimeMatch = mime.match(/^(video|audio)\/(\w+);\s*codecs="([^"]+)"/);
+    const ext = mimeMatch?.[2] ?? "mp4";
+    const codecs = mimeMatch?.[3] ?? "";
+    const isVideo = mime.startsWith("video");
+    const isAudio = mime.startsWith("audio");
+
+    const format: Format = {
+      format_id: String(raw.itag),
+      url,
+      ext: isAudio && ext === "mp4" ? "m4a" : ext,
+      vcodec: isVideo ? codecs.split(",")[0]?.trim() : "none",
+      acodec: isAudio ? codecs : (isVideo && codecs.includes(",") ? codecs.split(",")[1]?.trim() : undefined),
+      width: raw.width,
+      height: raw.height,
+      fps: raw.fps,
+      tbr: raw.bitrate ? Math.round(raw.bitrate / 1000) : undefined,
+      filesize: raw.contentLength ? parseInt(raw.contentLength, 10) : undefined,
+      format_note: raw.qualityLabel ?? raw.quality ?? undefined,
+      audio_channels: raw.audioChannels,
+      http_headers: {
+        "Origin": "https://www.youtube.com",
+        "Referer": "https://www.youtube.com/",
+        "User-Agent": USER_AGENT,
+      },
+    };
+
+    if (raw.width && raw.height) {
+      format.resolution = `${raw.width}x${raw.height}`;
+    }
+
+    return format;
   }
 }

package/src/extractors/youtube/innertube.ts

@@ -122,6 +122,13 @@ const CLIENTS: Record<string, ClientContext> = {
     apiKey: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
     clientId: 85,
   },
+  IOS: {
+    clientName: "IOS",
+    clientVersion: "19.45.4",
+    userAgent: "com.google.ios.youtube/19.45.4 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X;)",
+    apiKey: "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc",
+    clientId: 5,
+  },
 };
 
 const PLAYER_ENDPOINT = "https://www.youtube.com/youtubei/v1/player";
@@ -130,22 +137,28 @@ const BROWSE_ENDPOINT = "https://www.youtube.com/youtubei/v1/browse";
 export class InnerTubeClient {
   private clientName: string;
   private context: ClientContext;
+  private signatureTimestamp: number;
 
-  constructor(clientName: "WEB" | "ANDROID" | "TVHTML5_EMBED" = "WEB") {
+  constructor(clientName: "WEB" | "ANDROID" | "TVHTML5_EMBED" | "IOS" = "WEB", signatureTimestamp = 20073) {
     this.clientName = clientName;
     this.context = CLIENTS[clientName];
+    this.signatureTimestamp = signatureTimestamp;
   }
 
-  async getPlayerResponse(videoId: string, embedUrl?: string): Promise<PlayerResponse> {
-    const body = this.buildPlayerBody(videoId, embedUrl);
+  async getPlayerResponse(videoId: string, embedUrl?: string, visitorData?: string): Promise<PlayerResponse> {
+    const body = this.buildPlayerBody(videoId, embedUrl, visitorData);
+    const headers: Record<string, string> = {
+      "Content-Type": "application/json",
+      "User-Agent": this.context.userAgent,
+      "X-YouTube-Client-Name": String(this.context.clientId ?? 1),
+      "X-YouTube-Client-Version": this.context.clientVersion,
+    };
+    if (visitorData) {
+      headers["X-Goog-Visitor-Id"] = visitorData;
+    }
     const response = await fetch(`${PLAYER_ENDPOINT}?key=${this.context.apiKey}&prettyPrint=false`, {
       method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        "User-Agent": this.context.userAgent,
-        "X-YouTube-Client-Name": String(this.context.clientId ?? 1),
-        "X-YouTube-Client-Version": this.context.clientVersion,
-      },
+      headers,
       body: JSON.stringify(body),
     });
 
@@ -193,26 +206,37 @@ export class InnerTubeClient {
     return response.json() as Promise<BrowseResponse>;
   }
 
-  private buildPlayerBody(videoId: string, embedUrl?: string): Record<string, unknown> {
+  private buildPlayerBody(videoId: string, embedUrl?: string, visitorData?: string): Record<string, unknown> {
+    const clientContext: Record<string, unknown> = {
+      clientName: this.context.clientName,
+      clientVersion: this.context.clientVersion,
+      hl: "en",
+      gl: "US",
+    };
+    if (visitorData) {
+      clientContext.visitorData = visitorData;
+    }
     const body: Record<string, unknown> = {
       videoId,
       context: {
-        client: {
-          clientName: this.context.clientName,
-          clientVersion: this.context.clientVersion,
-          hl: "en",
-          gl: "US",
-        },
+        client: clientContext,
       },
       playbackContext: {
         contentPlaybackContext: {
-          signatureTimestamp: 20073,
+          signatureTimestamp: this.signatureTimestamp,
         },
       },
       contentCheckOk: true,
       racyCheckOk: true,
     };
 
+    if (this.clientName === "IOS") {
+      clientContext.deviceMake = "Apple";
+      clientContext.deviceModel = "iPhone16,2";
+      clientContext.osName = "iPhone";
+      clientContext.osVersion = "17.5.1.21F90";
+    }
+
     if (this.clientName === "TVHTML5_EMBED" && embedUrl) {
       (body.context as Record<string, unknown>).thirdParty = {
         embedUrl,
@@ -310,8 +334,12 @@ export class InnerTubeClient {
     return { subtitles, automatic_captions };
   }
 
-  static withClient(clientName: "WEB" | "ANDROID" | "TVHTML5_EMBED"): InnerTubeClient {
-    return new InnerTubeClient(clientName);
+  setSignatureTimestamp(sts: number): void {
+    this.signatureTimestamp = sts;
+  }
+
+  static withClient(clientName: "WEB" | "ANDROID" | "TVHTML5_EMBED" | "IOS", signatureTimestamp?: number): InnerTubeClient {
+    return new InnerTubeClient(clientName, signatureTimestamp);
   }
 }