getraw 0.2.2 → 0.3.0

package/src/extractors/youtube/player.ts

@@ -0,0 +1,139 @@
+import { analyzePlayerJs, getNsigProcessorFn } from "./js-analyzer";
+import type { PlayerScriptResult } from "./js-analyzer";
+
+const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36";
+
+let cachedPlayerUrl: string | null = null;
+let cachedScript: PlayerScriptResult | null = null;
+
+async function getPlayerUrl(pageHtml?: string): Promise<string> {
+  if (cachedPlayerUrl) return cachedPlayerUrl;
+
+  if (pageHtml) {
+    const match = pageHtml.match(/\/s\/player\/([a-zA-Z0-9_-]+)\/[^"]+?base\.js/);
+    if (match) {
+      cachedPlayerUrl = `https://www.youtube.com${match[0]}`;
+      return cachedPlayerUrl;
+    }
+  }
+
+  const resp = await fetch("https://www.youtube.com/iframe_api", {
+    headers: { "User-Agent": USER_AGENT },
+  });
+  if (!resp.ok) throw new Error(`Failed to fetch iframe_api: ${resp.status}`);
+  const text = await resp.text();
+  const match = text.match(/player\\\/([a-zA-Z0-9_-]+)\\\//);
+  if (!match) throw new Error("Could not extract player ID from iframe_api");
+  cachedPlayerUrl = `https://www.youtube.com/s/player/${match[1]}/player_ias.vflset/en_US/base.js`;
+  return cachedPlayerUrl;
+}
+
+async function getPlayerScript(pageHtml?: string): Promise<PlayerScriptResult> {
+  if (cachedScript) return cachedScript;
+
+  const playerUrl = await getPlayerUrl(pageHtml);
+  const resp = await fetch(playerUrl, {
+    headers: { "User-Agent": USER_AGENT },
+  });
+  if (!resp.ok) throw new Error(`Failed to fetch player JS: ${resp.status}`);
+  const playerJs = await resp.text();
+
+  cachedScript = analyzePlayerJs(playerJs);
+  return cachedScript;
+}
+
+export async function getSignatureTimestamp(pageHtml?: string): Promise<number> {
+  const script = await getPlayerScript(pageHtml);
+  return script.signatureTimestamp;
+}
+
+async function evalPlayerScript(
+  script: PlayerScriptResult,
+  n?: string,
+  sp?: string,
+  s?: string,
+): Promise<{ sig?: string; n?: string }> {
+  const code = `${script.output}\n${getNsigProcessorFn(n, sp, s)}`;
+  const fn = new Function(code);
+  const result: unknown = fn();
+  if (typeof result !== "object" || result === null) {
+    throw new Error("Got invalid result from player script evaluation");
+  }
+  return result as { sig?: string; n?: string };
+}
+
+export async function decipherStreamUrl(
+  rawUrl: string | undefined,
+  signatureCipher: string | undefined,
+  pageHtml?: string,
+): Promise<string | null> {
+  if (!rawUrl && !signatureCipher) return null;
+
+  const script = await getPlayerScript(pageHtml);
+
+  let urlString: string;
+  let sig: string | undefined;
+  let sp: string | undefined;
+
+  if (signatureCipher) {
+    const params = new URLSearchParams(signatureCipher);
+    urlString = params.get("url") ?? "";
+    sig = params.get("s") ?? undefined;
+    sp = params.get("sp") ?? "signature";
+    if (!urlString) return null;
+  } else if (rawUrl) {
+    urlString = rawUrl;
+  } else {
+    return null;
+  }
+
+  const urlObj = new URL(urlString);
+  const n = urlObj.searchParams.get("n") ?? undefined;
+
+  const needsEval = sig !== undefined || n !== undefined;
+
+  if (needsEval && script.hasNsigFunction) {
+    const result = await evalPlayerScript(script, n, sp, sig);
+
+    if (typeof result.sig === "string" && sp) {
+      urlObj.searchParams.set(sp, result.sig);
+    }
+
+    if (typeof result.n === "string") {
+      if (!result.n.startsWith("enhanced_except_")) {
+        urlObj.searchParams.set("n", result.n);
+      }
+    }
+  }
+
+  const client = urlObj.searchParams.get("c");
+  const CLIENT_VERSIONS: Record<string, string> = {
+    WEB: "2.20250615.01.00",
+    MWEB: "2.20250614.01.00",
+    WEB_REMIX: "1.20250611.01.00",
+    WEB_KIDS: "2.20250612.00.00",
+    TVHTML5: "7.20250612.16.00",
+    TVHTML5_SIMPLY: "2.0",
+    TVHTML5_SIMPLY_EMBEDDED_PLAYER: "2.0",
+    WEB_EMBEDDED_PLAYER: "2.20250613.01.00",
+  };
+  if (client && CLIENT_VERSIONS[client]) {
+    urlObj.searchParams.set("cver", CLIENT_VERSIONS[client]);
+  }
+
+  return urlObj.toString();
+}
+
+export function clearPlayerCache(): void {
+  cachedPlayerUrl = null;
+  cachedScript = null;
+}
+
+export function setPageHtmlForPlayerExtraction(html: string): void {
+  if (!cachedPlayerUrl) {
+    const match = html.match(/\/s\/player\/([a-zA-Z0-9_-]+)\/[^"]+?base\.js/);
+    if (match) {
+      cachedPlayerUrl = `https://www.youtube.com${match[0]}`;
+    }
+  }
+}

package/src/extractors/youtube/sabr-download.ts

@@ -0,0 +1,155 @@
+import { SabrStream } from "googlevideo/sabr-stream";
+import { buildSabrFormat } from "googlevideo/utils";
+import { Innertube, Platform } from "youtubei.js";
+import type { Format } from "../../core/types";
+import { writeFileSync, appendFileSync, unlinkSync, existsSync } from "fs";
+
+// Bun-native eval for youtubei.js player script execution
+Platform.shim.eval = async (data: { output: string }) => {
+  const code = data.output + "\nreturn { ...exportedVars };";
+  return new Function(code)();
+};
+
+let _yt: Awaited<ReturnType<typeof Innertube.create>> | null = null;
+
+async function getYt() {
+  if (!_yt) {
+    _yt = await Innertube.create();
+  }
+  return _yt;
+}
+
+export interface SabrDownloadResult {
+  videoPath: string | null;
+  audioPath: string | null;
+  selectedVideoItag: number | null;
+  selectedAudioItag: number | null;
+}
+
+export async function downloadViaSabr(
+  videoId: string,
+  outputPath: string,
+  options: {
+    videoQuality?: string;
+    onProgress?: (bytes: number, type: "video" | "audio") => void;
+  } = {},
+): Promise<SabrDownloadResult> {
+  const yt = await getYt();
+  const info = await yt.getInfo(videoId);
+
+  const streamingUrl = await yt.session.player?.decipher(
+    info.streaming_data?.server_abr_streaming_url,
+  );
+
+  const uConfig =
+    (info as unknown as Record<string, unknown>).page?.[0]?.player_config
+      ?.media_common_config?.media_ustreamer_request_config
+      ?.video_playback_ustreamer_config;
+
+  if (!streamingUrl) {
+    throw new Error("No SABR streaming URL available");
+  }
+
+  const sabrFormats =
+    info.streaming_data?.adaptive_formats?.map((f: unknown) =>
+      buildSabrFormat(f),
+    ) ?? [];
+
+  if (sabrFormats.length === 0) {
+    throw new Error("No adaptive formats available for SABR");
+  }
+
+  const sabr = new SabrStream({
+    formats: sabrFormats,
+    serverAbrStreamingUrl: streamingUrl,
+    videoPlaybackUstreamerConfig: uConfig,
+    durationMs: (info.basic_info.duration ?? 0) * 1000,
+    clientInfo: {
+      clientName: 1,
+      clientVersion: yt.session.context.client.clientVersion,
+    },
+  });
+
+  const { videoStream, audioStream, selectedFormats } = await sabr.start({
+    videoQuality: options.videoQuality ?? "720p",
+    audioQuality: "AUDIO_QUALITY_MEDIUM",
+  });
+
+  const videoPath = outputPath.replace(/\.[^.]+$/, ".video.mp4");
+  const audioPath = outputPath.replace(/\.[^.]+$/, ".audio.m4a");
+
+  // Download video stream
+  if (existsSync(videoPath)) unlinkSync(videoPath);
+  const videoReader = videoStream.getReader();
+  let videoBytes = 0;
+  while (true) {
+    const { done, value } = await videoReader.read();
+    if (done) break;
+    appendFileSync(videoPath, value);
+    videoBytes += value.byteLength;
+    options.onProgress?.(videoBytes, "video");
+  }
+
+  // Download audio stream
+  if (existsSync(audioPath)) unlinkSync(audioPath);
+  const audioReader = audioStream.getReader();
+  let audioBytes = 0;
+  while (true) {
+    const { done, value } = await audioReader.read();
+    if (done) break;
+    appendFileSync(audioPath, value);
+    audioBytes += value.byteLength;
+    options.onProgress?.(audioBytes, "audio");
+  }
+
+  return {
+    videoPath: videoBytes > 0 ? videoPath : null,
+    audioPath: audioBytes > 0 ? audioPath : null,
+    selectedVideoItag: selectedFormats.videoFormat?.itag ?? null,
+    selectedAudioItag: selectedFormats.audioFormat?.itag ?? null,
+  };
+}
+
+export async function getSabrFormats(videoId: string): Promise<Format[]> {
+  const yt = await getYt();
+  const info = await yt.getInfo(videoId);
+
+  const adaptiveFormats = info.streaming_data?.adaptive_formats ?? [];
+  const formats: Format[] = [];
+
+  for (const f of adaptiveFormats) {
+    const raw = f as unknown as Record<string, unknown>;
+    const mime = String(raw.mime_type ?? "");
+    const mimeMatch = mime.match(/^(video|audio)\/(\w+);\s*codecs="([^"]+)"/);
+    const ext = mimeMatch?.[2] ?? "mp4";
+    const codecs = mimeMatch?.[3] ?? "";
+    const isVideo = mime.startsWith("video");
+    const isAudio = mime.startsWith("audio");
+
+    formats.push({
+      format_id: String(raw.itag ?? ""),
+      url: "sabr://requires-sabr-download",
+      ext: isAudio && ext === "mp4" ? "m4a" : ext,
+      vcodec: isVideo ? codecs.split(",")[0]?.trim() : "none",
+      acodec: isAudio
+        ? codecs
+        : isVideo && codecs.includes(",")
+          ? codecs.split(",")[1]?.trim()
+          : undefined,
+      width: (raw.width as number) ?? undefined,
+      height: (raw.height as number) ?? undefined,
+      fps: (raw.fps as number) ?? undefined,
+      tbr: raw.bitrate
+        ? Math.round((raw.bitrate as number) / 1000)
+        : undefined,
+      filesize: raw.content_length
+        ? parseInt(String(raw.content_length), 10)
+        : undefined,
+      format_note: String(raw.quality_label ?? raw.quality ?? ""),
+      audio_channels: (raw.audio_channels as number) ?? undefined,
+      protocol: "sabr",
+    });
+  }
+
+  return formats;
+}

package/scripts/patch-youtubei.js

@@ -1,14 +0,0 @@
-#!/usr/bin/env node
-// Patches youtubei.js to use Bun-native JS evaluation instead of the default stub
-import { writeFileSync } from "fs";
-import { resolve } from "path";
-
-const evalPath = resolve("node_modules/youtubei.js/dist/src/platform/jsruntime/default.js");
-const evalCode = `export default async function evaluate(data) {
-  const fn = new Function(data.output);
-  return fn();
-}
-`;
-
-writeFileSync(evalPath, evalCode);
-console.log("Patched youtubei.js jsruntime for Bun-native evaluation");

package/video/.hyperframes/expanded-prompt.md

@@ -1,173 +0,0 @@
-# getraw — Product Video Expansion
-
-## Style Block
-
-- **BG:** #0c0c0f (near-black, blue undertone)
-- **FG:** #e8e8ec (off-white)
-- **Accent:** #00ff88 (terminal green)
-- **Surface:** #161620
-- **Muted:** #6b6b80
-- **Headline:** Space Grotesk 700, -0.03em tracking
-- **Code:** JetBrains Mono 400
-- **Labels:** JetBrains Mono 500 uppercase, 0.08em tracking
-
-## Rhythm Declaration
-
-`hook-PUNCH-breathe-BUILD-PEAK-CTA`
-
-6 scenes, ~30 seconds total. Fast hook, slam the name, breathe with the problem, build features, peak with the stats, close with install command.
-
-## Global Rules
-
-- Primary transition: glitch (0.3s) — 60% of transitions
-- Accent transition: staggered blocks (0.25s) — topic changes
-- All decoratives have ambient motion (scan-line drift, grid pulse, cursor blink)
-- Terminal green (#00ff88) used ONLY for: commands, active highlights, key stats
-- Everything else in off-white or muted
-
----
-
-## Scene 1: Hook (0s–5s)
-
-**Concept:** A terminal cursor blinks on black. A command types out character by character: `$ getraw "https://youtube.com/watch?v=..."`. The moment Enter is hit, the screen EXPLODES with data — format listings cascading down like a matrix waterfall. The viewer's reaction: "what is this tool?"
-
-**Mood:** Cinematic hacker. The opening frame of a tech thriller.
-
-**Depth layers:**
-- BG: #0c0c0f solid + subtle scan-line overlay at 8% opacity drifting upward + faint grid dots at 5% pulsing
-- MG: Terminal text, typing animation, cursor blink
-- FG: Faint terminal frame border (1px #2a2a3a), timestamp label top-right "v0.1.0"
-
-**Animation choreography:**
-- Cursor BLINKS twice (0.5s interval) at t=0.2
-- Command TYPES character-by-character at 40ms/char starting t=0.8
-- On "enter" at t=3.0: format data CASCADES in from top, staggered 30ms per line
-- Scan-lines drift upward continuously
-
-**Transition out:** Glitch, 0.3s, power3.inOut
-
----
-
-## Scene 2: Name Drop (5s–10s)
-
-**Concept:** The word "getraw" SLAMS into frame at massive scale — 160px, terminal green, filling 70% of the width. Below it, a subtitle types on: "yt-dlp replacement. Built in Bun." The green glows softly, casting light on the dark surface. This is the brand moment.
-
-**Mood:** Impact. Like a logo sting but for a CLI tool.
-
-**Depth layers:**
-- BG: Radial glow from center (#00ff88 at 12% opacity, scale breathing 1.0→1.05) + ghost text "MEDIA DOWNLOADER" at 4% opacity, 200px, behind the title
-- MG: "getraw" headline + subtitle line
-- FG: Two horizontal rules (top and bottom of frame, #2a2a3a, scaleX from 0), version badge "v0.1.0" bottom-right
-
-**Animation choreography:**
-- "getraw" SLAMS from y:80 with scale overshoot (1.05→1.0), expo.out, 0.5s at t=0.3
-- Subtitle TYPES on character-by-character at t=1.2, JetBrains Mono, muted color
-- Horizontal rules DRAW from center outward (scaleX: 0→1) at t=0.5, 0.6s
-- Radial glow BREATHES continuously (scale 1.0↔1.05, 3s loop)
-
-**Transition out:** Staggered blocks, 0.25s — signals topic change
-
----
-
-## Scene 3: The Problem (10s–15s)
-
-**Concept:** Split frame. Left side: "yt-dlp" with a list of pain points stacking up — "Python dependency", "External JS runtime", "36K line interpreter", "Slow startup". Each appears with a red-ish muted strike. Right side: blank, waiting. The viewer feels the weight of the problem before the solution appears.
-
-**Mood:** Tension. Editorial comparison. The "before" that makes the "after" land.
-
-**Depth layers:**
-- BG: Subtle vertical divider line at center (1px #2a2a3a) + grid dots left half at 5%
-- MG: "yt-dlp" label top-left + stacking pain points + right side empty space
-- FG: Small "THE PROBLEM" label top-center in muted, monospace uppercase
-
-**Animation choreography:**
-- "THE PROBLEM" label FADES in at t=0.2, subtle
-- "yt-dlp" SLIDES in from left at t=0.4
-- Pain points STACK one by one, each DROPPING from y:-20 with stagger 0.3s starting t=0.8
-- Each pain point gets a subtle strikethrough line that DRAWS across after landing
-- Right side stays intentionally empty — tension
-
-**Transition out:** Glitch, 0.3s
-
----
-
-## Scene 4: The Solution (15s–21s)
-
-**Concept:** The empty right side from scene 3 is now the full frame. "getraw" in green, and below it, three feature cards STAGGER in: "Native JS execution", "30+ site extractors", "Bun-powered CLI". Each card has a small icon-like label and a one-liner. Clean, confident, no clutter. The solution is elegant.
-
-**Mood:** Confidence. Clean resolve after tension.
-
-**Depth layers:**
-- BG: Radial glow bottom-left (#00ff88 at 10%) + faint circuit-board pattern at 3% opacity
-- MG: "getraw" label + three feature cards in surface color (#161620) with border
-- FG: Small arrow indicators (→) next to each card, accent green, appearing with stagger
-
-**Animation choreography:**
-- "getraw" SLIDES in from left at t=0.2, accent green, smaller (64px)
-- Feature cards CASCADE from right, stagger 0.15s, each from x:40 + opacity:0, expo.out
-- Arrow indicators POP in with scale overshoot 0.15s after their card lands
-- Circuit pattern DRIFTS slowly rightward
-
-**Transition out:** Glitch, 0.3s
-
----
-
-## Scene 5: Stats (21s–26s)
-
-**Concept:** Three big numbers SLAM in simultaneously: "30+" sites, "386" tests, "50ms" startup. Each in massive type (120px), terminal green. Below each number, a label in muted monospace. This is the proof. Numbers don't lie.
-
-**Mood:** Peak energy. Data as spectacle.
-
-**Depth layers:**
-- BG: Three vertical accent lines (#00ff88 at 8%) behind each stat, full height, subtle pulse
-- MG: Three stat columns with numbers + labels
-- FG: "BENCHMARKS" label top-center, monospace uppercase muted + scan-line overlay intensified to 12%
-
-**Animation choreography:**
-- "BENCHMARKS" label FADES in at t=0.1
-- All three numbers SLAM simultaneously from y:60, scale:1.1→1.0, expo.out, 0.4s at t=0.3
-- Labels TYPE on below each number, stagger 0.1s starting t=0.8
-- Accent lines PULSE once on number impact (opacity 8%→15%→8%, 0.6s)
-
-**Transition out:** Staggered blocks, 0.3s — wind down
-
----
-
-## Scene 6: CTA / Install (26s–30s)
-
-**Concept:** Back to terminal. A clean command prompt: `$ bun install -g getraw`. Below it, the npm badge and GitHub link. The cursor blinks at the end. Simple. The viewer knows exactly what to do next.
-
-**Mood:** Resolution. Clear call to action. The cursor blink is the mic drop.
-
-**Depth layers:**
-- BG: Subtle scan-lines returning + faint radial glow center (#00ff88 at 6%)
-- MG: Install command + npm/GitHub info
-- FG: Terminal frame border, "getraw.dev" bottom-center (aspirational), cursor blinking
-
-**Animation choreography:**
-- Terminal frame DRAWS in (border animation, 0.4s) at t=0.1
-- Command TYPES on at t=0.5, 50ms/char, green text
-- npm line FADES in at t=2.0, muted
-- GitHub URL FADES in at t=2.3, muted
-- Cursor BLINKS indefinitely (well, 4 blinks with calculated repeat count)
-
-**Transition out:** Final scene — elements fade to black over 0.8s. Last thing visible: the blinking cursor.
-
----
-
-## Recurring Motifs
-
-- Terminal cursor blink (scenes 1, 6)
-- Scan-line overlay (all scenes, varying intensity)
-- Character-by-character typing (scenes 1, 2, 6)
-- Accent green used only for active/important elements
-
-## Negative Prompt
-
-- No gradient text
-- No cyan or purple — green only
-- No rounded corners > 8px
-- No web-UI card shadows
-- No centered-everything layouts
-- No Inter, Roboto, or banned fonts
-- No pure #000 or #fff

package/video/design.md

@@ -1,82 +0,0 @@
----
-name: getraw
-colors:
-  primary: "#0c0c0f"
-  on-primary: "#e8e8ec"
-  accent: "#00ff88"
-  accent-dim: "#00cc6a"
-  surface: "#161620"
-  surface-border: "#2a2a3a"
-  muted: "#6b6b80"
-typography:
-  headline:
-    fontFamily: Space Grotesk
-    fontSize: 5rem
-    fontWeight: 700
-    letterSpacing: -0.03em
-  body:
-    fontFamily: JetBrains Mono
-    fontSize: 1.5rem
-    fontWeight: 400
-  label:
-    fontFamily: JetBrains Mono
-    fontSize: 0.875rem
-    fontWeight: 500
-    textTransform: uppercase
-    letterSpacing: 0.08em
-rounded:
-  none: 0px
-  sm: 4px
-  md: 8px
-spacing:
-  sm: 8px
-  md: 16px
-  lg: 40px
-motion:
-  energy: high
-  easing:
-    entry: "expo.out"
-    exit: "power3.in"
-    ambient: "sine.inOut"
-  duration:
-    entrance: 0.4
-    hold: 1.8
-    transition: 0.4
-  atmosphere:
-    - terminal-cursor
-    - scan-lines
-    - grid-dots
-  transition: glitch
----
-
-## Overview
-
-getraw is a fast media downloader CLI built in Bun/TypeScript — a yt-dlp replacement with native JS execution. The visual identity is terminal-native: dark background, green accent (terminal green), monospace type for code, and sharp geometric motion. Feels like watching a hacker tool come alive.
-
-## Colors
-
-- **Primary (#0c0c0f):** Near-black background with slight blue undertone
-- **On-primary (#e8e8ec):** Off-white text, not pure white
-- **Accent (#00ff88):** Terminal green — the signature color. Used for highlights, commands, active states
-- **Surface (#161620):** Slightly elevated panels, code blocks
-- **Muted (#6b6b80):** Comments, secondary info
-
-## Typography
-
-- **Headlines:** Space Grotesk 700 — geometric, techy, not generic
-- **Code/Body:** JetBrains Mono 400 — the developer's monospace
-- **Labels:** JetBrains Mono 500 uppercase — structural metadata
-
-## Do's and Don'ts
-
-### Do
-- Use terminal-style animations (typing effect, cursor blink, line-by-line reveal)
-- Show real CLI commands and output
-- Use the green accent sparingly but boldly
-- Let the dark background breathe
-
-### Don't
-- Use gradients on text
-- Use rounded corners larger than 8px
-- Use decorative serif fonts
-- Use bright colors other than the accent green