getraw 0.1.0

package/src/extractors/youtube/signature.ts

@@ -0,0 +1,163 @@
+const playerCache = new Map<string, string>();
+const sigFuncCache = new Map<string, (sig: string) => string>();
+
+export async function fetchPlayerJs(playerUrl: string): Promise<string> {
+  const cached = playerCache.get(playerUrl);
+  if (cached) return cached;
+
+  const fullUrl = playerUrl.startsWith("//")
+    ? `https:${playerUrl}`
+    : playerUrl.startsWith("/")
+      ? `https://www.youtube.com${playerUrl}`
+      : playerUrl;
+
+  const response = await fetch(fullUrl);
+  if (!response.ok) {
+    throw new Error(`Failed to fetch player JS: ${response.status}`);
+  }
+
+  const js = await response.text();
+  playerCache.set(playerUrl, js);
+  return js;
+}
+
+export function extractSignatureFunction(playerJs: string): (sig: string) => string {
+  const cached = sigFuncCache.get(playerJs.slice(0, 100));
+  if (cached) return cached;
+
+  const funcNameMatch = playerJs.match(
+    /\b[cs]\s*&&\s*[adf]\.set\([^,]+,\s*encodeURIComponent\(([a-zA-Z0-9$]+)\(/
+  ) ?? playerJs.match(
+    /\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+,\s*encodeURIComponent\(([a-zA-Z0-9$]+)\(/
+  ) ?? playerJs.match(
+    /\bm=([a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)/
+  ) ?? playerJs.match(
+    /\bc\s*&&\s*d\.set\([^,]+,\s*(?:encodeURIComponent\s*\()([a-zA-Z0-9$]+)\(/
+  ) ?? playerJs.match(
+    /\bc\s*&&\s*[a-z]\.set\([^,]+,\s*([a-zA-Z0-9$]+)\(/
+  ) ?? playerJs.match(
+    /\bc\s*&&\s*[a-z]\.set\([^,]+,\s*encodeURIComponent\(([a-zA-Z0-9$]+)\(/
+  );
+
+  if (!funcNameMatch) {
+    throw new Error("Could not find signature function name in player JS");
+  }
+
+  const funcName = funcNameMatch[1];
+  const escapedName = funcName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+  const funcBodyMatch = playerJs.match(
+    new RegExp(`${escapedName}=function\\(a\\)\\{a=a\\.split\\(""\\);([^}]+)\\}`)
+  );
+
+  if (!funcBodyMatch) {
+    throw new Error(`Could not find signature function body for ${funcName}`);
+  }
+
+  const funcBody = funcBodyMatch[1];
+  const operations = parseSignatureOperations(funcBody, playerJs);
+
+  const fn = (sig: string): string => {
+    const arr = sig.split("");
+    for (const op of operations) {
+      op(arr);
+    }
+    return arr.join("");
+  };
+
+  sigFuncCache.set(playerJs.slice(0, 100), fn);
+  return fn;
+}
+
+type SigOperation = (arr: string[]) => void;
+
+function parseSignatureOperations(funcBody: string, playerJs: string): SigOperation[] {
+  const helperMatch = funcBody.match(/([a-zA-Z0-9$]+)\./);
+  if (!helperMatch) return [];
+
+  const helperName = helperMatch[1];
+  const escapedHelper = helperName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+  const helperObjMatch = playerJs.match(
+    new RegExp(`var ${escapedHelper}=\\{([\\s\\S]*?)\\};`)
+  );
+
+  if (!helperObjMatch) return [];
+
+  const helperBody = helperObjMatch[1];
+
+  const methodMap = new Map<string, "reverse" | "splice" | "swap">();
+
+  const methodRegex = /([a-zA-Z0-9$]+):function\(([^)]*)\)\{([^}]+)\}/g;
+  let methodMatch: RegExpExecArray | null;
+  while ((methodMatch = methodRegex.exec(helperBody)) !== null) {
+    const name = methodMatch[1];
+    const body = methodMatch[3];
+
+    if (body.includes("reverse")) {
+      methodMap.set(name, "reverse");
+    } else if (body.includes("splice")) {
+      methodMap.set(name, "splice");
+    } else {
+      methodMap.set(name, "swap");
+    }
+  }
+
+  const operations: SigOperation[] = [];
+  const callRegex = new RegExp(
+    `${escapedHelper}\\.([a-zA-Z0-9$]+)\\(a,(\\d+)\\)`,
+    "g"
+  );
+
+  let callMatch: RegExpExecArray | null;
+  while ((callMatch = callRegex.exec(funcBody)) !== null) {
+    const method = callMatch[1];
+    const arg = parseInt(callMatch[2], 10);
+    const type = methodMap.get(method);
+
+    switch (type) {
+      case "reverse":
+        operations.push((arr) => arr.reverse());
+        break;
+      case "splice":
+        operations.push((arr) => arr.splice(0, arg));
+        break;
+      case "swap":
+        operations.push((arr) => {
+          const idx = arg % arr.length;
+          const tmp = arr[0];
+          arr[0] = arr[idx];
+          arr[idx] = tmp;
+        });
+        break;
+    }
+  }
+
+  return operations;
+}
+
+export function decipherSignatureUrl(
+  signatureCipher: string,
+  playerJs: string,
+): string {
+  const params = new URLSearchParams(signatureCipher);
+  const url = params.get("url");
+  const sig = params.get("s");
+  const sp = params.get("sp") ?? "signature";
+
+  if (!url || !sig) {
+    throw new Error("Missing url or signature in signatureCipher");
+  }
+
+  const sigFunc = extractSignatureFunction(playerJs);
+  const decipheredSig = sigFunc(sig);
+
+  const finalUrl = new URL(url);
+  finalUrl.searchParams.set(sp, decipheredSig);
+  return finalUrl.toString();
+}
+
+export function clearCache(): void {
+  playerCache.clear();
+  sigFuncCache.clear();
+}

package/src/networking/client.ts

@@ -0,0 +1,311 @@
+import { CookieJar } from "./cookies";
+import { getRoundRobinUserAgent } from "./user-agents";
+import { createProxyAgent, type ProxyAgent } from "./proxy";
+
+export interface CacheEntry {
+  body: string;
+  headers: Record<string, string>;
+  status: number;
+  expiresAt: number;
+}
+
+export interface RateLimiter {
+  requestsPerSecond: number;
+  lastRequestTime: number;
+  queue: Array<() => void>;
+}
+
+export interface HttpClientOptions {
+  userAgent?: string;
+  rotateUserAgent?: boolean;
+  referer?: string;
+  timeout?: number;
+  maxRetries?: number;
+  retryDelay?: number;
+  maxRedirects?: number;
+  requestsPerSecond?: number;
+  cacheTtl?: number;
+  cookieJar?: CookieJar;
+  proxyUrl?: string;
+  headers?: Record<string, string>;
+}
+
+export interface RequestOptions {
+  method?: string;
+  headers?: Record<string, string>;
+  body?: string | Uint8Array;
+  timeout?: number;
+  maxRetries?: number;
+  noCache?: boolean;
+  followRedirects?: boolean;
+}
+
+export interface HttpResponse {
+  status: number;
+  headers: Record<string, string>;
+  body: string;
+  url: string;
+  ok: boolean;
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function isRetryable(status: number): boolean {
+  return status === 429 || status === 502 || status === 503 || status === 504;
+}
+
+export class HttpClient {
+  private options: Required<HttpClientOptions>;
+  private cache: Map<string, CacheEntry> = new Map();
+  private rateLimiter: RateLimiter;
+  private proxyAgent: ProxyAgent | undefined;
+
+  constructor(options: HttpClientOptions = {}) {
+    this.options = {
+      userAgent: options.userAgent ?? getRoundRobinUserAgent(),
+      rotateUserAgent: options.rotateUserAgent ?? false,
+      referer: options.referer ?? "",
+      timeout: options.timeout ?? 30_000,
+      maxRetries: options.maxRetries ?? 3,
+      retryDelay: options.retryDelay ?? 1_000,
+      maxRedirects: options.maxRedirects ?? 10,
+      requestsPerSecond: options.requestsPerSecond ?? 0,
+      cacheTtl: options.cacheTtl ?? 0,
+      cookieJar: options.cookieJar ?? new CookieJar(),
+      proxyUrl: options.proxyUrl ?? "",
+      headers: options.headers ?? {},
+    };
+
+    this.rateLimiter = {
+      requestsPerSecond: this.options.requestsPerSecond,
+      lastRequestTime: 0,
+      queue: [],
+    };
+
+    if (this.options.proxyUrl) {
+      this.proxyAgent = createProxyAgent(this.options.proxyUrl);
+    }
+  }
+
+  private async waitForRateLimit(): Promise<void> {
+    if (this.rateLimiter.requestsPerSecond <= 0) return;
+    const minInterval = 1000 / this.rateLimiter.requestsPerSecond;
+    const now = Date.now();
+    const elapsed = now - this.rateLimiter.lastRequestTime;
+    if (elapsed < minInterval) {
+      await sleep(minInterval - elapsed);
+    }
+    this.rateLimiter.lastRequestTime = Date.now();
+  }
+
+  private getCacheKey(url: string, method: string): string {
+    return `${method}:${url}`;
+  }
+
+  private getFromCache(key: string): CacheEntry | undefined {
+    const entry = this.cache.get(key);
+    if (!entry) return undefined;
+    if (Date.now() > entry.expiresAt) {
+      this.cache.delete(key);
+      return undefined;
+    }
+    return entry;
+  }
+
+  private setCache(key: string, entry: Omit<CacheEntry, "expiresAt">): void {
+    if (this.options.cacheTtl <= 0) return;
+    this.cache.set(key, {
+      ...entry,
+      expiresAt: Date.now() + this.options.cacheTtl * 1000,
+    });
+  }
+
+  private buildHeaders(
+    url: string,
+    extra: Record<string, string> = {},
+  ): Record<string, string> {
+    const ua = this.options.rotateUserAgent
+      ? getRoundRobinUserAgent()
+      : this.options.userAgent;
+
+    const headers: Record<string, string> = {
+      "User-Agent": ua,
+      ...this.options.headers,
+      ...extra,
+    };
+
+    if (this.options.referer) {
+      headers["Referer"] = this.options.referer;
+    }
+
+    const cookieHeader = this.options.cookieJar.getCookieHeader(url);
+    if (cookieHeader) {
+      headers["Cookie"] = cookieHeader;
+    }
+
+    if (this.proxyAgent) {
+      const authHeader = this.proxyAgent.getAuthHeader();
+      if (authHeader) {
+        headers["Proxy-Authorization"] = authHeader;
+      }
+    }
+
+    return headers;
+  }
+
+  async request(url: string, opts: RequestOptions = {}): Promise<HttpResponse> {
+    const method = opts.method ?? "GET";
+    const cacheKey = this.getCacheKey(url, method);
+
+    if (!opts.noCache && method === "GET" && this.options.cacheTtl > 0) {
+      const cached = this.getFromCache(cacheKey);
+      if (cached) {
+        return {
+          status: cached.status,
+          headers: cached.headers,
+          body: cached.body,
+          url,
+          ok: cached.status >= 200 && cached.status < 300,
+        };
+      }
+    }
+
+    await this.waitForRateLimit();
+
+    const timeout = opts.timeout ?? this.options.timeout;
+    const maxRetries = opts.maxRetries ?? this.options.maxRetries;
+
+    let redirectUrl = url;
+    let redirectCount = 0;
+
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+      if (attempt > 0) {
+        const delay = this.options.retryDelay * Math.pow(2, attempt - 1);
+        await sleep(delay);
+        await this.waitForRateLimit();
+      }
+
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), timeout);
+
+      try {
+        const fetchOpts: RequestInit = {
+          method,
+          headers: this.buildHeaders(redirectUrl, opts.headers ?? {}),
+          signal: controller.signal,
+          redirect: "manual",
+        };
+
+        if (opts.body) {
+          fetchOpts.body = opts.body;
+        }
+
+        const response = await fetch(redirectUrl, fetchOpts);
+        clearTimeout(timer);
+
+        const isRedirect =
+          response.status >= 300 &&
+          response.status < 400 &&
+          response.headers.get("location");
+
+        if (
+          isRedirect &&
+          (opts.followRedirects !== false) &&
+          redirectCount < this.options.maxRedirects
+        ) {
+          const location = response.headers.get("location")!;
+          redirectUrl = new URL(location, redirectUrl).toString();
+          redirectCount++;
+          attempt = -1;
+          continue;
+        }
+
+        if (isRetryable(response.status) && attempt < maxRetries) {
+          const retryAfter = response.headers.get("retry-after");
+          if (retryAfter) {
+            await sleep(parseInt(retryAfter, 10) * 1000);
+          }
+          continue;
+        }
+
+        const body = await response.text();
+        const headers: Record<string, string> = {};
+        response.headers.forEach((value, key) => {
+          headers[key] = value;
+        });
+
+        const result: HttpResponse = {
+          status: response.status,
+          headers,
+          body,
+          url: redirectUrl,
+          ok: response.status >= 200 && response.status < 300,
+        };
+
+        if (method === "GET" && result.ok) {
+          this.setCache(cacheKey, {
+            body: result.body,
+            headers: result.headers,
+            status: result.status,
+          });
+        }
+
+        return result;
+      } catch (err) {
+        clearTimeout(timer);
+        if (attempt === maxRetries) {
+          throw new Error(
+            `HTTP request failed after ${maxRetries + 1} attempts: ${err instanceof Error ? err.message : String(err)}`,
+          );
+        }
+      }
+    }
+
+    throw new Error(`HTTP request failed: exhausted retries for ${url}`);
+  }
+
+  async get(url: string, opts: Omit<RequestOptions, "method"> = {}): Promise<HttpResponse> {
+    return this.request(url, { ...opts, method: "GET" });
+  }
+
+  async post(
+    url: string,
+    body: string | Uint8Array,
+    opts: Omit<RequestOptions, "method" | "body"> = {},
+  ): Promise<HttpResponse> {
+    return this.request(url, { ...opts, method: "POST", body });
+  }
+
+  async getJson<T>(url: string, opts: Omit<RequestOptions, "method"> = {}): Promise<T> {
+    const res = await this.get(url, {
+      ...opts,
+      headers: { Accept: "application/json", ...opts.headers },
+    });
+    if (!res.ok) {
+      throw new Error(`HTTP ${res.status} fetching JSON from ${url}`);
+    }
+    return JSON.parse(res.body) as T;
+  }
+
+  async getText(url: string, opts: Omit<RequestOptions, "method"> = {}): Promise<string> {
+    const res = await this.get(url, opts);
+    if (!res.ok) {
+      throw new Error(`HTTP ${res.status} fetching text from ${url}`);
+    }
+    return res.body;
+  }
+
+  clearCache(): void {
+    this.cache.clear();
+  }
+
+  setCookieJar(jar: CookieJar): void {
+    this.options.cookieJar = jar;
+  }
+
+  getCookieJar(): CookieJar {
+    return this.options.cookieJar;
+  }
+}

package/src/networking/cookies.ts

@@ -0,0 +1,138 @@
+export interface Cookie {
+  domain: string;
+  includeSubdomains: boolean;
+  path: string;
+  secure: boolean;
+  expires: number;
+  name: string;
+  value: string;
+}
+
+export interface BrowserCookieExtractor {
+  browser: string;
+  extract(domain?: string): Promise<Cookie[]>;
+}
+
+function parseBool(val: string): boolean {
+  return val.toUpperCase() === "TRUE";
+}
+
+export function parseNetscapeCookieFile(content: string): Cookie[] {
+  const cookies: Cookie[] = [];
+  for (const raw of content.split("\n")) {
+    const line = raw.trim();
+    if (!line || line.startsWith("#")) continue;
+    const parts = line.split("\t");
+    if (parts.length < 7) continue;
+    const [domain, includeSubdomains, path, secure, expires, name, value] =
+      parts;
+    cookies.push({
+      domain,
+      includeSubdomains: parseBool(includeSubdomains),
+      path,
+      secure: parseBool(secure),
+      expires: parseInt(expires, 10),
+      name,
+      value: value ?? "",
+    });
+  }
+  return cookies;
+}
+
+export function serializeNetscapeCookieFile(cookies: Cookie[]): string {
+  const header = "# Netscape HTTP Cookie File\n# Generated by dlpx\n";
+  const lines = cookies.map((c) =>
+    [
+      c.domain,
+      c.includeSubdomains ? "TRUE" : "FALSE",
+      c.path,
+      c.secure ? "TRUE" : "FALSE",
+      c.expires,
+      c.name,
+      c.value,
+    ].join("\t"),
+  );
+  return header + lines.join("\n") + "\n";
+}
+
+function domainMatches(cookieDomain: string, requestHost: string): boolean {
+  const cd = cookieDomain.startsWith(".") ? cookieDomain.slice(1) : cookieDomain;
+  return requestHost === cd || requestHost.endsWith(`.${cd}`);
+}
+
+export class CookieJar {
+  private cookies: Cookie[] = [];
+
+  load(content: string): void {
+    const parsed = parseNetscapeCookieFile(content);
+    for (const c of parsed) {
+      this.set(c);
+    }
+  }
+
+  set(cookie: Cookie): void {
+    const idx = this.cookies.findIndex(
+      (c) =>
+        c.domain === cookie.domain &&
+        c.path === cookie.path &&
+        c.name === cookie.name,
+    );
+    if (idx >= 0) {
+      this.cookies[idx] = cookie;
+    } else {
+      this.cookies.push(cookie);
+    }
+  }
+
+  get(domain: string, path: string, name: string): Cookie | undefined {
+    const now = Math.floor(Date.now() / 1000);
+    return this.cookies.find(
+      (c) =>
+        c.name === name &&
+        c.path === path &&
+        domainMatches(c.domain, domain) &&
+        (c.expires === 0 || c.expires > now),
+    );
+  }
+
+  getForUrl(url: string): Cookie[] {
+    const now = Math.floor(Date.now() / 1000);
+    const parsed = new URL(url);
+    const host = parsed.hostname;
+    const path = parsed.pathname;
+    const isSecure = parsed.protocol === "https:";
+
+    return this.cookies.filter(
+      (c) =>
+        domainMatches(c.domain, host) &&
+        path.startsWith(c.path) &&
+        (!c.secure || isSecure) &&
+        (c.expires === 0 || c.expires > now),
+    );
+  }
+
+  getCookieHeader(url: string): string {
+    return this.getForUrl(url)
+      .map((c) => `${c.name}=${c.value}`)
+      .join("; ");
+  }
+
+  removeExpired(): void {
+    const now = Math.floor(Date.now() / 1000);
+    this.cookies = this.cookies.filter(
+      (c) => c.expires === 0 || c.expires > now,
+    );
+  }
+
+  serialize(): string {
+    return serializeNetscapeCookieFile(this.cookies);
+  }
+
+  size(): number {
+    return this.cookies.length;
+  }
+
+  clear(): void {
+    this.cookies = [];
+  }
+}