getraw 0.1.0

package/src/extractors/bilibili/index.ts

@@ -0,0 +1,233 @@
+import { BaseExtractor, ExtractorError } from "../../core/types";
+import type { InfoDict, Format, Thumbnail } from "../../core/types";
+
+const VALID_URL = /https?:\/\/(?:www\.)?bilibili\.com\/video\/(BV[\w]+|av(\d+))/;
+
+const QUALITY_MAP: Record<number, string> = {
+  127: "8K",
+  126: "Dolby Vision",
+  125: "HDR",
+  120: "4K",
+  116: "1080p60",
+  112: "1080p+",
+  80: "1080p",
+  64: "720p",
+  32: "480p",
+  16: "360p",
+};
+
+const QUALITY_PREFERENCE: Record<number, number> = {
+  127: 10,
+  126: 9,
+  125: 8,
+  120: 7,
+  116: 6,
+  112: 5,
+  80: 4,
+  64: 3,
+  32: 2,
+  16: 1,
+};
+
+interface VideoViewData {
+  code: number;
+  message: string;
+  data: {
+    bvid: string;
+    aid: number;
+    cid: number;
+    title: string;
+    desc: string;
+    owner: { name: string; mid: number };
+    stat: { view: number; like: number; coin: number; reply: number };
+    pic: string;
+    duration: number;
+    pubdate: number;
+    pages?: Array<{ cid: number; page: number; part: string; duration: number }>;
+  };
+}
+
+interface DashStream {
+  id: number;
+  baseUrl: string;
+  base_url: string;
+  backupUrl?: string[];
+  backup_url?: string[];
+  bandwidth: number;
+  mimeType: string;
+  mime_type: string;
+  codecs: string;
+  width?: number;
+  height?: number;
+  frameRate?: string;
+  frame_rate?: string;
+  sar?: string;
+}
+
+interface PlayUrlData {
+  code: number;
+  message: string;
+  data: {
+    quality: number;
+    accept_quality: number[];
+    accept_description: string[];
+    dash?: {
+      video: DashStream[];
+      audio: DashStream[];
+    };
+    durl?: Array<{ url: string; size: number; order: number }>;
+  };
+}
+
+const BILIBILI_HEADERS = {
+  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+  Referer: "https://www.bilibili.com",
+  Origin: "https://www.bilibili.com",
+};
+
+function bvToAv(bvid: string): number {
+  const TABLE = "fZodR9XQDSUm21yCkr6zBqiveYah8bt4xsWpHnJE7jL5VG3guMTKNPAwcF";
+  const tr: Record<string, number> = {};
+  for (let i = 0; i < TABLE.length; i++) tr[TABLE[i]] = i;
+  const s = [11, 10, 3, 8, 4, 6];
+  const xor = 177451812;
+  const add = 8728348608;
+  let r = 0n;
+  for (let i = 0; i < 6; i++) r += BigInt(tr[bvid[s[i]]]) * 58n ** BigInt(i);
+  return Number((r - BigInt(add)) ^ BigInt(xor));
+}
+
+export class BilibiliExtractor extends BaseExtractor {
+  readonly _VALID_URL = VALID_URL;
+  readonly _NAME = "bilibili";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const match = VALID_URL.exec(url);
+    if (!match) throw new ExtractorError(`bilibili: invalid URL: ${url}`);
+
+    const idPart = match[1];
+    let bvid: string;
+    let aid: number;
+
+    if (idPart.startsWith("BV")) {
+      bvid = idPart;
+      aid = bvToAv(bvid);
+    } else {
+      aid = parseInt(match[2], 10);
+      bvid = `av${aid}`;
+    }
+
+    const viewUrl = `https://api.bilibili.com/x/web-interface/view?${idPart.startsWith("BV") ? "bvid=" + bvid : "aid=" + aid}`;
+    const viewResp = await fetch(viewUrl, { headers: BILIBILI_HEADERS });
+    if (!viewResp.ok) throw new ExtractorError(`bilibili: view API failed: ${viewResp.status}`);
+
+    const viewData = (await viewResp.json()) as VideoViewData;
+    if (viewData.code !== 0) throw new ExtractorError(`bilibili: ${viewData.message}`);
+
+    const video = viewData.data;
+    const cid = video.cid;
+
+    const playParams = new URLSearchParams({
+      bvid: video.bvid,
+      cid: String(cid),
+      qn: "127",
+      fnval: "4048",
+      fnver: "0",
+      fourk: "1",
+    });
+
+    const playResp = await fetch(`https://api.bilibili.com/x/player/playurl?${playParams}`, {
+      headers: BILIBILI_HEADERS,
+    });
+    if (!playResp.ok) throw new ExtractorError(`bilibili: playurl API failed: ${playResp.status}`);
+
+    const playData = (await playResp.json()) as PlayUrlData;
+    if (playData.code !== 0) throw new ExtractorError(`bilibili: playurl: ${playData.message}`);
+
+    const formats: Format[] = [];
+
+    if (playData.data.dash) {
+      const { video: videoStreams, audio: audioStreams } = playData.data.dash;
+
+      const bestAudio = audioStreams.reduce<DashStream | null>((best, a) => {
+        if (!best || a.bandwidth > best.bandwidth) return a;
+        return best;
+      }, null);
+
+      for (const vs of videoStreams) {
+        const streamUrl = vs.baseUrl || vs.base_url;
+        const qualityNote = QUALITY_MAP[vs.id] ?? `qn${vs.id}`;
+        const mime = vs.mimeType || vs.mime_type;
+        const ext = mime.includes("mp4") ? "mp4" : "webm";
+        const fps = vs.frameRate || vs.frame_rate ? parseFloat(vs.frameRate ?? vs.frame_rate ?? "0") : undefined;
+
+        formats.push({
+          format_id: `dash-video-${vs.id}`,
+          url: streamUrl,
+          ext,
+          vcodec: vs.codecs,
+          acodec: "none",
+          width: vs.width,
+          height: vs.height,
+          fps,
+          tbr: Math.round(vs.bandwidth / 1000),
+          format_note: qualityNote,
+          quality: QUALITY_PREFERENCE[vs.id] ?? 0,
+          protocol: "https",
+          http_headers: BILIBILI_HEADERS,
+        });
+      }
+
+      if (bestAudio) {
+        const audioUrl = bestAudio.baseUrl || bestAudio.base_url;
+        const mime = bestAudio.mimeType || bestAudio.mime_type;
+        const ext = mime.includes("mp4") ? "m4a" : "ogg";
+        formats.push({
+          format_id: "dash-audio-best",
+          url: audioUrl,
+          ext,
+          vcodec: "none",
+          acodec: bestAudio.codecs,
+          abr: Math.round(bestAudio.bandwidth / 1000),
+          format_note: "audio",
+          quality: 0,
+          protocol: "https",
+          http_headers: BILIBILI_HEADERS,
+        });
+      }
+    } else if (playData.data.durl) {
+      for (const [i, seg] of playData.data.durl.entries()) {
+        formats.push({
+          format_id: `flv-${i}`,
+          url: seg.url,
+          ext: "flv",
+          filesize: seg.size,
+          quality: QUALITY_PREFERENCE[playData.data.quality] ?? 0,
+          format_note: QUALITY_MAP[playData.data.quality] ?? `qn${playData.data.quality}`,
+          http_headers: BILIBILI_HEADERS,
+        });
+      }
+    }
+
+    const uploadDate = new Date(video.pubdate * 1000).toISOString().slice(0, 10).replace(/-/g, "");
+    const thumbnails: Thumbnail[] = [{ url: video.pic }];
+
+    return {
+      id: video.bvid,
+      title: video.title,
+      description: video.desc,
+      uploader: video.owner.name,
+      uploader_id: String(video.owner.mid),
+      uploader_url: `https://space.bilibili.com/${video.owner.mid}`,
+      duration: video.duration,
+      view_count: video.stat.view,
+      like_count: video.stat.like,
+      upload_date: uploadDate,
+      timestamp: video.pubdate,
+      thumbnails,
+      formats,
+      webpage_url: url,
+      _type: "video",
+    };
+  }
+}

package/src/extractors/bilibili/wbi.ts

@@ -0,0 +1,60 @@
+import { createHash } from "node:crypto";
+
+const MIXIN_KEY_ENC_TAB = [
+  46, 47, 18, 2, 53, 8, 23, 32, 15, 50, 10, 31, 58, 3, 45, 35, 27, 43, 5, 49, 33, 9, 42, 19, 29,
+  28, 14, 39, 12, 38, 41, 13, 37, 48, 7, 16, 24, 55, 40, 61, 26, 17, 0, 1, 60, 51, 30, 4, 22, 25,
+  54, 21, 56, 59, 6, 63, 57, 62, 11, 36, 20, 34, 44, 52,
+];
+
+interface NavData {
+  data: {
+    wbi_img: {
+      img_url: string;
+      sub_url: string;
+    };
+  };
+}
+
+function extractKey(url: string): string {
+  return url.split("/").pop()?.replace(/\.\w+$/, "") ?? "";
+}
+
+function getMixinKey(imgKey: string, subKey: string): string {
+  const raw = imgKey + subKey;
+  return MIXIN_KEY_ENC_TAB.map((i) => raw[i]).join("").slice(0, 32);
+}
+
+function filterNonPrintable(str: string): string {
+  return str.replace(/[!'"()\\s]/g, "");
+}
+
+export async function fetchMixinKey(): Promise<string> {
+  const resp = await fetch("https://api.bilibili.com/x/web-interface/nav", {
+    headers: {
+      "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+      Referer: "https://www.bilibili.com",
+    },
+  });
+  if (!resp.ok) throw new Error(`wbi: nav request failed: ${resp.status}`);
+  const data = (await resp.json()) as NavData;
+  const imgKey = extractKey(data.data.wbi_img.img_url);
+  const subKey = extractKey(data.data.wbi_img.sub_url);
+  return getMixinKey(imgKey, subKey);
+}
+
+export function signWbi(params: Record<string, string | number>, mixinKey: string): Record<string, string> {
+  const wts = Math.floor(Date.now() / 1000);
+  const signed: Record<string, string> = {};
+  for (const [k, v] of Object.entries(params)) {
+    signed[k] = filterNonPrintable(String(v));
+  }
+  signed["wts"] = String(wts);
+
+  const query = Object.keys(signed)
+    .sort()
+    .map((k) => `${k}=${signed[k]}`)
+    .join("&");
+
+  const wRid = createHash("md5").update(query + mixinKey).digest("hex");
+  return { ...signed, w_rid: wRid };
+}

package/src/extractors/coub.ts

@@ -0,0 +1,137 @@
+import { BaseExtractor, ExtractorError } from "../core/types";
+import type { InfoDict, Format, Thumbnail } from "../core/types";
+
+interface CoubMediaSource {
+  url?: string;
+  size?: number;
+}
+
+interface CoubFileVersions {
+  html5?: {
+    video?: Record<string, CoubMediaSource>;
+    audio?: Record<string, CoubMediaSource>;
+  };
+  mobile?: {
+    video?: string[];
+    audio?: string;
+  };
+}
+
+interface CoubData {
+  id?: number;
+  title?: string;
+  description?: string;
+  views_count?: number;
+  likes_count?: number;
+  created_at?: string;
+  duration?: number;
+  file_versions?: CoubFileVersions;
+  image_versions?: {
+    template?: string;
+    versions?: string[];
+  };
+  channel?: {
+    permalink?: string;
+    title?: string;
+  };
+  tags?: Array<{ title?: string }>;
+}
+
+export class CoubExtractor extends BaseExtractor {
+  readonly _VALID_URL = /https?:\/\/(?:www\.)?coub\.com\/(?:view|embed)\/([a-zA-Z0-9_-]+)/;
+  readonly _NAME = "coub";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const match = url.match(this._VALID_URL);
+    if (!match) throw new ExtractorError(`Invalid Coub URL: ${url}`);
+    const coubId = match[1];
+
+    const apiUrl = `https://coub.com/api/v2/coubs/${coubId}`;
+    const response = await fetch(apiUrl, {
+      headers: { "User-Agent": "Mozilla/5.0" },
+    });
+
+    if (!response.ok) {
+      throw new ExtractorError(`Coub API error: ${response.status}`);
+    }
+
+    const data = (await response.json()) as CoubData;
+
+    const formats: Format[] = [];
+    const html5 = data.file_versions?.html5;
+
+    const qualityOrder: Record<string, number> = {
+      higher: 4,
+      high: 3,
+      med: 2,
+      low: 1,
+    };
+
+    const heightMap: Record<string, number> = {
+      higher: 1080,
+      high: 720,
+      med: 360,
+      low: 240,
+    };
+
+    if (html5?.video) {
+      for (const [quality, source] of Object.entries(html5.video)) {
+        if (source.url) {
+          formats.push({
+            format_id: `video-${quality}`,
+            url: source.url,
+            ext: "mp4",
+            vcodec: "h264",
+            acodec: "none",
+            height: heightMap[quality],
+            filesize: source.size,
+            quality: qualityOrder[quality] ?? 0,
+            format_note: `video-only (${quality}); audio is separate and requires merge`,
+          });
+        }
+      }
+    }
+
+    if (html5?.audio) {
+      for (const [quality, source] of Object.entries(html5.audio)) {
+        if (source.url) {
+          formats.push({
+            format_id: `audio-${quality}`,
+            url: source.url,
+            ext: "mp4",
+            vcodec: "none",
+            acodec: "aac",
+            filesize: source.size,
+            quality: qualityOrder[quality] ?? 0,
+            format_note: `audio-only (${quality})`,
+          });
+        }
+      }
+    }
+
+    const thumbnails: Thumbnail[] = [];
+    const imgTemplate = data.image_versions?.template;
+    if (imgTemplate) {
+      for (const version of data.image_versions?.versions ?? []) {
+        thumbnails.push({ url: imgTemplate.replace("%{version}", version) });
+      }
+    }
+
+    return {
+      id: String(data.id ?? coubId),
+      title: data.title ?? coubId,
+      description: data.description,
+      duration: data.duration,
+      view_count: data.views_count,
+      like_count: data.likes_count,
+      uploader: data.channel?.title,
+      uploader_id: data.channel?.permalink,
+      upload_date: data.created_at?.slice(0, 10).replace(/-/g, ""),
+      thumbnails,
+      formats,
+      tags: data.tags?.map((t) => t.title ?? "").filter(Boolean),
+      webpage_url: url,
+      extractor: this._NAME,
+    };
+  }
+}

package/src/extractors/dailymotion.ts

@@ -0,0 +1,99 @@
+import { BaseExtractor, ExtractorError } from "../core/types";
+import type { InfoDict, Format, Thumbnail } from "../core/types";
+
+interface DailymotionQuality {
+  type: string;
+  url: string;
+}
+
+interface DailymotionMetadata {
+  id: string;
+  title: string;
+  description?: string;
+  duration?: number;
+  owner?: { screenname?: string; id?: string };
+  created_time?: number;
+  views_total?: number;
+  likes_total?: number;
+  thumbnail_url?: string;
+  qualities?: Record<string, DailymotionQuality[]>;
+}
+
+export class DailymotionExtractor extends BaseExtractor {
+  readonly _VALID_URL = /https?:\/\/(?:www\.)?dailymotion\.com\/video\/([a-zA-Z0-9]+)/;
+  readonly _NAME = "dailymotion";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const match = url.match(this._VALID_URL);
+    if (!match) throw new ExtractorError(`Invalid Dailymotion URL: ${url}`);
+    const videoId = match[1];
+
+    const apiUrl = `https://www.dailymotion.com/player/metadata/video/${videoId}?app=com.dailymotion.neon`;
+    const response = await fetch(apiUrl, {
+      headers: { "User-Agent": "Mozilla/5.0" },
+    });
+
+    if (!response.ok) {
+      throw new ExtractorError(`Dailymotion API error: ${response.status}`);
+    }
+
+    const data = (await response.json()) as DailymotionMetadata;
+
+    const formats: Format[] = [];
+    const qualities = data.qualities ?? {};
+
+    const qualityMap: Record<string, number> = {
+      "2160": 2160,
+      "1440": 1440,
+      "1080": 1080,
+      "720": 720,
+      "480": 480,
+      "380": 380,
+      "240": 240,
+      "144": 144,
+    };
+
+    for (const [qualityKey, streams] of Object.entries(qualities)) {
+      for (const stream of streams) {
+        if (stream.type === "video/mp4" && stream.url) {
+          const height = qualityMap[qualityKey];
+          formats.push({
+            format_id: `mp4-${qualityKey}`,
+            url: stream.url,
+            ext: "mp4",
+            height,
+            resolution: height ? `${height}p` : qualityKey,
+            quality: height ?? 0,
+          });
+        } else if ((stream.type === "application/x-mpegURL" || stream.type === "application/vnd.apple.mpegurl") && stream.url) {
+          formats.push({
+            format_id: `hls-${qualityKey}`,
+            url: stream.url,
+            ext: "mp4",
+            protocol: "m3u8",
+          });
+        }
+      }
+    }
+
+    const thumbnails: Thumbnail[] = data.thumbnail_url
+      ? [{ url: data.thumbnail_url }]
+      : [];
+
+    return {
+      id: videoId,
+      title: data.title,
+      description: data.description,
+      duration: data.duration,
+      uploader: data.owner?.screenname,
+      uploader_id: data.owner?.id,
+      timestamp: data.created_time,
+      view_count: data.views_total,
+      like_count: data.likes_total,
+      thumbnails,
+      formats,
+      webpage_url: url,
+      extractor: this._NAME,
+    };
+  }
+}

package/src/extractors/dropbox.ts

@@ -0,0 +1,52 @@
+import { BaseExtractor, ExtractorError } from "../core/types";
+import type { InfoDict } from "../core/types";
+
+export class DropboxExtractor extends BaseExtractor {
+  readonly _VALID_URL = /https?:\/\/(?:www\.)?dropbox\.com\/(?:s|sh|scl\/fo)\/[^?#]+/;
+  readonly _NAME = "dropbox";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const urlObj = new URL(url);
+
+    urlObj.searchParams.set("dl", "1");
+    urlObj.searchParams.delete("rlkey");
+
+    const directUrl = urlObj.toString();
+
+    const headResponse = await fetch(directUrl, {
+      method: "HEAD",
+      redirect: "follow",
+      headers: { "User-Agent": "Mozilla/5.0" },
+    });
+
+    if (!headResponse.ok && headResponse.status !== 302 && headResponse.status !== 301) {
+      throw new ExtractorError(`Dropbox: could not access file (status ${headResponse.status})`);
+    }
+
+    const finalUrl = headResponse.url || directUrl;
+    const pathname = urlObj.pathname;
+    const filename = pathname.split("/").pop() ?? "file";
+    const ext = filename.includes(".") ? filename.split(".").pop()!.toLowerCase() : "mp4";
+    const title = filename.replace(/\.[^.]+$/, "").replace(/[_-]/g, " ");
+
+    const contentLength = headResponse.headers.get("content-length");
+    const filesize = contentLength ? parseInt(contentLength) : undefined;
+
+    return {
+      id: urlObj.searchParams.get("id") ?? pathname.split("/").slice(-2, -1)[0] ?? "dropbox",
+      title,
+      url: finalUrl,
+      ext,
+      formats: [
+        {
+          format_id: "direct",
+          url: finalUrl,
+          ext,
+          filesize,
+        },
+      ],
+      webpage_url: url,
+      extractor: this._NAME,
+    };
+  }
+}

package/src/extractors/generic.ts

@@ -0,0 +1,118 @@
+import { BaseExtractor } from "../core/types";
+import type { InfoDict, Format } from "../core/types";
+
+const MEDIA_EXTENSIONS = /\.(mp4|webm|mkv|flv|avi|mov|wmv|mp3|aac|flac|opus|ogg|wav|m4a)(\?|$)/i;
+const MANIFEST_EXTENSIONS = /\.(m3u8|mpd)(\?|$)/i;
+
+export class GenericExtractor extends BaseExtractor {
+  readonly _VALID_URL = /^https?:\/\/.+/;
+  readonly _NAME = "generic";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    if (MEDIA_EXTENSIONS.test(url)) {
+      return this.extractDirectMedia(url);
+    }
+
+    if (MANIFEST_EXTENSIONS.test(url)) {
+      return this.extractManifest(url);
+    }
+
+    return this.extractFromPage(url);
+  }
+
+  private extractDirectMedia(url: string): InfoDict {
+    const urlObj = new URL(url);
+    const filename = urlObj.pathname.split("/").pop() ?? "media";
+    const ext = filename.split(".").pop() ?? "mp4";
+    const title = filename.replace(/\.[^.]+$/, "").replace(/[_-]/g, " ");
+
+    return {
+      id: this.generateId(url),
+      title,
+      url,
+      ext,
+      formats: [
+        {
+          format_id: "direct",
+          url,
+          ext,
+        },
+      ],
+    };
+  }
+
+  private extractManifest(url: string): InfoDict {
+    const isHLS = /\.m3u8/i.test(url);
+    const ext = isHLS ? "mp4" : "webm";
+    const protocol = isHLS ? "m3u8" : "dash";
+
+    return {
+      id: this.generateId(url),
+      title: "Media",
+      url,
+      ext,
+      formats: [
+        {
+          format_id: protocol,
+          url,
+          ext,
+          protocol,
+        },
+      ],
+    };
+  }
+
+  private async extractFromPage(url: string): Promise<InfoDict> {
+    const response = await fetch(url);
+    const html = await response.text();
+
+    const formats: Format[] = [];
+
+    const mediaMatches = html.matchAll(
+      /(?:src|href)=["']([^"']*?\.(mp4|webm|m3u8|mpd)(?:\?[^"']*?)?)["']/gi,
+    );
+    for (const match of mediaMatches) {
+      const mediaUrl = new URL(match[1], url).href;
+      const ext = match[2].toLowerCase();
+      formats.push({
+        format_id: `generic-${formats.length}`,
+        url: mediaUrl,
+        ext: ext === "m3u8" ? "mp4" : ext === "mpd" ? "webm" : ext,
+        protocol: ext === "m3u8" ? "m3u8" : ext === "mpd" ? "dash" : undefined,
+      });
+    }
+
+    const ogVideo = html.match(
+      /<meta[^>]+property=["']og:video(?::url)?["'][^>]+content=["']([^"']+)["']/i,
+    );
+    if (ogVideo) {
+      const videoUrl = new URL(ogVideo[1], url).href;
+      formats.push({
+        format_id: "og-video",
+        url: videoUrl,
+        ext: "mp4",
+      });
+    }
+
+    const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
+    const ogTitle = html.match(
+      /<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i,
+    );
+
+    return {
+      id: this.generateId(url),
+      title: ogTitle?.[1] ?? titleMatch?.[1] ?? "Unknown",
+      webpage_url: url,
+      formats,
+    };
+  }
+
+  private generateId(url: string): string {
+    let hash = 0;
+    for (let i = 0; i < url.length; i++) {
+      const char = url.charCodeAt(i);
+      hash = ((hash << 5) - hash + char) | 0;
+    }
+    return Math.abs(hash).toString(36);
+  }
+}