getraw 0.1.0

package/src/extractors/soundcloud/index.ts

@@ -0,0 +1,161 @@
+import { BaseExtractor, ExtractorError } from "../../core/types";
+import type { InfoDict, Format, Thumbnail } from "../../core/types";
+
+interface SoundCloudTranscoding {
+  url: string;
+  preset: string;
+  duration?: number;
+  format: {
+    protocol: string;
+    mime_type: string;
+  };
+  quality?: string;
+}
+
+interface SoundCloudUser {
+  id: number;
+  username: string;
+  permalink_url?: string;
+}
+
+interface SoundCloudTrack {
+  id: number;
+  title: string;
+  description?: string;
+  duration?: number;
+  playback_count?: number;
+  likes_count?: number;
+  comment_count?: number;
+  created_at?: string;
+  genre?: string;
+  tag_list?: string;
+  permalink_url?: string;
+  user?: SoundCloudUser;
+  artwork_url?: string;
+  waveform_url?: string;
+  media?: {
+    transcodings?: SoundCloudTranscoding[];
+  };
+}
+
+interface StreamResponse {
+  url: string;
+}
+
+const CLIENT_ID_PATTERN =
+  /,client_id:"([a-zA-Z0-9_-]{32,})"/;
+
+async function extractClientId(pageUrl: string): Promise<string> {
+  const pageResponse = await fetch(pageUrl, {
+    headers: { "User-Agent": "Mozilla/5.0" },
+  });
+  if (!pageResponse.ok) {
+    throw new ExtractorError(`SoundCloud page fetch failed: ${pageResponse.status}`);
+  }
+  const html = await pageResponse.text();
+
+  const scriptMatches = html.matchAll(/<script[^>]+src="(https:\/\/a-v2\.sndcdn\.com\/assets\/[^"]+\.js)"/g);
+  const scriptUrls: string[] = [];
+  for (const match of scriptMatches) {
+    scriptUrls.push(match[1]);
+  }
+
+  for (const scriptUrl of scriptUrls.slice(-3)) {
+    const scriptResponse = await fetch(scriptUrl);
+    if (!scriptResponse.ok) continue;
+    const scriptText = await scriptResponse.text();
+    const match = CLIENT_ID_PATTERN.exec(scriptText);
+    if (match) return match[1];
+  }
+
+  throw new ExtractorError("Could not extract SoundCloud client_id from JS bundle");
+}
+
+export class SoundCloudExtractor extends BaseExtractor {
+  readonly _VALID_URL =
+    /^https?:\/\/(?:(?:www|m)\.)?soundcloud\.com\/([^/]+)\/(?!sets\/)([^/?#]+)/;
+  readonly _NAME = "soundcloud";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const clientId = await extractClientId(url);
+
+    const resolveUrl = `https://api-v2.soundcloud.com/resolve?url=${encodeURIComponent(url)}&client_id=${clientId}`;
+    const resolveResponse = await fetch(resolveUrl, {
+      headers: { "User-Agent": "Mozilla/5.0" },
+    });
+    if (!resolveResponse.ok) {
+      throw new ExtractorError(`SoundCloud resolve failed: ${resolveResponse.status}`);
+    }
+
+    const track = (await resolveResponse.json()) as SoundCloudTrack;
+
+    if (!track.id) {
+      throw new ExtractorError("Could not resolve SoundCloud track");
+    }
+
+    const transcodings = track.media?.transcodings ?? [];
+    const formats: Format[] = [];
+
+    for (const transcoding of transcodings) {
+      const streamResponse = await fetch(
+        `${transcoding.url}?client_id=${clientId}`,
+        { headers: { "User-Agent": "Mozilla/5.0" } },
+      );
+      if (!streamResponse.ok) continue;
+
+      const stream = (await streamResponse.json()) as StreamResponse;
+      if (!stream.url) continue;
+
+      const isHLS = transcoding.format.protocol === "hls";
+      const isOpus = transcoding.preset.includes("opus");
+      const isAac = transcoding.preset.includes("aac") || transcoding.format.mime_type.includes("aac");
+
+      formats.push({
+        format_id: `${transcoding.format.protocol}-${transcoding.preset}`,
+        url: stream.url,
+        ext: isHLS ? (isOpus ? "opus" : "m4a") : "mp3",
+        protocol: isHLS ? "m3u8" : "https",
+        acodec: isOpus ? "opus" : isAac ? "aac" : "mp3",
+        vcodec: "none",
+        abr: isOpus ? 64 : 128,
+        format_note: transcoding.preset,
+        quality: isOpus ? 0 : 1,
+      });
+    }
+
+    if (formats.length === 0) {
+      throw new ExtractorError("No playable formats found for this SoundCloud track");
+    }
+
+    formats.sort((a, b) => (b.quality ?? 0) - (a.quality ?? 0));
+
+    const thumbnails: Thumbnail[] = [];
+    if (track.artwork_url) {
+      thumbnails.push({ url: track.artwork_url.replace("-large", "-t500x500"), preference: 1 });
+      thumbnails.push({ url: track.artwork_url });
+    }
+
+    const tags = track.tag_list
+      ? track.tag_list.match(/"[^"]+"|[^ ]+/g)?.map((t) => t.replace(/"/g, "")) ?? []
+      : [];
+
+    return {
+      id: String(track.id),
+      title: track.title,
+      description: track.description,
+      uploader: track.user?.username,
+      uploader_id: track.user ? String(track.user.id) : undefined,
+      uploader_url: track.user?.permalink_url,
+      duration: track.duration ? Math.round(track.duration / 1000) : undefined,
+      view_count: track.playback_count,
+      like_count: track.likes_count,
+      comment_count: track.comment_count,
+      upload_date: track.created_at?.replace(/-/g, "").slice(0, 8),
+      webpage_url: track.permalink_url ?? url,
+      categories: track.genre ? [track.genre] : undefined,
+      tags,
+      thumbnails,
+      formats,
+    };
+  }
+}

package/src/extractors/soundcloud/playlist.ts

@@ -0,0 +1,129 @@
+import { BaseExtractor, ExtractorError } from "../../core/types";
+import type { InfoDict } from "../../core/types";
+
+interface SoundCloudUser {
+  id: number;
+  username: string;
+  permalink_url?: string;
+}
+
+interface SoundCloudTrackRef {
+  id: number;
+  title?: string;
+  permalink_url?: string;
+}
+
+interface SoundCloudPlaylist {
+  id: number;
+  title: string;
+  description?: string;
+  duration?: number;
+  track_count?: number;
+  likes_count?: number;
+  created_at?: string;
+  permalink_url?: string;
+  user?: SoundCloudUser;
+  artwork_url?: string;
+  tracks?: SoundCloudTrackRef[];
+  is_album?: boolean;
+}
+
+interface SoundCloudTracksResponse {
+  collection: SoundCloudTrackRef[];
+  next_href?: string;
+}
+
+const CLIENT_ID_PATTERN = /,client_id:"([a-zA-Z0-9_-]{32,})"/;
+
+async function extractClientId(pageUrl: string): Promise<string> {
+  const pageResponse = await fetch(pageUrl, {
+    headers: { "User-Agent": "Mozilla/5.0" },
+  });
+  if (!pageResponse.ok) {
+    throw new ExtractorError(`SoundCloud page fetch failed: ${pageResponse.status}`);
+  }
+  const html = await pageResponse.text();
+  const scriptMatches = html.matchAll(/<script[^>]+src="(https:\/\/a-v2\.sndcdn\.com\/assets\/[^"]+\.js)"/g);
+  const scriptUrls: string[] = [];
+  for (const match of scriptMatches) {
+    scriptUrls.push(match[1]);
+  }
+  for (const scriptUrl of scriptUrls.slice(-3)) {
+    const scriptResponse = await fetch(scriptUrl);
+    if (!scriptResponse.ok) continue;
+    const scriptText = await scriptResponse.text();
+    const match = CLIENT_ID_PATTERN.exec(scriptText);
+    if (match) return match[1];
+  }
+  throw new ExtractorError("Could not extract SoundCloud client_id from JS bundle");
+}
+
+async function fetchAllTracks(
+  playlistId: number,
+  clientId: string,
+): Promise<SoundCloudTrackRef[]> {
+  const allTracks: SoundCloudTrackRef[] = [];
+  let nextHref: string | null =
+    `https://api-v2.soundcloud.com/playlists/${playlistId}/tracks?client_id=${clientId}&limit=50`;
+
+  while (nextHref) {
+    const response = await fetch(nextHref, {
+      headers: { "User-Agent": "Mozilla/5.0" },
+    });
+    if (!response.ok) break;
+    const data = (await response.json()) as SoundCloudTracksResponse;
+    allTracks.push(...(data.collection ?? []));
+    nextHref = data.next_href
+      ? `${data.next_href}&client_id=${clientId}`
+      : null;
+  }
+
+  return allTracks;
+}
+
+export class SoundCloudPlaylistExtractor extends BaseExtractor {
+  readonly _VALID_URL =
+    /^https?:\/\/(?:(?:www|m)\.)?soundcloud\.com\/([^/]+)\/sets\/([^/?#]+)/;
+  readonly _NAME = "soundcloud:playlist";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const clientId = await extractClientId(url);
+
+    const resolveUrl = `https://api-v2.soundcloud.com/resolve?url=${encodeURIComponent(url)}&client_id=${clientId}`;
+    const resolveResponse = await fetch(resolveUrl, {
+      headers: { "User-Agent": "Mozilla/5.0" },
+    });
+    if (!resolveResponse.ok) {
+      throw new ExtractorError(`SoundCloud resolve failed: ${resolveResponse.status}`);
+    }
+
+    const playlist = (await resolveResponse.json()) as SoundCloudPlaylist;
+    if (!playlist.id) throw new ExtractorError("Could not resolve SoundCloud playlist");
+
+    const tracks = await fetchAllTracks(playlist.id, clientId);
+
+    const entries: InfoDict[] = tracks.map((track, idx) => ({
+      id: String(track.id),
+      title: track.title ?? `Track ${idx + 1}`,
+      webpage_url: track.permalink_url ?? url,
+      url: track.permalink_url ?? url,
+      _type: "url" as const,
+      playlist_index: idx + 1,
+    }));
+
+    return {
+      id: String(playlist.id),
+      title: playlist.title,
+      description: playlist.description,
+      uploader: playlist.user?.username,
+      uploader_id: playlist.user ? String(playlist.user.id) : undefined,
+      uploader_url: playlist.user?.permalink_url,
+      like_count: playlist.likes_count,
+      upload_date: playlist.created_at?.replace(/-/g, "").slice(0, 8),
+      webpage_url: playlist.permalink_url ?? url,
+      _type: "playlist",
+      entries,
+      playlist_count: tracks.length,
+    };
+  }
+}

package/src/extractors/spotify.ts

@@ -0,0 +1,97 @@
+import { BaseExtractor, ExtractorError } from "../core/types";
+import type { InfoDict, Thumbnail } from "../core/types";
+
+export class SpotifyExtractor extends BaseExtractor {
+  readonly _VALID_URL = /https?:\/\/open\.spotify\.com\/episode\/([a-zA-Z0-9]+)/;
+  readonly _NAME = "spotify";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const match = url.match(this._VALID_URL);
+    if (!match) throw new ExtractorError(`Invalid Spotify episode URL: ${url}`);
+    const episodeId = match[1];
+
+    const response = await fetch(url, {
+      headers: {
+        "User-Agent": "Mozilla/5.0",
+        "Accept-Language": "en-US,en;q=0.9",
+      },
+    });
+
+    if (!response.ok) {
+      throw new ExtractorError(`Spotify fetch error: ${response.status}`);
+    }
+
+    const html = await response.text();
+
+    const nextDataMatch = html.match(/<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/);
+    let audioPreviewUrl: string | undefined;
+    let title = "Spotify Podcast Episode";
+    let description: string | undefined;
+    let duration: number | undefined;
+    let uploader: string | undefined;
+    let thumbnails: Thumbnail[] = [];
+
+    if (nextDataMatch) {
+      try {
+        const nextData = JSON.parse(nextDataMatch[1]) as Record<string, unknown>;
+        const props = (nextData.props as Record<string, unknown>)?.pageProps as Record<string, unknown> | undefined;
+        const episode = props?.episode as Record<string, unknown> | undefined;
+
+        if (episode) {
+          title = (episode.name as string) ?? title;
+          description = episode.description as string | undefined;
+          duration = typeof episode.duration_ms === "number"
+            ? Math.round(episode.duration_ms / 1000)
+            : undefined;
+          audioPreviewUrl = episode.audio_preview_url as string | undefined;
+
+          const show = episode.show as Record<string, unknown> | undefined;
+          uploader = show?.name as string | undefined;
+
+          const images = episode.images as Array<{ url: string; width?: number; height?: number }> | undefined;
+          if (images) {
+            thumbnails = images.map((img) => ({
+              url: img.url,
+              width: img.width,
+              height: img.height,
+            }));
+          }
+        }
+      } catch {
+      }
+    }
+
+    if (!audioPreviewUrl) {
+      const previewMatch = html.match(/"audio_preview_url"\s*:\s*"([^"]+)"/);
+      if (previewMatch) audioPreviewUrl = previewMatch[1];
+    }
+
+    if (!audioPreviewUrl) {
+      throw new ExtractorError(
+        "Spotify: no audio preview URL found. Note: full podcast audio requires Spotify auth. Only 30-second previews are available without DRM.",
+      );
+    }
+
+    return {
+      id: episodeId,
+      title,
+      description,
+      duration,
+      uploader,
+      thumbnails,
+      url: audioPreviewUrl,
+      ext: "mp3",
+      formats: [
+        {
+          format_id: "preview-mp3",
+          url: audioPreviewUrl,
+          ext: "mp3",
+          acodec: "mp3",
+          format_note: "30-second preview only (full episode requires Spotify auth)",
+        },
+      ],
+      webpage_url: url,
+      extractor: this._NAME,
+    };
+  }
+}

package/src/extractors/streamable.ts

@@ -0,0 +1,121 @@
+import { BaseExtractor, ExtractorError } from "../core/types";
+import type { InfoDict, Format, Thumbnail } from "../core/types";
+
+interface StreamableVideoSource {
+  url?: string;
+  width?: number;
+  height?: number;
+  bitrate?: number;
+  size?: number;
+}
+
+interface StreamablePageData {
+  title?: string;
+  status?: number;
+  percent?: number;
+  thumbnail_url?: string;
+  files?: Record<string, StreamableVideoSource>;
+  url?: string;
+  duration?: number;
+}
+
+export class StreamableExtractor extends BaseExtractor {
+  readonly _VALID_URL = /https?:\/\/(?:www\.)?streamable\.com\/([a-zA-Z0-9]+)/;
+  readonly _NAME = "streamable";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const match = url.match(this._VALID_URL);
+    if (!match) throw new ExtractorError(`Invalid Streamable URL: ${url}`);
+    const videoId = match[1];
+
+    const response = await fetch(url, {
+      headers: { "User-Agent": "Mozilla/5.0" },
+    });
+
+    if (!response.ok) {
+      throw new ExtractorError(`Streamable fetch error: ${response.status}`);
+    }
+
+    const html = await response.text();
+
+    let pageData: StreamablePageData = {};
+
+    const reactDataMatch = html.match(/window\.__reactData__\s*=\s*(\{[\s\S]*?});\s*<\/script>/);
+    if (reactDataMatch) {
+      try {
+        const reactData = JSON.parse(reactDataMatch[1]) as Record<string, unknown>;
+        const videoKey = Object.keys(reactData).find((k) => (reactData[k] as Record<string, unknown>)?.files);
+        if (videoKey) pageData = reactData[videoKey] as StreamablePageData;
+      } catch {
+      }
+    }
+
+    if (!pageData.files) {
+      const nextDataMatch = html.match(/<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/);
+      if (nextDataMatch) {
+        try {
+          const nextData = JSON.parse(nextDataMatch[1]) as Record<string, unknown>;
+          const props = (nextData.props as Record<string, unknown>)?.pageProps as Record<string, unknown> | undefined;
+          pageData = (props?.video ?? props?.clip ?? {}) as StreamablePageData;
+        } catch {
+        }
+      }
+    }
+
+    if (!pageData.files) {
+      const jsonMatch = html.match(/"files"\s*:\s*(\{[^}]+\})/);
+      if (jsonMatch) {
+        try {
+          pageData.files = JSON.parse(jsonMatch[1]) as Record<string, StreamableVideoSource>;
+        } catch {
+        }
+      }
+    }
+
+    const files = pageData.files ?? {};
+    const formats: Format[] = [];
+
+    const qualityMap: Record<string, number> = {
+      mp4: 1,
+      "mp4-mobile": 2,
+      original: 3,
+    };
+
+    for (const [key, source] of Object.entries(files)) {
+      if (!source.url) continue;
+
+      const fullUrl = source.url.startsWith("//")
+        ? `https:${source.url}`
+        : source.url;
+
+      formats.push({
+        format_id: key,
+        url: fullUrl,
+        ext: "mp4",
+        width: source.width,
+        height: source.height,
+        tbr: source.bitrate,
+        filesize: source.size,
+        quality: qualityMap[key] ?? 0,
+        resolution: source.height ? `${source.height}p` : undefined,
+      });
+    }
+
+    const thumbnails: Thumbnail[] = pageData.thumbnail_url
+      ? [{ url: pageData.thumbnail_url.startsWith("//") ? `https:${pageData.thumbnail_url}` : pageData.thumbnail_url }]
+      : [];
+
+    const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
+    const title = pageData.title ?? titleMatch?.[1]?.replace(" - Streamable", "").trim() ?? videoId;
+
+    return {
+      id: videoId,
+      title,
+      thumbnails,
+      formats,
+      duration: pageData.duration,
+      webpage_url: url,
+      extractor: this._NAME,
+    };
+  }
+}

package/src/extractors/ted.ts

@@ -0,0 +1,151 @@
+import { BaseExtractor, ExtractorError } from "../core/types";
+import type { InfoDict, Format, Thumbnail, Subtitle } from "../core/types";
+
+interface TEDVideoResource {
+  bitrate?: number;
+  file?: string;
+  quality?: string;
+  height?: number;
+  width?: number;
+}
+
+interface TEDPlayerData {
+  resources?: {
+    h264?: TEDVideoResource[];
+    hls?: { stream?: string };
+  };
+  duration?: number;
+  thumb?: string;
+}
+
+interface TEDTalkData {
+  id?: number;
+  slug?: string;
+  title?: string;
+  description?: string;
+  duration?: number;
+  viewedCount?: number;
+  publishedAt?: string;
+  speakers?: Array<{ firstname?: string; lastname?: string }>;
+  playerData?: TEDPlayerData | string;
+  subtitledDownloads?: Record<string, { high?: string; low?: string; name?: string }>;
+  image?: { url?: string };
+  canonicalUrl?: string;
+}
+
+interface TEDNextData {
+  props?: {
+    pageProps?: {
+      talkData?: TEDTalkData;
+      videoData?: TEDTalkData;
+    };
+  };
+}
+
+export class TEDExtractor extends BaseExtractor {
+  readonly _VALID_URL = /https?:\/\/(?:www\.)?ted\.com\/talks\/([a-zA-Z0-9_]+)/;
+  readonly _NAME = "ted";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const match = url.match(this._VALID_URL);
+    if (!match) throw new ExtractorError(`Invalid TED URL: ${url}`);
+    const talkSlug = match[1];
+
+    const response = await fetch(url, {
+      headers: { "User-Agent": "Mozilla/5.0" },
+    });
+
+    if (!response.ok) {
+      throw new ExtractorError(`TED fetch error: ${response.status}`);
+    }
+
+    const html = await response.text();
+
+    const nextDataMatch = html.match(/<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/);
+    if (!nextDataMatch) {
+      throw new ExtractorError("TED: could not find __NEXT_DATA__");
+    }
+
+    let nextData: TEDNextData;
+    try {
+      nextData = JSON.parse(nextDataMatch[1]) as TEDNextData;
+    } catch {
+      throw new ExtractorError("TED: failed to parse __NEXT_DATA__");
+    }
+
+    const talkData = nextData.props?.pageProps?.talkData ?? nextData.props?.pageProps?.videoData;
+    if (!talkData) throw new ExtractorError("TED: no talk data found");
+
+    let playerData: TEDPlayerData | undefined;
+    if (typeof talkData.playerData === "string") {
+      try {
+        playerData = JSON.parse(talkData.playerData) as TEDPlayerData;
+      } catch {
+      }
+    } else if (talkData.playerData && typeof talkData.playerData === "object") {
+      playerData = talkData.playerData;
+    }
+
+    const formats: Format[] = [];
+
+    const h264Resources = playerData?.resources?.h264 ?? [];
+    for (const resource of h264Resources) {
+      if (!resource.file) continue;
+      formats.push({
+        format_id: `mp4-${resource.bitrate ?? resource.quality ?? "unknown"}`,
+        url: resource.file,
+        ext: "mp4",
+        vcodec: "h264",
+        tbr: resource.bitrate,
+        height: resource.height,
+        width: resource.width,
+        quality: resource.height ?? resource.bitrate ?? 0,
+        resolution: resource.height ? `${resource.height}p` : undefined,
+      });
+    }
+
+    const hlsStream = playerData?.resources?.hls?.stream;
+    if (hlsStream) {
+      formats.push({
+        format_id: "hls",
+        url: hlsStream,
+        ext: "mp4",
+        protocol: "m3u8",
+        quality: -1,
+      });
+    }
+
+    const subtitles: Record<string, Subtitle[]> = {};
+    const subtitledDownloads = talkData.subtitledDownloads ?? {};
+    for (const [lang, subtitleData] of Object.entries(subtitledDownloads)) {
+      const subUrl = subtitleData.high ?? subtitleData.low;
+      if (subUrl) {
+        subtitles[lang] = [{ url: subUrl, ext: "srt", name: subtitleData.name }];
+      }
+    }
+
+    const thumbnails: Thumbnail[] = [];
+    const thumbUrl = playerData?.thumb ?? talkData.image?.url;
+    if (thumbUrl) thumbnails.push({ url: thumbUrl });
+
+    const speakerNames = (talkData.speakers ?? [])
+      .map((s) => [s.firstname, s.lastname].filter(Boolean).join(" "))
+      .filter(Boolean);
+
+    return {
+      id: String(talkData.id ?? talkSlug),
+      title: talkData.title ?? talkSlug,
+      description: talkData.description,
+      duration: talkData.duration ?? playerData?.duration,
+      view_count: talkData.viewedCount,
+      uploader: speakerNames[0],
+      upload_date: talkData.publishedAt?.slice(0, 10).replace(/-/g, ""),
+      thumbnails,
+      formats,
+      subtitles,
+      tags: [],
+      webpage_url: url,
+      extractor: this._NAME,
+    };
+  }
+}