getraw 0.1.0

package/src/extractors/peertube.ts

@@ -0,0 +1,143 @@
+import { BaseExtractor, ExtractorError } from "../core/types";
+import type { InfoDict, Format, Thumbnail, Subtitle } from "../core/types";
+
+interface PeerTubeFile {
+  fileUrl?: string;
+  fileDownloadUrl?: string;
+  torrentUrl?: string;
+  resolution?: { id?: number; label?: string };
+  size?: number;
+  fps?: number;
+  width?: number;
+  height?: number;
+}
+
+interface PeerTubeStreamingPlaylist {
+  playlistUrl?: string;
+  type?: number;
+}
+
+interface PeerTubeVideo {
+  uuid?: string;
+  name?: string;
+  description?: string;
+  duration?: number;
+  views?: number;
+  likes?: number;
+  dislikes?: number;
+  publishedAt?: string;
+  thumbnailUrl?: string;
+  previewUrl?: string;
+  isLive?: boolean;
+  account?: { displayName?: string; name?: string; url?: string };
+  channel?: { displayName?: string; name?: string; url?: string };
+  files?: PeerTubeFile[];
+  streamingPlaylists?: PeerTubeStreamingPlaylist[];
+  captions?: Array<{ language?: { id?: string; label?: string }; captionPath?: string; fileUrl?: string }>;
+}
+
+const PEERTUBE_INSTANCE_REGEX = /https?:\/\/([^/]+)\/(?:videos\/watch|w)\/([a-zA-Z0-9-]+)/;
+const PEERTUBE_EMBED_REGEX = /https?:\/\/([^/]+)\/videos\/embed\/([a-zA-Z0-9-]+)/;
+
+export class PeerTubeExtractor extends BaseExtractor {
+  readonly _VALID_URL = /https?:\/\/[^/]+\/(?:videos\/(?:watch|embed)|w)\/[a-zA-Z0-9-]+/;
+  readonly _NAME = "peertube";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const match = url.match(PEERTUBE_INSTANCE_REGEX) ?? url.match(PEERTUBE_EMBED_REGEX);
+    if (!match) throw new ExtractorError(`Invalid PeerTube URL: ${url}`);
+
+    const instance = match[1];
+    const videoId = match[2];
+    const apiUrl = `https://${instance}/api/v1/videos/${videoId}`;
+
+    const response = await fetch(apiUrl, {
+      headers: { "User-Agent": "Mozilla/5.0" },
+    });
+
+    if (!response.ok) {
+      throw new ExtractorError(`PeerTube API error: ${response.status} for ${apiUrl}`);
+    }
+
+    const data = (await response.json()) as PeerTubeVideo;
+
+    const formats: Format[] = [];
+
+    for (const file of data.files ?? []) {
+      const fileUrl = file.fileUrl ?? file.fileDownloadUrl;
+      if (!fileUrl) continue;
+
+      const height = file.resolution?.id;
+      formats.push({
+        format_id: `mp4-${file.resolution?.label ?? height ?? "unknown"}`,
+        url: fileUrl,
+        ext: "mp4",
+        height,
+        fps: file.fps,
+        filesize: file.size,
+        resolution: file.resolution?.label,
+        quality: height ?? 0,
+      });
+    }
+
+    for (const playlist of data.streamingPlaylists ?? []) {
+      if (playlist.playlistUrl) {
+        formats.push({
+          format_id: "hls",
+          url: playlist.playlistUrl,
+          ext: "mp4",
+          protocol: "m3u8",
+          quality: -1,
+        });
+      }
+    }
+
+    const thumbnails: Thumbnail[] = [];
+    if (data.thumbnailUrl) {
+      thumbnails.push({
+        url: data.thumbnailUrl.startsWith("http")
+          ? data.thumbnailUrl
+          : `https://${instance}${data.thumbnailUrl}`,
+      });
+    }
+    if (data.previewUrl) {
+      thumbnails.push({
+        url: data.previewUrl.startsWith("http")
+          ? data.previewUrl
+          : `https://${instance}${data.previewUrl}`,
+        preference: 1,
+      });
+    }
+
+    const subtitles: Record<string, Subtitle[]> = {};
+    for (const caption of data.captions ?? []) {
+      const lang = caption.language?.id ?? "und";
+      const captionUrl = caption.fileUrl ?? (caption.captionPath
+        ? `https://${instance}${caption.captionPath}`
+        : undefined);
+      if (captionUrl) {
+        subtitles[lang] = [{ url: captionUrl, ext: "vtt", name: caption.language?.label }];
+      }
+    }
+
+    return {
+      id: data.uuid ?? videoId,
+      title: data.name ?? videoId,
+      description: data.description,
+      duration: data.duration,
+      view_count: data.views,
+      like_count: data.likes,
+      uploader: data.account?.displayName ?? data.account?.name,
+      uploader_url: data.account?.url,
+      channel: data.channel?.displayName ?? data.channel?.name,
+      channel_url: data.channel?.url,
+      upload_date: data.publishedAt?.slice(0, 10).replace(/-/g, ""),
+      thumbnails,
+      formats,
+      subtitles,
+      live_status: data.isLive ? "is_live" : "not_live",
+      webpage_url: url,
+      extractor: this._NAME,
+    };
+  }
+}

package/src/extractors/reddit/gallery.ts

@@ -0,0 +1,124 @@
+import { BaseExtractor, ExtractorError } from "../../core/types";
+import type { InfoDict, Format } from "../../core/types";
+
+interface GalleryItem {
+  media_id: string;
+  id: number;
+  caption?: string;
+}
+
+interface GalleryMedia {
+  [mediaId: string]: {
+    e: string;
+    m?: string;
+    p?: Array<{ u: string; x: number; y: number }>;
+    s?: { u?: string; mp4?: string; gif?: string; x?: number; y?: number };
+    id: string;
+  };
+}
+
+interface RedditGalleryPost {
+  id: string;
+  title: string;
+  author: string;
+  url: string;
+  score?: number;
+  created_utc?: number;
+  gallery_data?: { items?: GalleryItem[] };
+  media_metadata?: GalleryMedia;
+}
+
+interface RedditApiResponse {
+  data: {
+    children: Array<{ data: RedditGalleryPost }>;
+  };
+}
+
+export class RedditGalleryExtractor extends BaseExtractor {
+  readonly _VALID_URL =
+    /^https?:\/\/(?:www\.|old\.)?reddit\.com\/(?:r\/[^/]+\/comments\/[^/]+|gallery\/[^/]+)/;
+  readonly _NAME = "reddit:gallery";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const normalized = url.replace(/(?:www\.|old\.)?reddit\.com/, "www.reddit.com");
+    const jsonUrl = normalized.replace(/\?.*$/, "").replace(/\/$/, "") + ".json";
+
+    const response = await fetch(jsonUrl, {
+      headers: { "User-Agent": "dlpx/1.0" },
+    });
+    if (!response.ok) {
+      throw new ExtractorError(`Reddit API returned ${response.status}`);
+    }
+
+    const data = (await response.json()) as RedditApiResponse[];
+    const post = data?.[0]?.data?.children?.[0]?.data;
+    if (!post) {
+      throw new ExtractorError("Could not parse Reddit gallery response");
+    }
+
+    if (!post.gallery_data || !post.media_metadata) {
+      throw new ExtractorError("No gallery data found in this post");
+    }
+
+    const items = post.gallery_data.items ?? [];
+    const metadata = post.media_metadata;
+
+    const entries: InfoDict[] = items.map((item, idx) => {
+      const media = metadata[item.media_id];
+      const formats: Format[] = [];
+
+      if (media) {
+        if (media.e === "AnimatedImage" && media.s?.mp4) {
+          formats.push({
+            format_id: "mp4",
+            url: media.s.mp4.replace(/&amp;/g, "&"),
+            ext: "mp4",
+            width: media.s.x,
+            height: media.s.y,
+          });
+        } else if (media.e === "Image" && media.s?.u) {
+          formats.push({
+            format_id: "image",
+            url: media.s.u.replace(/&amp;/g, "&"),
+            ext: media.m?.split("/")?.[1] ?? "jpg",
+            width: media.s.x,
+            height: media.s.y,
+          });
+        }
+
+        if (media.p && media.p.length > 0) {
+          for (const preview of media.p) {
+            formats.push({
+              format_id: `preview-${preview.x}x${preview.y}`,
+              url: preview.u.replace(/&amp;/g, "&"),
+              ext: "jpg",
+              width: preview.x,
+              height: preview.y,
+            });
+          }
+        }
+      }
+
+      return {
+        id: item.media_id,
+        title: item.caption ?? `${post.title} [${idx + 1}/${items.length}]`,
+        webpage_url: url,
+        uploader: post.author,
+        playlist_index: idx + 1,
+        formats,
+      };
+    });
+
+    return {
+      id: post.id,
+      title: post.title,
+      webpage_url: url,
+      uploader: post.author,
+      timestamp: post.created_utc,
+      like_count: post.score,
+      _type: "playlist",
+      entries,
+      playlist_count: entries.length,
+    };
+  }
+}

package/src/extractors/reddit/index.ts

@@ -0,0 +1,203 @@
+import { BaseExtractor, ExtractorError } from "../../core/types";
+import type { InfoDict, Format, Thumbnail } from "../../core/types";
+
+interface RedditVideoData {
+  dash_url?: string;
+  fallback_url?: string;
+  width?: number;
+  height?: number;
+  duration?: number;
+  is_gif?: boolean;
+}
+
+interface RedditPostData {
+  id: string;
+  title: string;
+  author: string;
+  url: string;
+  thumbnail?: string;
+  subreddit?: string;
+  score?: number;
+  created_utc?: number;
+  is_video?: boolean;
+  secure_media?: {
+    reddit_video?: RedditVideoData;
+  };
+  media?: {
+    reddit_video?: RedditVideoData;
+  };
+  preview?: {
+    images?: Array<{
+      source?: { url: string; width?: number; height?: number };
+    }>;
+  };
+}
+
+interface RedditApiResponse {
+  data: {
+    children: Array<{
+      data: RedditPostData;
+    }>;
+  };
+}
+
+function normalizeRedditUrl(url: string): string {
+  const urlObj = new URL(url);
+  if (urlObj.hostname === "v.redd.it") {
+    return url;
+  }
+  urlObj.hostname = "www.reddit.com";
+  return urlObj.href;
+}
+
+async function fetchAudioUrlFromDash(dashUrl: string): Promise<string | null> {
+  try {
+    const response = await fetch(dashUrl);
+    if (!response.ok) return null;
+    const text = await response.text();
+    const audioMatch = text.match(/<AdaptationSet[^>]*mimeType="audio[^"]*"[\s\S]*?<BaseURL>([^<]+)<\/BaseURL>/i);
+    if (audioMatch) {
+      const base = new URL(dashUrl);
+      return new URL(audioMatch[1], base.origin + base.pathname.replace(/[^/]+$/, "")).href;
+    }
+    const audioInitMatch = text.match(/initialization="([^"]*audio[^"]*)"/i);
+    if (audioInitMatch) {
+      const base = new URL(dashUrl);
+      return new URL(audioInitMatch[1], base.origin + base.pathname.replace(/[^/]+$/, "")).href;
+    }
+  } catch {
+    return null;
+  }
+  return null;
+}
+
+export class RedditExtractor extends BaseExtractor {
+  readonly _VALID_URL =
+    /^https?:\/\/(?:www\.|old\.)?reddit\.com\/r\/[^/]+\/comments\/[^/]+|^https?:\/\/v\.redd\.it\/[^/]+/;
+  readonly _NAME = "reddit";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    if (/^https?:\/\/v\.redd\.it\//.test(url)) {
+      return this._extractVReddIt(url);
+    }
+    return this._extractRedditPost(url);
+  }
+
+  private async _extractVReddIt(url: string): Promise<InfoDict> {
+    const jsonUrl = url.endsWith("/") ? url + ".json" : url + "/.json";
+    const response = await fetch(jsonUrl, {
+      headers: { "User-Agent": "dlpx/1.0" },
+    });
+    if (!response.ok) {
+      throw new ExtractorError(`Reddit API returned ${response.status}`);
+    }
+    const data = (await response.json()) as RedditApiResponse | RedditApiResponse[];
+    const apiData = Array.isArray(data) ? data[0] : data;
+    const post = apiData?.data?.children?.[0]?.data;
+    if (!post) {
+      throw new ExtractorError("Could not parse Reddit API response");
+    }
+    return this._buildInfoFromPost(url, post);
+  }
+
+  private async _extractRedditPost(url: string): Promise<InfoDict> {
+    const normalized = normalizeRedditUrl(url);
+    const jsonUrl = normalized.replace(/\?.*$/, "").replace(/\/$/, "") + ".json";
+    const response = await fetch(jsonUrl, {
+      headers: { "User-Agent": "dlpx/1.0" },
+    });
+    if (!response.ok) {
+      throw new ExtractorError(`Reddit API returned ${response.status}`);
+    }
+    const data = (await response.json()) as RedditApiResponse[];
+    const post = data?.[0]?.data?.children?.[0]?.data;
+    if (!post) {
+      throw new ExtractorError("Could not parse Reddit API response");
+    }
+    return this._buildInfoFromPost(url, post);
+  }
+
+  private async _buildInfoFromPost(url: string, post: RedditPostData): Promise<InfoDict> {
+    const videoData = post.secure_media?.reddit_video ?? post.media?.reddit_video;
+
+    if (!videoData) {
+      throw new ExtractorError("No Reddit video found in this post");
+    }
+
+    const formats: Format[] = [];
+
+    if (videoData.fallback_url) {
+      formats.push({
+        format_id: "mp4-video-only",
+        url: videoData.fallback_url,
+        ext: "mp4",
+        vcodec: "h264",
+        acodec: "none",
+        width: videoData.width,
+        height: videoData.height,
+        format_note: "video only",
+      });
+    }
+
+    if (videoData.dash_url) {
+      let audioUrl: string | null = null;
+      if (videoData.fallback_url) {
+        audioUrl = videoData.fallback_url.replace(/DASH_\d+\.mp4/, "DASH_audio.mp4");
+      }
+      if (!audioUrl) {
+        audioUrl = await fetchAudioUrlFromDash(videoData.dash_url);
+      }
+
+      formats.push({
+        format_id: "dash",
+        url: videoData.dash_url,
+        ext: "mp4",
+        protocol: "dash",
+        width: videoData.width,
+        height: videoData.height,
+        format_note: "DASH manifest",
+      });
+
+      if (audioUrl && videoData.fallback_url) {
+        formats.push({
+          format_id: "mp4-with-audio",
+          url: videoData.fallback_url,
+          ext: "mp4",
+          vcodec: "h264",
+          acodec: "aac",
+          width: videoData.width,
+          height: videoData.height,
+          format_note: "video+audio (merged)",
+          http_headers: { "User-Agent": "dlpx/1.0" },
+        });
+      }
+    }
+
+    const thumbnails: Thumbnail[] = [];
+    if (post.thumbnail && post.thumbnail !== "default" && post.thumbnail !== "self") {
+      thumbnails.push({ url: post.thumbnail });
+    }
+    const previewSource = post.preview?.images?.[0]?.source;
+    if (previewSource?.url) {
+      thumbnails.push({
+        url: previewSource.url.replace(/&amp;/g, "&"),
+        width: previewSource.width,
+        height: previewSource.height,
+        preference: 1,
+      });
+    }
+
+    return {
+      id: post.id,
+      title: post.title,
+      webpage_url: url,
+      uploader: post.author,
+      duration: videoData.duration,
+      timestamp: post.created_utc,
+      like_count: post.score,
+      formats,
+      thumbnails,
+      extractor: this._NAME,
+    };
+  }
+}

package/src/extractors/rumble.ts

@@ -0,0 +1,127 @@
+import { BaseExtractor, ExtractorError } from "../core/types";
+import type { InfoDict, Format, Thumbnail } from "../core/types";
+
+interface RumbleVideoSource {
+  url: string;
+  w?: number;
+  h?: number;
+  fps?: number;
+  bitrate?: number;
+}
+
+interface RumbleVideoInfo {
+  title?: string;
+  description?: string;
+  author?: string;
+  pubDate?: string;
+  duration?: number;
+  mainAncestorId?: string;
+  ua?: Record<string, Record<string, RumbleVideoSource>>;
+  t?: Array<{ i?: string; u?: string }>;
+}
+
+export class RumbleExtractor extends BaseExtractor {
+  readonly _VALID_URL = /https?:\/\/(?:www\.)?rumble\.com\/(?:v[a-zA-Z0-9]+-[^/?]+\.html|embed\/([a-zA-Z0-9]+))/;
+  readonly _NAME = "rumble";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const response = await fetch(url, {
+      headers: { "User-Agent": "Mozilla/5.0" },
+    });
+
+    if (!response.ok) {
+      throw new ExtractorError(`Rumble fetch error: ${response.status}`);
+    }
+
+    const html = await response.text();
+
+    const embedIdMatch = html.match(/rumble\.com\/embed\/([a-zA-Z0-9]+)/);
+    let embedHtml = html;
+
+    if (embedIdMatch && !url.includes("/embed/")) {
+      const embedUrl = `https://rumble.com/embed/${embedIdMatch[1]}/`;
+      const embedResponse = await fetch(embedUrl, {
+        headers: { "User-Agent": "Mozilla/5.0" },
+      });
+      embedHtml = await embedResponse.text();
+    }
+
+    const configMatch = embedHtml.match(/RumblePlayer\.render\([^,]+,\s*(\{[\s\S]*?\})\s*\)/);
+    const jsonMatch = embedHtml.match(/"video"\s*:\s*(\{[\s\S]*?\})\s*(?:,\s*"|\})/);
+
+    let videoData: RumbleVideoInfo = {};
+
+    if (configMatch) {
+      try {
+        videoData = JSON.parse(configMatch[1]) as RumbleVideoInfo;
+      } catch {
+      }
+    }
+
+    if (!videoData.ua && jsonMatch) {
+      try {
+        videoData = JSON.parse(jsonMatch[1]) as RumbleVideoInfo;
+      } catch {
+      }
+    }
+
+    if (!videoData.ua) {
+      const scriptMatch = embedHtml.match(/<script[^>]*>\s*var\s+videoConfig\s*=\s*(\{[\s\S]*?\});\s*<\/script>/);
+      if (scriptMatch) {
+        try {
+          videoData = JSON.parse(scriptMatch[1]) as RumbleVideoInfo;
+        } catch {
+        }
+      }
+    }
+
+    const ua = videoData.ua ?? {};
+    const formats: Format[] = [];
+
+    for (const [formatKey, sourceMap] of Object.entries(ua)) {
+      for (const [, source] of Object.entries(sourceMap)) {
+        if (source.url) {
+          const isHLS = source.url.includes(".m3u8");
+          formats.push({
+            format_id: formatKey,
+            url: source.url,
+            ext: isHLS ? "mp4" : "mp4",
+            protocol: isHLS ? "m3u8" : undefined,
+            width: source.w,
+            height: source.h,
+            fps: source.fps,
+            tbr: source.bitrate,
+            resolution: source.h ? `${source.h}p` : undefined,
+            quality: source.h ?? 0,
+          });
+        }
+      }
+    }
+
+    const thumbnails: Thumbnail[] = [];
+    if (videoData.t) {
+      for (const thumb of videoData.t) {
+        if (thumb.u) thumbnails.push({ url: thumb.u });
+      }
+    }
+
+    const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
+    const ogTitle = html.match(/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i);
+    const title = videoData.title ?? ogTitle?.[1] ?? titleMatch?.[1] ?? "Rumble Video";
+
+    const idMatch = url.match(/\/(v[a-zA-Z0-9]+)-/) ?? url.match(/embed\/([a-zA-Z0-9]+)/);
+    const id = idMatch?.[1] ?? "unknown";
+
+    return {
+      id,
+      title,
+      description: videoData.description,
+      uploader: videoData.author,
+      duration: videoData.duration,
+      thumbnails,
+      formats,
+      webpage_url: url,
+      extractor: this._NAME,
+    };
+  }
+}