getraw 0.1.0

package/src/extractors/google-drive.ts

@@ -0,0 +1,106 @@
+import { BaseExtractor, ExtractorError } from "../core/types";
+import type { InfoDict, Format } from "../core/types";
+
+const GDRIVE_VALID_URL = /https?:\/\/(?:docs\.google\.com\/(?:file\/d\/|open\?id=)|drive\.google\.com\/(?:file\/d\/|open\?id=)|drive\.google\.com\/uc\?(?:.*&)?id=)([a-zA-Z0-9_-]+)/;
+
+export class GoogleDriveExtractor extends BaseExtractor {
+  readonly _VALID_URL = GDRIVE_VALID_URL;
+  readonly _NAME = "google-drive";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const match = url.match(this._VALID_URL);
+    if (!match) throw new ExtractorError(`Invalid Google Drive URL: ${url}`);
+    const fileId = match[1];
+
+    const downloadUrl = `https://drive.google.com/uc?export=download&id=${fileId}`;
+
+    const response = await fetch(downloadUrl, {
+      headers: { "User-Agent": "Mozilla/5.0" },
+      redirect: "follow",
+    });
+
+    if (!response.ok) {
+      throw new ExtractorError(`Google Drive: fetch error ${response.status}`);
+    }
+
+    const contentType = response.headers.get("content-type") ?? "";
+
+    if (contentType.includes("text/html")) {
+      const html = await response.text();
+      return this.handleVirusScanPage(html, fileId, url);
+    }
+
+    const contentDisposition = response.headers.get("content-disposition") ?? "";
+    const filenameMatch = contentDisposition.match(/filename\*?=(?:UTF-8'')?["']?([^"';\n]+)["']?/i);
+    const filename = filenameMatch?.[1]
+      ? decodeURIComponent(filenameMatch[1].trim())
+      : `gdrive_${fileId}`;
+    const ext = filename.includes(".") ? filename.split(".").pop()!.toLowerCase() : "mp4";
+    const title = filename.replace(/\.[^.]+$/, "");
+
+    const contentLength = response.headers.get("content-length");
+    const filesize = contentLength ? parseInt(contentLength) : undefined;
+
+    const resolvedUrl = response.url || downloadUrl;
+    return this.buildInfo(fileId, title, ext, resolvedUrl, filesize, url);
+  }
+
+  private async handleVirusScanPage(html: string, fileId: string, originalUrl: string): Promise<InfoDict> {
+    const confirmMatch = html.match(/[?&]confirm=([0-9A-Za-z_-]+)/);
+    if (!confirmMatch) {
+      const idMatch = html.match(/id=([a-zA-Z0-9_-]+)/);
+      const foundId = idMatch?.[1] ?? fileId;
+      const directUrl = `https://drive.google.com/uc?export=download&id=${foundId}`;
+      return this.buildInfo(fileId, `gdrive_${fileId}`, "mp4", directUrl, undefined, originalUrl);
+    }
+
+    const confirmToken = confirmMatch[1];
+    const confirmedUrl = `https://drive.google.com/uc?export=download&id=${fileId}&confirm=${confirmToken}`;
+
+    const confirmedResponse = await fetch(confirmedUrl, {
+      headers: { "User-Agent": "Mozilla/5.0" },
+      redirect: "follow",
+    });
+
+    if (!confirmedResponse.ok) {
+      throw new ExtractorError(`Google Drive: confirmed download failed with status ${confirmedResponse.status}`);
+    }
+
+    const contentDisposition = confirmedResponse.headers.get("content-disposition") ?? "";
+    const filenameMatch = contentDisposition.match(/filename\*?=(?:UTF-8'')?["']?([^"';\n]+)["']?/i);
+    const filename = filenameMatch?.[1]
+      ? decodeURIComponent(filenameMatch[1].trim())
+      : `gdrive_${fileId}`;
+    const ext = filename.includes(".") ? filename.split(".").pop()!.toLowerCase() : "mp4";
+    const title = filename.replace(/\.[^.]+$/, "");
+
+    const contentLength = confirmedResponse.headers.get("content-length");
+    const filesize = contentLength ? parseInt(contentLength) : undefined;
+
+    return this.buildInfo(fileId, title, ext, confirmedUrl, filesize, originalUrl);
+  }
+
+  private buildInfo(id: string, title: string, ext: string, directUrl: string, filesize: number | undefined, webpageUrl: string): InfoDict {
+    const formats: Format[] = [
+      {
+        format_id: "direct",
+        url: directUrl,
+        ext,
+        filesize,
+        http_headers: {
+          "User-Agent": "Mozilla/5.0",
+        },
+      },
+    ];
+
+    return {
+      id,
+      title,
+      url: directUrl,
+      ext,
+      formats,
+      webpage_url: webpageUrl,
+      extractor: this._NAME,
+    };
+  }
+}

package/src/extractors/imgur.ts

@@ -0,0 +1,156 @@
+import { BaseExtractor, ExtractorError } from "../core/types";
+import type { InfoDict, Format, Thumbnail } from "../core/types";
+
+interface ImgurImage {
+  id?: string;
+  title?: string;
+  description?: string;
+  type?: string;
+  mp4?: string;
+  link?: string;
+  width?: number;
+  height?: number;
+  size?: number;
+  animated?: boolean;
+}
+
+interface ImgurApiResponse {
+  data?: ImgurImage | ImgurImage[] | { images?: ImgurImage[]; title?: string; description?: string; id?: string };
+  success?: boolean;
+  status?: number;
+}
+
+const IMGUR_VALID_URL = /https?:\/\/(?:i\.)?imgur\.com\/(?:a\/|gallery\/)?([a-zA-Z0-9]+)(?:\.[a-zA-Z]+)?/;
+
+export class ImgurExtractor extends BaseExtractor {
+  readonly _VALID_URL = IMGUR_VALID_URL;
+  readonly _NAME = "imgur";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const match = url.match(this._VALID_URL);
+    if (!match) throw new ExtractorError(`Invalid Imgur URL: ${url}`);
+    const itemId = match[1];
+
+    if (url.includes("i.imgur.com") && /\.(gif|gifv|mp4|webm)$/i.test(url)) {
+      return this.extractDirectMedia(url, itemId);
+    }
+
+    const isAlbum = url.includes("/a/") || url.includes("/gallery/");
+    return isAlbum
+      ? this.extractAlbum(url, itemId)
+      : this.extractSingle(url, itemId);
+  }
+
+  private extractDirectMedia(url: string, id: string): InfoDict {
+    const mp4Url = url.replace(/\.gifv?$/i, ".mp4");
+    return {
+      id,
+      title: id,
+      url: mp4Url,
+      ext: "mp4",
+      formats: [{ format_id: "mp4", url: mp4Url, ext: "mp4" }],
+      webpage_url: url,
+      extractor: this._NAME,
+    };
+  }
+
+  private async extractSingle(url: string, id: string): Promise<InfoDict> {
+    const apiUrl = `https://api.imgur.com/3/image/${id}`;
+    const response = await fetch(apiUrl, {
+      headers: {
+        Authorization: "Client-ID 546c25a59c58ad7",
+        "User-Agent": "Mozilla/5.0",
+      },
+    });
+
+    if (!response.ok) {
+      return this.extractDirectMedia(`https://i.imgur.com/${id}.mp4`, id);
+    }
+
+    const data = (await response.json()) as ImgurApiResponse;
+    const image = data.data as ImgurImage | undefined;
+
+    if (!image) throw new ExtractorError("Imgur: no image data in response");
+
+    return this.buildImageInfo(image, url);
+  }
+
+  private async extractAlbum(url: string, id: string): Promise<InfoDict> {
+    const apiUrl = `https://api.imgur.com/3/album/${id}/images`;
+    const response = await fetch(apiUrl, {
+      headers: {
+        Authorization: "Client-ID 546c25a59c58ad7",
+        "User-Agent": "Mozilla/5.0",
+      },
+    });
+
+    if (!response.ok) {
+      throw new ExtractorError(`Imgur album API error: ${response.status}`);
+    }
+
+    const data = (await response.json()) as ImgurApiResponse;
+    const images = data.data as ImgurImage[] | undefined;
+
+    if (!images || !Array.isArray(images)) {
+      throw new ExtractorError("Imgur: no album images in response");
+    }
+
+    const videoImages = images.filter((img) => img.animated || img.type?.includes("video") || img.mp4);
+
+    if (videoImages.length === 0) {
+      throw new ExtractorError("Imgur: album contains no video/animated content");
+    }
+
+    if (videoImages.length === 1) {
+      return this.buildImageInfo(videoImages[0], url);
+    }
+
+    const entries = videoImages.map((img, i) => ({
+      ...this.buildImageInfo(img, `https://imgur.com/${img.id ?? ""}`),
+      playlist_index: i + 1,
+    }));
+
+    return {
+      id,
+      title: `Imgur Album ${id}`,
+      entries,
+      _type: "playlist" as const,
+      playlist_count: entries.length,
+      webpage_url: url,
+      extractor: this._NAME,
+    };
+  }
+
+  private buildImageInfo(image: ImgurImage, url: string): InfoDict {
+    const id = image.id ?? "unknown";
+    const formats: Format[] = [];
+    const thumbnails: Thumbnail[] = [];
+
+    const mp4Url = image.mp4 ?? `https://i.imgur.com/${id}.mp4`;
+
+    if (image.animated || image.type?.includes("gif") || image.mp4) {
+      formats.push({
+        format_id: "mp4",
+        url: mp4Url,
+        ext: "mp4",
+        width: image.width,
+        height: image.height,
+        filesize: image.size,
+      });
+    }
+
+    if (image.link) {
+      thumbnails.push({ url: image.link, width: image.width, height: image.height });
+    }
+
+    return {
+      id,
+      title: image.title ?? id,
+      description: image.description,
+      thumbnails,
+      formats,
+      webpage_url: url,
+      extractor: this._NAME,
+    };
+  }
+}

package/src/extractors/instagram/index.ts

@@ -0,0 +1,263 @@
+import { BaseExtractor, ExtractorError } from "../../core/types";
+import type { InfoDict, Format, Thumbnail } from "../../core/types";
+
+const VALID_URL =
+  /https?:\/\/(?:www\.)?instagram\.com\/(?:p|reel|reels)\/([A-Za-z0-9_-]+)/;
+
+interface IGMediaNode {
+  __typename?: string;
+  id?: string;
+  shortcode?: string;
+  video_url?: string;
+  display_url?: string;
+  is_video?: boolean;
+  dimensions?: { width?: number; height?: number };
+  accessibility_caption?: string;
+  edge_media_to_caption?: { edges?: Array<{ node?: { text?: string } }> };
+  owner?: {
+    id?: string;
+    username?: string;
+    full_name?: string;
+    profile_pic_url?: string;
+  };
+  taken_at_timestamp?: number;
+  video_view_count?: number;
+  edge_liked_by?: { count?: number };
+  edge_media_to_comment?: { count?: number };
+  video_duration?: number;
+  edge_sidecar_to_children?: {
+    edges?: Array<{ node?: IGMediaNode }>;
+  };
+  thumbnail_src?: string;
+  thumbnail_resources?: Array<{
+    src?: string;
+    config_width?: number;
+    config_height?: number;
+  }>;
+}
+
+interface IGSharedData {
+  entry_data?: {
+    PostPage?: Array<{
+      graphql?: { shortcode_media?: IGMediaNode };
+    }>;
+  };
+  config?: { viewer?: { id?: string } };
+}
+
+interface IGAdditionalData {
+  graphql?: { shortcode_media?: IGMediaNode };
+}
+
+export class InstagramExtractor extends BaseExtractor {
+  readonly _VALID_URL = VALID_URL;
+  readonly _NAME = "instagram";
+
+  private readonly _headers: Record<string, string> = {
+    "User-Agent":
+      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    Accept:
+      "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+    "Sec-Fetch-Dest": "document",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-Site": "none",
+  };
+
+  private extractSharedData(html: string): IGMediaNode | null {
+    const sharedDataMatch = html.match(
+      /window\._sharedData\s*=\s*(\{[\s\S]*?\});(?:\s*<\/script>|\s*window\.)/,
+    );
+    if (sharedDataMatch) {
+      try {
+        const sharedData = JSON.parse(sharedDataMatch[1]) as IGSharedData;
+        const media =
+          sharedData?.entry_data?.PostPage?.[0]?.graphql?.shortcode_media;
+        if (media) return media;
+      } catch {
+      }
+    }
+
+    const additionalDataMatch = html.match(
+      /window\.__additionalDataLoaded\s*\(\s*['"][^'"]+['"]\s*,\s*(\{[\s\S]*?\})\s*\)\s*;/,
+    );
+    if (additionalDataMatch) {
+      try {
+        const additionalData = JSON.parse(
+          additionalDataMatch[1],
+        ) as IGAdditionalData;
+        const media = additionalData?.graphql?.shortcode_media;
+        if (media) return media;
+      } catch {
+      }
+    }
+
+    const scriptMatches = html.matchAll(
+      /<script[^>]*type="application\/json"[^>]*>([\s\S]*?)<\/script>/g,
+    );
+    for (const scriptMatch of scriptMatches) {
+      try {
+        const data = JSON.parse(scriptMatch[1]) as Record<string, unknown>;
+        if (data && typeof data === "object" && "shortcode_media" in data) {
+          return (data as { shortcode_media: IGMediaNode }).shortcode_media;
+        }
+      } catch {
+      }
+    }
+
+    return null;
+  }
+
+  private buildInfoFromMedia(
+    media: IGMediaNode,
+    shortcode: string,
+    url: string,
+  ): InfoDict {
+    const formats: Format[] = [];
+    const thumbnails: Thumbnail[] = [];
+    const entries: InfoDict[] = [];
+
+    const description =
+      media.edge_media_to_caption?.edges?.[0]?.node?.text ?? undefined;
+
+    const uploadDate = media.taken_at_timestamp
+      ? new Date(media.taken_at_timestamp * 1000)
+          .toISOString()
+          .slice(0, 10)
+          .replace(/-/g, "")
+      : undefined;
+
+    if (media.thumbnail_resources) {
+      for (const thumb of media.thumbnail_resources) {
+        if (thumb.src) {
+          thumbnails.push({
+            url: thumb.src,
+            width: thumb.config_width,
+            height: thumb.config_height,
+          });
+        }
+      }
+    }
+    if (media.thumbnail_src) {
+      thumbnails.push({ url: media.thumbnail_src });
+    }
+    if (media.display_url) {
+      thumbnails.push({ url: media.display_url, preference: 1 });
+    }
+
+    if (media.edge_sidecar_to_children?.edges) {
+      for (const edge of media.edge_sidecar_to_children.edges) {
+        const node = edge.node;
+        if (!node) continue;
+        const entryFormats: Format[] = [];
+        const entryThumbs: Thumbnail[] = [];
+
+        if (node.is_video && node.video_url) {
+          entryFormats.push({
+            format_id: "mp4",
+            url: node.video_url,
+            ext: "mp4",
+            width: node.dimensions?.width,
+            height: node.dimensions?.height,
+          });
+        }
+        if (node.display_url) {
+          entryThumbs.push({ url: node.display_url });
+        }
+
+        entries.push({
+          id: node.id ?? node.shortcode ?? "unknown",
+          title: description?.slice(0, 100) ?? `Instagram media`,
+          description,
+          formats: entryFormats,
+          thumbnails: entryThumbs,
+          _type: node.is_video ? "video" : "url",
+          url: node.is_video ? node.video_url : node.display_url,
+        });
+      }
+    }
+
+    if (entries.length === 0 && media.is_video && media.video_url) {
+      formats.push({
+        format_id: "mp4",
+        url: media.video_url,
+        ext: "mp4",
+        width: media.dimensions?.width,
+        height: media.dimensions?.height,
+      });
+    }
+
+    const base: InfoDict = {
+      id: media.id ?? shortcode,
+      title: description?.slice(0, 100) ?? `Instagram post ${shortcode}`,
+      description,
+      uploader: media.owner?.full_name ?? media.owner?.username,
+      uploader_id: media.owner?.username,
+      uploader_url: media.owner?.username
+        ? `https://www.instagram.com/${media.owner.username}/`
+        : undefined,
+      channel_id: media.owner?.id,
+      upload_date: uploadDate,
+      timestamp: media.taken_at_timestamp,
+      view_count: media.video_view_count,
+      like_count: media.edge_liked_by?.count,
+      comment_count: media.edge_media_to_comment?.count,
+      duration: media.video_duration,
+      thumbnails,
+      webpage_url: url,
+    };
+
+    if (entries.length > 1) {
+      return { ...base, _type: "playlist", entries, playlist_count: entries.length };
+    }
+
+    return { ...base, formats, _type: "video" };
+  }
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const match = VALID_URL.exec(url);
+    if (!match) throw new ExtractorError(`instagram: invalid URL: ${url}`);
+    const shortcode = match[1];
+
+    const apiUrl = `https://www.instagram.com/p/${shortcode}/?__a=1&__d=dis`;
+    const apiResp = await fetch(apiUrl, {
+      headers: {
+        ...this._headers,
+        Accept: "application/json, text/javascript, */*; q=0.01",
+        "X-Requested-With": "XMLHttpRequest",
+      },
+    });
+
+    if (apiResp.ok) {
+      try {
+        const data = (await apiResp.json()) as IGAdditionalData;
+        const media = data?.graphql?.shortcode_media;
+        if (media) {
+          return this.buildInfoFromMedia(media, shortcode, url);
+        }
+      } catch {
+      }
+    }
+
+    const pageResp = await fetch(`https://www.instagram.com/p/${shortcode}/`, {
+      headers: this._headers,
+    });
+
+    if (!pageResp.ok) {
+      throw new ExtractorError(
+        `instagram: page fetch failed: ${pageResp.status} ${pageResp.statusText}`,
+      );
+    }
+
+    const html = await pageResp.text();
+    const media = this.extractSharedData(html);
+
+    if (!media) {
+      throw new ExtractorError(
+        `instagram: could not extract media data for post ${shortcode}`,
+      );
+    }
+
+    return this.buildInfoFromMedia(media, shortcode, url);
+  }
+}

package/src/extractors/instagram/reels.ts

@@ -0,0 +1,166 @@
+import { BaseExtractor, ExtractorError } from "../../core/types";
+import type { InfoDict, Format, Thumbnail } from "../../core/types";
+
+const VALID_URL = /https?:\/\/(?:www\.)?instagram\.com\/reels?\/?(?:\?.*)?$/;
+
+const REELS_DOC_ID = "8845758582119845";
+
+interface ReelsEdgeNode {
+  id?: string;
+  shortcode?: string;
+  is_video?: boolean;
+  video_url?: string;
+  display_url?: string;
+  dimensions?: { width?: number; height?: number };
+  video_view_count?: number;
+  video_duration?: number;
+  taken_at_timestamp?: number;
+  edge_liked_by?: { count?: number };
+  edge_media_to_comment?: { count?: number };
+  edge_media_to_caption?: { edges?: Array<{ node?: { text?: string } }> };
+  owner?: {
+    id?: string;
+    username?: string;
+    full_name?: string;
+  };
+}
+
+interface ReelsResponse {
+  data?: {
+    xdt_api__v1__clips__home__connection_v2?: {
+      edges?: Array<{ node?: { media?: ReelsEdgeNode } }>;
+      page_info?: { end_cursor?: string; has_next_page?: boolean };
+    };
+  };
+}
+
+export class InstagramReelsExtractor extends BaseExtractor {
+  readonly _VALID_URL = VALID_URL;
+  readonly _NAME = "instagram:reels";
+
+  private readonly _headers: Record<string, string> = {
+    "User-Agent":
+      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    Accept: "*/*",
+    "Accept-Language": "en-US,en;q=0.5",
+    "Content-Type": "application/x-www-form-urlencoded",
+    "X-FB-Friendly-Name": "PolarisClipsHomePageQuery",
+    "X-IG-App-ID": "936619743392459",
+    Origin: "https://www.instagram.com",
+    Referer: "https://www.instagram.com/reels/",
+  };
+
+  private buildEntryFromNode(node: ReelsEdgeNode): InfoDict | null {
+    if (!node.is_video || !node.video_url) return null;
+
+    const shortcode = node.shortcode ?? node.id ?? "unknown";
+    const description = node.edge_media_to_caption?.edges?.[0]?.node?.text;
+    const formats: Format[] = [
+      {
+        format_id: "mp4",
+        url: node.video_url,
+        ext: "mp4",
+        width: node.dimensions?.width,
+        height: node.dimensions?.height,
+      },
+    ];
+    const thumbnails: Thumbnail[] = node.display_url
+      ? [{ url: node.display_url }]
+      : [];
+    const uploadDate = node.taken_at_timestamp
+      ? new Date(node.taken_at_timestamp * 1000)
+          .toISOString()
+          .slice(0, 10)
+          .replace(/-/g, "")
+      : undefined;
+
+    return {
+      id: node.id ?? shortcode,
+      title: description?.slice(0, 100) ?? `Instagram Reel ${shortcode}`,
+      description,
+      url: `https://www.instagram.com/reel/${shortcode}/`,
+      webpage_url: `https://www.instagram.com/reel/${shortcode}/`,
+      uploader: node.owner?.full_name ?? node.owner?.username,
+      uploader_id: node.owner?.username,
+      uploader_url: node.owner?.username
+        ? `https://www.instagram.com/${node.owner.username}/`
+        : undefined,
+      channel_id: node.owner?.id,
+      timestamp: node.taken_at_timestamp,
+      upload_date: uploadDate,
+      view_count: node.video_view_count,
+      like_count: node.edge_liked_by?.count,
+      comment_count: node.edge_media_to_comment?.count,
+      duration: node.video_duration,
+      formats,
+      thumbnails,
+      _type: "video",
+    };
+  }
+
+  private async fetchReelsPage(cursor?: string): Promise<ReelsResponse> {
+    const variables: Record<string, unknown> = {
+      surface: "REELS_TAB",
+      has_threaded_comments: true,
+    };
+    if (cursor) variables["after"] = cursor;
+
+    const body = new URLSearchParams({
+      doc_id: REELS_DOC_ID,
+      variables: JSON.stringify(variables),
+      server_timestamps: "true",
+    });
+
+    const resp = await fetch("https://www.instagram.com/graphql/query", {
+      method: "POST",
+      headers: this._headers,
+      body: body.toString(),
+    });
+
+    if (!resp.ok) {
+      throw new ExtractorError(
+        `instagram:reels: GraphQL request failed: ${resp.status}`,
+      );
+    }
+
+    return (await resp.json()) as ReelsResponse;
+  }
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const entries: InfoDict[] = [];
+    let cursor: string | undefined = undefined;
+    let hasNextPage = true;
+    let pageCount = 0;
+    const maxPages = 5;
+
+    while (hasNextPage && pageCount < maxPages) {
+      const data = await this.fetchReelsPage(cursor);
+      const connection =
+        data?.data?.xdt_api__v1__clips__home__connection_v2;
+      const edges = connection?.edges ?? [];
+
+      for (const edge of edges) {
+        const media = edge.node?.media;
+        if (!media) continue;
+        const entry = this.buildEntryFromNode(media);
+        if (entry) entries.push(entry);
+      }
+
+      const pageInfo = connection?.page_info;
+      hasNextPage = pageInfo?.has_next_page ?? false;
+      cursor = pageInfo?.end_cursor;
+      pageCount++;
+
+      if (edges.length === 0) break;
+    }
+
+    return {
+      id: "instagram-reels",
+      title: "Instagram Reels",
+      webpage_url: url,
+      _type: "playlist",
+      entries,
+      playlist_count: entries.length,
+    };
+  }
+}