getraw 0.1.0

package/src/extractors/twitch/live.ts

@@ -0,0 +1,132 @@
+import { BaseExtractor, ExtractorError } from "../../core/types";
+import type { InfoDict, Format, Thumbnail } from "../../core/types";
+
+const TWITCH_CLIENT_ID = "kimne78kx3ncx6brgo4mv6wki5h1ko";
+const GQL_ENDPOINT = "https://gql.twitch.tv/gql";
+
+interface StreamAccessToken {
+  value: string;
+  signature: string;
+}
+
+interface StreamNode {
+  id?: string;
+  title?: string;
+  viewersCount?: number;
+  previewImageURL?: string;
+  broadcaster?: { displayName?: string; login?: string; id?: string };
+  game?: { name?: string };
+}
+
+interface GQLResponse<T> {
+  data: T;
+  errors?: Array<{ message: string }>;
+}
+
+async function gqlRequest<T>(query: object): Promise<T> {
+  const response = await fetch(GQL_ENDPOINT, {
+    method: "POST",
+    headers: {
+      "Client-ID": TWITCH_CLIENT_ID,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify(query),
+  });
+  if (!response.ok) {
+    throw new ExtractorError(`Twitch GQL request failed: ${response.status}`);
+  }
+  const result = (await response.json()) as GQLResponse<T>;
+  if (result.errors?.length) {
+    throw new ExtractorError(`Twitch GQL error: ${result.errors[0].message}`);
+  }
+  return result.data;
+}
+
+export class TwitchLiveExtractor extends BaseExtractor {
+  readonly _VALID_URL =
+    /^https?:\/\/(?:www\.)?twitch\.tv\/([a-zA-Z0-9_]+)(?:\/)?(?:\?.*)?$/;
+  readonly _NAME = "twitch:live";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const match = this._VALID_URL.exec(url);
+    if (!match) throw new ExtractorError("Invalid Twitch channel URL");
+    const login = match[1].toLowerCase();
+
+    const [tokenData, streamData] = await Promise.all([
+      gqlRequest<{ streamPlaybackAccessToken: StreamAccessToken }>({
+        operationName: "PlaybackAccessToken",
+        variables: {
+          isLive: true,
+          login,
+          isVod: false,
+          vodID: "",
+          playerType: "site",
+        },
+        extensions: {
+          persistedQuery: {
+            version: 1,
+            sha256Hash: "0828119ded1c13477966434e15800ff57ddacf13ba1911c129dc2200705b0712",
+          },
+        },
+      }),
+      gqlRequest<{ user?: { stream?: StreamNode } }>({
+        operationName: "StreamMetadata",
+        variables: { channelLogin: login },
+        extensions: {
+          persistedQuery: {
+            version: 1,
+            sha256Hash: "1c719a40e481453e5c48d9bb585d971b8b372f8ebb105b17076722264dfa5b3e",
+          },
+        },
+      }),
+    ]);
+
+    const token = tokenData.streamPlaybackAccessToken;
+    if (!token) throw new ExtractorError("Could not get stream access token — channel may be offline");
+
+    const hlsUrl =
+      `https://usher.twitchapps.com/api/channel/hls/${login}.m3u8?` +
+      new URLSearchParams({
+        sig: token.signature,
+        token: token.value,
+        allow_source: "true",
+        allow_audio_only: "true",
+        allow_spectre: "true",
+        p: String(Math.floor(Math.random() * 999999)),
+        platform: "web",
+        play_session_id: crypto.randomUUID().replace(/-/g, ""),
+        supported_codecs: "avc1",
+      });
+
+    const stream = streamData?.user?.stream;
+
+    const formats: Format[] = [
+      {
+        format_id: "hls-live",
+        url: hlsUrl,
+        ext: "mp4",
+        protocol: "m3u8",
+        http_headers: { "Client-ID": TWITCH_CLIENT_ID },
+        format_note: "live stream",
+      },
+    ];
+
+    const thumbnails: Thumbnail[] = [];
+    if (stream?.previewImageURL) {
+      thumbnails.push({ url: stream.previewImageURL });
+    }
+
+    return {
+      id: login,
+      title: stream?.title ?? `${login} live stream`,
+      webpage_url: url,
+      uploader: login,
+      uploader_id: login,
+      view_count: stream?.viewersCount,
+      live_status: "is_live",
+      categories: stream?.game?.name ? [stream.game.name] : undefined,
+      formats,
+      thumbnails,
+    };
+  }
+}

package/src/extractors/twitter/index.ts

@@ -0,0 +1,140 @@
+import { BaseExtractor, ExtractorError } from "../../core/types";
+import type { InfoDict, Format, Thumbnail } from "../../core/types";
+
+const VALID_URL = /https?:\/\/(?:www\.)?(?:twitter|x)\.com\/\w+\/status\/(\d+)/;
+
+interface VideoVariant {
+  content_type: string;
+  url: string;
+  bitrate?: number;
+}
+
+interface VideoInfo {
+  variants: VideoVariant[];
+  duration_millis?: number;
+}
+
+interface MediaDetail {
+  type: string;
+  video_info?: VideoInfo;
+  media_url_https?: string;
+  original_info?: { width: number; height: number };
+}
+
+interface TweetResult {
+  id_str: string;
+  full_text: string;
+  user?: { name: string; screen_name: string; id_str: string };
+  created_at?: string;
+  favorite_count?: number;
+  retweet_count?: number;
+  views?: { count?: string };
+  mediaDetails?: MediaDetail[];
+}
+
+export class TwitterExtractor extends BaseExtractor {
+  readonly _VALID_URL = VALID_URL;
+  readonly _NAME = "twitter";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const match = VALID_URL.exec(url);
+    if (!match) throw new ExtractorError(`twitter: invalid URL: ${url}`);
+    const tweetId = match[1];
+
+    const apiUrl = `https://cdn.syndication.twimg.com/tweet-result?id=${tweetId}&lang=en&features=tfw_timeline_list%3A%3Btfw_follower_count_sunset%3Atrue&token=0`;
+    const resp = await fetch(apiUrl, {
+      headers: {
+        "User-Agent": "Mozilla/5.0 (compatible; dlpx/1.0)",
+        Accept: "application/json",
+        Referer: "https://platform.twitter.com/",
+        Origin: "https://platform.twitter.com",
+      },
+    });
+
+    if (!resp.ok) {
+      throw new ExtractorError(`twitter: API request failed: ${resp.status} ${resp.statusText}`);
+    }
+
+    const data = (await resp.json()) as TweetResult;
+
+    if (!data || !data.id_str) {
+      throw new ExtractorError(`twitter: tweet not found or protected: ${tweetId}`);
+    }
+
+    const formats: Format[] = [];
+    const thumbnails: Thumbnail[] = [];
+
+    if (data.mediaDetails && data.mediaDetails.length > 0) {
+      for (const media of data.mediaDetails) {
+        if (media.media_url_https) {
+          thumbnails.push({
+            url: media.media_url_https,
+            width: media.original_info?.width,
+            height: media.original_info?.height,
+          });
+        }
+
+        if (media.type === "video" || media.type === "animated_gif") {
+          const videoInfo = media.video_info;
+          if (videoInfo?.variants) {
+            const mp4Variants = videoInfo.variants
+              .filter((v) => v.content_type === "video/mp4")
+              .sort((a, b) => (b.bitrate ?? 0) - (a.bitrate ?? 0));
+
+            for (const variant of mp4Variants) {
+              formats.push({
+                format_id: `mp4-${variant.bitrate ?? 0}`,
+                url: variant.url,
+                ext: "mp4",
+                tbr: variant.bitrate ? variant.bitrate / 1000 : undefined,
+                vcodec: "h264",
+                acodec: mp4Variants.length > 0 ? "aac" : undefined,
+              });
+            }
+
+            const hlsVariant = videoInfo.variants.find(
+              (v) => v.content_type === "application/x-mpegURL",
+            );
+            if (hlsVariant) {
+              formats.push({
+                format_id: "hls",
+                url: hlsVariant.url,
+                ext: "mp4",
+                protocol: "m3u8",
+              });
+            }
+          }
+        }
+      }
+    }
+
+    const uploadDate = data.created_at
+      ? new Date(data.created_at).toISOString().slice(0, 10).replace(/-/g, "")
+      : undefined;
+
+    const viewCount = data.views?.count ? parseInt(data.views.count, 10) : undefined;
+
+    const duration = data.mediaDetails?.[0]?.video_info?.duration_millis
+      ? data.mediaDetails[0].video_info!.duration_millis / 1000
+      : undefined;
+
+    return {
+      id: tweetId,
+      title: data.full_text ? data.full_text.slice(0, 100) : `Tweet ${tweetId}`,
+      description: data.full_text,
+      uploader: data.user?.name,
+      uploader_id: data.user?.screen_name,
+      uploader_url: data.user?.screen_name
+        ? `https://twitter.com/${data.user.screen_name}`
+        : undefined,
+      upload_date: uploadDate,
+      view_count: viewCount,
+      like_count: data.favorite_count,
+      duration,
+      formats,
+      thumbnails,
+      webpage_url: url,
+      _type: "video",
+    };
+  }
+}

package/src/extractors/twitter/spaces.ts

@@ -0,0 +1,200 @@
+import { BaseExtractor, ExtractorError } from "../../core/types";
+import type { InfoDict, Format } from "../../core/types";
+
+const VALID_URL = /https?:\/\/(?:www\.)?(?:twitter|x)\.com\/i\/spaces\/([A-Za-z0-9]+)/;
+
+const BEARER_TOKEN =
+  "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs=";
+
+interface GuestTokenResponse {
+  guest_token: string;
+}
+
+interface AudioSpaceMetadata {
+  rest_id?: string;
+  metadata?: {
+    title?: string;
+    creator_results?: {
+      result?: {
+        legacy?: { name?: string; screen_name?: string };
+        rest_id?: string;
+      };
+    };
+    started_at?: number;
+    state?: string;
+  };
+  sharings?: unknown;
+}
+
+interface AudioSpaceResponse {
+  data?: {
+    audioSpace?: AudioSpaceMetadata;
+  };
+}
+
+interface LiveVideoStreamStatus {
+  source?: { location?: string; noRedirectPlaybackUrl?: string };
+  sessionId?: string;
+}
+
+export class TwitterSpacesExtractor extends BaseExtractor {
+  readonly _VALID_URL = VALID_URL;
+  readonly _NAME = "twitter:spaces";
+
+  private async getGuestToken(): Promise<string> {
+    const resp = await fetch("https://api.x.com/1.1/guest/activate.json", {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${BEARER_TOKEN}`,
+        "Content-Type": "application/x-www-form-urlencoded",
+        "User-Agent": "TwitterAndroid/9.99.0-release.0",
+      },
+    });
+
+    if (!resp.ok) {
+      throw new ExtractorError(
+        `twitter:spaces: failed to get guest token: ${resp.status}`,
+      );
+    }
+
+    const data = (await resp.json()) as GuestTokenResponse;
+    if (!data.guest_token) {
+      throw new ExtractorError("twitter:spaces: no guest token in response");
+    }
+    return data.guest_token;
+  }
+
+  private async getSpaceMetadata(
+    spaceId: string,
+    guestToken: string,
+  ): Promise<AudioSpaceMetadata> {
+    const variables = JSON.stringify({
+      id: spaceId,
+      isMetatagsQuery: false,
+      withReplays: true,
+      withListeners: true,
+    });
+
+    const features = JSON.stringify({
+      spaces_2022_h2_spaces_communities: true,
+      spaces_2022_h2_clipping: true,
+      creator_subscriptions_tweet_preview_api_enabled: true,
+    });
+
+    const url =
+      `https://api.x.com/graphql/kY7JFQmAeBaVp4UBdrK-wA/AudioSpaceById?` +
+      `variables=${encodeURIComponent(variables)}&features=${encodeURIComponent(features)}`;
+
+    const resp = await fetch(url, {
+      headers: {
+        Authorization: `Bearer ${BEARER_TOKEN}`,
+        "x-guest-token": guestToken,
+        "User-Agent": "TwitterAndroid/9.99.0-release.0",
+        "Content-Type": "application/json",
+      },
+    });
+
+    if (!resp.ok) {
+      throw new ExtractorError(
+        `twitter:spaces: GraphQL request failed: ${resp.status}`,
+      );
+    }
+
+    const data = (await resp.json()) as AudioSpaceResponse;
+    const space = data?.data?.audioSpace;
+    if (!space) {
+      throw new ExtractorError(
+        `twitter:spaces: space not found or unavailable: ${spaceId}`,
+      );
+    }
+    return space;
+  }
+
+  private async getStreamUrl(
+    mediaKey: string,
+    guestToken: string,
+  ): Promise<string> {
+    const resp = await fetch(
+      `https://twitter.com/i/api/1.1/live_video_stream/status/${mediaKey}`,
+      {
+        headers: {
+          Authorization: `Bearer ${BEARER_TOKEN}`,
+          "x-guest-token": guestToken,
+          "User-Agent": "TwitterAndroid/9.99.0-release.0",
+        },
+      },
+    );
+
+    if (!resp.ok) {
+      throw new ExtractorError(
+        `twitter:spaces: stream status request failed: ${resp.status}`,
+      );
+    }
+
+    const data = (await resp.json()) as LiveVideoStreamStatus;
+    const location =
+      data?.source?.noRedirectPlaybackUrl ?? data?.source?.location;
+    if (!location) {
+      throw new ExtractorError(
+        "twitter:spaces: no stream location in response",
+      );
+    }
+    return location;
+  }
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const match = VALID_URL.exec(url);
+    if (!match) throw new ExtractorError(`twitter:spaces: invalid URL: ${url}`);
+    const spaceId = match[1];
+
+    const guestToken = await this.getGuestToken();
+    const space = await this.getSpaceMetadata(spaceId, guestToken);
+
+    const metadata = space.metadata;
+    const creatorLegacy = metadata?.creator_results?.result?.legacy;
+    const creatorId = metadata?.creator_results?.result?.rest_id;
+
+    const formats: Format[] = [];
+
+    if (space.rest_id) {
+      try {
+        const streamUrl = await this.getStreamUrl(space.rest_id, guestToken);
+        formats.push({
+          format_id: "hls-audio",
+          url: streamUrl,
+          ext: "m4a",
+          protocol: "m3u8",
+          acodec: "aac",
+          vcodec: "none",
+        });
+      } catch {
+      }
+    }
+
+    const startedAt = metadata?.started_at;
+
+    return {
+      id: spaceId,
+      title: metadata?.title ?? `Twitter Space ${spaceId}`,
+      uploader: creatorLegacy?.name,
+      uploader_id: creatorLegacy?.screen_name,
+      uploader_url: creatorLegacy?.screen_name
+        ? `https://twitter.com/${creatorLegacy.screen_name}`
+        : undefined,
+      channel_id: creatorId,
+      timestamp: startedAt,
+      upload_date: startedAt
+        ? new Date(startedAt).toISOString().slice(0, 10).replace(/-/g, "")
+        : undefined,
+      live_status:
+        metadata?.state === "Running"
+          ? "is_live"
+          : metadata?.state === "Ended"
+            ? "was_live"
+            : "not_live",
+      formats,
+      webpage_url: url,
+      _type: "video",
+    };
+  }
+}

package/src/extractors/vimeo/index.ts

@@ -0,0 +1,187 @@
+import { BaseExtractor, ExtractorError } from "../../core/types";
+import type { InfoDict, Format, Thumbnail } from "../../core/types";
+
+interface VimeoHLSCDN {
+  url: string;
+  avc_url?: string;
+}
+
+interface VimeoDashCDN {
+  url: string;
+}
+
+interface VimeoProgressiveFile {
+  quality: string;
+  mime: string;
+  width?: number;
+  height?: number;
+  fps?: number;
+  url: string;
+  size?: number;
+}
+
+interface VimeoConfig {
+  video: {
+    id: number;
+    title: string;
+    description?: string;
+    duration?: number;
+    owner?: {
+      name?: string;
+      url?: string;
+      account_type?: string;
+    };
+    thumbs?: Record<string, string>;
+    embed_code?: string;
+    width?: number;
+    height?: number;
+    live_event?: { status?: string };
+  };
+  request: {
+    files: {
+      hls?: { cdns?: Record<string, VimeoHLSCDN>; default_cdn?: string };
+      dash?: { cdns?: Record<string, VimeoDashCDN>; default_cdn?: string };
+      progressive?: VimeoProgressiveFile[];
+    };
+    cookie?: Record<string, string>;
+  };
+}
+
+function extractVimeoId(url: string): string | null {
+  const patterns = [
+    /vimeo\.com\/(\d+)/,
+    /player\.vimeo\.com\/video\/(\d+)/,
+    /vimeo\.com\/channels\/[^/]+\/(\d+)/,
+    /vimeo\.com\/groups\/[^/]+\/videos\/(\d+)/,
+  ];
+  for (const pattern of patterns) {
+    const match = pattern.exec(url);
+    if (match) return match[1];
+  }
+  return null;
+}
+
+export class VimeoExtractor extends BaseExtractor {
+  readonly _VALID_URL =
+    /^https?:\/\/(?:(?:www|player)\.)?vimeo\.com\/(?:video\/|channels\/[^/]+\/|groups\/[^/]+\/videos\/)?(\d+)/;
+  readonly _NAME = "vimeo";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const videoId = extractVimeoId(url);
+    if (!videoId) throw new ExtractorError("Could not extract Vimeo video ID");
+
+    const configUrl = `https://player.vimeo.com/video/${videoId}/config`;
+    const response = await fetch(configUrl, {
+      headers: {
+        Referer: `https://vimeo.com/${videoId}`,
+        "User-Agent":
+          "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+      },
+    });
+
+    if (!response.ok) {
+      throw new ExtractorError(`Vimeo config request failed: ${response.status}`);
+    }
+
+    const config = (await response.json()) as VimeoConfig;
+    const video = config.video;
+    const files = config.request?.files;
+
+    if (!files) throw new ExtractorError("No media files found in Vimeo config");
+
+    const formats: Format[] = [];
+
+    if (files.progressive && files.progressive.length > 0) {
+      for (const prog of files.progressive) {
+        formats.push({
+          format_id: `http-${prog.quality}`,
+          url: prog.url,
+          ext: prog.mime?.split("/")?.[1] ?? "mp4",
+          width: prog.width,
+          height: prog.height,
+          fps: prog.fps,
+          filesize: prog.size,
+          format_note: prog.quality,
+          vcodec: "h264",
+          acodec: "aac",
+          quality: prog.height ?? 0,
+        });
+      }
+    }
+
+    if (files.hls?.cdns) {
+      const cdns = files.hls.cdns;
+      const defaultCdn = files.hls.default_cdn;
+      const cdnNames = defaultCdn
+        ? [defaultCdn, ...Object.keys(cdns).filter((k) => k !== defaultCdn)]
+        : Object.keys(cdns);
+
+      for (const cdnName of cdnNames) {
+        const cdn = cdns[cdnName];
+        const hlsUrl = cdn.avc_url ?? cdn.url;
+        if (hlsUrl) {
+          formats.push({
+            format_id: `hls-${cdnName}`,
+            url: hlsUrl,
+            ext: "mp4",
+            protocol: "m3u8",
+            format_note: `HLS (${cdnName})`,
+            source_preference: cdnName === defaultCdn ? 1 : 0,
+          });
+        }
+      }
+    }
+
+    if (files.dash?.cdns) {
+      const cdns = files.dash.cdns;
+      const defaultCdn = files.dash.default_cdn;
+      const cdnNames = defaultCdn
+        ? [defaultCdn, ...Object.keys(cdns).filter((k) => k !== defaultCdn)]
+        : Object.keys(cdns);
+
+      for (const cdnName of cdnNames) {
+        const cdn = cdns[cdnName];
+        if (cdn.url) {
+          formats.push({
+            format_id: `dash-${cdnName}`,
+            url: cdn.url,
+            ext: "mp4",
+            protocol: "dash",
+            format_note: `DASH (${cdnName})`,
+            source_preference: cdnName === defaultCdn ? 1 : 0,
+          });
+        }
+      }
+    }
+
+    if (formats.length === 0) {
+      throw new ExtractorError("No playable formats found for this Vimeo video");
+    }
+
+    const thumbnails: Thumbnail[] = [];
+    if (video.thumbs) {
+      for (const [size, thumbUrl] of Object.entries(video.thumbs)) {
+        const dim = parseInt(size, 10);
+        thumbnails.push({
+          url: thumbUrl,
+          width: isNaN(dim) ? undefined : dim,
+          id: size,
+        });
+      }
+    }
+
+    return {
+      id: videoId,
+      title: video.title,
+      description: video.description,
+      uploader: video.owner?.name,
+      uploader_url: video.owner?.url,
+      duration: video.duration,
+      webpage_url: `https://vimeo.com/${videoId}`,
+      width: video.width,
+      height: video.height,
+      thumbnails,
+      formats,
+    };
+  }
+}