getraw 0.1.0

package/src/extractors/tiktok/index.ts

@@ -0,0 +1,207 @@
+import { BaseExtractor, ExtractorError } from "../../core/types";
+import type { InfoDict, Format, Thumbnail } from "../../core/types";
+
+const VALID_URL =
+  /https?:\/\/(?:www\.)?(?:tiktok\.com\/@[\w.]+\/video\/(\d+)|vm\.tiktok\.com\/([\w]+))/;
+
+interface TikTokAuthor {
+  uniqueId?: string;
+  nickname?: string;
+  id?: string;
+  avatarThumb?: string;
+}
+
+interface TikTokMusic {
+  title?: string;
+  authorName?: string;
+  id?: string;
+}
+
+interface TikTokVideo {
+  playAddr?: string;
+  downloadAddr?: string;
+  width?: number;
+  height?: number;
+  duration?: number;
+  bitrate?: number;
+  format?: string;
+  codecType?: string;
+  cover?: string;
+  dynamicCover?: string;
+  originCover?: string;
+}
+
+interface TikTokStats {
+  diggCount?: number;
+  shareCount?: number;
+  commentCount?: number;
+  playCount?: number;
+}
+
+interface TikTokItemStruct {
+  id?: string;
+  desc?: string;
+  createTime?: number;
+  author?: TikTokAuthor;
+  music?: TikTokMusic;
+  video?: TikTokVideo;
+  stats?: TikTokStats;
+  textExtra?: Array<{ hashtagName?: string }>;
+}
+
+interface TikTokRehydrationData {
+  __DEFAULT_SCOPE__?: {
+    "webapp.video-detail"?: {
+      itemInfo?: {
+        itemStruct?: TikTokItemStruct;
+      };
+    };
+  };
+}
+
+export class TikTokExtractor extends BaseExtractor {
+  readonly _VALID_URL = VALID_URL;
+  readonly _NAME = "tiktok";
+
+  private readonly _headers: Record<string, string> = {
+    "User-Agent":
+      "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",
+    Accept:
+      "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+    Referer: "https://www.tiktok.com/",
+  };
+
+  private async resolveShortUrl(url: string): Promise<string> {
+    const resp = await fetch(url, {
+      headers: this._headers,
+      redirect: "follow",
+    });
+    return resp.url;
+  }
+
+  private extractRehydrationData(html: string): TikTokRehydrationData | null {
+    const match = html.match(
+      /<script[^>]+id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>([\s\S]*?)<\/script>/,
+    );
+    if (!match) return null;
+
+    try {
+      return JSON.parse(match[1]) as TikTokRehydrationData;
+    } catch {
+      return null;
+    }
+  }
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    let resolvedUrl = url;
+
+    if (/vm\.tiktok\.com/.test(url)) {
+      resolvedUrl = await this.resolveShortUrl(url);
+    }
+
+    const match = VALID_URL.exec(resolvedUrl);
+    const videoId = match?.[1] ?? match?.[2] ?? "unknown";
+
+    const resp = await fetch(resolvedUrl, { headers: this._headers });
+    if (!resp.ok) {
+      throw new ExtractorError(
+        `tiktok: page fetch failed: ${resp.status} ${resp.statusText}`,
+      );
+    }
+
+    const html = await resp.text();
+    const rehydrationData = this.extractRehydrationData(html);
+
+    if (!rehydrationData) {
+      throw new ExtractorError(
+        "tiktok: could not extract __UNIVERSAL_DATA_FOR_REHYDRATION__ from page",
+      );
+    }
+
+    const itemStruct =
+      rehydrationData?.__DEFAULT_SCOPE__?.["webapp.video-detail"]?.itemInfo
+        ?.itemStruct;
+
+    if (!itemStruct) {
+      throw new ExtractorError("tiktok: could not find video item data");
+    }
+
+    const formats: Format[] = [];
+    const thumbnails: Thumbnail[] = [];
+    const video = itemStruct.video;
+
+    if (video) {
+      if (video.downloadAddr) {
+        formats.push({
+          format_id: "download",
+          url: video.downloadAddr,
+          ext: video.format ?? "mp4",
+          width: video.width,
+          height: video.height,
+          tbr: video.bitrate ? video.bitrate / 1000 : undefined,
+          vcodec: video.codecType,
+          quality: 10,
+          http_headers: {
+            Referer: "https://www.tiktok.com/",
+            "User-Agent": this._headers["User-Agent"],
+          },
+        });
+      }
+
+      if (video.playAddr && video.playAddr !== video.downloadAddr) {
+        formats.push({
+          format_id: "play",
+          url: video.playAddr,
+          ext: video.format ?? "mp4",
+          width: video.width,
+          height: video.height,
+          tbr: video.bitrate ? video.bitrate / 1000 : undefined,
+          vcodec: video.codecType,
+          quality: 5,
+          http_headers: {
+            Referer: "https://www.tiktok.com/",
+            "User-Agent": this._headers["User-Agent"],
+          },
+        });
+      }
+
+      if (video.originCover) thumbnails.push({ url: video.originCover, preference: 2 });
+      if (video.dynamicCover) thumbnails.push({ url: video.dynamicCover, preference: 1 });
+      if (video.cover) thumbnails.push({ url: video.cover, preference: 0 });
+    }
+
+    const author = itemStruct.author;
+    const stats = itemStruct.stats;
+    const tags = itemStruct.textExtra
+      ?.filter((t) => t.hashtagName)
+      .map((t) => t.hashtagName as string);
+
+    const createTime = itemStruct.createTime;
+
+    return {
+      id: itemStruct.id ?? videoId,
+      title: itemStruct.desc ?? `TikTok video ${videoId}`,
+      description: itemStruct.desc,
+      uploader: author?.nickname,
+      uploader_id: author?.uniqueId,
+      uploader_url: author?.uniqueId
+        ? `https://www.tiktok.com/@${author.uniqueId}`
+        : undefined,
+      channel_id: author?.id,
+      timestamp: createTime,
+      upload_date: createTime
+        ? new Date(createTime * 1000).toISOString().slice(0, 10).replace(/-/g, "")
+        : undefined,
+      duration: video?.duration,
+      view_count: stats?.playCount,
+      like_count: stats?.diggCount,
+      comment_count: stats?.commentCount,
+      formats,
+      thumbnails,
+      tags,
+      webpage_url: resolvedUrl,
+      _type: "video",
+    };
+  }
+}

package/src/extractors/tiktok/user.ts

@@ -0,0 +1,176 @@
+import { BaseExtractor, ExtractorError } from "../../core/types";
+import type { InfoDict } from "../../core/types";
+
+const VALID_URL = /https?:\/\/(?:www\.)?tiktok\.com\/@([\w.]+)(?:\/?(?:\?.*)?)?$/;
+
+interface TikTokUserVideo {
+  id?: string;
+  desc?: string;
+  createTime?: number;
+  video?: { cover?: string };
+}
+
+interface TikTokUserListResponse {
+  itemList?: TikTokUserVideo[];
+  cursor?: string;
+  hasMore?: boolean;
+  minCursor?: number;
+  maxCursor?: number;
+}
+
+interface TikTokUserRehydration {
+  __DEFAULT_SCOPE__?: {
+    "webapp.user-detail"?: {
+      userInfo?: {
+        user?: {
+          id?: string;
+          uniqueId?: string;
+          nickname?: string;
+          signature?: string;
+        };
+        stats?: {
+          videoCount?: number;
+        };
+      };
+    };
+  };
+}
+
+export class TikTokUserExtractor extends BaseExtractor {
+  readonly _VALID_URL = VALID_URL;
+  readonly _NAME = "tiktok:user";
+
+  private readonly _headers: Record<string, string> = {
+    "User-Agent":
+      "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",
+    Accept:
+      "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+    Referer: "https://www.tiktok.com/",
+  };
+
+  private extractUserData(html: string): TikTokUserRehydration | null {
+    const match = html.match(
+      /<script[^>]+id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>([\s\S]*?)<\/script>/,
+    );
+    if (!match) return null;
+    try {
+      return JSON.parse(match[1]) as TikTokUserRehydration;
+    } catch {
+      return null;
+    }
+  }
+
+  private async fetchUserVideos(
+    userId: string,
+    cursor: string = "0",
+    count: number = 35,
+  ): Promise<TikTokUserListResponse> {
+    const params = new URLSearchParams({
+      userId,
+      count: String(count),
+      cursor,
+      cookie_enabled: "true",
+      screen_width: "1920",
+      screen_height: "1080",
+      browser_language: "en-US",
+      browser_platform: "Win32",
+      browser_name: "Mozilla",
+      browser_version: "5.0",
+      browser_online: "true",
+      timezone_name: "America/New_York",
+      priority_region: "",
+      referer: "",
+    });
+
+    const resp = await fetch(
+      `https://www.tiktok.com/api/post/item_list/?${params.toString()}`,
+      {
+        headers: {
+          ...this._headers,
+          Accept: "application/json, text/plain, */*",
+        },
+      },
+    );
+
+    if (!resp.ok) {
+      throw new ExtractorError(
+        `tiktok:user: video list request failed: ${resp.status}`,
+      );
+    }
+
+    return (await resp.json()) as TikTokUserListResponse;
+  }
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const match = VALID_URL.exec(url);
+    if (!match) throw new ExtractorError(`tiktok:user: invalid URL: ${url}`);
+    const username = match[1];
+
+    const resp = await fetch(url, { headers: this._headers });
+    if (!resp.ok) {
+      throw new ExtractorError(
+        `tiktok:user: page fetch failed: ${resp.status}`,
+      );
+    }
+
+    const html = await resp.text();
+    const userData = this.extractUserData(html);
+
+    const userInfo =
+      userData?.__DEFAULT_SCOPE__?.["webapp.user-detail"]?.userInfo;
+    const user = userInfo?.user;
+
+    if (!user?.id) {
+      throw new ExtractorError(
+        `tiktok:user: could not find user data for @${username}`,
+      );
+    }
+
+    const userId = user.id;
+    const entries: InfoDict[] = [];
+    let cursor = "0";
+    let hasMore = true;
+    let pageCount = 0;
+    const maxPages = 20;
+
+    while (hasMore && pageCount < maxPages) {
+      const data = await this.fetchUserVideos(userId, cursor);
+      const items = data.itemList ?? [];
+
+      for (const item of items) {
+        if (!item.id) continue;
+        entries.push({
+          id: item.id,
+          title: item.desc ?? `TikTok video ${item.id}`,
+          url: `https://www.tiktok.com/@${username}/video/${item.id}`,
+          webpage_url: `https://www.tiktok.com/@${username}/video/${item.id}`,
+          _type: "url",
+          uploader: user.nickname,
+          uploader_id: username,
+          thumbnails: item.video?.cover ? [{ url: item.video.cover }] : [],
+          timestamp: item.createTime,
+        });
+      }
+
+      hasMore = data.hasMore ?? false;
+      cursor = data.maxCursor?.toString() ?? String(Number(cursor) + items.length);
+      pageCount++;
+
+      if (items.length === 0) break;
+    }
+
+    return {
+      id: userId,
+      title: `${user.nickname ?? username}'s TikTok videos`,
+      description: user.signature,
+      uploader: user.nickname,
+      uploader_id: username,
+      uploader_url: url,
+      webpage_url: url,
+      _type: "playlist",
+      entries,
+      playlist_count: entries.length,
+    };
+  }
+}

package/src/extractors/twitch/clips.ts

@@ -0,0 +1,125 @@
+import { BaseExtractor, ExtractorError } from "../../core/types";
+import type { InfoDict, Format, Thumbnail } from "../../core/types";
+
+const TWITCH_CLIENT_ID = "kimne78kx3ncx6brgo4mv6wki5h1ko";
+const GQL_ENDPOINT = "https://gql.twitch.tv/gql";
+
+interface ClipAccessToken {
+  value: string;
+  signature: string;
+}
+
+interface ClipVideoQuality {
+  frameRate: number;
+  quality: string;
+  sourceURL: string;
+}
+
+interface ClipNode {
+  id: string;
+  slug: string;
+  title: string;
+  durationSeconds?: number;
+  viewCount?: number;
+  createdAt?: string;
+  broadcaster?: { displayName?: string; login?: string; id?: string };
+  curator?: { displayName?: string; login?: string };
+  thumbnailURL?: string;
+  videoQualities?: ClipVideoQuality[];
+  playbackAccessToken?: ClipAccessToken;
+}
+
+interface GQLResponse<T> {
+  data: T;
+  errors?: Array<{ message: string }>;
+}
+
+async function gqlRequest<T>(query: object): Promise<T> {
+  const response = await fetch(GQL_ENDPOINT, {
+    method: "POST",
+    headers: {
+      "Client-ID": TWITCH_CLIENT_ID,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify(query),
+  });
+  if (!response.ok) {
+    throw new ExtractorError(`Twitch GQL request failed: ${response.status}`);
+  }
+  const result = (await response.json()) as GQLResponse<T>;
+  if (result.errors?.length) {
+    throw new ExtractorError(`Twitch GQL error: ${result.errors[0].message}`);
+  }
+  return result.data;
+}
+
+export class TwitchClipExtractor extends BaseExtractor {
+  readonly _VALID_URL =
+    /^https?:\/\/(?:www\.)?twitch\.tv\/(?:[^/]+)\/clip\/([^/?#]+)|^https?:\/\/clips\.twitch\.tv\/([^/?#]+)/;
+  readonly _NAME = "twitch:clip";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const match = this._VALID_URL.exec(url);
+    if (!match) throw new ExtractorError("Invalid Twitch clip URL");
+    const slug = match[1] ?? match[2];
+
+    const data = await gqlRequest<{ clip: ClipNode }>({
+      operationName: "VideoAccessToken_Clip",
+      variables: { slug },
+      extensions: {
+        persistedQuery: {
+          version: 1,
+          sha256Hash: "36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11",
+        },
+      },
+    });
+
+    const clip = data?.clip;
+    if (!clip) throw new ExtractorError("Could not find Twitch clip");
+
+    const token = clip.playbackAccessToken;
+    const qualities = clip.videoQualities ?? [];
+
+    const formats: Format[] = qualities.map((q) => {
+      const qualityParams = token
+        ? `?sig=${encodeURIComponent(token.signature)}&token=${encodeURIComponent(token.value)}`
+        : "";
+      const heightMap: Record<string, number> = {
+        "1080": 1080,
+        "720": 720,
+        "480": 480,
+        "360": 360,
+        "160": 160,
+      };
+      return {
+        format_id: `clip-${q.quality}`,
+        url: q.sourceURL + qualityParams,
+        ext: "mp4",
+        fps: q.frameRate,
+        height: heightMap[q.quality],
+        format_note: `${q.quality}p`,
+        http_headers: { "Client-ID": TWITCH_CLIENT_ID },
+      };
+    });
+
+    const thumbnails: Thumbnail[] = [];
+    if (clip.thumbnailURL) {
+      thumbnails.push({ url: clip.thumbnailURL });
+    }
+
+    return {
+      id: clip.id,
+      title: clip.title,
+      webpage_url: url,
+      uploader: clip.broadcaster?.displayName,
+      uploader_id: clip.broadcaster?.login,
+      channel: clip.broadcaster?.displayName,
+      channel_id: clip.broadcaster?.id,
+      duration: clip.durationSeconds,
+      view_count: clip.viewCount,
+      upload_date: clip.createdAt?.replace(/-/g, "").slice(0, 8),
+      formats,
+      thumbnails,
+    };
+  }
+}

package/src/extractors/twitch/index.ts

@@ -0,0 +1,136 @@
+import { BaseExtractor, ExtractorError } from "../../core/types";
+import type { InfoDict, Format, Thumbnail } from "../../core/types";
+
+const TWITCH_CLIENT_ID = "kimne78kx3ncx6brgo4mv6wki5h1ko";
+const GQL_ENDPOINT = "https://gql.twitch.tv/gql";
+
+interface GQLResponse<T> {
+  data: T;
+  errors?: Array<{ message: string }>;
+}
+
+interface PlaybackAccessToken {
+  value: string;
+  signature: string;
+}
+
+interface VideoNode {
+  id: string;
+  title: string;
+  description?: string;
+  publishedAt?: string;
+  lengthSeconds?: number;
+  viewCount?: number;
+  owner?: { displayName?: string; login?: string; id?: string };
+  previewThumbnailURL?: string;
+  thumbnailURLs?: string[];
+}
+
+async function gqlRequest<T>(query: object): Promise<T> {
+  const response = await fetch(GQL_ENDPOINT, {
+    method: "POST",
+    headers: {
+      "Client-ID": TWITCH_CLIENT_ID,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify(query),
+  });
+  if (!response.ok) {
+    throw new ExtractorError(`Twitch GQL request failed: ${response.status}`);
+  }
+  const result = (await response.json()) as GQLResponse<T>;
+  if (result.errors?.length) {
+    throw new ExtractorError(`Twitch GQL error: ${result.errors[0].message}`);
+  }
+  return result.data;
+}
+
+export class TwitchVODExtractor extends BaseExtractor {
+  readonly _VALID_URL = /^https?:\/\/(?:www\.)?twitch\.tv\/videos\/(\d+)/;
+  readonly _NAME = "twitch:vod";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const match = this._VALID_URL.exec(url);
+    if (!match) throw new ExtractorError("Invalid Twitch VOD URL");
+    const videoId = match[1];
+
+    const [tokenData, metaData] = await Promise.all([
+      gqlRequest<{ videoPlaybackAccessToken: PlaybackAccessToken }>({
+        operationName: "PlaybackAccessToken",
+        variables: {
+          isLive: false,
+          login: "",
+          isVod: true,
+          vodID: videoId,
+          playerType: "site",
+        },
+        extensions: {
+          persistedQuery: {
+            version: 1,
+            sha256Hash: "0828119ded1c13477966434e15800ff57ddacf13ba1911c129dc2200705b0712",
+          },
+        },
+      }),
+      gqlRequest<{ video: VideoNode }>({
+        operationName: "VideoMetadata",
+        variables: { channelLogin: "", videoID: videoId },
+        extensions: {
+          persistedQuery: {
+            version: 1,
+            sha256Hash: "49b5b8f268cdeb259d75b58dcb0c1a748e3b575003448a2333dc5cdafd49adad",
+          },
+        },
+      }),
+    ]);
+
+    const token = tokenData.videoPlaybackAccessToken;
+    if (!token) throw new ExtractorError("Could not get Twitch VOD access token");
+
+    const hlsUrl =
+      `https://usher.twitchapps.com/vod/${videoId}?` +
+      new URLSearchParams({
+        sig: token.signature,
+        token: token.value,
+        allow_source: "true",
+        allow_audio_only: "true",
+        allow_spectre: "true",
+        p: String(Math.floor(Math.random() * 999999)),
+        platform: "web",
+        play_session_id: crypto.randomUUID().replace(/-/g, ""),
+        supported_codecs: "avc1",
+      });
+
+    const video = metaData?.video;
+
+    const formats: Format[] = [
+      {
+        format_id: "hls",
+        url: hlsUrl,
+        ext: "mp4",
+        protocol: "m3u8",
+        http_headers: { "Client-ID": TWITCH_CLIENT_ID },
+      },
+    ];
+
+    const thumbnails: Thumbnail[] = [];
+    if (video?.previewThumbnailURL) {
+      thumbnails.push({ url: video.previewThumbnailURL });
+    }
+
+    return {
+      id: videoId,
+      title: video?.title ?? `Twitch VOD ${videoId}`,
+      description: video?.description,
+      uploader: video?.owner?.displayName,
+      uploader_id: video?.owner?.login,
+      channel: video?.owner?.displayName,
+      channel_id: video?.owner?.id,
+      duration: video?.lengthSeconds,
+      view_count: video?.viewCount,
+      upload_date: video?.publishedAt?.replace(/-/g, "").slice(0, 8),
+      webpage_url: url,
+      formats,
+      thumbnails,
+    };
+  }
+}