getraw 0.1.0

package/src/extractors/kick/clips.ts

@@ -0,0 +1,91 @@
+import { BaseExtractor, ExtractorError } from "../../core/types";
+import type { InfoDict, Format, Thumbnail } from "../../core/types";
+
+const VALID_URL = /https?:\/\/(?:www\.)?kick\.com\/([^/?#]+)\/clips\/([^/?#]+)/;
+
+interface KickClip {
+  id: string;
+  title: string;
+  clip_url: string;
+  thumbnail_url: string;
+  duration: number;
+  view_count: number;
+  created_at: string;
+  channel: {
+    slug: string;
+    user_id: number;
+    user: { username: string };
+  };
+  category?: { name: string };
+}
+
+interface KickClipResponse {
+  clip: KickClip;
+}
+
+const KICK_HEADERS = {
+  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+  Accept: "application/json",
+  Referer: "https://kick.com",
+};
+
+export class KickClipsExtractor extends BaseExtractor {
+  readonly _VALID_URL = VALID_URL;
+  readonly _NAME = "kick:clips";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const match = VALID_URL.exec(url);
+    if (!match) throw new ExtractorError(`kick:clips: invalid URL: ${url}`);
+
+    const channelSlug = match[1];
+    const clipId = match[2];
+
+    const apiUrl = `https://kick.com/api/v2/clips/${clipId}`;
+    const resp = await fetch(apiUrl, { headers: KICK_HEADERS });
+
+    if (!resp.ok) {
+      throw new ExtractorError(`kick:clips: API request failed: ${resp.status} ${resp.statusText}`);
+    }
+
+    const data = (await resp.json()) as KickClipResponse;
+    const clip = data.clip;
+
+    if (!clip) {
+      throw new ExtractorError(`kick:clips: clip not found: ${clipId}`);
+    }
+
+    const formats: Format[] = [
+      {
+        format_id: "mp4",
+        url: clip.clip_url,
+        ext: "mp4",
+        protocol: "https",
+      },
+    ];
+
+    const thumbnails: Thumbnail[] = clip.thumbnail_url
+      ? [{ url: clip.thumbnail_url }]
+      : [];
+
+    const uploadDate = clip.created_at
+      ? new Date(clip.created_at).toISOString().slice(0, 10).replace(/-/g, "")
+      : undefined;
+
+    return {
+      id: clip.id,
+      title: clip.title,
+      channel: channelSlug,
+      channel_url: `https://kick.com/${channelSlug}`,
+      uploader: clip.channel?.user?.username,
+      uploader_id: String(clip.channel?.user_id ?? ""),
+      duration: clip.duration,
+      view_count: clip.view_count,
+      upload_date: uploadDate,
+      thumbnails,
+      formats,
+      categories: clip.category ? [clip.category.name] : undefined,
+      webpage_url: url,
+      _type: "video",
+    };
+  }
+}

package/src/extractors/kick/index.ts

@@ -0,0 +1,118 @@
+import { BaseExtractor, ExtractorError } from "../../core/types";
+import type { InfoDict, Format, Thumbnail } from "../../core/types";
+
+const VALID_URL = /https?:\/\/(?:www\.)?kick\.com\/video\/([a-zA-Z0-9_-]+)/;
+
+interface KickVodData {
+  id: string;
+  slug: string;
+  title: string;
+  channel: {
+    slug: string;
+    user_id: number;
+    user: { username: string };
+  };
+  duration: number;
+  views: number;
+  created_at: string;
+  thumbnail: { responsive?: string; url?: string } | string | null;
+  source: string;
+  categories?: Array<{ name: string }>;
+  start_time?: string;
+}
+
+interface KickVodResponse {
+  data: {
+    id: string;
+    slug: string;
+    title: string;
+    channel: {
+      slug: string;
+      user_id: number;
+      user: { username: string };
+    };
+    duration: number;
+    views: number;
+    created_at: string;
+    thumbnail: { responsive?: string; url?: string } | string | null;
+    source: string;
+    categories?: Array<{ name: string }>;
+    start_time?: string;
+  } | null;
+}
+
+const KICK_HEADERS = {
+  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+  Accept: "application/json",
+  Referer: "https://kick.com",
+};
+
+export class KickExtractor extends BaseExtractor {
+  readonly _VALID_URL = VALID_URL;
+  readonly _NAME = "kick";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const match = VALID_URL.exec(url);
+    if (!match) throw new ExtractorError(`kick: invalid URL: ${url}`);
+
+    const videoId = match[1];
+
+    const apiUrl = `https://kick.com/api/v1/video/${videoId}`;
+    const resp = await fetch(apiUrl, { headers: KICK_HEADERS });
+
+    if (!resp.ok) {
+      throw new ExtractorError(`kick: API request failed: ${resp.status} ${resp.statusText}`);
+    }
+
+    const response = (await resp.json()) as KickVodResponse;
+    const vod = response.data;
+
+    if (!vod) {
+      throw new ExtractorError(`kick: VOD not found: ${videoId}`);
+    }
+
+    const hlsUrl = vod.source;
+    if (!hlsUrl) throw new ExtractorError(`kick: no source URL for VOD: ${videoId}`);
+
+    const formats: Format[] = [
+      {
+        format_id: "hls",
+        url: hlsUrl,
+        ext: "mp4",
+        protocol: "m3u8",
+        http_headers: KICK_HEADERS,
+      },
+    ];
+
+    let thumbnailUrl: string | undefined;
+    if (typeof vod.thumbnail === "string") {
+      thumbnailUrl = vod.thumbnail;
+    } else if (vod.thumbnail && typeof vod.thumbnail === "object") {
+      thumbnailUrl = vod.thumbnail.responsive ?? vod.thumbnail.url;
+    }
+
+    const thumbnails: Thumbnail[] = thumbnailUrl ? [{ url: thumbnailUrl }] : [];
+
+    const uploadDate = vod.created_at
+      ? new Date(vod.created_at).toISOString().slice(0, 10).replace(/-/g, "")
+      : undefined;
+
+    return {
+      id: vod.id,
+      title: vod.title,
+      channel: vod.channel.slug,
+      channel_url: `https://kick.com/${vod.channel.slug}`,
+      uploader: vod.channel.user.username,
+      uploader_id: String(vod.channel.user_id),
+      uploader_url: `https://kick.com/${vod.channel.slug}`,
+      duration: vod.duration,
+      view_count: vod.views,
+      upload_date: uploadDate,
+      thumbnails,
+      formats,
+      categories: vod.categories?.map((c) => c.name),
+      webpage_url: url,
+      _type: "video",
+    };
+  }
+}

package/src/extractors/kick/live.ts

@@ -0,0 +1,89 @@
+import { BaseExtractor, ExtractorError } from "../../core/types";
+import type { InfoDict, Format, Thumbnail } from "../../core/types";
+
+const VALID_URL = /https?:\/\/(?:www\.)?kick\.com\/([^/?#]+)(?:\/(?!video|clips).*)?$/;
+
+interface KickLivestreamData {
+  id: number;
+  slug: string;
+  is_live: boolean;
+  playback_url: string;
+  session_title: string;
+  viewer_count?: number;
+  thumbnail?: { responsive?: string; url?: string } | null;
+  category?: { name: string };
+  started_at: string | null;
+  user: { username: string; id: number };
+}
+
+interface KickChannelResponse {
+  id: number;
+  slug: string;
+  livestream: KickLivestreamData | null;
+  user: { username: string; id: number };
+}
+
+const KICK_HEADERS = {
+  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+  Accept: "application/json",
+  Referer: "https://kick.com",
+};
+
+export class KickLiveExtractor extends BaseExtractor {
+  readonly _VALID_URL = VALID_URL;
+  readonly _NAME = "kick:live";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const match = VALID_URL.exec(url);
+    if (!match) throw new ExtractorError(`kick:live: invalid URL: ${url}`);
+
+    const channelSlug = match[1];
+
+    const apiUrl = `https://kick.com/api/v2/channels/${channelSlug}`;
+    const resp = await fetch(apiUrl, { headers: KICK_HEADERS });
+
+    if (!resp.ok) {
+      throw new ExtractorError(`kick:live: channel API failed: ${resp.status} ${resp.statusText}`);
+    }
+
+    const channel = (await resp.json()) as KickChannelResponse;
+    const live = channel.livestream;
+
+    if (!live || !live.is_live) {
+      throw new ExtractorError(`kick:live: ${channelSlug} is not currently live`);
+    }
+
+    const formats: Format[] = [
+      {
+        format_id: "hls",
+        url: live.playback_url,
+        ext: "mp4",
+        protocol: "m3u8",
+        http_headers: KICK_HEADERS,
+      },
+    ];
+
+    const thumbnailUrl = live.thumbnail?.responsive ?? live.thumbnail?.url;
+    const thumbnails: Thumbnail[] = thumbnailUrl ? [{ url: thumbnailUrl }] : [];
+
+    const releaseTimestamp = live.started_at ? new Date(live.started_at).getTime() / 1000 : undefined;
+
+    return {
+      id: String(live.id),
+      title: live.session_title,
+      channel: channelSlug,
+      channel_id: String(channel.id),
+      channel_url: `https://kick.com/${channelSlug}`,
+      uploader: channel.user.username,
+      uploader_id: String(channel.user.id),
+      view_count: live.viewer_count,
+      release_timestamp: releaseTimestamp,
+      thumbnails,
+      formats,
+      categories: live.category ? [live.category.name] : undefined,
+      live_status: "is_live",
+      webpage_url: url,
+      _type: "video",
+    };
+  }
+}

package/src/extractors/niconico/index.ts

@@ -0,0 +1,209 @@
+import { BaseExtractor, ExtractorError } from "../../core/types";
+import type { InfoDict, Format, Thumbnail } from "../../core/types";
+
+const VALID_URL = /https?:\/\/(?:www\.)?nicovideo\.jp\/watch\/((?:sm|nm)\d+)/;
+
+interface NicoVideoData {
+  id: string;
+  title: string;
+  description: string;
+  owner?: { id: number; nickname: string };
+  user?: { id: number; nickname: string };
+  count: { view: number; comment: number; mylist: number; like: number };
+  duration: number;
+  thumbnail: { url: string; largeUrl?: string };
+  registeredAt: string;
+  tags: Array<{ name: string }>;
+  channel?: { id: string; name: string };
+}
+
+interface NicoApiData {
+  meta: { status: number };
+  data: {
+    video: NicoVideoData;
+    media: {
+      domand?: {
+        videos: Array<{
+          id: string;
+          isAvailable: boolean;
+          label: string;
+          bitRate: number;
+          width: number;
+          height: number;
+          qualityLevel: number;
+        }>;
+        audios: Array<{
+          id: string;
+          isAvailable: boolean;
+          label: string;
+          bitRate: number;
+          samplingRate: number;
+          qualityLevel: number;
+        }>;
+        accessRightKey: string;
+      };
+    };
+  };
+}
+
+interface DmsSession {
+  meta: { status: number; message: string };
+  data: {
+    contentUrl: string;
+    created: string;
+    expireTime: string;
+  };
+}
+
+const NICONICO_HEADERS = {
+  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+  Accept: "application/json",
+  "Accept-Language": "ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7",
+  Referer: "https://www.nicovideo.jp",
+};
+
+export class NiconicoExtractor extends BaseExtractor {
+  readonly _VALID_URL = VALID_URL;
+  readonly _NAME = "niconico";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const match = VALID_URL.exec(url);
+    if (!match) throw new ExtractorError(`niconico: invalid URL: ${url}`);
+
+    const videoId = match[1];
+
+    const watchResp = await fetch(`https://www.nicovideo.jp/watch/${videoId}`, {
+      headers: {
+        ...NICONICO_HEADERS,
+        Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+      },
+    });
+
+    if (!watchResp.ok) {
+      throw new ExtractorError(`niconico: watch page request failed: ${watchResp.status}`);
+    }
+
+    const html = await watchResp.text();
+    const cookies = watchResp.headers.get("set-cookie") ?? "";
+
+    const dataMatch = html.match(/id="js-initial-watch-data"\s+data-api-data="([^"]+)"/);
+    if (!dataMatch) {
+      throw new ExtractorError(`niconico: could not find watch data in page for ${videoId}`);
+    }
+
+    const apiData = JSON.parse(
+      dataMatch[1].replace(/&quot;/g, '"').replace(/&amp;/g, "&"),
+    ) as NicoApiData;
+
+    if (apiData.meta.status !== 200) {
+      throw new ExtractorError(`niconico: API returned non-200 status for ${videoId}`);
+    }
+
+    const video = apiData.data.video;
+    const domand = apiData.data.media.domand;
+
+    const formats: Format[] = [];
+
+    if (domand) {
+      const availableVideos = domand.videos.filter((v) => v.isAvailable);
+      const availableAudios = domand.audios.filter((a) => a.isAvailable);
+
+      if (availableVideos.length === 0) {
+        throw new ExtractorError(`niconico: no available video streams for ${videoId}`);
+      }
+
+      const bestVideo = availableVideos.reduce((best, v) =>
+        v.qualityLevel > best.qualityLevel ? v : best,
+      );
+      const bestAudio = availableAudios.reduce<typeof availableAudios[0] | null>((best, a) => {
+        if (!best || a.qualityLevel > best.qualityLevel) return a;
+        return best;
+      }, null);
+
+      const sessionBody = {
+        outputs: [[bestVideo.id, bestAudio?.id ?? availableAudios[0]?.id]],
+        accessRightKey: domand.accessRightKey,
+      };
+
+      const sessionResp = await fetch(
+        `https://nvapi.nicovideo.jp/v1/watch/${videoId}/access-rights/hls?actionTrackId=dlpx_${Date.now()}`,
+        {
+          method: "POST",
+          headers: {
+            ...NICONICO_HEADERS,
+            "Content-Type": "application/json",
+            "X-Frontend-Id": "6",
+            "X-Frontend-Version": "0",
+            Cookie: cookies,
+          },
+          body: JSON.stringify(sessionBody),
+        },
+      );
+
+      if (sessionResp.ok) {
+        const sessionData = (await sessionResp.json()) as DmsSession;
+        if (sessionData.meta.status === 201 && sessionData.data?.contentUrl) {
+          formats.push({
+            format_id: "hls",
+            url: sessionData.data.contentUrl,
+            ext: "mp4",
+            protocol: "m3u8",
+            vcodec: bestVideo.id.includes("h264") ? "avc1" : "hev1",
+            width: bestVideo.width,
+            height: bestVideo.height,
+            tbr: Math.round(bestVideo.bitRate / 1000),
+            format_note: bestVideo.label,
+            http_headers: NICONICO_HEADERS,
+          });
+        }
+      }
+    }
+
+    if (formats.length === 0) {
+      formats.push({
+        format_id: "fallback",
+        url: `https://www.nicovideo.jp/watch/${videoId}`,
+        ext: "mp4",
+        protocol: "https",
+        format_note: "webpage fallback",
+      });
+    }
+
+    const thumbnails: Thumbnail[] = [];
+    if (video.thumbnail.largeUrl) thumbnails.push({ url: video.thumbnail.largeUrl });
+    thumbnails.push({ url: video.thumbnail.url });
+
+    const uploadDate = video.registeredAt
+      ? new Date(video.registeredAt).toISOString().slice(0, 10).replace(/-/g, "")
+      : undefined;
+
+    const uploader = video.owner?.nickname ?? video.user?.nickname;
+    const uploaderId = video.owner?.id ?? video.user?.id;
+
+    return {
+      id: videoId,
+      title: video.title,
+      description: video.description,
+      uploader,
+      uploader_id: uploaderId ? String(uploaderId) : undefined,
+      uploader_url: uploaderId
+        ? `https://www.nicovideo.jp/user/${uploaderId}`
+        : undefined,
+      channel: video.channel?.name,
+      channel_id: video.channel?.id,
+      channel_url: video.channel?.id
+        ? `https://ch.nicovideo.jp/channel/${video.channel.id}`
+        : undefined,
+      duration: video.duration,
+      view_count: video.count.view,
+      comment_count: video.count.comment,
+      like_count: video.count.like,
+      upload_date: uploadDate,
+      thumbnails,
+      formats,
+      tags: video.tags.map((t) => t.name),
+      webpage_url: url,
+      _type: "video",
+    };
+  }
+}

package/src/extractors/odysee.ts

@@ -0,0 +1,126 @@
+import { BaseExtractor, ExtractorError } from "../core/types";
+import type { InfoDict, Format, Thumbnail } from "../core/types";
+
+interface OdyseeClaimValue {
+  stream?: {
+    source?: { media_type?: string; sd_hash?: string };
+    video?: { width?: number; height?: number; duration?: number };
+    audio?: { channel_count?: number };
+  };
+  title?: string;
+  description?: string;
+  thumbnail?: { url?: string };
+  tags?: string[];
+}
+
+interface OdyseeClaim {
+  claim_id?: string;
+  name?: string;
+  value?: OdyseeClaimValue;
+  channel_name?: string;
+  signing_channel?: { name?: string; claim_id?: string };
+  value_type?: string;
+}
+
+interface OdyseeProxyResponse {
+  result?: {
+    items?: OdyseeClaim[];
+    streaming_url?: string;
+  };
+  error?: { message?: string };
+}
+
+export class OdyseeExtractor extends BaseExtractor {
+  readonly _VALID_URL = /https?:\/\/(?:www\.)?(?:odysee\.com|lbry\.tv)\/@[^/]+:[^/]+\/([^/?#]+)/;
+  readonly _NAME = "odysee";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const urlObj = new URL(url);
+    const claimUrl = `lbry:/${urlObj.pathname}`;
+
+    const resolveResponse = await fetch("https://api.na-backend.odysee.com/api/v1/proxy", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "User-Agent": "Mozilla/5.0",
+      },
+      body: JSON.stringify({
+        method: "resolve",
+        params: { urls: [claimUrl] },
+      }),
+    });
+
+    if (!resolveResponse.ok) {
+      throw new ExtractorError(`Odysee resolve error: ${resolveResponse.status}`);
+    }
+
+    const resolveData = (await resolveResponse.json()) as { result?: Record<string, OdyseeClaim>; error?: { message?: string } };
+
+    const claims = resolveData.result ?? {};
+    const claim = Object.values(claims)[0];
+
+    if (!claim || !claim.claim_id) {
+      throw new ExtractorError("Odysee: could not resolve claim");
+    }
+
+    const streamResponse = await fetch("https://api.na-backend.odysee.com/api/v1/proxy", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "User-Agent": "Mozilla/5.0",
+      },
+      body: JSON.stringify({
+        method: "get",
+        params: { uri: claimUrl, save_file: false },
+      }),
+    });
+
+    if (!streamResponse.ok) {
+      throw new ExtractorError(`Odysee stream error: ${streamResponse.status}`);
+    }
+
+    const streamData = (await streamResponse.json()) as OdyseeProxyResponse;
+
+    if (streamData.error?.message) {
+      throw new ExtractorError(`Odysee: ${streamData.error.message}`);
+    }
+
+    const streamingUrl = streamData.result?.streaming_url;
+    if (!streamingUrl) {
+      throw new ExtractorError("Odysee: no streaming URL found");
+    }
+
+    const value = claim.value ?? {};
+    const isHLS = streamingUrl.includes(".m3u8");
+
+    const formats: Format[] = [
+      {
+        format_id: isHLS ? "hls" : "mp4",
+        url: streamingUrl,
+        ext: "mp4",
+        protocol: isHLS ? "m3u8" : undefined,
+        width: value.stream?.video?.width,
+        height: value.stream?.video?.height,
+      },
+    ];
+
+    const thumbnails: Thumbnail[] = value.thumbnail?.url
+      ? [{ url: value.thumbnail.url }]
+      : [];
+
+    const channelName = claim.signing_channel?.name ?? claim.channel_name;
+
+    return {
+      id: claim.claim_id,
+      title: value.title ?? claim.name ?? "Odysee Video",
+      description: value.description,
+      uploader: channelName,
+      duration: value.stream?.video?.duration,
+      thumbnails,
+      formats,
+      tags: value.tags,
+      webpage_url: url,
+      extractor: this._NAME,
+    };
+  }
+}