getraw 0.1.0

package/src/core/types.ts

@@ -0,0 +1,237 @@
+export interface Thumbnail {
+  url: string;
+  width?: number;
+  height?: number;
+  id?: string;
+  preference?: number;
+}
+
+export interface Subtitle {
+  url: string;
+  ext: string;
+  name?: string;
+  data?: string;
+}
+
+export interface Chapter {
+  title: string;
+  start_time: number;
+  end_time: number;
+}
+
+export interface Format {
+  format_id: string;
+  url: string;
+  ext: string;
+  protocol?: string;
+  width?: number;
+  height?: number;
+  resolution?: string;
+  fps?: number;
+  vcodec?: string;
+  acodec?: string;
+  vbr?: number;
+  abr?: number;
+  tbr?: number;
+  filesize?: number;
+  filesize_approx?: number;
+  format_note?: string;
+  quality?: number;
+  language?: string;
+  audio_channels?: number;
+  dynamic_range?: string;
+  has_drm?: boolean;
+  http_headers?: Record<string, string>;
+  source_preference?: number;
+  container?: string;
+}
+
+export interface InfoDict {
+  id: string;
+  title: string;
+  url?: string;
+  ext?: string;
+  formats?: Format[];
+  thumbnails?: Thumbnail[];
+  subtitles?: Record<string, Subtitle[]>;
+  automatic_captions?: Record<string, Subtitle[]>;
+  chapters?: Chapter[];
+  description?: string;
+  uploader?: string;
+  uploader_id?: string;
+  uploader_url?: string;
+  channel?: string;
+  channel_id?: string;
+  channel_url?: string;
+  duration?: number;
+  view_count?: number;
+  like_count?: number;
+  comment_count?: number;
+  upload_date?: string;
+  timestamp?: number;
+  age_limit?: number;
+  webpage_url?: string;
+  categories?: string[];
+  tags?: string[];
+  live_status?: "is_live" | "was_live" | "is_upcoming" | "not_live";
+  release_timestamp?: number;
+  playlist?: string;
+  playlist_index?: number;
+  playlist_count?: number;
+  entries?: InfoDict[];
+  requested_formats?: Format[];
+  filename?: string;
+  _type?: "video" | "playlist" | "url" | "url_transparent";
+  extractor?: string;
+  extractor_key?: string;
+  http_headers?: Record<string, string>;
+}
+
+export interface DownloadProgress {
+  downloaded_bytes: number;
+  total_bytes: number | null;
+  speed: number | null;
+  eta: number | null;
+  percent: number | null;
+  status: "downloading" | "finished" | "error";
+  filename: string;
+}
+
+export interface Options {
+  format: string;
+  output: string;
+  extractAudio: boolean;
+  audioFormat: string;
+  audioQuality: string;
+  writeSubs: boolean;
+  subLangs: string;
+  listFormats: boolean;
+  dumpJson: boolean;
+  quiet: boolean;
+  verbose: boolean;
+  noProgress: boolean;
+  retries: number;
+  rateLimit: number | null;
+  proxy: string | null;
+  cookies: string | null;
+  userAgent: string;
+  referer: string | null;
+  embedThumbnail: boolean;
+  embedSubs: boolean;
+  mergeOutputFormat: string | null;
+  paths: { home: string; temp: string };
+  ffmpegLocation: string | null;
+  version: boolean;
+  help: boolean;
+  urls: string[];
+}
+
+export abstract class BaseExtractor {
+  abstract readonly _VALID_URL: RegExp;
+  abstract readonly _NAME: string;
+
+  canHandle(url: string): boolean {
+    return this._VALID_URL.test(url);
+  }
+
+  async extract(url: string): Promise<InfoDict> {
+    if (!this.canHandle(url)) {
+      throw new ExtractorError(`URL not supported by ${this._NAME}: ${url}`);
+    }
+    try {
+      const info = await this._real_extract(url);
+      info.extractor = this._NAME;
+      info.extractor_key = this.constructor.name;
+      return info;
+    } catch (err) {
+      if (err instanceof ExtractorError) throw err;
+      throw new ExtractorError(
+        `${this._NAME}: extraction failed: ${err instanceof Error ? err.message : String(err)}`,
+        { cause: err },
+      );
+    }
+  }
+
+  protected abstract _real_extract(url: string): Promise<InfoDict>;
+}
+
+export abstract class Downloader {
+  abstract readonly protocol: string;
+
+  abstract download(
+    url: string,
+    filepath: string,
+    options: DownloadOptions,
+  ): Promise<void>;
+
+  abstract canHandle(protocol: string): boolean;
+}
+
+export interface DownloadOptions {
+  headers?: Record<string, string>;
+  rateLimit?: number | null;
+  retries?: number;
+  onProgress?: (progress: DownloadProgress) => void;
+}
+
+export abstract class PostProcessor {
+  abstract readonly _NAME: string;
+
+  abstract run(info: InfoDict, filepath: string): Promise<PostProcessResult>;
+}
+
+export interface PostProcessResult {
+  filepath: string;
+  info: InfoDict;
+  files_to_delete: string[];
+}
+
+export class ExtractorError extends Error {
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
+    this.name = "ExtractorError";
+  }
+}
+
+export class DownloadError extends Error {
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
+    this.name = "DownloadError";
+  }
+}
+
+export class PostProcessError extends Error {
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
+    this.name = "PostProcessError";
+  }
+}
+
+export const DEFAULT_OPTIONS: Options = {
+  format: "bv*+ba/b",
+  output: "%(title)s [%(id)s].%(ext)s",
+  extractAudio: false,
+  audioFormat: "mp3",
+  audioQuality: "5",
+  writeSubs: false,
+  subLangs: "en",
+  listFormats: false,
+  dumpJson: false,
+  quiet: false,
+  verbose: false,
+  noProgress: false,
+  retries: 3,
+  rateLimit: null,
+  proxy: null,
+  cookies: null,
+  userAgent: "dlpx/0.0.0",
+  referer: null,
+  embedThumbnail: false,
+  embedSubs: false,
+  mergeOutputFormat: null,
+  paths: { home: ".", temp: "" },
+  ffmpegLocation: null,
+  version: false,
+  help: false,
+  urls: [],
+};

package/src/downloaders/base.ts

@@ -0,0 +1,25 @@
+import { Downloader } from "../core/types";
+import { HttpDownloader } from "./http";
+import { HlsDownloader } from "./hls";
+import { DashDownloader } from "./dash";
+
+export { Downloader };
+
+const downloaders: Downloader[] = [
+  new HttpDownloader(),
+  new HlsDownloader(),
+  new DashDownloader(),
+];
+
+export function registerDownloader(downloader: Downloader): void {
+  downloaders.push(downloader);
+}
+
+export function getDownloader(protocol: string): Downloader | null {
+  for (const dl of downloaders) {
+    if (dl.canHandle(protocol)) {
+      return dl;
+    }
+  }
+  return null;
+}

package/src/downloaders/dash.ts

@@ -0,0 +1,287 @@
+import { Downloader, DownloadError } from "../core/types";
+import type { DownloadOptions } from "../core/types";
+import { FragmentDownloader } from "./fragment";
+import type { Segment } from "./fragment";
+import { logger } from "../core/logger";
+
+interface DashRepresentation {
+  id: string;
+  bandwidth?: number;
+  width?: number;
+  height?: number;
+  codecs?: string;
+  mimeType?: string;
+  segments?: DashSegmentInfo;
+  initialization?: string;
+  segmentList?: DashSegmentList;
+  segmentTemplate?: DashSegmentTemplate;
+}
+
+interface DashAdaptationSet {
+  id?: string;
+  contentType?: string;
+  mimeType?: string;
+  representations: DashRepresentation[];
+  segments?: DashSegmentInfo;
+  segmentTemplate?: DashSegmentTemplate;
+}
+
+interface DashPeriod {
+  id?: string;
+  adaptationSets: DashAdaptationSet[];
+}
+
+interface DashMpd {
+  periods: DashPeriod[];
+}
+
+interface DashSegmentInfo {
+  list: Array<{ uri: string; timeline: Array<{ start: number; end: number; uri: string }> }>;
+  timeline: Array<{ start: number; end: number; uri: string }>;
+  initialization?: { sourceURL?: string };
+}
+
+interface DashSegmentList {
+  initialization?: { sourceURL?: string };
+  segmentURLs: Array<{ media?: string }>;
+}
+
+interface DashSegmentTemplate {
+  initialization?: string;
+  media?: string;
+  timescale?: number;
+  duration?: number;
+  startNumber?: number;
+  segmentTimeline?: Array<{ t?: number; d: number; r?: number }>;
+}
+
+export class DashDownloader extends Downloader {
+  readonly protocol = "dash";
+
+  canHandle(protocol: string): boolean {
+    return protocol === "dash" || protocol === "mpd";
+  }
+
+  async download(
+    url: string,
+    filepath: string,
+    options: DownloadOptions,
+  ): Promise<void> {
+    const retries = options.retries ?? 3;
+    let lastError: Error | null = null;
+
+    for (let attempt = 1; attempt <= retries; attempt++) {
+      try {
+        await this.downloadDash(url, filepath, options);
+        return;
+      } catch (err) {
+        lastError = err instanceof Error ? err : new Error(String(err));
+        if (attempt < retries) {
+          const delay = 1000 * Math.pow(2, attempt - 1);
+          logger.warn(`DASH download failed (attempt ${attempt}/${retries}): ${lastError.message}`);
+          await new Promise((r) => setTimeout(r, delay));
+        }
+      }
+    }
+
+    throw new DownloadError(`DASH download failed after ${retries} attempts: ${lastError?.message}`);
+  }
+
+  private async fetchMpd(url: string, headers: Record<string, string>): Promise<string> {
+    const resp = await fetch(url, { headers });
+    if (!resp.ok) {
+      throw new DownloadError(`Failed to fetch MPD: HTTP ${resp.status}`);
+    }
+    return resp.text();
+  }
+
+  private async downloadDash(url: string, filepath: string, options: DownloadOptions): Promise<void> {
+    const { parse } = await import("mpd-parser");
+    const headers = { ...options.headers };
+
+    const mpdText = await this.fetchMpd(url, headers);
+    const parsed = parse(mpdText, { manifestUri: url }) as DashMpd;
+
+    if (!parsed.periods || parsed.periods.length === 0) {
+      throw new DownloadError("MPD has no periods");
+    }
+
+    const allSegments: Segment[] = [];
+    let globalIdx = 0;
+
+    for (const period of parsed.periods) {
+      const periodSegments = this.extractPeriodSegments(period, url, globalIdx);
+      globalIdx += periodSegments.length;
+      allSegments.push(...periodSegments);
+    }
+
+    if (allSegments.length === 0) {
+      throw new DownloadError("No segments found in MPD");
+    }
+
+    const tempDir = `/tmp/dlpx-dash-${Date.now()}`;
+    const fragmenter = new FragmentDownloader();
+    await fragmenter.downloadSegments(allSegments, filepath, {
+      ...options,
+      concurrency: 8,
+      tempDir,
+    });
+  }
+
+  private extractPeriodSegments(period: DashPeriod, baseUrl: string, startIdx: number): Segment[] {
+    const { videoSet, audioSet } = this.selectAdaptationSets(period.adaptationSets);
+
+    const segments: Segment[] = [];
+    let idx = startIdx;
+
+    if (videoSet) {
+      const best = this.selectBestRepresentation(videoSet.representations);
+      if (best) {
+        const segs = this.buildSegments(best, videoSet, baseUrl, idx);
+        idx += segs.length;
+        segments.push(...segs);
+        logger.debug(`DASH: selected video representation ${best.id} (${best.bandwidth ?? "?"} bps)`);
+      }
+    }
+
+    if (audioSet) {
+      const best = this.selectBestRepresentation(audioSet.representations);
+      if (best) {
+        const segs = this.buildSegments(best, audioSet, baseUrl, idx);
+        segments.push(...segs);
+        logger.debug(`DASH: selected audio representation ${best.id} (${best.bandwidth ?? "?"} bps)`);
+      }
+    }
+
+    return segments;
+  }
+
+  private selectAdaptationSets(sets: DashAdaptationSet[]): {
+    videoSet: DashAdaptationSet | null;
+    audioSet: DashAdaptationSet | null;
+  } {
+    let videoSet: DashAdaptationSet | null = null;
+    let audioSet: DashAdaptationSet | null = null;
+
+    for (const set of sets) {
+      const mime = set.mimeType ?? set.contentType ?? "";
+      if (mime.startsWith("video") && !videoSet) {
+        videoSet = set;
+      } else if (mime.startsWith("audio") && !audioSet) {
+        audioSet = set;
+      }
+    }
+
+    if (!videoSet && sets.length > 0) {
+      videoSet = sets[0];
+    }
+
+    return { videoSet, audioSet };
+  }
+
+  private selectBestRepresentation(reps: DashRepresentation[]): DashRepresentation | null {
+    if (!reps || reps.length === 0) return null;
+    return reps.reduce((a, b) => (b.bandwidth ?? 0) > (a.bandwidth ?? 0) ? b : a);
+  }
+
+  private buildSegments(
+    rep: DashRepresentation,
+    set: DashAdaptationSet,
+    baseUrl: string,
+    startIdx: number,
+  ): Segment[] {
+    const segments: Segment[] = [];
+    let idx = startIdx;
+
+    const template = rep.segmentTemplate ?? set.segmentTemplate;
+    if (template) {
+      return this.buildFromTemplate(template, rep, baseUrl, startIdx);
+    }
+
+    const segList = rep.segmentList;
+    if (segList) {
+      if (segList.initialization?.sourceURL) {
+        segments.push({
+          url: resolveUrl(baseUrl, segList.initialization.sourceURL),
+          index: idx++,
+          isInit: true,
+        });
+      }
+      for (const s of segList.segmentURLs) {
+        if (s.media) {
+          segments.push({ url: resolveUrl(baseUrl, s.media), index: idx++ });
+        }
+      }
+      return segments;
+    }
+
+    const info = rep.segments ?? set.segments;
+    if (info) {
+      if (info.initialization?.sourceURL) {
+        segments.push({
+          url: resolveUrl(baseUrl, info.initialization.sourceURL),
+          index: idx++,
+          isInit: true,
+        });
+      }
+      for (const entry of info.timeline ?? []) {
+        if (entry.uri) {
+          segments.push({ url: resolveUrl(baseUrl, entry.uri), index: idx++ });
+        }
+      }
+    }
+
+    return segments;
+  }
+
+  private buildFromTemplate(
+    template: DashSegmentTemplate,
+    rep: DashRepresentation,
+    baseUrl: string,
+    startIdx: number,
+  ): Segment[] {
+    const segments: Segment[] = [];
+    let idx = startIdx;
+
+    if (template.initialization) {
+      const initUrl = interpolateTemplate(template.initialization, rep.id, 0, 0);
+      segments.push({ url: resolveUrl(baseUrl, initUrl), index: idx++, isInit: true });
+    }
+
+    const mediaTemplate = template.media;
+    if (!mediaTemplate) return segments;
+
+    if (template.segmentTimeline && template.segmentTimeline.length > 0) {
+      const timescale = template.timescale ?? 1;
+      let t = 0;
+      let segNum = template.startNumber ?? 1;
+
+      for (const entry of template.segmentTimeline) {
+        if (entry.t !== undefined) t = entry.t;
+        const repeat = (entry.r ?? 0) + 1;
+        for (let r = 0; r < repeat; r++) {
+          const segUrl = interpolateTemplate(mediaTemplate, rep.id, segNum, t);
+          segments.push({ url: resolveUrl(baseUrl, segUrl), index: idx++ });
+          t += entry.d;
+          segNum++;
+        }
+      }
+    } else if (template.duration && template.timescale) {
+      logger.warn("DASH: duration-based SegmentTemplate without SegmentTimeline — segment count unknown, skipping");
+    }
+
+    return segments;
+  }
+}
+
+function interpolateTemplate(template: string, repId: string, number: number, time: number): string {
+  return template
+    .replace("$RepresentationID$", repId)
+    .replace(/\$Number(%0\d+d)?\$/g, (_, fmt) => fmt ? number.toString().padStart(parseInt(fmt.slice(2)), "0") : String(number))
+    .replace(/\$Time(%0\d+d)?\$/g, (_, fmt) => fmt ? time.toString().padStart(parseInt(fmt.slice(2)), "0") : String(time));
+}
+
+function resolveUrl(base: string, relative: string): string {
+  if (/^https?:\/\//i.test(relative)) return relative;
+  return new URL(relative, base).toString();
+}