getraw 0.1.0

package/src/downloaders/fragment.ts

@@ -0,0 +1,226 @@
+import { DownloadError } from "../core/types";
+import type { DownloadOptions, DownloadProgress } from "../core/types";
+import { logger } from "../core/logger";
+
+export interface Segment {
+  url: string;
+  index: number;
+  byteRange?: { start: number; end: number };
+  key?: { method: string; uri: string; iv?: string };
+  isInit?: boolean;
+}
+
+export interface FragmentDownloadOptions extends DownloadOptions {
+  concurrency?: number;
+  tempDir?: string;
+}
+
+export class FragmentDownloader {
+  private completedSegments = new Set<number>();
+
+  async downloadSegments(
+    segments: Segment[],
+    outputPath: string,
+    options: FragmentDownloadOptions,
+  ): Promise<void> {
+    const concurrency = options.concurrency ?? 8;
+    const tempDir = options.tempDir ?? "/tmp/dlpx-fragments";
+
+    await Bun.$`mkdir -p ${tempDir}`.quiet();
+
+    const stateFile = `${tempDir}/completed.json`;
+    try {
+      const stateData = await Bun.file(stateFile).text();
+      const state = JSON.parse(stateData) as { completed: number[] };
+      this.completedSegments = new Set(state.completed);
+    } catch {
+      this.completedSegments = new Set();
+    }
+
+    const pending = segments.filter((s) => !this.completedSegments.has(s.index));
+    const total = segments.length;
+    let done = this.completedSegments.size;
+
+    const keyCache = new Map<string, Uint8Array>();
+
+    const downloadSegment = async (seg: Segment): Promise<void> => {
+      const segPath = `${tempDir}/seg-${seg.index.toString().padStart(6, "0")}`;
+      const retries = options.retries ?? 3;
+
+      for (let attempt = 1; attempt <= retries; attempt++) {
+        try {
+          const headers: Record<string, string> = { ...options.headers };
+          if (seg.byteRange) {
+            headers["Range"] = `bytes=${seg.byteRange.start}-${seg.byteRange.end}`;
+          }
+
+          const resp = await fetch(seg.url, { headers });
+          if (!resp.ok) {
+            throw new DownloadError(`HTTP ${resp.status} on segment ${seg.index}`);
+          }
+
+          let data = new Uint8Array(await resp.arrayBuffer());
+
+          if (seg.key && seg.key.method === "AES-128") {
+            data = await this.decryptAes128(data, seg.key, seg.index, keyCache, options.headers);
+          }
+
+          await Bun.write(segPath, data);
+          this.completedSegments.add(seg.index);
+          await Bun.write(
+            stateFile,
+            JSON.stringify({ completed: Array.from(this.completedSegments) }),
+          );
+
+          done++;
+          const percent = (done / total) * 100;
+          if (options.onProgress) {
+            const progress: DownloadProgress = {
+              downloaded_bytes: done,
+              total_bytes: total,
+              speed: null,
+              eta: null,
+              percent,
+              status: "downloading",
+              filename: outputPath,
+            };
+            options.onProgress(progress);
+          }
+
+          return;
+        } catch (err) {
+          if (attempt === retries) throw err;
+          const delay = 1000 * Math.pow(2, attempt - 1);
+          logger.warn(`Segment ${seg.index} attempt ${attempt} failed, retrying in ${delay}ms`);
+          await sleep(delay);
+        }
+      }
+    };
+
+    await runConcurrent(pending, concurrency, downloadSegment);
+
+    await this.concatenateSegments(segments, tempDir, outputPath);
+
+    try {
+      await Bun.$`rm -rf ${tempDir}`.quiet();
+    } catch {
+      // ignore cleanup errors
+    }
+
+    if (options.onProgress) {
+      options.onProgress({
+        downloaded_bytes: total,
+        total_bytes: total,
+        speed: null,
+        eta: null,
+        percent: 100,
+        status: "finished",
+        filename: outputPath,
+      });
+    }
+  }
+
+  private async concatenateSegments(
+    segments: Segment[],
+    tempDir: string,
+    outputPath: string,
+  ): Promise<void> {
+    const writer = Bun.file(outputPath).writer();
+    const ordered = segments.slice().sort((a, b) => a.index - b.index);
+
+    for (const seg of ordered) {
+      const segPath = `${tempDir}/seg-${seg.index.toString().padStart(6, "0")}`;
+      try {
+        const data = await Bun.file(segPath).arrayBuffer();
+        writer.write(new Uint8Array(data));
+      } catch (err) {
+        throw new DownloadError(`Missing segment ${seg.index}: ${err instanceof Error ? err.message : String(err)}`);
+      }
+    }
+
+    await writer.end();
+  }
+
+  private async decryptAes128(
+    data: Uint8Array,
+    key: { method: string; uri: string; iv?: string },
+    segmentIndex: number,
+    keyCache: Map<string, Uint8Array>,
+    headers?: Record<string, string>,
+  ): Promise<Uint8Array> {
+    let keyBytes = keyCache.get(key.uri);
+    if (!keyBytes) {
+      const resp = await fetch(key.uri, { headers: headers ?? {} });
+      if (!resp.ok) {
+        throw new DownloadError(`Failed to fetch AES key: HTTP ${resp.status}`);
+      }
+      keyBytes = new Uint8Array(await resp.arrayBuffer());
+      keyCache.set(key.uri, keyBytes);
+    }
+
+    const iv = key.iv
+      ? hexToBytes(key.iv.replace("0x", "").replace("0X", ""))
+      : segmentIndexToIv(segmentIndex);
+
+    const cryptoKey = await crypto.subtle.importKey(
+      "raw",
+      keyBytes,
+      { name: "AES-CBC" },
+      false,
+      ["decrypt"],
+    );
+
+    const decrypted = await crypto.subtle.decrypt(
+      { name: "AES-CBC", iv },
+      cryptoKey,
+      data,
+    );
+
+    return new Uint8Array(decrypted);
+  }
+}
+
+function hexToBytes(hex: string): Uint8Array {
+  const bytes = new Uint8Array(hex.length / 2);
+  for (let i = 0; i < hex.length; i += 2) {
+    bytes[i / 2] = parseInt(hex.substring(i, i + 2), 16);
+  }
+  return bytes;
+}
+
+function segmentIndexToIv(index: number): Uint8Array {
+  const iv = new Uint8Array(16);
+  const view = new DataView(iv.buffer);
+  view.setUint32(12, index, false);
+  return iv;
+}
+
+async function runConcurrent<T>(
+  items: T[],
+  concurrency: number,
+  fn: (item: T) => Promise<void>,
+): Promise<void> {
+  const queue = [...items];
+  const workers: Promise<void>[] = [];
+
+  const worker = async (): Promise<void> => {
+    while (queue.length > 0) {
+      const item = queue.shift();
+      if (item !== undefined) {
+        await fn(item);
+      }
+    }
+  };
+
+  for (let i = 0; i < Math.min(concurrency, items.length); i++) {
+    workers.push(worker());
+  }
+
+  await Promise.all(workers);
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export { runConcurrent, sleep };

package/src/downloaders/hls.ts

@@ -0,0 +1,170 @@
+import { Downloader, DownloadError } from "../core/types";
+import type { DownloadOptions } from "../core/types";
+import { FragmentDownloader } from "./fragment";
+import type { Segment } from "./fragment";
+import { logger } from "../core/logger";
+
+interface HlsKey {
+  method: string;
+  uri: string;
+  iv?: string;
+}
+
+interface HlsSegmentRaw {
+  uri: string;
+  key?: HlsKey;
+  byteRange?: { length: number; offset?: number };
+  map?: { uri: string; byteRange?: { length: number; offset?: number } };
+}
+
+interface HlsVariant {
+  uri: string;
+  bandwidth?: number;
+  resolution?: { width: number; height: number };
+  codecs?: string;
+}
+
+interface HlsMasterPlaylist {
+  isMasterPlaylist: true;
+  variants: HlsVariant[];
+}
+
+interface HlsMediaPlaylist {
+  isMasterPlaylist: false;
+  segments: HlsSegmentRaw[];
+  endList?: boolean;
+}
+
+type HlsPlaylist = HlsMasterPlaylist | HlsMediaPlaylist;
+
+export class HlsDownloader extends Downloader {
+  readonly protocol = "m3u8";
+
+  canHandle(protocol: string): boolean {
+    return protocol === "m3u8" || protocol === "hls";
+  }
+
+  async download(
+    url: string,
+    filepath: string,
+    options: DownloadOptions,
+  ): Promise<void> {
+    const retries = options.retries ?? 3;
+    let lastError: Error | null = null;
+
+    for (let attempt = 1; attempt <= retries; attempt++) {
+      try {
+        await this.downloadHls(url, filepath, options);
+        return;
+      } catch (err) {
+        lastError = err instanceof Error ? err : new Error(String(err));
+        if (attempt < retries) {
+          const delay = 1000 * Math.pow(2, attempt - 1);
+          logger.warn(`HLS download failed (attempt ${attempt}/${retries}): ${lastError.message}`);
+          await new Promise((r) => setTimeout(r, delay));
+        }
+      }
+    }
+
+    throw new DownloadError(`HLS download failed after ${retries} attempts: ${lastError?.message}`);
+  }
+
+  private async fetchManifest(url: string, headers: Record<string, string>): Promise<string> {
+    const resp = await fetch(url, { headers });
+    if (!resp.ok) {
+      throw new DownloadError(`Failed to fetch manifest: HTTP ${resp.status}`);
+    }
+    return resp.text();
+  }
+
+  private async downloadHls(url: string, filepath: string, options: DownloadOptions): Promise<void> {
+    const { parse } = await import("hls-parser");
+    const headers = { ...options.headers };
+
+    const manifestText = await this.fetchManifest(url, headers);
+    const parsed = parse(manifestText) as HlsPlaylist;
+
+    if (parsed.isMasterPlaylist) {
+      const master = parsed as HlsMasterPlaylist;
+      if (master.variants.length === 0) {
+        throw new DownloadError("HLS master playlist has no variants");
+      }
+      const best = master.variants.reduce((a, b) =>
+        (b.bandwidth ?? 0) > (a.bandwidth ?? 0) ? b : a,
+      );
+      const mediaUrl = resolveUrl(url, best.uri);
+      logger.debug(`HLS: selected variant ${mediaUrl} (bandwidth: ${best.bandwidth ?? "unknown"})`);
+
+      const mediaText = await this.fetchManifest(mediaUrl, headers);
+      const mediaParsed = parse(mediaText) as HlsMediaPlaylist;
+      await this.downloadMediaPlaylist(mediaParsed, mediaUrl, filepath, options);
+    } else {
+      await this.downloadMediaPlaylist(parsed as HlsMediaPlaylist, url, filepath, options);
+    }
+  }
+
+  private async downloadMediaPlaylist(
+    playlist: HlsMediaPlaylist,
+    baseUrl: string,
+    filepath: string,
+    options: DownloadOptions,
+  ): Promise<void> {
+    const rawSegments = playlist.segments;
+    if (!rawSegments || rawSegments.length === 0) {
+      throw new DownloadError("HLS media playlist has no segments");
+    }
+
+    const segments: Segment[] = [];
+    let idx = 0;
+
+    for (const raw of rawSegments) {
+      if (raw.map) {
+        const mapUrl = resolveUrl(baseUrl, raw.map.uri);
+        segments.push({
+          url: mapUrl,
+          index: idx++,
+          isInit: true,
+          byteRange: raw.map.byteRange
+            ? { start: raw.map.byteRange.offset ?? 0, end: (raw.map.byteRange.offset ?? 0) + raw.map.byteRange.length - 1 }
+            : undefined,
+        });
+      }
+
+      const segUrl = resolveUrl(baseUrl, raw.uri);
+      const seg: Segment = {
+        url: segUrl,
+        index: idx++,
+      };
+
+      if (raw.key) {
+        seg.key = {
+          method: raw.key.method,
+          uri: resolveUrl(baseUrl, raw.key.uri),
+          iv: raw.key.iv,
+        };
+      }
+
+      if (raw.byteRange) {
+        seg.byteRange = {
+          start: raw.byteRange.offset ?? 0,
+          end: (raw.byteRange.offset ?? 0) + raw.byteRange.length - 1,
+        };
+      }
+
+      segments.push(seg);
+    }
+
+    const tempDir = `/tmp/dlpx-hls-${Date.now()}`;
+    const fragmenter = new FragmentDownloader();
+    await fragmenter.downloadSegments(segments, filepath, {
+      ...options,
+      concurrency: 8,
+      tempDir,
+    });
+  }
+}
+
+function resolveUrl(base: string, relative: string): string {
+  if (/^https?:\/\//i.test(relative)) return relative;
+  return new URL(relative, base).toString();
+}

package/src/downloaders/http.ts

@@ -0,0 +1,260 @@
+import { Downloader, DownloadError } from "../core/types";
+import type { DownloadOptions, DownloadProgress } from "../core/types";
+import { logger } from "../core/logger";
+
+const CHUNK_SIZE = 8 * 1024 * 1024;
+const DEFAULT_CONCURRENCY = 4;
+
+export class HttpDownloader extends Downloader {
+  readonly protocol = "https";
+
+  canHandle(protocol: string): boolean {
+    return protocol === "http" || protocol === "https";
+  }
+
+  async download(
+    url: string,
+    filepath: string,
+    options: DownloadOptions,
+  ): Promise<void> {
+    const retries = options.retries ?? 3;
+    let lastError: Error | null = null;
+
+    for (let attempt = 1; attempt <= retries; attempt++) {
+      try {
+        await this.downloadAttempt(url, filepath, options, attempt > 1);
+        return;
+      } catch (err) {
+        lastError = err instanceof Error ? err : new Error(String(err));
+        if (attempt < retries) {
+          const delay = 1000 * Math.pow(2, attempt - 1);
+          logger.warn(`Download failed (attempt ${attempt}/${retries}): ${lastError.message}, retrying in ${delay}ms`);
+          await sleep(delay);
+        }
+      }
+    }
+
+    throw new DownloadError(
+      `Download failed after ${retries} attempts: ${lastError?.message}`,
+    );
+  }
+
+  private buildHeaders(options: DownloadOptions): Record<string, string> {
+    return { ...options.headers };
+  }
+
+  private async getContentLength(url: string, headers: Record<string, string>): Promise<number | null> {
+    try {
+      const resp = await fetch(url, { method: "HEAD", headers });
+      if (resp.ok) {
+        const cl = resp.headers.get("content-length");
+        const acceptRanges = resp.headers.get("accept-ranges");
+        if (cl && acceptRanges === "bytes") {
+          return parseInt(cl, 10);
+        }
+      }
+    } catch {
+      // fall through
+    }
+    return null;
+  }
+
+  private async downloadAttempt(
+    url: string,
+    filepath: string,
+    options: DownloadOptions,
+    isResume: boolean,
+  ): Promise<void> {
+    const headers = this.buildHeaders(options);
+
+    let existingBytes = 0;
+    if (isResume) {
+      try {
+        const file = Bun.file(filepath);
+        existingBytes = file.size;
+        if (existingBytes > 0) {
+          headers["Range"] = `bytes=${existingBytes}-`;
+        }
+      } catch {
+        existingBytes = 0;
+      }
+    }
+
+    const headHeaders = this.buildHeaders(options);
+    const contentLength = isResume ? null : await this.getContentLength(url, headHeaders);
+
+    if (contentLength && contentLength > CHUNK_SIZE * 2 && !isResume) {
+      await this.downloadConcurrent(url, filepath, contentLength, options);
+      return;
+    }
+
+    const response = await fetch(url, { headers });
+    if (!response.ok && response.status !== 206) {
+      throw new DownloadError(`HTTP ${response.status}: ${response.statusText}`);
+    }
+
+    const totalStr = response.headers.get("content-length");
+    const totalBytes = totalStr ? parseInt(totalStr, 10) + existingBytes : null;
+
+    const body = response.body;
+    if (!body) {
+      throw new DownloadError("Empty response body");
+    }
+
+    const writer = Bun.file(filepath).writer();
+    const reader = body.getReader();
+    let downloadedBytes = existingBytes;
+    const startTime = Date.now();
+    const rateLimit = options.rateLimit ?? null;
+
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+
+        writer.write(value);
+        downloadedBytes += value.byteLength;
+
+        if (rateLimit && rateLimit > 0) {
+          const elapsed = (Date.now() - startTime) / 1000;
+          const expectedTime = (downloadedBytes - existingBytes) / rateLimit;
+          if (elapsed < expectedTime) {
+            await sleep((expectedTime - elapsed) * 1000);
+          }
+        }
+
+        if (options.onProgress) {
+          const elapsed = (Date.now() - startTime) / 1000;
+          const speed = elapsed > 0 ? (downloadedBytes - existingBytes) / elapsed : 0;
+          const remaining = totalBytes ? (totalBytes - downloadedBytes) / (speed || 1) : null;
+          const percent = totalBytes ? (downloadedBytes / totalBytes) * 100 : null;
+
+          const progress: DownloadProgress = {
+            downloaded_bytes: downloadedBytes,
+            total_bytes: totalBytes,
+            speed,
+            eta: remaining,
+            percent,
+            status: "downloading",
+            filename: filepath,
+          };
+          options.onProgress(progress);
+        }
+      }
+
+      await writer.end();
+
+      if (options.onProgress) {
+        options.onProgress({
+          downloaded_bytes: downloadedBytes,
+          total_bytes: totalBytes,
+          speed: null,
+          eta: null,
+          percent: 100,
+          status: "finished",
+          filename: filepath,
+        });
+      }
+    } catch (err) {
+      await writer.end();
+      throw err;
+    }
+  }
+
+  private async downloadConcurrent(
+    url: string,
+    filepath: string,
+    totalBytes: number,
+    options: DownloadOptions,
+  ): Promise<void> {
+    const concurrency = DEFAULT_CONCURRENCY;
+    const chunks: Array<{ start: number; end: number; index: number }> = [];
+
+    for (let i = 0, idx = 0; i < totalBytes; i += CHUNK_SIZE, idx++) {
+      chunks.push({ start: i, end: Math.min(i + CHUNK_SIZE - 1, totalBytes - 1), index: idx });
+    }
+
+    const tempDir = `/tmp/dlpx-http-${Date.now()}`;
+    await Bun.$`mkdir -p ${tempDir}`.quiet();
+
+    const baseHeaders = this.buildHeaders(options);
+    let downloadedBytes = 0;
+    const startTime = Date.now();
+
+    const downloadChunk = async (chunk: { start: number; end: number; index: number }): Promise<void> => {
+      const headers = { ...baseHeaders, Range: `bytes=${chunk.start}-${chunk.end}` };
+      const retries = options.retries ?? 3;
+
+      for (let attempt = 1; attempt <= retries; attempt++) {
+        try {
+          const resp = await fetch(url, { headers });
+          if (!resp.ok && resp.status !== 206) {
+            throw new DownloadError(`HTTP ${resp.status} on chunk ${chunk.index}`);
+          }
+          const data = new Uint8Array(await resp.arrayBuffer());
+          await Bun.write(`${tempDir}/chunk-${chunk.index.toString().padStart(6, "0")}`, data);
+
+          downloadedBytes += data.byteLength;
+
+          if (options.onProgress) {
+            const elapsed = (Date.now() - startTime) / 1000;
+            const speed = elapsed > 0 ? downloadedBytes / elapsed : 0;
+            const remaining = speed > 0 ? (totalBytes - downloadedBytes) / speed : null;
+
+            options.onProgress({
+              downloaded_bytes: downloadedBytes,
+              total_bytes: totalBytes,
+              speed,
+              eta: remaining,
+              percent: (downloadedBytes / totalBytes) * 100,
+              status: "downloading",
+              filename: filepath,
+            });
+          }
+          return;
+        } catch (err) {
+          if (attempt === retries) throw err;
+          await sleep(1000 * Math.pow(2, attempt - 1));
+        }
+      }
+    };
+
+    const queue = [...chunks];
+    const workers: Promise<void>[] = [];
+    const worker = async (): Promise<void> => {
+      while (queue.length > 0) {
+        const chunk = queue.shift();
+        if (chunk !== undefined) await downloadChunk(chunk);
+      }
+    };
+    for (let i = 0; i < Math.min(concurrency, chunks.length); i++) {
+      workers.push(worker());
+    }
+    await Promise.all(workers);
+
+    const writer = Bun.file(filepath).writer();
+    for (const chunk of chunks) {
+      const data = await Bun.file(`${tempDir}/chunk-${chunk.index.toString().padStart(6, "0")}`).arrayBuffer();
+      writer.write(new Uint8Array(data));
+    }
+    await writer.end();
+
+    await Bun.$`rm -rf ${tempDir}`.quiet();
+
+    if (options.onProgress) {
+      options.onProgress({
+        downloaded_bytes: totalBytes,
+        total_bytes: totalBytes,
+        speed: null,
+        eta: null,
+        percent: 100,
+        status: "finished",
+        filename: filepath,
+      });
+    }
+  }
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}