getraw 0.1.0

package/src/utils/sanitize.ts

@@ -0,0 +1,36 @@
+const ILLEGAL_CHARS = /[<>:"/\\|?*\x00-\x1F]/g;
+const RESERVED_NAMES = new Set([
+  "CON", "PRN", "AUX", "NUL",
+  "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9",
+  "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9",
+]);
+
+export function sanitizeFilename(filename: string): string {
+  let sanitized = filename.replace(ILLEGAL_CHARS, "_");
+  sanitized = sanitized.replace(/\.+$/, "");
+  sanitized = sanitized.replace(/\s+/g, " ").trim();
+
+  if (sanitized.length === 0) {
+    sanitized = "download";
+  }
+
+  const parts = sanitized.split(".");
+  if (parts.length > 1) {
+    const name = parts.slice(0, -1).join(".");
+    const ext = parts[parts.length - 1];
+    if (RESERVED_NAMES.has(name.toUpperCase())) {
+      return `_${name}.${ext}`;
+    }
+    return `${name}.${ext}`;
+  }
+
+  if (RESERVED_NAMES.has(sanitized.toUpperCase())) {
+    return `_${sanitized}`;
+  }
+
+  return sanitized;
+}
+
+export function sanitizePath(path: string): string {
+  return path.split("/").map(sanitizeFilename).join("/");
+}

package/src/utils/traverse.ts

@@ -0,0 +1,68 @@
+type PathSegment = string | number | ((val: unknown) => boolean);
+type TraversePath = PathSegment[];
+
+export function traverse_obj(
+  obj: unknown,
+  ...paths: TraversePath[]
+): unknown {
+  for (const path of paths) {
+    const result = walkPath(obj, path);
+    if (result !== undefined) return result;
+  }
+  return undefined;
+}
+
+function walkPath(obj: unknown, path: TraversePath): unknown {
+  let current: unknown = obj;
+
+  for (const segment of path) {
+    if (current === null || current === undefined) return undefined;
+
+    if (typeof segment === "function") {
+      if (!segment(current)) return undefined;
+      continue;
+    }
+
+    if (typeof segment === "number") {
+      if (Array.isArray(current)) {
+        current = current[segment];
+      } else {
+        return undefined;
+      }
+      continue;
+    }
+
+    if (typeof segment === "string") {
+      if (segment === "...") {
+        return deepSearch(current, path.slice(path.indexOf(segment) + 1));
+      }
+
+      if (typeof current === "object" && current !== null) {
+        current = (current as Record<string, unknown>)[segment];
+      } else {
+        return undefined;
+      }
+      continue;
+    }
+  }
+
+  return current;
+}
+
+function deepSearch(obj: unknown, remainingPath: TraversePath): unknown {
+  if (obj === null || obj === undefined) return undefined;
+  if (typeof obj !== "object") return undefined;
+
+  const result = walkPath(obj, remainingPath);
+  if (result !== undefined) return result;
+
+  const values = Array.isArray(obj) ? obj : Object.values(obj);
+  for (const value of values) {
+    if (typeof value === "object" && value !== null) {
+      const found = deepSearch(value, remainingPath);
+      if (found !== undefined) return found;
+    }
+  }
+
+  return undefined;
+}

package/tests/core/format-sorter.test.ts

@@ -0,0 +1,96 @@
+import { describe, test, expect } from "bun:test";
+import { selectFormats, sortFormats, parseFormatString, formatFormatTable } from "../../src/core/format-sorter";
+import type { Format } from "../../src/core/types";
+
+const FORMATS: Format[] = [
+  { format_id: "251", url: "https://example.com/audio", ext: "webm", acodec: "opus", abr: 160, vcodec: "none" },
+  { format_id: "140", url: "https://example.com/audio2", ext: "m4a", acodec: "aac", abr: 128, vcodec: "none" },
+  { format_id: "244", url: "https://example.com/480", ext: "webm", vcodec: "vp9", acodec: "none", height: 480, width: 854, tbr: 750 },
+  { format_id: "247", url: "https://example.com/720", ext: "webm", vcodec: "vp9", acodec: "none", height: 720, width: 1280, tbr: 1500 },
+  { format_id: "248", url: "https://example.com/1080", ext: "webm", vcodec: "vp9", acodec: "none", height: 1080, width: 1920, tbr: 2500 },
+  { format_id: "22", url: "https://example.com/720av", ext: "mp4", vcodec: "h264", acodec: "aac", height: 720, width: 1280, tbr: 2000, abr: 192 },
+];
+
+describe("parseFormatString", () => {
+  test("parses simple best", () => {
+    const spec = parseFormatString("best");
+    expect(spec.type).toBe("single");
+    expect(spec.video?.best).toBe(true);
+  });
+
+  test("parses merge format", () => {
+    const spec = parseFormatString("bv*+ba");
+    expect(spec.type).toBe("merge");
+    expect(spec.video?.best).toBe(true);
+    expect(spec.audio?.best).toBe(true);
+    expect(spec.audio?.audioOnly).toBe(true);
+  });
+
+  test("parses fallback chain", () => {
+    const spec = parseFormatString("bv*+ba/b");
+    expect(spec.type).toBe("merge");
+    expect(spec.fallback).toBeDefined();
+    expect(spec.fallback?.type).toBe("single");
+  });
+
+  test("parses height filter", () => {
+    const spec = parseFormatString("1080p");
+    expect(spec.video?.height).toBe(1080);
+  });
+});
+
+describe("selectFormats", () => {
+  test("selects best video + best audio for bv*+ba", () => {
+    const selected = selectFormats(FORMATS, "bv*+ba");
+    expect(selected).toHaveLength(2);
+  });
+
+  test("selects single best format for 'best'", () => {
+    const selected = selectFormats(FORMATS, "best");
+    expect(selected).toHaveLength(1);
+  });
+
+  test("selects by format ID", () => {
+    const selected = selectFormats(FORMATS, "22");
+    expect(selected).toHaveLength(1);
+    expect(selected[0].format_id).toBe("22");
+  });
+
+  test("selects by height", () => {
+    const selected = selectFormats(FORMATS, "1080p");
+    expect(selected).toHaveLength(1);
+    expect(selected[0].height).toBe(1080);
+  });
+
+  test("falls back when primary fails", () => {
+    const selected = selectFormats(FORMATS, "4320p/best");
+    expect(selected).toHaveLength(1);
+  });
+
+  test("returns empty for impossible format", () => {
+    const selected = selectFormats(FORMATS, "nonexistent");
+    expect(selected).toHaveLength(0);
+  });
+});
+
+describe("sortFormats", () => {
+  test("sorts by quality ascending", () => {
+    const sorted = sortFormats(FORMATS);
+    const heights = sorted
+      .filter((f) => f.height !== undefined)
+      .map((f) => f.height);
+    for (let i = 1; i < heights.length; i++) {
+      expect(heights[i]!).toBeGreaterThanOrEqual(heights[i - 1]!);
+    }
+  });
+});
+
+describe("formatFormatTable", () => {
+  test("returns a formatted table string", () => {
+    const table = formatFormatTable(FORMATS);
+    expect(table).toContain("ID");
+    expect(table).toContain("EXT");
+    expect(table).toContain("251");
+    expect(table).toContain("248");
+  });
+});

package/tests/core/output-template.test.ts

@@ -0,0 +1,56 @@
+import { describe, test, expect } from "bun:test";
+import { renderTemplate, buildFilename } from "../../src/core/output-template";
+import type { InfoDict } from "../../src/core/types";
+
+const INFO: InfoDict = {
+  id: "abc123",
+  title: "Test Video - Special Edition",
+  ext: "mp4",
+  uploader: "TestChannel",
+  uploader_id: "UC12345",
+  duration: 300,
+  view_count: 1000000,
+  upload_date: "20240115",
+};
+
+describe("renderTemplate", () => {
+  test("renders basic title and id", () => {
+    const result = renderTemplate("%(title)s [%(id)s].%(ext)s", INFO);
+    expect(result).toBe("Test Video - Special Edition [abc123].mp4");
+  });
+
+  test("renders numeric fields", () => {
+    const result = renderTemplate("%(duration)d seconds", INFO);
+    expect(result).toBe("300 seconds");
+  });
+
+  test("handles missing fields", () => {
+    const result = renderTemplate("%(channel)s - %(title)s", INFO);
+    expect(result).toContain("NA");
+    expect(result).toContain("Test Video");
+  });
+
+  test("renders uploader", () => {
+    const result = renderTemplate("%(uploader)s - %(title)s", INFO);
+    expect(result).toBe("TestChannel - Test Video - Special Edition");
+  });
+});
+
+describe("buildFilename", () => {
+  test("sanitizes illegal characters", () => {
+    const infoWithBadTitle: InfoDict = {
+      ...INFO,
+      title: 'Video: "Test" <Special>',
+    };
+    const result = buildFilename("%(title)s.%(ext)s", infoWithBadTitle);
+    expect(result).not.toContain(":");
+    expect(result).not.toContain('"');
+    expect(result).not.toContain("<");
+    expect(result).not.toContain(">");
+  });
+
+  test("produces valid filename", () => {
+    const result = buildFilename("%(title)s [%(id)s].%(ext)s", INFO);
+    expect(result).toBe("Test Video - Special Edition [abc123].mp4");
+  });
+});

package/tests/core/types.test.ts

@@ -0,0 +1,79 @@
+import { describe, test, expect } from "bun:test";
+import {
+  BaseExtractor,
+  ExtractorError,
+  DownloadError,
+  PostProcessError,
+  DEFAULT_OPTIONS,
+} from "../../src/core/types";
+import type { InfoDict } from "../../src/core/types";
+
+class TestExtractor extends BaseExtractor {
+  readonly _VALID_URL = /^https:\/\/example\.com\/watch\?v=[\w-]+/;
+  readonly _NAME = "test";
+
+  protected async _real_extract(url: string): Promise<InfoDict> {
+    const id = new URL(url).searchParams.get("v") ?? "unknown";
+    return {
+      id,
+      title: "Test Video",
+      formats: [
+        { format_id: "720p", url: "https://example.com/720.mp4", ext: "mp4", height: 720 },
+        { format_id: "1080p", url: "https://example.com/1080.mp4", ext: "mp4", height: 1080 },
+      ],
+    };
+  }
+}
+
+describe("BaseExtractor", () => {
+  const extractor = new TestExtractor();
+
+  test("canHandle returns true for matching URLs", () => {
+    expect(extractor.canHandle("https://example.com/watch?v=abc123")).toBe(true);
+  });
+
+  test("canHandle returns false for non-matching URLs", () => {
+    expect(extractor.canHandle("https://other.com/video")).toBe(false);
+  });
+
+  test("extract returns InfoDict with extractor metadata", async () => {
+    const info = await extractor.extract("https://example.com/watch?v=test1");
+    expect(info.id).toBe("test1");
+    expect(info.title).toBe("Test Video");
+    expect(info.extractor).toBe("test");
+    expect(info.extractor_key).toBe("TestExtractor");
+    expect(info.formats).toHaveLength(2);
+  });
+
+  test("extract throws ExtractorError for non-matching URL", async () => {
+    await expect(extractor.extract("https://bad.com/video")).rejects.toThrow(ExtractorError);
+  });
+});
+
+describe("Error classes", () => {
+  test("ExtractorError has correct name", () => {
+    const err = new ExtractorError("test");
+    expect(err.name).toBe("ExtractorError");
+    expect(err.message).toBe("test");
+  });
+
+  test("DownloadError has correct name", () => {
+    const err = new DownloadError("test");
+    expect(err.name).toBe("DownloadError");
+  });
+
+  test("PostProcessError has correct name", () => {
+    const err = new PostProcessError("test");
+    expect(err.name).toBe("PostProcessError");
+  });
+});
+
+describe("DEFAULT_OPTIONS", () => {
+  test("has sensible defaults", () => {
+    expect(DEFAULT_OPTIONS.format).toBe("bv*+ba/b");
+    expect(DEFAULT_OPTIONS.retries).toBe(3);
+    expect(DEFAULT_OPTIONS.quiet).toBe(false);
+    expect(DEFAULT_OPTIONS.verbose).toBe(false);
+    expect(DEFAULT_OPTIONS.urls).toEqual([]);
+  });
+});

package/tests/unit/downloaders/dash.test.ts

@@ -0,0 +1,57 @@
+import { describe, it, expect } from "bun:test";
+import { DashDownloader } from "../../../src/downloaders/dash";
+import { DownloadError } from "../../../src/core/types";
+
+describe("DashDownloader", () => {
+  const downloader = new DashDownloader();
+
+  it("canHandle dash and mpd protocols", () => {
+    expect(downloader.canHandle("dash")).toBe(true);
+    expect(downloader.canHandle("mpd")).toBe(true);
+    expect(downloader.canHandle("http")).toBe(false);
+    expect(downloader.canHandle("m3u8")).toBe(false);
+  });
+
+  it("has protocol set to dash", () => {
+    expect(downloader.protocol).toBe("dash");
+  });
+
+  it("throws DownloadError when MPD fetch fails", async () => {
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = async () => new Response(null, { status: 404, statusText: "Not Found" });
+
+    await expect(
+      downloader.download("https://example.com/manifest.mpd", "/tmp/test.mp4", { retries: 1 }),
+    ).rejects.toThrow(DownloadError);
+
+    globalThis.fetch = originalFetch;
+  });
+
+  it("uses exponential backoff on retry", async () => {
+    const originalFetch = globalThis.fetch;
+
+    globalThis.fetch = async () =>
+      new Response(null, { status: 503, statusText: "Service Unavailable" });
+
+    const start = Date.now();
+    await expect(
+      downloader.download("https://example.com/manifest.mpd", "/tmp/test.mp4", { retries: 2 }),
+    ).rejects.toThrow(DownloadError);
+    const elapsed = Date.now() - start;
+
+    expect(elapsed).toBeGreaterThanOrEqual(1000);
+    globalThis.fetch = originalFetch;
+  });
+
+  it("throws DownloadError on non-ok MPD response", async () => {
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = async () =>
+      new Response("Forbidden", { status: 403, statusText: "Forbidden" });
+
+    await expect(
+      downloader.download("https://example.com/manifest.mpd", "/tmp/test.mp4", { retries: 1 }),
+    ).rejects.toThrow(DownloadError);
+
+    globalThis.fetch = originalFetch;
+  });
+});

package/tests/unit/downloaders/hls.test.ts

@@ -0,0 +1,120 @@
+import { describe, it, expect, beforeEach, afterEach } from "bun:test";
+import { HlsDownloader } from "../../../src/downloaders/hls";
+import { DownloadError } from "../../../src/core/types";
+
+const MEDIA_PLAYLIST = `#EXTM3U
+#EXT-X-VERSION:3
+#EXT-X-TARGETDURATION:10
+#EXTINF:10.0,
+seg0.ts
+#EXTINF:10.0,
+seg1.ts
+#EXTINF:10.0,
+seg2.ts
+#EXT-X-ENDLIST
+`;
+
+const MASTER_PLAYLIST = `#EXTM3U
+#EXT-X-STREAM-INF:BANDWIDTH=1500000,RESOLUTION=1280x720
+high/index.m3u8
+#EXT-X-STREAM-INF:BANDWIDTH=500000,RESOLUTION=640x360
+low/index.m3u8
+`;
+
+describe("HlsDownloader", () => {
+  const downloader = new HlsDownloader();
+
+  it("canHandle m3u8 and hls protocols", () => {
+    expect(downloader.canHandle("m3u8")).toBe(true);
+    expect(downloader.canHandle("hls")).toBe(true);
+    expect(downloader.canHandle("http")).toBe(false);
+    expect(downloader.canHandle("dash")).toBe(false);
+  });
+
+  it("has protocol set to m3u8", () => {
+    expect(downloader.protocol).toBe("m3u8");
+  });
+
+  it("throws DownloadError when manifest fetch fails", async () => {
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = async () => new Response(null, { status: 404, statusText: "Not Found" });
+
+    await expect(
+      downloader.download("https://example.com/stream.m3u8", "/tmp/test.ts", { retries: 1 }),
+    ).rejects.toThrow(DownloadError);
+
+    globalThis.fetch = originalFetch;
+  });
+
+  it("throws DownloadError when segment download fails", async () => {
+    const originalFetch = globalThis.fetch;
+    let callCount = 0;
+
+    globalThis.fetch = async (url: RequestInfo | URL) => {
+      callCount++;
+      const urlStr = String(url);
+      if (urlStr.includes("stream.m3u8")) {
+        return new Response(MEDIA_PLAYLIST, { status: 200 });
+      }
+      return new Response(null, { status: 500, statusText: "Server Error" });
+    };
+
+    await expect(
+      downloader.download("https://example.com/stream.m3u8", "/tmp/test.ts", { retries: 1 }),
+    ).rejects.toThrow(DownloadError);
+
+    globalThis.fetch = originalFetch;
+  });
+
+  it("selects highest bandwidth variant from master playlist", async () => {
+    const originalFetch = globalThis.fetch;
+    const fetchedUrls: string[] = [];
+
+    globalThis.fetch = async (url: RequestInfo | URL) => {
+      const urlStr = String(url);
+      fetchedUrls.push(urlStr);
+
+      if (urlStr.endsWith("master.m3u8")) {
+        return new Response(MASTER_PLAYLIST, { status: 200 });
+      }
+      if (urlStr.includes("high/index.m3u8")) {
+        return new Response(MEDIA_PLAYLIST, { status: 200 });
+      }
+      if (urlStr.includes("seg")) {
+        return new Response(new Uint8Array(100).fill(0), { status: 200 });
+      }
+      return new Response(null, { status: 404 });
+    };
+
+    await downloader.download("https://example.com/master.m3u8", "/tmp/hls-out.ts", {
+      retries: 1,
+    }).catch(() => {});
+
+    const fetchedHighVariant = fetchedUrls.some((u) => u.includes("high/index.m3u8"));
+    expect(fetchedHighVariant).toBe(true);
+
+    const fetchedLowVariant = fetchedUrls.some((u) => u.includes("low/index.m3u8"));
+    expect(fetchedLowVariant).toBe(false);
+
+    globalThis.fetch = originalFetch;
+  });
+
+  it("uses exponential backoff on retry", async () => {
+    const originalFetch = globalThis.fetch;
+    let callCount = 0;
+
+    globalThis.fetch = async () => {
+      callCount++;
+      return new Response(null, { status: 503, statusText: "Service Unavailable" });
+    };
+
+    const start = Date.now();
+    await expect(
+      downloader.download("https://example.com/stream.m3u8", "/tmp/test.ts", { retries: 2 }),
+    ).rejects.toThrow(DownloadError);
+    const elapsed = Date.now() - start;
+
+    expect(elapsed).toBeGreaterThanOrEqual(1000);
+    globalThis.fetch = originalFetch;
+  });
+});

package/tests/unit/downloaders/http.test.ts

@@ -0,0 +1,114 @@
+import { describe, it, expect } from "bun:test";
+import { HttpDownloader } from "../../../src/downloaders/http";
+import { DownloadError } from "../../../src/core/types";
+
+describe("HttpDownloader", () => {
+  const downloader = new HttpDownloader();
+
+  it("canHandle http and https protocols", () => {
+    expect(downloader.canHandle("http")).toBe(true);
+    expect(downloader.canHandle("https")).toBe(true);
+    expect(downloader.canHandle("m3u8")).toBe(false);
+    expect(downloader.canHandle("dash")).toBe(false);
+    expect(downloader.canHandle("ftp")).toBe(false);
+  });
+
+  it("has protocol set to https", () => {
+    expect(downloader.protocol).toBe("https");
+  });
+
+  it("throws DownloadError on HTTP error after retries", async () => {
+    const originalFetch = globalThis.fetch;
+    let calls = 0;
+    globalThis.fetch = async () => {
+      calls++;
+      return new Response(null, { status: 404, statusText: "Not Found" });
+    };
+
+    await expect(
+      downloader.download("https://example.com/file.mp4", "/tmp/test-http.mp4", {
+        retries: 2,
+      }),
+    ).rejects.toThrow(DownloadError);
+
+    expect(calls).toBeGreaterThan(0);
+    globalThis.fetch = originalFetch;
+  });
+
+  it("reports progress during download", async () => {
+    const originalFetch = globalThis.fetch;
+    const chunkData = new Uint8Array(1024).fill(1);
+
+    globalThis.fetch = async (_url: RequestInfo | URL, init?: RequestInit) => {
+      if (init?.method === "HEAD") {
+        return new Response(null, {
+          status: 200,
+          headers: { "content-length": "1024", "accept-ranges": "bytes" },
+        });
+      }
+      return new Response(chunkData, {
+        status: 200,
+        headers: { "content-length": "1024" },
+      });
+    };
+
+    const progressEvents: number[] = [];
+    await downloader.download("https://example.com/file.mp4", "/tmp/test-progress.mp4", {
+      retries: 1,
+      onProgress: (p) => {
+        if (p.percent !== null) progressEvents.push(p.percent);
+      },
+    });
+
+    expect(progressEvents.length).toBeGreaterThan(0);
+    expect(progressEvents[progressEvents.length - 1]).toBe(100);
+    globalThis.fetch = originalFetch;
+  });
+
+  it("sends Range header on second attempt (resume)", async () => {
+    const originalFetch = globalThis.fetch;
+    const rangeRequests: string[] = [];
+    let attempt = 0;
+
+    globalThis.fetch = async (_url: RequestInfo | URL, init?: RequestInit) => {
+      attempt++;
+      const headers = (init?.headers ?? {}) as Record<string, string>;
+      if (headers["Range"]) rangeRequests.push(headers["Range"]);
+      if (attempt === 1) {
+        throw new Error("Simulated network failure");
+      }
+      return new Response(new Uint8Array(512).fill(2), {
+        status: 206,
+        headers: { "content-length": "512" },
+      });
+    };
+
+    await downloader.download("https://example.com/file.mp4", "/tmp/resume-test.mp4", {
+      retries: 2,
+    }).catch(() => {});
+
+    globalThis.fetch = originalFetch;
+
+    expect(attempt).toBeGreaterThan(1);
+  });
+
+  it("accepts a rateLimit option without throwing", async () => {
+    const originalFetch = globalThis.fetch;
+    const data = new Uint8Array(128).fill(3);
+
+    globalThis.fetch = async () =>
+      new Response(data, {
+        status: 200,
+        headers: { "content-length": "128" },
+      });
+
+    await expect(
+      downloader.download("https://example.com/file.mp4", "/tmp/rate-test.mp4", {
+        retries: 1,
+        rateLimit: 1024 * 1024,
+      }),
+    ).resolves.toBeUndefined();
+
+    globalThis.fetch = originalFetch;
+  });
+});