offcourse 0.0.2 → 1.0.0

package/src/downloader/queue.test.ts

@@ -0,0 +1,199 @@
+import { describe, expect, it } from "vitest";
+import { AsyncQueue } from "./queue.js";
+
+describe("AsyncQueue", () => {
+  it("processes items in order", async () => {
+    const processed: string[] = [];
+    const queue = new AsyncQueue<string>({
+      concurrency: 1,
+      maxRetries: 0,
+    });
+
+    queue.add("1", "first");
+    queue.add("2", "second");
+    queue.add("3", "third");
+
+    await queue.process(async (item) => {
+      processed.push(item);
+    });
+
+    expect(processed).toEqual(["first", "second", "third"]);
+  });
+
+  it("tracks completed and failed counts", async () => {
+    const queue = new AsyncQueue<string>({
+      concurrency: 1,
+      maxRetries: 0,
+    });
+
+    queue.add("1", "success");
+    queue.add("2", "fail");
+    queue.add("3", "success");
+
+    const result = await queue.process(async (item) => {
+      if (item === "fail") {
+        throw new Error("Intentional failure");
+      }
+    });
+
+    expect(result.completed).toBe(2);
+    expect(result.failed).toBe(1);
+    expect(result.errors).toHaveLength(1);
+    expect(result.errors[0]?.id).toBe("2");
+  });
+
+  it("retries failed items up to maxRetries", async () => {
+    let attempts = 0;
+    const queue = new AsyncQueue<string>({
+      concurrency: 1,
+      maxRetries: 3,
+    });
+
+    queue.add("1", "retry-test");
+
+    await queue.process(async () => {
+      attempts++;
+      if (attempts < 3) {
+        throw new Error("Not yet");
+      }
+    });
+
+    expect(attempts).toBe(3);
+  });
+
+  it("adds multiple items with addAll", async () => {
+    const queue = new AsyncQueue<number>({
+      concurrency: 1,
+      maxRetries: 0,
+    });
+
+    queue.addAll([
+      { id: "a", data: 1 },
+      { id: "b", data: 2 },
+      { id: "c", data: 3 },
+    ]);
+
+    const sum = { value: 0 };
+    await queue.process(async (item) => {
+      sum.value += item;
+    });
+
+    expect(sum.value).toBe(6);
+  });
+
+  it("reports progress during processing", async () => {
+    const progressCalls: { completed: number; total: number }[] = [];
+
+    const queue = new AsyncQueue<string>({
+      concurrency: 1,
+      maxRetries: 0,
+      onProgress: (completed, total) => {
+        progressCalls.push({ completed, total });
+      },
+    });
+
+    queue.add("1", "a");
+    queue.add("2", "b");
+    queue.add("3", "c");
+
+    await queue.process(async () => {
+      // no-op
+    });
+
+    expect(progressCalls.length).toBeGreaterThan(0);
+    // Last call should have all completed
+    const lastCall = progressCalls.at(-1);
+    expect(lastCall?.completed).toBe(3);
+    expect(lastCall?.total).toBe(3);
+  });
+
+  it("returns correct status", async () => {
+    const queue = new AsyncQueue<string>({
+      concurrency: 1,
+      maxRetries: 0,
+    });
+
+    queue.add("1", "a");
+    queue.add("2", "b");
+
+    const beforeStatus = queue.getStatus();
+    expect(beforeStatus.pending).toBe(2);
+    expect(beforeStatus.completed).toBe(0);
+
+    await queue.process(async () => {});
+
+    const afterStatus = queue.getStatus();
+    expect(afterStatus.pending).toBe(0);
+    expect(afterStatus.completed).toBe(2);
+  });
+
+  it("handles concurrent processing", async () => {
+    const processing = new Set<string>();
+    let maxConcurrent = 0;
+
+    const queue = new AsyncQueue<string>({
+      concurrency: 2,
+      maxRetries: 0,
+    });
+
+    queue.addAll([
+      { id: "1", data: "a" },
+      { id: "2", data: "b" },
+      { id: "3", data: "c" },
+      { id: "4", data: "d" },
+    ]);
+
+    await queue.process(async (_item, id) => {
+      processing.add(id);
+      maxConcurrent = Math.max(maxConcurrent, processing.size);
+      // Simulate async work
+      await new Promise((r) => setTimeout(r, 10));
+      processing.delete(id);
+    });
+
+    expect(maxConcurrent).toBeLessThanOrEqual(2);
+  });
+
+  it("handles empty queue", async () => {
+    const queue = new AsyncQueue<string>({
+      concurrency: 1,
+      maxRetries: 0,
+    });
+
+    const result = await queue.process(async () => {});
+
+    expect(result.completed).toBe(0);
+    expect(result.failed).toBe(0);
+    expect(result.errors).toEqual([]);
+  });
+
+  it("captures error messages correctly", async () => {
+    const queue = new AsyncQueue<string>({
+      concurrency: 1,
+      maxRetries: 0,
+    });
+
+    queue.add("1", "test");
+
+    const result = await queue.process(async () => {
+      throw new Error("Specific error message");
+    });
+
+    expect(result.errors[0]?.error).toBe("Specific error message");
+  });
+
+  it("handles non-Error throws", async () => {
+    const queue = new AsyncQueue<string>({
+      concurrency: 1,
+      maxRetries: 0,
+    });
+
+    queue.add("1", "test");
+
+    const result = await queue.process(async () => {
+      throw "String error";
+    });
+
+    expect(result.errors[0]?.error).toBe("String error");
+  });
+});

package/src/downloader/queue.ts

@@ -0,0 +1,118 @@
+import PQueue from "p-queue";
+
+export interface QueueItem<T> {
+  id: string;
+  data: T;
+  status: "pending" | "processing" | "completed" | "failed";
+  error?: string;
+  retries: number;
+}
+
+export interface QueueOptions {
+  concurrency: number;
+  maxRetries: number;
+  onProgress?: ((completed: number, total: number, current?: string) => void) | undefined;
+}
+
+/**
+ * An async queue for processing items with concurrency control.
+ * Uses p-queue internally for battle-tested concurrency handling.
+ */
+export class AsyncQueue<T> {
+  private items: QueueItem<T>[] = [];
+  private readonly queue: PQueue;
+  private readonly maxRetries: number;
+  private readonly onProgress:
+    | ((completed: number, total: number, current?: string) => void)
+    | undefined;
+
+  constructor(options: QueueOptions) {
+    this.queue = new PQueue({ concurrency: options.concurrency });
+    this.maxRetries = options.maxRetries;
+    this.onProgress = options.onProgress;
+  }
+
+  /**
+   * Adds an item to the queue.
+   */
+  add(id: string, data: T): void {
+    this.items.push({
+      id,
+      data,
+      status: "pending",
+      retries: 0,
+    });
+  }
+
+  /**
+   * Adds multiple items to the queue.
+   */
+  addAll(items: { id: string; data: T }[]): void {
+    for (const item of items) {
+      this.add(item.id, item.data);
+    }
+  }
+
+  /**
+   * Processes all items in the queue using the provided handler.
+   */
+  async process(
+    handler: (item: T, id: string) => Promise<void>
+  ): Promise<{ completed: number; failed: number; errors: { id: string; error: string }[] }> {
+    const errors: { id: string; error: string }[] = [];
+
+    const processItem = async (item: QueueItem<T>): Promise<void> => {
+      item.status = "processing";
+
+      // maxRetries=0 means try once, maxRetries=3 means try up to 3 times total
+      const maxAttempts = Math.max(1, this.maxRetries);
+
+      for (let attempt = 0; attempt < maxAttempts; attempt++) {
+        try {
+          await handler(item.data, item.id);
+          item.status = "completed";
+          this.reportProgress();
+          return;
+        } catch (error) {
+          item.retries = attempt + 1;
+          if (attempt >= maxAttempts - 1) {
+            item.status = "failed";
+            item.error = error instanceof Error ? error.message : String(error);
+            errors.push({ id: item.id, error: item.error });
+            this.reportProgress();
+          }
+        }
+      }
+    };
+
+    // Add all items to the p-queue
+    await this.queue.addAll(this.items.map((item) => () => processItem(item)));
+
+    const completed = this.items.filter((i) => i.status === "completed").length;
+    const failed = this.items.filter((i) => i.status === "failed").length;
+
+    return { completed, failed, errors };
+  }
+
+  private reportProgress(): void {
+    if (!this.onProgress) return;
+
+    const completed = this.items.filter((i) => i.status === "completed").length;
+    const total = this.items.length;
+    const current = this.items.find((i) => i.status === "processing");
+
+    this.onProgress(completed, total, current?.id);
+  }
+
+  /**
+   * Gets the current queue status.
+   */
+  getStatus(): { pending: number; processing: number; completed: number; failed: number } {
+    return {
+      pending: this.items.filter((i) => i.status === "pending").length,
+      processing: this.items.filter((i) => i.status === "processing").length,
+      completed: this.items.filter((i) => i.status === "completed").length,
+      failed: this.items.filter((i) => i.status === "failed").length,
+    };
+  }
+}

package/src/downloader/vimeoDownloader.test.ts

@@ -0,0 +1,62 @@
+import { describe, expect, it } from "vitest";
+import { extractVimeoId } from "./vimeoDownloader.js";
+
+describe("extractVimeoId", () => {
+  it("extracts ID from standard vimeo.com URL", () => {
+    const url = "https://vimeo.com/123456789";
+    expect(extractVimeoId(url)).toBe("123456789");
+  });
+
+  it("extracts ID from vimeo.com/video URL", () => {
+    const url = "https://vimeo.com/video/123456789";
+    expect(extractVimeoId(url)).toBe("123456789");
+  });
+
+  it("extracts ID from player.vimeo.com URL", () => {
+    const url = "https://player.vimeo.com/video/987654321";
+    expect(extractVimeoId(url)).toBe("987654321");
+  });
+
+  it("extracts ID from channel URL", () => {
+    const url = "https://vimeo.com/channels/staffpicks/123456789";
+    expect(extractVimeoId(url)).toBe("123456789");
+  });
+
+  it("extracts ID from groups URL", () => {
+    const url = "https://vimeo.com/groups/shortfilms/videos/123456789";
+    expect(extractVimeoId(url)).toBe("123456789");
+  });
+
+  it("extracts ID from URL with query params", () => {
+    const url = "https://vimeo.com/123456789?share=copy&autoplay=1";
+    expect(extractVimeoId(url)).toBe("123456789");
+  });
+
+  it("extracts ID from URL with hash for unlisted videos", () => {
+    const url = "https://vimeo.com/123456789/abcdef1234";
+    expect(extractVimeoId(url)).toBe("123456789");
+  });
+
+  it("extracts ID from player URL with h parameter", () => {
+    const url = "https://player.vimeo.com/video/123456789?h=abcdef1234";
+    expect(extractVimeoId(url)).toBe("123456789");
+  });
+
+  it("returns null for non-Vimeo URL", () => {
+    expect(extractVimeoId("https://youtube.com/watch?v=abc123")).toBeNull();
+    expect(extractVimeoId("https://loom.com/embed/abc123")).toBeNull();
+  });
+
+  it("returns null for Vimeo homepage", () => {
+    expect(extractVimeoId("https://vimeo.com")).toBeNull();
+    expect(extractVimeoId("https://vimeo.com/")).toBeNull();
+  });
+
+  it("returns null for empty string", () => {
+    expect(extractVimeoId("")).toBeNull();
+  });
+
+  it("returns null for invalid string", () => {
+    expect(extractVimeoId("not-a-url")).toBeNull();
+  });
+});