@nadimtuhin/ytranscript 1.0.0

package/dist/index.js

@@ -0,0 +1,616 @@
+// src/lib/fetcher.ts
+function extractVideoId(input) {
+  if (/^[a-zA-Z0-9_-]{11}$/.test(input)) {
+    return input;
+  }
+  try {
+    const url = new URL(input);
+    if (url.hostname.includes("youtube.com")) {
+      const v = url.searchParams.get("v");
+      if (v && /^[a-zA-Z0-9_-]{11}$/.test(v))
+        return v;
+    }
+    if (url.hostname === "youtu.be") {
+      const id = url.pathname.slice(1).split("/")[0];
+      if (/^[a-zA-Z0-9_-]{11}$/.test(id))
+        return id;
+    }
+    if (url.pathname.startsWith("/embed/")) {
+      const id = url.pathname.split("/")[2];
+      if (/^[a-zA-Z0-9_-]{11}$/.test(id))
+        return id;
+    }
+  } catch {
+  }
+  return null;
+}
+async function fetchPlayerResponse(videoId, timeout) {
+  const controller = new AbortController;
+  const timeoutId = setTimeout(() => controller.abort(), timeout);
+  try {
+    const response = await fetch("https://www.youtube.com/youtubei/v1/player?prettyPrint=false", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "User-Agent": USER_AGENT
+      },
+      body: JSON.stringify({
+        context: {
+          client: {
+            clientName: "WEB",
+            clientVersion: "2.20240101.00.00"
+          }
+        },
+        videoId
+      }),
+      signal: controller.signal
+    });
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    }
+    return await response.json();
+  } finally {
+    clearTimeout(timeoutId);
+  }
+}
+async function fetchCaptionTrack(url, timeout) {
+  const controller = new AbortController;
+  const timeoutId = setTimeout(() => controller.abort(), timeout);
+  try {
+    const jsonUrl = `${url}&fmt=json3`;
+    const response = await fetch(jsonUrl, {
+      headers: { "User-Agent": USER_AGENT },
+      signal: controller.signal
+    });
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}`);
+    }
+    const data = await response.json();
+    const events = data.events || [];
+    const segments = [];
+    for (const event of events) {
+      if (!event.segs)
+        continue;
+      const text = event.segs.map((seg) => seg.utf8 || "").join("").trim();
+      if (!text)
+        continue;
+      segments.push({
+        text,
+        start: (event.tStartMs || 0) / 1000,
+        duration: (event.dDurationMs || 0) / 1000
+      });
+    }
+    return segments;
+  } finally {
+    clearTimeout(timeoutId);
+  }
+}
+function selectCaptionTrack(tracks, preferredLanguages, includeAutoGenerated) {
+  if (!tracks.length)
+    return null;
+  const manual = tracks.filter((t) => t.kind !== "asr");
+  const auto = tracks.filter((t) => t.kind === "asr");
+  const searchOrder = includeAutoGenerated ? [...manual, ...auto] : manual;
+  for (const lang of preferredLanguages) {
+    const match = searchOrder.find((t) => t.languageCode.toLowerCase().startsWith(lang.toLowerCase()));
+    if (match)
+      return match;
+  }
+  return searchOrder[0] || null;
+}
+async function fetchTranscript(videoId, options = {}) {
+  const { languages = ["en"], timeout = 30000, includeAutoGenerated = true } = options;
+  const playerResponse = await fetchPlayerResponse(videoId, timeout);
+  const captionTracks = playerResponse.captions?.playerCaptionsTracklistRenderer?.captionTracks;
+  if (!captionTracks?.length) {
+    throw new Error("No captions available for this video");
+  }
+  const selectedTrack = selectCaptionTrack(captionTracks, languages, includeAutoGenerated);
+  if (!selectedTrack) {
+    throw new Error("No suitable caption track found");
+  }
+  const segments = await fetchCaptionTrack(selectedTrack.baseUrl, timeout);
+  if (!segments.length) {
+    throw new Error("Caption track is empty");
+  }
+  const fullText = segments.map((s) => s.text).join(" ");
+  return {
+    videoId,
+    text: fullText,
+    segments,
+    language: selectedTrack.languageCode,
+    isAutoGenerated: selectedTrack.kind === "asr"
+  };
+}
+var USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
+// node_modules/yocto-queue/index.js
+class Node {
+  value;
+  next;
+  constructor(value) {
+    this.value = value;
+  }
+}
+
+class Queue {
+  #head;
+  #tail;
+  #size;
+  constructor() {
+    this.clear();
+  }
+  enqueue(value) {
+    const node = new Node(value);
+    if (this.#head) {
+      this.#tail.next = node;
+      this.#tail = node;
+    } else {
+      this.#head = node;
+      this.#tail = node;
+    }
+    this.#size++;
+  }
+  dequeue() {
+    const current = this.#head;
+    if (!current) {
+      return;
+    }
+    this.#head = this.#head.next;
+    this.#size--;
+    if (!this.#head) {
+      this.#tail = undefined;
+    }
+    return current.value;
+  }
+  peek() {
+    if (!this.#head) {
+      return;
+    }
+    return this.#head.value;
+  }
+  clear() {
+    this.#head = undefined;
+    this.#tail = undefined;
+    this.#size = 0;
+  }
+  get size() {
+    return this.#size;
+  }
+  *[Symbol.iterator]() {
+    let current = this.#head;
+    while (current) {
+      yield current.value;
+      current = current.next;
+    }
+  }
+  *drain() {
+    while (this.#head) {
+      yield this.dequeue();
+    }
+  }
+}
+
+// node_modules/p-limit/index.js
+function validateConcurrency(concurrency) {
+  if (!((Number.isInteger(concurrency) || concurrency === Number.POSITIVE_INFINITY) && concurrency > 0)) {
+    throw new TypeError("Expected `concurrency` to be a number from 1 and up");
+  }
+}
+function pLimit(concurrency) {
+  validateConcurrency(concurrency);
+  const queue = new Queue;
+  let activeCount = 0;
+  const resumeNext = () => {
+    if (activeCount < concurrency && queue.size > 0) {
+      queue.dequeue()();
+      activeCount++;
+    }
+  };
+  const next = () => {
+    activeCount--;
+    resumeNext();
+  };
+  const run = async (function_, resolve, arguments_) => {
+    const result = (async () => function_(...arguments_))();
+    resolve(result);
+    try {
+      await result;
+    } catch {
+    }
+    next();
+  };
+  const enqueue = (function_, resolve, arguments_) => {
+    new Promise((internalResolve) => {
+      queue.enqueue(internalResolve);
+    }).then(run.bind(undefined, function_, resolve, arguments_));
+    (async () => {
+      await Promise.resolve();
+      if (activeCount < concurrency) {
+        resumeNext();
+      }
+    })();
+  };
+  const generator = (function_, ...arguments_) => new Promise((resolve) => {
+    enqueue(function_, resolve, arguments_);
+  });
+  Object.defineProperties(generator, {
+    activeCount: {
+      get: () => activeCount
+    },
+    pendingCount: {
+      get: () => queue.size
+    },
+    clearQueue: {
+      value() {
+        queue.clear();
+      }
+    },
+    concurrency: {
+      get: () => concurrency,
+      set(newConcurrency) {
+        validateConcurrency(newConcurrency);
+        concurrency = newConcurrency;
+        queueMicrotask(() => {
+          while (activeCount < concurrency && queue.size > 0) {
+            resumeNext();
+          }
+        });
+      }
+    }
+  });
+  return generator;
+}
+
+// src/lib/processor.ts
+async function processVideos(videos, options = {}) {
+  const {
+    concurrency = DEFAULT_CONCURRENCY,
+    pauseAfter = DEFAULT_PAUSE_AFTER,
+    pauseDuration = DEFAULT_PAUSE_DURATION,
+    skipIds = new Set,
+    onProgress,
+    ...fetchOptions
+  } = options;
+  const toProcess = videos.filter((v) => !skipIds.has(v.videoId));
+  if (!toProcess.length) {
+    return [];
+  }
+  const limit = pLimit(concurrency);
+  const results = [];
+  let completed = 0;
+  const processOne = async (meta) => {
+    try {
+      const transcript = await fetchTranscript(meta.videoId, fetchOptions);
+      return { meta, transcript };
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "Unknown error";
+      return { meta, transcript: null, error: message };
+    }
+  };
+  const batches = [];
+  for (let i = 0;i < toProcess.length; i += pauseAfter) {
+    batches.push(toProcess.slice(i, i + pauseAfter));
+  }
+  for (let batchIndex = 0;batchIndex < batches.length; batchIndex++) {
+    const batch = batches[batchIndex];
+    const batchPromises = batch.map((meta) => limit(async () => {
+      const result = await processOne(meta);
+      completed++;
+      onProgress?.(completed, toProcess.length, result);
+      return result;
+    }));
+    const batchResults = await Promise.all(batchPromises);
+    results.push(...batchResults);
+    if (batchIndex < batches.length - 1 && pauseDuration > 0) {
+      await new Promise((resolve) => setTimeout(resolve, pauseDuration));
+    }
+  }
+  return results;
+}
+async function* streamVideos(videos, options = {}) {
+  const {
+    concurrency = DEFAULT_CONCURRENCY,
+    pauseAfter = DEFAULT_PAUSE_AFTER,
+    pauseDuration = DEFAULT_PAUSE_DURATION,
+    skipIds = new Set,
+    ...fetchOptions
+  } = options;
+  const toProcess = videos.filter((v) => !skipIds.has(v.videoId));
+  if (!toProcess.length) {
+    return;
+  }
+  const limit = pLimit(concurrency);
+  let processedInBatch = 0;
+  for (const meta of toProcess) {
+    const result = await limit(async () => {
+      try {
+        const transcript = await fetchTranscript(meta.videoId, fetchOptions);
+        return { meta, transcript };
+      } catch (error) {
+        const message = error instanceof Error ? error.message : "Unknown error";
+        return { meta, transcript: null, error: message };
+      }
+    });
+    yield result;
+    processedInBatch++;
+    if (processedInBatch >= pauseAfter) {
+      processedInBatch = 0;
+      if (pauseDuration > 0) {
+        await new Promise((resolve) => setTimeout(resolve, pauseDuration));
+      }
+    }
+  }
+}
+var DEFAULT_CONCURRENCY = 4;
+var DEFAULT_PAUSE_AFTER = 10;
+var DEFAULT_PAUSE_DURATION = 5000;
+// src/loaders/history.ts
+function extractVideoIdFromUrl(url) {
+  try {
+    const parsed = new URL(url);
+    if (parsed.hostname.includes("youtube.com")) {
+      return parsed.searchParams.get("v");
+    }
+    if (parsed.hostname === "youtu.be") {
+      return parsed.pathname.slice(1);
+    }
+  } catch {
+  }
+  return null;
+}
+async function loadWatchHistory(filePath) {
+  const file = Bun.file(filePath);
+  const text = await file.text();
+  const data = JSON.parse(text);
+  const results = [];
+  for (const item of data) {
+    const url = item.titleUrl;
+    if (!url)
+      continue;
+    const videoId = extractVideoIdFromUrl(url);
+    if (!videoId)
+      continue;
+    const channel = item.subtitles?.[0];
+    results.push({
+      videoId,
+      title: item.title,
+      url,
+      channel: channel ? { name: channel.name, url: channel.url } : undefined,
+      watchedAt: item.time,
+      source: "history"
+    });
+  }
+  return results;
+}
+// src/loaders/watchLater.ts
+function parseCSV(text) {
+  const lines = text.split("\n").filter((line) => line.trim());
+  if (lines.length < 2)
+    return [];
+  const headers = parseCSVLine(lines[0]);
+  const rows = [];
+  for (let i = 1;i < lines.length; i++) {
+    const values = parseCSVLine(lines[i]);
+    const row = {};
+    for (let j = 0;j < headers.length; j++) {
+      row[headers[j]] = values[j] || "";
+    }
+    rows.push(row);
+  }
+  return rows;
+}
+function parseCSVLine(line) {
+  const result = [];
+  let current = "";
+  let inQuotes = false;
+  for (let i = 0;i < line.length; i++) {
+    const char = line[i];
+    const nextChar = line[i + 1];
+    if (char === '"') {
+      if (inQuotes && nextChar === '"') {
+        current += '"';
+        i++;
+      } else {
+        inQuotes = !inQuotes;
+      }
+    } else if (char === "," && !inQuotes) {
+      result.push(current.trim());
+      current = "";
+    } else {
+      current += char;
+    }
+  }
+  result.push(current.trim());
+  return result;
+}
+async function loadWatchLater(filePath) {
+  const file = Bun.file(filePath);
+  const text = await file.text();
+  const rows = parseCSV(text);
+  const results = [];
+  for (const row of rows) {
+    const videoId = row["Video ID"] || row.video_id || row["Video Id"];
+    const addedAt = row["Playlist Video Creation Timestamp"] || row.added_at || row["Added At"];
+    if (!videoId)
+      continue;
+    results.push({
+      videoId,
+      watchedAt: addedAt,
+      source: "watch_later"
+    });
+  }
+  return results;
+}
+
+// src/loaders/index.ts
+function fromVideoIds(inputs) {
+  const results = [];
+  for (const input of inputs) {
+    const videoId = extractVideoId(input);
+    if (!videoId)
+      continue;
+    results.push({
+      videoId,
+      url: input.startsWith("http") ? input : `https://www.youtube.com/watch?v=${videoId}`,
+      source: "manual"
+    });
+  }
+  return results;
+}
+function mergeVideoSources(...sources) {
+  const seen = new Map;
+  for (const source of sources) {
+    for (const meta of source) {
+      if (!seen.has(meta.videoId)) {
+        seen.set(meta.videoId, meta);
+      }
+    }
+  }
+  return Array.from(seen.values());
+}
+async function loadProcessedIds(jsonlPath) {
+  const ids = new Set;
+  try {
+    const file = Bun.file(jsonlPath);
+    if (!await file.exists()) {
+      return ids;
+    }
+    const text = await file.text();
+    const lines = text.split("\n").filter((l) => l.trim());
+    for (const line of lines) {
+      try {
+        const record = JSON.parse(line);
+        if (record.meta?.videoId) {
+          ids.add(record.meta.videoId);
+        } else if (record.videoId) {
+          ids.add(record.videoId);
+        }
+      } catch {
+      }
+    }
+  } catch {
+  }
+  return ids;
+}
+// src/outputs/index.ts
+async function writeJsonl(results, options) {
+  const lines = results.map((r) => JSON.stringify(r));
+  const content = `${lines.join("\n")}\n`;
+  if (options.append) {
+    await Bun.write(options.path, content, { mode: "a" });
+  } else {
+    await Bun.write(options.path, content);
+  }
+}
+async function appendJsonl(result, path) {
+  const file = Bun.file(path);
+  const existing = await file.exists() ? await file.text() : "";
+  const newContent = `${existing + JSON.stringify(result)}\n`;
+  await Bun.write(path, newContent);
+}
+async function writeCsv(results, options) {
+  const headers = [
+    "video_id",
+    "source",
+    "title",
+    "url",
+    "channel_name",
+    "watched_at",
+    "language",
+    "is_auto_generated",
+    "transcript",
+    "error"
+  ];
+  const rows = results.map((r) => [
+    r.meta.videoId,
+    r.meta.source,
+    r.meta.title || "",
+    r.meta.url || "",
+    r.meta.channel?.name || "",
+    r.meta.watchedAt || "",
+    r.transcript?.language || "",
+    r.transcript?.isAutoGenerated ? "true" : "false",
+    r.transcript?.text || "",
+    r.error || ""
+  ]);
+  const csvContent = [
+    headers.join(","),
+    ...rows.map((row) => row.map((cell) => `"${String(cell).replace(/"/g, '""')}"`).join(","))
+  ].join("\n");
+  if (options.append) {
+    const file = Bun.file(options.path);
+    const existing = await file.exists() ? await file.text() : "";
+    const content = existing ? `${rows.map((row) => row.map((cell) => `"${String(cell).replace(/"/g, '""')}"`).join(",")).join("\n")}\n` : `${csvContent}\n`;
+    await Bun.write(options.path, existing + content);
+  } else {
+    await Bun.write(options.path, `${csvContent}\n`);
+  }
+}
+function formatSrt(transcript) {
+  const lines = [];
+  transcript.segments.forEach((segment, index) => {
+    const startTime = formatSrtTime(segment.start);
+    const endTime = formatSrtTime(segment.start + segment.duration);
+    lines.push(String(index + 1));
+    lines.push(`${startTime} --> ${endTime}`);
+    lines.push(segment.text);
+    lines.push("");
+  });
+  return lines.join("\n");
+}
+function formatVtt(transcript) {
+  const lines = ["WEBVTT", ""];
+  for (const segment of transcript.segments) {
+    const startTime = formatVttTime(segment.start);
+    const endTime = formatVttTime(segment.start + segment.duration);
+    lines.push(`${startTime} --> ${endTime}`);
+    lines.push(segment.text);
+    lines.push("");
+  }
+  return lines.join("\n");
+}
+function formatText(transcript, includeTimestamps = false) {
+  if (!includeTimestamps) {
+    return transcript.text;
+  }
+  return transcript.segments.map((s) => `[${formatTimestamp(s.start)}] ${s.text}`).join("\n");
+}
+function formatSrtTime(seconds) {
+  const hours = Math.floor(seconds / 3600);
+  const minutes = Math.floor(seconds % 3600 / 60);
+  const secs = Math.floor(seconds % 60);
+  const ms = Math.floor(seconds % 1 * 1000);
+  return `${pad(hours, 2)}:${pad(minutes, 2)}:${pad(secs, 2)},${pad(ms, 3)}`;
+}
+function formatVttTime(seconds) {
+  const hours = Math.floor(seconds / 3600);
+  const minutes = Math.floor(seconds % 3600 / 60);
+  const secs = Math.floor(seconds % 60);
+  const ms = Math.floor(seconds % 1 * 1000);
+  return `${pad(hours, 2)}:${pad(minutes, 2)}:${pad(secs, 2)}.${pad(ms, 3)}`;
+}
+function formatTimestamp(seconds) {
+  const minutes = Math.floor(seconds / 60);
+  const secs = Math.floor(seconds % 60);
+  return `${minutes}:${pad(secs, 2)}`;
+}
+function pad(num, size) {
+  return String(num).padStart(size, "0");
+}
+export {
+  writeJsonl,
+  writeCsv,
+  streamVideos,
+  processVideos,
+  mergeVideoSources,
+  loadWatchLater,
+  loadWatchHistory,
+  loadProcessedIds,
+  fromVideoIds,
+  formatVtt,
+  formatText,
+  formatSrt,
+  fetchTranscript,
+  extractVideoId,
+  appendJsonl
+};