@stephane-klein/youtube-to-markdown 0.1.0

package/src/markdown-stats.js

@@ -0,0 +1,200 @@
+import { readFile, readdir } from "node:fs/promises";
+import { join, relative } from "node:path";
+import { parse } from "yaml";
+import { DEFAULT_DIR } from "./vtt.js";
+
+const FRONTMATTER = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/;
+const LANGUAGE = /\.([^./]+)\.md$/;
+
+async function markdownFiles(dir, base = dir) {
+  const entries = await readdir(dir, { withFileTypes: true });
+  const files = [];
+  for (const entry of entries) {
+    const path = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...(await markdownFiles(path, base)));
+    } else if (entry.isFile() && entry.name.endsWith(".md")) {
+      files.push(relative(base, path));
+    }
+  }
+  return files;
+}
+
+const numberOrNull = (value) =>
+  typeof value === "number" && Number.isFinite(value) ? value : null;
+
+function safeParse(text) {
+  try {
+    return parse(text);
+  } catch {
+    return null;
+  }
+}
+
+function bump(map, key, { input, cached, output, cost }) {
+  const entry = map.get(key) ??
+    { files: 0, input: 0, cached: 0, output: 0, cost: 0, unknownCost: 0 };
+  entry.files++;
+  entry.input += input ?? 0;
+  entry.cached += cached ?? 0;
+  entry.output += output ?? 0;
+  if (cost === null) entry.unknownCost++;
+  else entry.cost += cost;
+  map.set(key, entry);
+}
+
+const int = new Intl.NumberFormat("en-US");
+const seconds = (value) => `${value.toFixed(1)}s`;
+const usd = (value) => `$${value.toFixed(6)}`;
+const row = (label, value) => `  ${label.padEnd(20)} ${value}`;
+
+export async function runMarkdownStats({ dir = DEFAULT_DIR } = {}) {
+  const stats = {
+    files: 0,
+    withFrontmatter: 0,
+    withoutFrontmatter: 0,
+    inputTokens: 0,
+    cachedInputTokens: 0,
+    outputTokens: 0,
+    cost: 0,
+    unknownCost: 0,
+    durationSum: 0,
+    durationCount: 0,
+    durationMin: null,
+    durationMax: null,
+    firstGeneratedAt: null,
+    lastGeneratedAt: null,
+    models: new Map(),
+    languages: new Map(),
+    folders: new Map(),
+  };
+
+  const names = (await markdownFiles(dir)).sort();
+
+  for (const name of names) {
+    stats.files++;
+
+    const match = FRONTMATTER.exec(await readFile(join(dir, name), "utf8"));
+    const parsed = match ? safeParse(match[1]) : null;
+    const data =
+      parsed && typeof parsed === "object" && "generated_at" in parsed
+        ? parsed
+        : null;
+
+    if (!data) {
+      stats.withoutFrontmatter++;
+      continue;
+    }
+    stats.withFrontmatter++;
+
+    const llm = data.llm ?? {};
+    const input = numberOrNull(llm.input_tokens);
+    const cached = numberOrNull(llm.cached_input_tokens);
+    const output = numberOrNull(llm.output_tokens);
+    const cost = numberOrNull(llm.estimated_cost_usd);
+    const duration = numberOrNull(llm.duration_seconds);
+
+    stats.inputTokens += input ?? 0;
+    stats.cachedInputTokens += cached ?? 0;
+    stats.outputTokens += output ?? 0;
+    if (cost === null) stats.unknownCost++;
+    else stats.cost += cost;
+
+    if (duration !== null) {
+      stats.durationSum += duration;
+      stats.durationCount++;
+      stats.durationMin = stats.durationMin === null
+        ? duration
+        : Math.min(stats.durationMin, duration);
+      stats.durationMax = stats.durationMax === null
+        ? duration
+        : Math.max(stats.durationMax, duration);
+    }
+
+    if (typeof data.generated_at === "string") {
+      const at = data.generated_at;
+      if (!stats.firstGeneratedAt || at < stats.firstGeneratedAt) {
+        stats.firstGeneratedAt = at;
+      }
+      if (!stats.lastGeneratedAt || at > stats.lastGeneratedAt) {
+        stats.lastGeneratedAt = at;
+      }
+    }
+
+    bump(stats.models, llm.model ?? "unknown", { input, cached, output, cost });
+    bump(stats.languages, LANGUAGE.exec(name)?.[1] ?? "unknown", {
+      input,
+      cached,
+      output,
+      cost,
+    });
+    const segments = name.split(/[\\/]/);
+    bump(stats.folders, segments.length > 1 ? segments[0] : "(root)", {
+      input,
+      cached,
+      output,
+      cost,
+    });
+  }
+
+  const lines = [
+    `${dir}/ — ${int.format(stats.files)} file(s)`,
+    row("with frontmatter", int.format(stats.withFrontmatter)),
+    row("without frontmatter", int.format(stats.withoutFrontmatter)),
+    "",
+    "tokens",
+    row("input", int.format(stats.inputTokens)),
+    row("cached input", int.format(stats.cachedInputTokens)),
+    row("output", int.format(stats.outputTokens)),
+    row("total", int.format(stats.inputTokens + stats.outputTokens)),
+    "",
+    "cost",
+    row("estimated", usd(stats.cost)),
+    row("unknown prices", `${int.format(stats.unknownCost)} file(s)`),
+    "",
+    "processing time",
+    row("total", seconds(stats.durationSum)),
+    row(
+      "mean",
+      seconds(
+        stats.durationCount ? stats.durationSum / stats.durationCount : 0,
+      ),
+    ),
+    row(
+      "min / max",
+      stats.durationCount
+        ? `${seconds(stats.durationMin)} / ${seconds(stats.durationMax)}`
+        : "—",
+    ),
+    "",
+    "generated_at",
+    row("first", stats.firstGeneratedAt ?? "—"),
+    row("last", stats.lastGeneratedAt ?? "—"),
+  ];
+
+  const breakdown = (title, map) => {
+    lines.push("", title);
+    if (map.size === 0) {
+      lines.push("  (none)");
+      return;
+    }
+    for (const key of [...map.keys()].sort()) {
+      const entry = map.get(key);
+      const unknown = entry.unknownCost > 0
+        ? `  (+${int.format(entry.unknownCost)} unknown)`
+        : "";
+      lines.push(
+        `  ${key.padEnd(16)} ${int.format(entry.files).padStart(4)} file(s)  ` +
+          `${int.format(entry.input).padStart(9)} in / ` +
+          `${int.format(entry.output).padStart(9)} out  ` +
+          `${usd(entry.cost)}${unknown}`,
+      );
+    }
+  };
+
+  breakdown("models", stats.models);
+  breakdown("languages", stats.languages);
+  breakdown("folders", stats.folders);
+
+  console.log(lines.join("\n"));
+}

package/src/organize-files.js

@@ -0,0 +1,137 @@
+import { mkdir, readdir, rename } from "node:fs/promises";
+import { basename, dirname, join, relative } from "node:path";
+import {
+  DEFAULT_DIR,
+  ORPHANS_DIR,
+  assertFolderSlug,
+  exists,
+  primaryTitle,
+  slugify,
+} from "./vtt.js";
+
+const MARKER = /^(.*)\.([^.]+)\.vtt\.missing$/;
+const VTT = /^(.*)\.([^.]+)\.vtt$/;
+const MD = /^(.*)\.([^.]+)\.md$/;
+
+function prefixOf(name) {
+  for (const pattern of [MARKER, VTT, MD]) {
+    const match = pattern.exec(name);
+    if (match) return match[1];
+  }
+  return null;
+}
+
+async function walk(dir) {
+  const entries = await readdir(dir, { withFileTypes: true });
+  const files = [];
+  for (const entry of entries) {
+    const path = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (entry.name === ORPHANS_DIR) continue;
+      files.push(...(await walk(path)));
+    } else if (entry.isFile()) {
+      files.push(path);
+    }
+  }
+  return files;
+}
+
+function foldersByPrefix(feed) {
+  const targets = new Map();
+  for (const channel of feed) {
+    const folder = assertFolderSlug(channel.folder_slug) ?? "";
+    for (const video of channel.videos ?? []) {
+      const slug = slugify(primaryTitle(video));
+      if (!video.date || !slug) continue;
+      const prefix = `${video.date}_${slug}`;
+      const folders = targets.get(prefix) ?? new Set();
+      folders.add(folder);
+      targets.set(prefix, folders);
+    }
+  }
+  return targets;
+}
+
+export async function runReorganize({
+  feed = [],
+  dir = DEFAULT_DIR,
+  dryRun = false,
+} = {}) {
+  const targets = foldersByPrefix(feed);
+  const files = (await exists(dir)) ? (await walk(dir)).sort() : [];
+  const claimed = new Set();
+
+  const moves = [];
+  const conflicts = [];
+  let inPlace = 0;
+
+  for (const file of files) {
+    const name = basename(file);
+    const folders = targets.get(prefixOf(name));
+
+    let target;
+    let orphan = false;
+    if (!folders || folders.size === 0) {
+      target = join(dir, ORPHANS_DIR, relative(dir, file));
+      orphan = true;
+    } else if (folders.size > 1) {
+      conflicts.push({
+        file,
+        reason: `ambiguous folder_slug (${[...folders].sort().join(", ")})`,
+      });
+      continue;
+    } else {
+      const folder = [...folders][0];
+      target = folder ? join(dir, folder, name) : join(dir, name);
+    }
+
+    if (target === file) {
+      inPlace++;
+      continue;
+    }
+    if (claimed.has(target) || (await exists(target))) {
+      conflicts.push({
+        file,
+        reason: `target already exists: ${relative(dir, target)}`,
+      });
+      continue;
+    }
+
+    claimed.add(target);
+    moves.push({ from: file, to: target, orphan });
+  }
+
+  const width = String(moves.length).length;
+  let moved = 0;
+  let orphans = 0;
+  for (const [index, move] of moves.entries()) {
+    const prefix = `[${String(index + 1).padStart(width)}/${moves.length}]`;
+    const status = move.orphan ? "orphan" : "moved";
+    if (move.orphan) orphans++;
+    else moved++;
+    if (dryRun) {
+      console.error(
+        `${prefix} ${status.padEnd(9)} ${relative(dir, move.to)} (dry run)`,
+      );
+      continue;
+    }
+    await mkdir(dirname(move.to), { recursive: true });
+    await rename(move.from, move.to);
+    console.error(`${prefix} ${status.padEnd(9)} ${relative(dir, move.to)}`);
+  }
+
+  for (const { file, reason } of conflicts) {
+    console.error(`conflict  ${relative(dir, file)} (${reason})`);
+  }
+
+  if (dryRun) {
+    console.error(`dry run: no file was moved`);
+    console.error(
+      `summary: ${moved} to move, ${orphans} to orphan, ${inPlace} already in place, ${conflicts.length} conflict`,
+    );
+    return;
+  }
+  console.error(
+    `summary: ${moved} moved, ${orphans} orphan, ${inPlace} already in place, ${conflicts.length} conflict`,
+  );
+}

package/src/vtt.js

@@ -0,0 +1,137 @@
+import {
+  copyFile,
+  mkdtemp,
+  readdir,
+  rm,
+  stat,
+  writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { isAbsolute, join, resolve } from "node:path";
+import { runYtDlp } from "./ytdlp.js";
+
+export const DEFAULT_DIR = resolve("contents");
+export const ORPHANS_DIR = "_orphans";
+
+export function assertFolderSlug(folderSlug) {
+  if (folderSlug === undefined || folderSlug === null || folderSlug === "") {
+    return null;
+  }
+  if (typeof folderSlug !== "string") {
+    throw new Error("folder_slug must be a string");
+  }
+
+  const segments = folderSlug.split(/[\\/]+/).filter(Boolean);
+  if (isAbsolute(folderSlug) || segments.includes("..")) {
+    throw new Error(
+      `folder_slug must be a relative path inside contents_path: ${folderSlug}`,
+    );
+  }
+  if (segments.length === 0) {
+    throw new Error("folder_slug must not be empty");
+  }
+  if (segments[0] === ORPHANS_DIR) {
+    throw new Error(`folder_slug must not be "${ORPHANS_DIR}" (reserved)`);
+  }
+
+  return segments.join("/");
+}
+
+export const slugify = (text) =>
+  text
+    .normalize("NFD")
+    .replace(/[\u0300-\u036f]/g, "")
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+
+export const primaryTitle = (video) => {
+  const title = video.title;
+  if (typeof title === "string") return title;
+  return title?.fr ?? title?.en ?? "";
+};
+
+export const titleForLang = (title, lang) => {
+  if (typeof title === "string") return title;
+  return title?.[lang] ?? title?.fr ?? title?.en ?? "";
+};
+
+export async function exists(path) {
+  try {
+    await stat(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function findVtt(dir) {
+  const entries = await readdir(dir, { withFileTypes: true });
+  const entry = entries.find(
+    (item) => item.isFile && item.name.endsWith(".vtt"),
+  );
+  return entry ? join(dir, entry.name) : null;
+}
+
+export async function writeMarker(path, url) {
+  await writeFile(path, `${url}\n${new Date().toISOString()}\n`);
+}
+
+export async function downloadVtt(url, lang, dest) {
+  const tmp = await mkdtemp(join(tmpdir(), "yt-to-md-"));
+  let failed = false;
+  try {
+    for (const mode of ["--write-subs", "--write-auto-subs"]) {
+      const { success } = await runYtDlp(
+        [
+          "--skip-download",
+          "--sleep-requests",
+          "2",
+          mode,
+          "--sub-langs",
+          `^${lang}$`,
+          "--sub-format",
+          "vtt",
+          "-o",
+          join(tmp, "sub.%(ext)s"),
+          url,
+        ],
+        { allowFailure: true },
+      );
+      if (!success) failed = true;
+      const vtt = await findVtt(tmp);
+      if (vtt) {
+        await copyFile(vtt, dest);
+        return "downloaded";
+      }
+    }
+    return failed ? "error" : "missing";
+  } finally {
+    await rm(tmp, { recursive: true, force: true });
+  }
+}
+
+export function videoJobs(video, dir = DEFAULT_DIR, folderSlug = null) {
+  const folder = assertFolderSlug(folderSlug);
+  const langs = Object.entries(video.download_vtt ?? {})
+    .filter(([, on]) => on)
+    .map(([lang]) => lang);
+  if (langs.length === 0 || !video.url) return [];
+
+  const slug = slugify(primaryTitle(video));
+  if (!slug) return [];
+
+  return langs.map((lang) => {
+    const name = `${video.date}_${slug}.${lang}`;
+    const base = folder ? join(dir, folder, name) : join(dir, name);
+    return {
+      url: video.url,
+      lang,
+      title: primaryTitle(video),
+      titles: video.title,
+      base,
+      vtt: `${base}.vtt`,
+      marker: `${base}.vtt.missing`,
+    };
+  });
+}

package/src/ytdlp.js

@@ -0,0 +1,25 @@
+import { spawn } from "node:child_process";
+
+const JS_RUNTIME = ["--js-runtimes", "node"];
+
+export function runYtDlp(args, { stderr = "ignore", allowFailure = false } = {}) {
+  return new Promise((resolve, reject) => {
+    const child = spawn(
+      "yt-dlp",
+      ["--no-warnings", ...JS_RUNTIME, ...args],
+      { stdio: ["ignore", "pipe", stderr] },
+    );
+
+    const chunks = [];
+    child.stdout.on("data", (chunk) => chunks.push(chunk));
+    child.on("error", reject);
+    child.on("close", (code) => {
+      const stdout = Buffer.concat(chunks).toString("utf8");
+      if (code === 0 || allowFailure) {
+        resolve({ success: code === 0, code, stdout });
+      } else {
+        reject(new Error(`yt-dlp a échoué (code ${code}) : ${args.join(" ")}`));
+      }
+    });
+  });
+}