@stephane-klein/youtube-to-markdown 0.3.0 → 0.4.0

package/README.md

@@ -147,8 +147,8 @@ variables, then the settings written in the YAML file, then the built-in
 defaults. The API key stays a secret: `--api-key` or `OPENAI_API_KEY`. The
 settings are `x_opencode_session`, `model_id`, `openaiapi_endpoint`,
 `concurrency` and `ytdlp_concurrency` (plus the optional `api_key`,
-`retry_delays` and `max_output_tokens`). I only set the source URLs, and the
-`extract-video-metadata` command fills the rest:
+`retry_delays`, `max_output_tokens` and `chunk_target_tokens`). I only set the
+source URLs, and the `extract-video-metadata` command fills the rest:
 
 `x_opencode_session` is the base of the `X-OpenCode-Session` header; it is
 optional and specific to OpenCode. A stable per-video suffix is appended to it,
@@ -302,6 +302,25 @@ from the server is honored when longer than the schedule. Each retry is shown in
 the phase output; after the last attempt the file is reported as an error and is
 retried on the next run.
 
+### Output budget and long transcripts
+
+The model must rewrite the whole transcript, so the command asks it for roughly
+twice the input size (`maxOutputTokens`). That budget is capped per call: the
+command reads the model's advertised output limit from the pricing catalogs and
+uses it as the ceiling; if the catalog has no entry it falls back to 32768
+tokens. `--max-output-tokens` (or `YT_TO_MD_MAX_OUTPUT_TOKENS`) overrides the
+ceiling by hand.
+
+When the projected output would exceed the ceiling — a long video on a model
+with a small output limit — the transcript is split into chunks that each fit
+the budget, generated independently, then stitched together: immediately
+repeated headings are dropped and each junction is smoothed by a small extra
+LLM call that rewrites the beginning of the next section so it flows from the
+previous one. `--chunk-target-tokens` (or `YT_TO_MD_CHUNK_TARGET_TOKENS`,
+default 8000) sets the target input size of each chunk. The generation of a
+chunk is guaranteed to stop with `stop`; otherwise the file is reported as an
+error and nothing is written. The frontmatter records how many chunks were used.
+
 ### Generated frontmatter
 
 Each generated file starts with a YAML frontmatter describing the run:
@@ -313,6 +332,7 @@ video_title: Pourquoi π est-il si fou ? Relativité 1
 generated_at: 2026-09-19T10:12:33.456Z
 llm:
   model: mimo-v2.5
+  chunks: 1
   duration_seconds: 87.1
   input_tokens: 4428
   cached_input_tokens: 0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stephane-klein/youtube-to-markdown",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Collect YouTube channel videos into a YAML config with yt-dlp, download transcripts and turn them into Markdown prose with an LLM.",
   "type": "module",
   "license": "MIT",
@@ -22,7 +22,8 @@
     "access": "public"
   },
   "scripts": {
-    "youtube-to-markdown": "node src/cli.js"
+    "youtube-to-markdown": "node src/cli.js",
+    "test": "node --test"
   },
   "dependencies": {
     "@ai-sdk/openai-compatible": "~3.0.51",

package/src/chunk.js

@@ -0,0 +1,68 @@
+export const CHARS_PER_TOKEN = 3.5;
+
+export function estimateTokens(text) {
+  return Math.ceil(`${text ?? ""}`.length / CHARS_PER_TOKEN);
+}
+
+export function chunkTranscript(transcript, { targetTokens }) {
+  const target = Math.max(1, Math.floor(Number(targetTokens) || 1));
+  const lines = `${transcript ?? ""}`.split("\n");
+  const chunks = [];
+  let current = [];
+  let currentTokens = 0;
+
+  for (const line of lines) {
+    const lineTokens = estimateTokens(line) + 1;
+    if (current.length > 0 && currentTokens + lineTokens > target) {
+      chunks.push(current.join("\n"));
+      current = [];
+      currentTokens = 0;
+    }
+    current.push(line);
+    currentTokens += lineTokens;
+  }
+
+  if (current.length > 0) chunks.push(current.join("\n"));
+  return chunks.length > 0 ? chunks : [""];
+}
+
+export function splitBlocks(markdown) {
+  return `${markdown ?? ""}`.split(/\n{2,}/);
+}
+
+export function firstBlock(markdown) {
+  return splitBlocks(markdown)[0]?.trim() ?? "";
+}
+
+export function lastBlock(markdown) {
+  const blocks = splitBlocks(markdown).filter((block) => block.trim());
+  return blocks[blocks.length - 1]?.trim() ?? "";
+}
+
+export function replaceFirstBlock(markdown, replacement) {
+  const blocks = splitBlocks(markdown);
+  if (blocks.length === 0) return replacement;
+  blocks[0] = replacement;
+  return blocks.join("\n\n");
+}
+
+export function dedupeAdjacentHeadings(markdown) {
+  const lines = `${markdown ?? ""}`.split("\n");
+  const out = [];
+  let lastHeading = null;
+
+  for (const line of lines) {
+    const heading = line.match(/^\s*#{1,6}\s+(.*\S)\s*$/);
+    if (heading) {
+      const key = heading[1].trim().toLowerCase();
+      if (key === lastHeading) continue;
+      lastHeading = key;
+      out.push(line);
+      continue;
+    }
+    if (line.trim() !== "") lastHeading = null;
+    out.push(line);
+  }
+
+  return out.join("\n").replace(/\n{3,}/g, "\n\n");
+}

package/src/cli.js

@@ -23,6 +23,7 @@ const SETTING_KEYS = [
   "ytdlpConcurrency",
   "retryDelays",
   "maxOutputTokens",
+  "chunkTargetTokens",
 ];
 
 const handle = (run) => async (argv) => {
@@ -116,6 +117,10 @@ const cli = yargs(hideBin(process.argv))
           describe: "Override the computed output token budget",
           type: "number",
         })
+        .option("chunk-target-tokens", {
+          describe: "Target input tokens per chunk when a transcript is split (default: 8000)",
+          type: "number",
+        })
         .option("force", {
           describe: "Regenerate every Markdown file",
           type: "boolean",

package/src/config.js

@@ -14,6 +14,7 @@ const YAML_KEYS = {
   ytdlpConcurrency: "ytdlp_concurrency",
   retryDelays: "retry_delays",
   maxOutputTokens: "max_output_tokens",
+  chunkTargetTokens: "chunk_target_tokens",
 };
 
 export function settingsFromConfig(config) {
@@ -34,6 +35,7 @@ const ENV = {
   ytdlpConcurrency: "YT_TO_MD_YTDLP_CONCURRENCY",
   retryDelays: "YT_TO_MD_RETRY_DELAYS",
   maxOutputTokens: "YT_TO_MD_MAX_OUTPUT_TOKENS",
+  chunkTargetTokens: "YT_TO_MD_CHUNK_TARGET_TOKENS",
 };
 
 export function loadEnv(env = process.env) {

package/src/generate-markdown.js

@@ -5,6 +5,14 @@ import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
 import { APICallError, generateText } from "ai";
 import { Listr, ListrLogger, ProcessOutput } from "listr2";
 import { stringify } from "yaml";
+import {
+  chunkTranscript,
+  dedupeAdjacentHeadings,
+  estimateTokens,
+  firstBlock,
+  lastBlock,
+  replaceFirstBlock,
+} from "./chunk.js";
 import { loadPricing } from "./pricing.js";
 import {
   DEFAULT_DIR,
@@ -26,6 +34,15 @@ const SYSTEM = [
   "- Output only the Markdown body.",
 ].join("\n");
 
+const SMOOTH_SYSTEM = [
+  "You improve the transition between two consecutive Markdown sections that were written independently.",
+  "- You are given the end of the previous section and the beginning of the next one.",
+  "- Rewrite ONLY the beginning of the next section so it flows naturally from the previous one.",
+  "- Keep the same language, the same facts and roughly the same length.",
+  "- Keep the Markdown formatting of the beginning.",
+  "- Output only the rewritten beginning, without any preamble or code fences.",
+].join("\n");
+
 const WIDTH = 80;
 
 function cachedInputTokens(usage) {
@@ -50,20 +67,26 @@ function estimateCost(price, usage) {
 
 const OUTPUT_FACTOR = 2;
 const OUTPUT_FLOOR = 2048;
-const OUTPUT_CEILING = 32768;
+const DEFAULT_CEILING = 32768;
+const DEFAULT_CHUNK_TARGET_TOKENS = 8000;
 
-function outputBudget(transcript, override) {
-  if (override !== undefined && override !== null && override !== "") {
-    return Number(override);
-  }
+function isSet(value) {
+  return value !== undefined && value !== null && value !== "";
+}
 
-  const inputTokens = Math.ceil(transcript.length / 3.5);
+function plannedBudget(transcript, ceiling) {
+  const inputTokens = estimateTokens(transcript);
   return Math.min(
-    OUTPUT_CEILING,
+    ceiling,
     Math.max(OUTPUT_FLOOR, Math.ceil(inputTokens * OUTPUT_FACTOR)),
   );
 }
 
+function budgetForPart(text, { maxOutputTokens, ceiling }) {
+  if (isSet(maxOutputTokens)) return Number(maxOutputTokens);
+  return plannedBudget(text, ceiling);
+}
+
 function metricsLine(modelId, usage, elapsed, cost) {
   return [
     modelId,
@@ -77,13 +100,14 @@ function metricsLine(modelId, usage, elapsed, cost) {
     .join("  ");
 }
 
-function frontmatter(job, { modelId, usage, elapsed, cost, finishReason, price }) {
+function frontmatter(job, { modelId, usage, elapsed, cost, finishReason, price, chunks }) {
   const data = {
     source_url: job.url,
     video_title: titleForLang(job.titles, job.lang),
     generated_at: new Date().toISOString(),
     llm: {
       model: modelId,
+      chunks: chunks ?? 1,
       duration_seconds: Number(elapsed.toFixed(1)),
       input_tokens: usage?.inputTokens ?? null,
       cached_input_tokens: usage ? cachedInputTokens(usage) : null,
@@ -242,36 +266,40 @@ function videoSession(base, job) {
   return `${base || "youtube-to-markdown"}-${id}`;
 }
 
-async function toMarkdown(
-  { model, retryDelays, maxOutputTokens, session },
-  title,
-  transcript,
-  onRetry,
+function cleanBody(text) {
+  return text
+    .trim()
+    .replace(/^```[a-z]*\n?/, "")
+    .replace(/\n?```$/, "")
+    .trim();
+}
+
+async function generateOnce(
+  { model, retryDelays, session, limiter },
+  { system, prompt, maxOutputTokens, onRetry },
 ) {
   const totalAttempts = retryDelays.length + 1;
 
   for (let attempt = 1;; attempt++) {
     const started = performance.now();
     try {
-      const { text, usage, finishReason } = await generateText({
-        model,
-        system: SYSTEM,
-        prompt: `Title: ${title}\n\nTranscript:\n${transcript}`,
-        temperature: 0.2,
-        reasoning: "low",
-        maxOutputTokens: outputBudget(transcript, maxOutputTokens),
-        maxRetries: 0,
-        headers: { "X-OpenCode-Session": session },
-      });
+      const call = () =>
+        generateText({
+          model,
+          system,
+          prompt,
+          temperature: 0.2,
+          reasoning: "low",
+          maxOutputTokens,
+          maxRetries: 0,
+          headers: { "X-OpenCode-Session": session },
+        });
+      const { text, usage, finishReason } = await (
+        limiter ? limiter(call) : call()
+      );
       const elapsed = (performance.now() - started) / 1000;
 
-      const body = text
-        .trim()
-        .replace(/^```[a-z]*\n?/, "")
-        .replace(/\n?```$/, "")
-        .trim();
-
-      return { body, usage, finishReason, elapsed };
+      return { text, usage, finishReason, elapsed };
     } catch (error) {
       if (!isRetryable(error) || attempt > retryDelays.length) {
         throw new Error(describeError(error, attempt), { cause: error });
@@ -283,6 +311,182 @@ async function toMarkdown(
   }
 }
 
+function chunkPrompt(title, transcript, index, total) {
+  if (total <= 1) return `Title: ${title}\n\nTranscript:\n${transcript}`;
+  return [
+    `Title: ${title}`,
+    "",
+    `This is part ${index + 1} of ${total} of the transcript.`,
+    "Output only the Markdown body for this part, without the main title.",
+    "",
+    `Transcript (part ${index + 1}/${total}):`,
+    transcript,
+  ].join("\n");
+}
+
+function smoothPrompt(previousTail, nextHead) {
+  return [
+    "End of the previous section:",
+    "",
+    previousTail,
+    "",
+    "Beginning of the next section:",
+    "",
+    nextHead,
+  ].join("\n");
+}
+
+function sumUsage(usages) {
+  const present = usages.filter(Boolean);
+  if (present.length === 0) return null;
+
+  const total = {
+    inputTokens: 0,
+    outputTokens: 0,
+    totalTokens: 0,
+    inputTokenDetails: { noCacheTokens: 0, cacheReadTokens: 0 },
+    outputTokenDetails: { textTokens: 0, reasoningTokens: 0 },
+  };
+
+  for (const usage of present) {
+    total.inputTokens += usage.inputTokens ?? 0;
+    total.outputTokens += usage.outputTokens ?? 0;
+    total.totalTokens += usage.totalTokens ?? 0;
+    total.inputTokenDetails.noCacheTokens +=
+      usage.inputTokenDetails?.noCacheTokens ?? 0;
+    total.inputTokenDetails.cacheReadTokens +=
+      usage.inputTokenDetails?.cacheReadTokens ?? 0;
+    total.outputTokenDetails.textTokens +=
+      usage.outputTokenDetails?.textTokens ?? 0;
+    total.outputTokenDetails.reasoningTokens +=
+      usage.outputTokenDetails?.reasoningTokens ?? 0;
+  }
+
+  return total;
+}
+
+function createLimiter(limit) {
+  const max = Math.max(1, Number(limit) || 1);
+  let active = 0;
+  const queue = [];
+
+  const pump = () => {
+    while (active < max && queue.length > 0) {
+      active++;
+      const { task, resolve, reject } = queue.shift();
+      Promise.resolve()
+        .then(task)
+        .then(resolve, reject)
+        .finally(() => {
+          active--;
+          pump();
+        });
+    }
+  };
+
+  return (task) =>
+    new Promise((resolve, reject) => {
+      queue.push({ task, resolve, reject });
+      pump();
+    });
+}
+
+async function toMarkdown(
+  {
+    model,
+    retryDelays,
+    maxOutputTokens,
+    ceiling,
+    chunkTargetTokens,
+    session,
+    limiter,
+  },
+  title,
+  transcript,
+  onRetry,
+  onProgress,
+) {
+  const budgetCeiling = isSet(maxOutputTokens)
+    ? Number(maxOutputTokens)
+    : ceiling;
+  const desired = Math.max(
+    OUTPUT_FLOOR,
+    Math.ceil(estimateTokens(transcript) * OUTPUT_FACTOR),
+  );
+  const parts =
+    desired > budgetCeiling
+      ? chunkTranscript(transcript, { targetTokens: chunkTargetTokens })
+      : [transcript];
+
+  const total = parts.length;
+  const bodies = new Array(total);
+  const usages = [];
+  let elapsed = 0;
+  let finishReason = "stop";
+
+  await Promise.all(
+    parts.map(async (part, index) => {
+      onProgress?.(index + 1, total);
+      const result = await generateOnce(
+        { model, retryDelays, session, limiter },
+        {
+          system: SYSTEM,
+          prompt: chunkPrompt(title, part, index, total),
+          maxOutputTokens: budgetForPart(part, { maxOutputTokens, ceiling }),
+          onRetry,
+        },
+      );
+      bodies[index] = cleanBody(result.text);
+      usages.push(result.usage);
+      elapsed += result.elapsed;
+      if (result.finishReason !== "stop" && finishReason === "stop") {
+        finishReason = result.finishReason;
+      }
+    }),
+  );
+
+  if (finishReason === "stop" && total > 1) {
+    for (let index = 0; index + 1 < total; index++) {
+      const previousTail = lastBlock(bodies[index]);
+      const nextHead = firstBlock(bodies[index + 1]);
+      if (!previousTail || !nextHead) continue;
+
+      const smoothing = await generateOnce(
+        { model, retryDelays, session, limiter },
+        {
+          system: SMOOTH_SYSTEM,
+          prompt: smoothPrompt(previousTail, nextHead),
+          maxOutputTokens: plannedBudget(nextHead, budgetCeiling),
+          onRetry,
+        },
+      );
+      usages.push(smoothing.usage);
+      elapsed += smoothing.elapsed;
+      if (smoothing.finishReason === "stop") {
+        const rewritten = cleanBody(smoothing.text);
+        if (rewritten) {
+          bodies[index + 1] = replaceFirstBlock(bodies[index + 1], rewritten);
+        }
+      }
+    }
+  }
+
+  const body = dedupeAdjacentHeadings(
+    bodies
+      .join("\n\n")
+      .replace(/\n{3,}/g, "\n\n")
+      .trim(),
+  );
+
+  return {
+    body,
+    usage: sumUsage(usages),
+    finishReason,
+    elapsed,
+    chunks: total,
+  };
+}
+
 async function forEachConcurrent(items, limit, worker) {
   let next = 0;
   await Promise.all(
@@ -305,6 +509,7 @@ export async function runGenerateMarkdown({
   ytdlpConcurrency = 2,
   retryDelays = "10,30,60",
   maxOutputTokens,
+  chunkTargetTokens = DEFAULT_CHUNK_TARGET_TOKENS,
   force = false,
   dir = DEFAULT_DIR,
 } = {}) {
@@ -316,6 +521,7 @@ export async function runGenerateMarkdown({
   const RETRY_DELAYS = parseRetryDelays(retryDelays);
   const llmConcurrency = Number(concurrency);
   const vttConcurrency = Number(ytdlpConcurrency);
+  const limiter = createLimiter(llmConcurrency);
 
   const model = createModel({ modelId, endpoint, apiKey: key, session: sessionBase });
 
@@ -369,6 +575,15 @@ export async function runGenerateMarkdown({
   }
 
   const prices = pending.length > 0 ? await loadPricing({ endpoint }) : null;
+  const modelOutputLimit = prices?.outputLimitAt(modelId) ?? null;
+  const ceiling = isSet(maxOutputTokens)
+    ? Number(maxOutputTokens)
+    : isSet(modelOutputLimit)
+      ? Number(modelOutputLimit)
+      : DEFAULT_CEILING;
+  console.error(
+    `output budget: up to ${ceiling} tokens per call, chunk target ${chunkTargetTokens} tokens`,
+  );
 
   const downloadTask = {
     title: "Download transcripts",
@@ -446,12 +661,15 @@ export async function runGenerateMarkdown({
         try {
           const transcript = vttToText(await readFile(job.vtt, "utf8"));
           const price = prices ? prices.priceAt(modelId) : null;
-          const { body, usage, finishReason, elapsed } = await toMarkdown(
+          const { body, usage, finishReason, elapsed, chunks } = await toMarkdown(
             {
               model,
               retryDelays: RETRY_DELAYS,
               maxOutputTokens,
+              ceiling,
+              chunkTargetTokens,
               session: videoSession(sessionBase, job),
+              limiter,
             },
             job.title,
             transcript,
@@ -459,6 +677,11 @@ export async function runGenerateMarkdown({
               task.output = `retrying in ${Math.round(waitMs / 1000)}s ` +
                 `(attempt ${nextAttempt}/${totalAttempts})  ${name}`;
             },
+            (part, totalParts) => {
+              if (totalParts > 1) {
+                task.output = `generating part ${part}/${totalParts}  ${name}`;
+              }
+            },
           );
 
           stats.totalSeconds += elapsed;
@@ -482,6 +705,7 @@ export async function runGenerateMarkdown({
             cost,
             finishReason,
             price,
+            chunks,
           });
           await mkdir(dirname(md), { recursive: true });
           await writeFile(
@@ -518,3 +742,13 @@ export async function runGenerateMarkdown({
     }`,
   );
 }
+
+export {
+  budgetForPart,
+  chunkPrompt,
+  cleanBody,
+  createLimiter,
+  plannedBudget,
+  smoothPrompt,
+  sumUsage,
+};

package/src/pricing.js

@@ -173,5 +173,13 @@ export async function loadPricing({ endpoint } = {}) {
       if (entry) return litellmPrice(entry, at);
       return modelsDevPrice(provider.models?.[modelId]);
     },
+    outputLimitAt(modelId) {
+      const entry = litellm?.get(modelId);
+      const limit =
+        entry?.max_output_tokens ??
+        provider.models?.[modelId]?.limit?.output ??
+        entry?.max_tokens;
+      return typeof limit === "number" && limit > 0 ? limit : null;
+    },
   };
 }