@stephane-klein/youtube-to-markdown 0.5.0 → 0.9.0

package/README.md

@@ -184,8 +184,9 @@ variables, then the settings written in the YAML file, then the built-in
 defaults. The API key stays a secret: `--api-key` or `OPENAI_API_KEY`. The
 settings are `x_opencode_session`, `model_id`, `openaiapi_endpoint`,
 `concurrency` and `ytdlp_concurrency` (plus the optional `api_key`,
-`retry_delays`, `max_output_tokens` and `chunk_target_tokens`). I only set the
-source URLs, and the `extract-video-metadata` command fills the rest:
+`retry_delays`, `max_output_tokens`, `chunk_target_tokens` and `reasoning`). I
+only set the source URLs, and the `extract-video-metadata` command fills the
+rest:
 
 `x_opencode_session` is the base of the `X-OpenCode-Session` header; it is
 optional and specific to OpenCode. A stable per-video suffix is appended to it,
@@ -335,9 +336,49 @@ only the last 20 events on screen, so it stays bounded even with a large feed.
 Retryable API errors (HTTP 408, 409, 429 or >= 500) are retried with growing
 delays — 10 s, 30 s, 60 s by default, configurable with `--retry-delays`
 (comma-separated seconds, or `YT_TO_MD_RETRY_DELAYS`). A `Retry-After` header
-from the server is honored when longer than the schedule. Each retry is shown in
-the phase output; after the last attempt the file is reported as an error and is
-retried on the next run.
+from the server is honored when longer than the schedule. Each retry line shows
+the reason reported by the API (status and response body), for example:
+
+```
+retrying in 30s (attempt 3/4) HTTP 429 rate limited  david-cayla/....fr.md
+```
+
+After the last attempt the file is reported as an error and is retried on the
+next run. At the end of the run, a `failures:` block lists each failed file with
+its reason, so the cause stays visible even after the phase output has scrolled.
+The command exits with a non-zero status when at least one file failed.
+
+### Empty output
+
+A model can finish with `finish_reason: stop` but return no text at all, for
+example when it spends its whole response on internal reasoning. Such a
+generation is reported as an error and no file is written:
+
+```
+error (empty output: part 1/1, finish=stop, text_tokens=1, reasoning_tokens=4001)  ....fr.md
+```
+
+The `failures:` block repeats the reason at the end of the run. Add
+`--debug-llm` (or `YT_TO_MD_DEBUG_LLM=1`) to log the request's `max_tokens` and
+`reasoning_effort`, plus the size of the provider's `content` and
+`reasoning_content`, which helps diagnose provider-side empty answers.
+
+### Reasoning effort
+
+The command asks the model for a reasoning effort of `low` by default. Set
+`reasoning` in the YAML file, `YT_TO_MD_REASONING`, or `--reasoning` to change
+it. The accepted values are `provider-default`, `none`, `minimal`, `low`,
+`medium`, `high` and `xhigh`, as exposed by ai-sdk. `none` disables reasoning:
+on a DeepSeek endpoint it sends the provider-specific `thinking: disabled`
+option, elsewhere it sends `reasoning: none`. `provider-default` leaves the
+choice to the provider.
+
+```yaml
+reasoning: "none"
+```
+
+The `doctor` command shows the effective value and applies it to its test
+request, so it reflects the real configuration.
 
 ### Output budget and long transcripts
 
@@ -356,7 +397,8 @@ LLM call that rewrites the beginning of the next section so it flows from the
 previous one. `--chunk-target-tokens` (or `YT_TO_MD_CHUNK_TARGET_TOKENS`,
 default 8000) sets the target input size of each chunk. The generation of a
 chunk is guaranteed to stop with `stop`; otherwise the file is reported as an
-error and nothing is written. The frontmatter records how many chunks were used.
+error and nothing is written. The same happens when a part comes back with an
+empty body. The frontmatter records how many chunks were used.
 
 ### Generated frontmatter
 
@@ -374,8 +416,11 @@ llm:
   input_tokens: 4428
   cached_input_tokens: 0
   output_tokens: 3262
+  reasoning_tokens: 0
+  text_tokens: 3262
   estimated_cost_usd: 0.000927
   finish_reason: stop
+  raw_finish_reason: stop
   pricing:
     source: models.dev
     peak: null

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stephane-klein/youtube-to-markdown",
-  "version": "0.5.0",
+  "version": "0.9.0",
   "description": "Collect YouTube channel videos into a YAML config with yt-dlp, download transcripts and turn them into Markdown prose with an LLM.",
   "type": "module",
   "license": "MIT",

package/src/cli.js

@@ -24,6 +24,7 @@ const SETTING_KEYS = [
   "retryDelays",
   "maxOutputTokens",
   "chunkTargetTokens",
+  "reasoning",
 ];
 
 const handle = (run) => async (argv) => {
@@ -121,6 +122,15 @@ const cli = yargs(hideBin(process.argv))
           describe: "Target input tokens per chunk when a transcript is split (default: 8000)",
           type: "number",
         })
+        .option("reasoning", {
+          describe: "Reasoning effort: provider-default, none, minimal, low, medium, high or xhigh (default: low)",
+          type: "string",
+        })
+        .option("debug-llm", {
+          describe: "Print LLM request and response diagnostics",
+          type: "boolean",
+          default: false,
+        })
         .option("force", {
           describe: "Regenerate every Markdown file",
           type: "boolean",
@@ -128,7 +138,13 @@ const cli = yargs(hideBin(process.argv))
         }),
     handle(async (argv) => {
       const { feed, settings, dir } = project(argv);
-      await runGenerateMarkdown({ feed, ...settings, dir, force: argv.force });
+      await runGenerateMarkdown({
+        feed,
+        ...settings,
+        dir,
+        force: argv.force,
+        debugLlm: argv.debugLlm,
+      });
     }),
   )
   .command(
@@ -175,6 +191,10 @@ const cli = yargs(hideBin(process.argv))
           describe: "Value of the X-OpenCode-Session header",
           type: "string",
         })
+        .option("reasoning", {
+          describe: "Reasoning effort to test (default: low)",
+          type: "string",
+        })
         .option("llm", {
           describe: "Run a minimal LLM request to test the API access",
           type: "boolean",
@@ -193,6 +213,7 @@ const cli = yargs(hideBin(process.argv))
           endpoint: argv.endpoint,
           apiKey: argv.apiKey,
           session: argv.session,
+          reasoning: argv.reasoning,
         },
         llm: argv.llm,
         timeoutMs: argv.timeout * 1000,

package/src/config.js

@@ -15,6 +15,7 @@ const YAML_KEYS = {
   retryDelays: "retry_delays",
   maxOutputTokens: "max_output_tokens",
   chunkTargetTokens: "chunk_target_tokens",
+  reasoning: "reasoning",
 };
 
 export function settingsFromConfig(config) {
@@ -36,6 +37,8 @@ const ENV = {
   retryDelays: "YT_TO_MD_RETRY_DELAYS",
   maxOutputTokens: "YT_TO_MD_MAX_OUTPUT_TOKENS",
   chunkTargetTokens: "YT_TO_MD_CHUNK_TARGET_TOKENS",
+  reasoning: "YT_TO_MD_REASONING",
+  debugLlm: "YT_TO_MD_DEBUG_LLM",
 };
 
 export function loadEnv(env = process.env) {

package/src/doctor.js

@@ -2,7 +2,12 @@ import { access, constants } from "node:fs/promises";
 import { dirname, resolve } from "node:path";
 import { generateText } from "ai";
 import { loadConfig, settingsFromConfig } from "./config.js";
-import { createModel, describeError } from "./generate-markdown.js";
+import {
+  createModel,
+  DEFAULT_REASONING,
+  describeError,
+  reasoningOptions,
+} from "./generate-markdown.js";
 import { loadPricing } from "./pricing.js";
 import { exists } from "./vtt.js";
 import { runYtDlp } from "./ytdlp.js";
@@ -124,11 +129,13 @@ async function checkLlm({ llm, settings, key, timeoutMs }) {
       session: settings.session,
     });
     const started = performance.now();
+    const reasoning = settings.reasoning ?? DEFAULT_REASONING;
     const { usage } = await generateText({
       model,
       prompt: "ping",
       maxOutputTokens: 8,
       maxRetries: 0,
+      ...reasoningOptions(reasoning, settings.endpoint),
       abortSignal: AbortSignal.timeout(timeoutMs),
     });
     const elapsed = (performance.now() - started) / 1000;
@@ -218,6 +225,12 @@ export async function runDoctor({
   if (settings.session) {
     rows.push({ label: "session", value: "set", status: "info", note: null });
   }
+  rows.push({
+    label: "reasoning",
+    value: settings.reasoning ?? DEFAULT_REASONING,
+    status: "info",
+    note: null,
+  });
   rows.push({ label: "pricing", ...(await checkPricing({ settings })) });
   const llmRow = await checkLlm({ llm, settings, key, timeoutMs });
   rows.push({ label: "llm", ...llmRow });

package/src/generate-markdown.js

@@ -68,6 +68,7 @@ function estimateCost(price, usage) {
 const OUTPUT_FACTOR = 2;
 const OUTPUT_FLOOR = 2048;
 const DEFAULT_CEILING = 32768;
+const DEFAULT_REASONING = "low";
 const DEFAULT_CHUNK_TARGET_TOKENS = 8000;
 
 function isSet(value) {
@@ -88,19 +89,33 @@ function budgetForPart(text, { maxOutputTokens, ceiling }) {
 }
 
 function metricsLine(modelId, usage, elapsed, cost) {
+  const reasoning = usage?.outputTokenDetails?.reasoningTokens;
   return [
     modelId,
     `llm ${elapsed.toFixed(1)}s`,
     usage
       ? `${usage.inputTokens ?? 0} in / ${usage.outputTokens ?? 0} out`
       : "usage ?",
+    reasoning ? `${reasoning} reasoning` : null,
     cost === null ? null : `~$${cost.toFixed(6)}`,
   ]
     .filter(Boolean)
     .join("  ");
 }
 
-function frontmatter(job, { modelId, usage, elapsed, cost, finishReason, price, chunks }) {
+function frontmatter(
+  job,
+  {
+    modelId,
+    usage,
+    elapsed,
+    cost,
+    finishReason,
+    rawFinishReason,
+    price,
+    chunks,
+  },
+) {
   const data = {
     source_url: job.url,
     video_title: titleForLang(job.titles, job.lang),
@@ -112,8 +127,11 @@ function frontmatter(job, { modelId, usage, elapsed, cost, finishReason, price,
       input_tokens: usage?.inputTokens ?? null,
       cached_input_tokens: usage ? cachedInputTokens(usage) : null,
       output_tokens: usage?.outputTokens ?? null,
+      reasoning_tokens: usage?.outputTokenDetails?.reasoningTokens ?? null,
+      text_tokens: usage?.outputTokenDetails?.textTokens ?? null,
       estimated_cost_usd: cost === null ? null : Number(cost.toFixed(6)),
       finish_reason: finishReason,
+      raw_finish_reason: rawFinishReason ?? null,
       pricing: price ? { source: price.source, peak: price.peak } : null,
     },
   };
@@ -121,6 +139,26 @@ function frontmatter(job, { modelId, usage, elapsed, cost, finishReason, price,
   return `---\n${stringify(data, { lineWidth: 0 }).trimEnd()}\n---\n\n`;
 }
 
+export function emptyOutputReason({
+  emptyParts,
+  chunks,
+  finishReason,
+  rawFinishReason,
+  usage,
+}) {
+  const raw =
+    rawFinishReason && rawFinishReason !== finishReason
+      ? `/${rawFinishReason}`
+      : "";
+  const text = usage?.outputTokenDetails?.textTokens ?? null;
+  const reasoning = usage?.outputTokenDetails?.reasoningTokens ?? null;
+  return (
+    `empty output: part ${emptyParts.join(",")}/${chunks}, ` +
+    `finish=${finishReason}${raw}, text_tokens=${text}, ` +
+    `reasoning_tokens=${reasoning}`
+  );
+}
+
 const isHeading = (text) => /^#{1,6}\s/.test(text);
 
 const isStructured = (text) =>
@@ -205,13 +243,16 @@ function retryDelayMs(error, fallbackMs) {
   return Math.max(headerMs, fallbackMs);
 }
 
-export function describeError(error, attempts) {
-  const suffix = ` after ${attempts} attempt(s)`;
-  if (!APICallError.isInstance(error)) return `${error.message}${suffix}`;
+export function describeCause(error) {
+  if (!APICallError.isInstance(error)) return `${error.message}`;
 
   const status = error.statusCode ? `HTTP ${error.statusCode}` : "API error";
   const body = error.responseBody?.trim().replace(/\s+/g, " ").slice(0, 200);
-  return `${status} ${error.message}${suffix}${body ? `: ${body}` : ""}`;
+  return `${status} ${error.message}${body ? `: ${body}` : ""}`;
+}
+
+export function describeError(error, attempts) {
+  return `${describeCause(error)} after ${attempts} attempt(s)`;
 }
 
 function parseRetryDelays(value) {
@@ -258,6 +299,44 @@ export function createModel({ modelId, endpoint, apiKey, session }) {
   return provider(modelId);
 }
 
+export function reasoningOptions(reasoning, endpoint = "") {
+  if (reasoning === "none") {
+    return /deepseek/i.test(endpoint)
+      ? {
+          providerOptions: {
+            opencode: { thinking: { type: "disabled" } },
+          },
+        }
+      : { reasoning: "none" };
+  }
+  return reasoning ? { reasoning } : {};
+}
+
+function logLlmDebug({
+  maxOutputTokens,
+  reasoning,
+  finishReason,
+  rawFinishReason,
+  text,
+  reasoningText,
+  warnings,
+}) {
+  console.error(
+    `llm debug: max_tokens=${maxOutputTokens ?? "-"} ` +
+      `reasoning=${reasoning ?? "provider-default"} ` +
+      `finish=${finishReason} raw=${rawFinishReason ?? "-"} ` +
+      `text_chars=${text?.length ?? 0} ` +
+      `reasoning_chars=${reasoningText?.length ?? 0}`,
+  );
+  if (warnings?.length) {
+    console.error(`llm debug: warnings=${JSON.stringify(warnings)}`);
+  }
+  if (!`${text ?? ""}`.trim() && reasoningText) {
+    const preview = reasoningText.replace(/\s+/g, " ").slice(0, 300);
+    console.error(`llm debug: empty text, reasoning preview: ${preview}`);
+  }
+}
+
 function videoSession(base, job) {
   const id = createHash("sha256")
     .update(`${job.url}#${job.lang}`)
@@ -275,7 +354,16 @@ function cleanBody(text) {
 }
 
 async function generateOnce(
-  { model, retryDelays, session, limiter, priceAt },
+  {
+    model,
+    retryDelays,
+    session,
+    limiter,
+    priceAt,
+    reasoning,
+    endpoint,
+    debug,
+  },
   { system, prompt, maxOutputTokens, onRetry },
 ) {
   const totalAttempts = retryDelays.length + 1;
@@ -291,26 +379,52 @@ async function generateOnce(
           system,
           prompt,
           temperature: 0.2,
-          reasoning: "low",
+          ...reasoningOptions(reasoning, endpoint),
           maxOutputTokens,
           maxRetries: 0,
           headers: { "X-OpenCode-Session": session },
         });
       };
-      const { text, usage, finishReason } = await (
-        limiter ? limiter(call) : call()
-      );
+      const result = await (limiter ? limiter(call) : call());
+      const {
+        text,
+        usage,
+        finishReason,
+        rawFinishReason,
+        reasoningText,
+        warnings,
+      } = result;
       const elapsed = (performance.now() - started) / 1000;
       const price = priceAt ? priceAt(at) : null;
       const cost = estimateCost(price, usage);
 
-      return { text, usage, finishReason, elapsed, price, cost };
+      if (debug) {
+        logLlmDebug({
+          maxOutputTokens,
+          reasoning,
+          finishReason,
+          rawFinishReason,
+          text,
+          reasoningText,
+          warnings,
+        });
+      }
+
+      return {
+        text,
+        usage,
+        finishReason,
+        rawFinishReason,
+        elapsed,
+        price,
+        cost,
+      };
     } catch (error) {
       if (!isRetryable(error) || attempt > retryDelays.length) {
         throw new Error(describeError(error, attempt), { cause: error });
       }
       const waitMs = retryDelayMs(error, retryDelays[attempt - 1]);
-      onRetry?.(attempt + 1, totalAttempts, waitMs);
+      onRetry?.(attempt + 1, totalAttempts, waitMs, error);
       await delay(waitMs);
     }
   }
@@ -406,6 +520,9 @@ async function toMarkdown(
     session,
     limiter,
     priceAt,
+    reasoning,
+    endpoint,
+    debug,
   },
   title,
   transcript,
@@ -429,6 +546,7 @@ async function toMarkdown(
   const usages = [];
   let elapsed = 0;
   let finishReason = "stop";
+  let rawFinishReason;
   let cost = 0;
   let costKnown = true;
   let price = null;
@@ -437,7 +555,16 @@ async function toMarkdown(
     parts.map(async (part, index) => {
       onProgress?.(index + 1, total);
       const result = await generateOnce(
-        { model, retryDelays, session, limiter, priceAt },
+        {
+          model,
+          retryDelays,
+          session,
+          limiter,
+          priceAt,
+          reasoning,
+          endpoint,
+          debug,
+        },
         {
           system: SYSTEM,
           prompt: chunkPrompt(title, part, index, total),
@@ -451,8 +578,12 @@ async function toMarkdown(
       if (result.cost === null) costKnown = false;
       else cost += result.cost;
       if (price === null && result.price) price = result.price;
+      if (rawFinishReason === undefined) {
+        rawFinishReason = result.rawFinishReason;
+      }
       if (result.finishReason !== "stop" && finishReason === "stop") {
         finishReason = result.finishReason;
+        rawFinishReason = result.rawFinishReason;
       }
     }),
   );
@@ -464,7 +595,16 @@ async function toMarkdown(
       if (!previousTail || !nextHead) continue;
 
       const smoothing = await generateOnce(
-        { model, retryDelays, session, limiter, priceAt },
+        {
+          model,
+          retryDelays,
+          session,
+          limiter,
+          priceAt,
+          reasoning,
+          endpoint,
+          debug,
+        },
         {
           system: SMOOTH_SYSTEM,
           prompt: smoothPrompt(previousTail, nextHead),
@@ -492,11 +632,16 @@ async function toMarkdown(
       .replace(/\n{3,}/g, "\n\n")
       .trim(),
   );
+  const emptyParts = bodies.flatMap((part, index) =>
+    part ? [] : [index + 1],
+  );
 
   return {
     body,
     usage: sumUsage(usages),
     finishReason,
+    rawFinishReason,
+    emptyParts,
     elapsed,
     chunks: total,
     cost: costKnown ? cost : null,
@@ -527,7 +672,9 @@ export async function runGenerateMarkdown({
   retryDelays = "10,30,60",
   maxOutputTokens,
   chunkTargetTokens = DEFAULT_CHUNK_TARGET_TOKENS,
+  reasoning = DEFAULT_REASONING,
   force = false,
+  debugLlm = false,
   dir = DEFAULT_DIR,
 } = {}) {
   const key = apiKey ?? process.env.OPENAI_API_KEY;
@@ -535,6 +682,7 @@ export async function runGenerateMarkdown({
   if (!modelId) throw new Error("model is not set (--model)");
   if (!endpoint) throw new Error("endpoint is not set (--endpoint)");
 
+  const debug = debugLlm === true || /^(1|true|yes|on)$/i.test(`${debugLlm}`);
   const RETRY_DELAYS = parseRetryDelays(retryDelays);
   const llmConcurrency = Number(concurrency);
   const vttConcurrency = Number(ytdlpConcurrency);
@@ -557,6 +705,7 @@ export async function runGenerateMarkdown({
     already: 0,
     skipped: 0,
     errors: 0,
+    failures: [],
     totalSeconds: 0,
     totalInput: 0,
     totalOutput: 0,
@@ -678,30 +827,43 @@ export async function runGenerateMarkdown({
         const name = relative(dir, md);
         try {
           const transcript = vttToText(await readFile(job.vtt, "utf8"));
-          const { body, usage, finishReason, elapsed, chunks, cost, price } =
-            await toMarkdown(
-              {
-                model,
-                retryDelays: RETRY_DELAYS,
-                maxOutputTokens,
-                ceiling,
-                chunkTargetTokens,
-                session: videoSession(sessionBase, job),
-                limiter,
-                priceAt,
-              },
-              job.title,
-              transcript,
-              (nextAttempt, totalAttempts, waitMs) => {
-                task.output = `retrying in ${Math.round(waitMs / 1000)}s ` +
-                  `(attempt ${nextAttempt}/${totalAttempts})  ${name}`;
-              },
-              (part, totalParts) => {
-                if (totalParts > 1) {
-                  task.output = `generating part ${part}/${totalParts}  ${name}`;
-                }
-              },
-            );
+          const {
+            body,
+            usage,
+            finishReason,
+            rawFinishReason,
+            emptyParts,
+            elapsed,
+            chunks,
+            cost,
+            price,
+          } = await toMarkdown(
+            {
+              model,
+              retryDelays: RETRY_DELAYS,
+              maxOutputTokens,
+              ceiling,
+              chunkTargetTokens,
+              session: videoSession(sessionBase, job),
+              limiter,
+              priceAt,
+              reasoning,
+              endpoint,
+              debug,
+            },
+            job.title,
+            transcript,
+            (nextAttempt, totalAttempts, waitMs, error) => {
+              task.output = `retrying in ${Math.round(waitMs / 1000)}s ` +
+                `(attempt ${nextAttempt}/${totalAttempts}) ` +
+                `${describeCause(error)}  ${name}`;
+            },
+            (part, totalParts) => {
+              task.output = totalParts > 1
+                ? `generating part ${part}/${totalParts}  ${name}`
+                : `generating  ${name}`;
+            },
+          );
 
           stats.totalSeconds += elapsed;
           stats.totalInput += usage?.inputTokens ?? 0;
@@ -712,16 +874,35 @@ export async function runGenerateMarkdown({
 
           if (finishReason !== "stop") {
             stats.errors++;
+            stats.failures.push({
+              name,
+              reason: `finishReason=${finishReason}`,
+            });
             task.output = `error (finishReason=${finishReason})  ${name}`;
             return;
           }
 
+          if (emptyParts.length > 0) {
+            const reason = emptyOutputReason({
+              emptyParts,
+              chunks,
+              finishReason,
+              rawFinishReason,
+              usage,
+            });
+            stats.errors++;
+            stats.failures.push({ name, reason });
+            task.output = `error (${reason})  ${name}`;
+            return;
+          }
+
           const front = frontmatter(job, {
             modelId,
             usage,
             elapsed,
             cost,
             finishReason,
+            rawFinishReason,
             price,
             chunks,
           });
@@ -734,6 +915,7 @@ export async function runGenerateMarkdown({
           task.output = `generated  ${name}  ${metrics}`;
         } catch (error) {
           stats.errors++;
+          stats.failures.push({ name, reason: error.message });
           task.output = `error (${error.message})  ${name}`;
         } finally {
           const upToDate = stats.already + stats.generated;
@@ -759,6 +941,15 @@ export async function runGenerateMarkdown({
       stats.costKnown ? `, ~$${stats.totalCost.toFixed(6)}` : ""
     }`,
   );
+
+  if (stats.failures.length > 0) {
+    console.error("\nfailures:");
+    for (const failure of stats.failures) {
+      console.error(`  - ${failure.name}: ${failure.reason}`);
+    }
+  }
+
+  if (stats.errors > 0) process.exitCode = 1;
 }
 
 export {
@@ -766,7 +957,9 @@ export {
   chunkPrompt,
   cleanBody,
   createLimiter,
+  DEFAULT_REASONING,
   estimateCost,
+  metricsLine,
   plannedBudget,
   smoothPrompt,
   sumUsage,