@stephane-klein/youtube-to-markdown 0.4.0 → 0.5.0

package/README.md

@@ -13,6 +13,37 @@ channel or playlist and then to download its transcripts.
 In the final step, I use an LLM through a standard OpenAI-compatible API to
 reconstruct a complete text from the video transcripts.
 
+## AI usage in this project
+
+This project was developed using:
+
+- Harness: [OpenCode](https://opencode.ai)
+- Models: [DeepSeek-V4.1-Flash](https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash) via [OpenCode Go](https://opencode.ai/go)
+- I mostly vibe coded this project: I made the implementation, library and architecture choices, but I reviewed the code very little.
+
+The figures below cover the period from the start of the project to version 0.4.0
+([de39801](https://github.com/stephane-klein/youtube-to-markdown/commit/de39801f123fa1cfaeaa3b9cfed5071968977474)).
+
+Metrics:
+
+- 16 OpenCode sessions, 1426 interactions;
+- 11 h of human time on this project, against 4 h of AI generation time over the same period, so a human/AI time ratio of 2.75 (11 h / 4 h);
+- Cost: $2.4958;
+- Tokens:
+  - input: 7,047,916
+  - output: 421,770
+  - cache read: 228,841,600
+  - total: 236,311,286
+
+Per agent:
+
+| agent   | interactions | share | output tokens | active hours | active share | generation hours | cost   |
+|:--------|-------------:|------:|--------------:|-------------:|-------------:|-----------------:|-------:|
+| build   |          962 |   68% |       259,706 |         3.71 |          38% |             1.93 | $1.385 |
+| plan    |          450 |   32% |       155,136 |         5.91 |          61% |             2.10 | $1.098 |
+| explore |           14 |    1% |         6,928 |         0.02 |           0% |             0.02 | $0.013 |
+
+
 ## Requirements
 
 - Node.js 22 or later;
@@ -129,7 +160,13 @@ can gate a CI job.
 ```sh
 $ youtube-to-markdown extract-video-metadata
 → https://www.youtube.com/@le_science4all
+  exact dates 50/2217  2.4s elapsed  ~1m 44s left
+  exact dates 100/2217  4.8s elapsed  ~1m 42s left
+  exact dates 2217/2217  1m 46s elapsed
+  done in 1m 46s
 → https://www.youtube.com/@MonsieurPhi
+  done in 12.3s
+summary: 2 sources, 2678 videos, 2m 05s elapsed
 ```
 
 The command is idempotent: a new run only adds the missing videos.
@@ -348,10 +385,10 @@ llm:
 `video_title` is the original video title in the file's language and
 `generated_at` is the UTC time of the generation. `pricing.source` names the
 catalog the rates came from and `pricing.peak` tells whether the peak rate
-applied (`null` when the source has no peak/off-peak pricing). Existing files
-are left untouched: run `youtube-to-markdown generate-markdown --force` to
-regenerate every file, which is also the way to add the frontmatter to files
-generated before it existed.
+applied to the first call (`null` when the source has no peak/off-peak pricing).
+Existing files are left untouched: run `youtube-to-markdown generate-markdown
+--force` to regenerate every file, which is also the way to add the frontmatter
+to files generated before it existed.
 
 ### Pricing
 
@@ -364,6 +401,10 @@ catalogs are cached for one day under `$XDG_CACHE_HOME/youtube-to-markdown`
 (`~/.cache/youtube-to-markdown` by default). When a catalog or the model is
 unavailable, `estimated_cost_usd` is `null` and the run continues.
 
+The rate is sampled at the start of every LLM call, so a run that straddles a
+peak/off-peak boundary is priced with the right rate on each side. The
+`pricing` block in the frontmatter describes the first call of the file.
+
 ## Organize the files into folders
 
 A channel may set `folder_slug` to keep its files in their own subdirectory of

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stephane-klein/youtube-to-markdown",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "Collect YouTube channel videos into a YAML config with yt-dlp, download transcripts and turn them into Markdown prose with an LLM.",
   "type": "module",
   "license": "MIT",

package/src/extract-video-metadata.js

@@ -1,9 +1,12 @@
 import { writeFile } from "node:fs/promises";
 import { stringify } from "yaml";
+import { formatDuration, progressLine } from "./progress.js";
 import { runYtDlp } from "./ytdlp.js";
 
 const DATE_CHUNK = 50;
 
+const count = (value, word) => `${value} ${word}${value === 1 ? "" : "s"}`;
+
 const toIso = (raw) =>
   /^\d{8}$/.test(raw)
     ? `${raw.slice(0, 4)}-${raw.slice(4, 6)}-${raw.slice(6, 8)}`
@@ -61,12 +64,10 @@ const videoId = (url) => /[?&]v=([\w-]+)/.exec(url)?.[1] ?? url;
 
 async function fetchExactDates(urls) {
   const dates = new Map();
+  const startedAt = performance.now();
 
   for (let i = 0; i < urls.length; i += DATE_CHUNK) {
     const chunk = urls.slice(i, i + DATE_CHUNK);
-    console.error(
-      `  exact dates ${Math.min(i + DATE_CHUNK, urls.length)}/${urls.length}`,
-    );
     const out = await ytDlp(
       [
         "--skip-download",
@@ -84,6 +85,15 @@ async function fetchExactDates(urls) {
       const iso = toIso(raw ?? "");
       if (id && iso) dates.set(id, iso);
     }
+
+    console.error(
+      progressLine({
+        label: "exact dates",
+        done: Math.min(i + DATE_CHUNK, urls.length),
+        total: urls.length,
+        startedAt,
+      }),
+    );
   }
 
   return dates;
@@ -180,9 +190,14 @@ export async function runExtract({
   config = {},
   forceDates = false,
 } = {}) {
+  const startedAt = performance.now();
+  let sourceCount = 0;
+  let videoCount = 0;
+
   for (const channel of config.feed ?? []) {
     if (typeof channel.url !== "string" || !channel.url) continue;
 
+    const channelStartedAt = performance.now();
     const videos = channel.videos ?? [];
     const knownDates = forceDates
       ? new Map()
@@ -197,7 +212,17 @@ export async function runExtract({
     channel.videos = mergeVideos(videos, fetched.videos, {
       sort: !fetched.playlist,
     });
+
+    sourceCount += 1;
+    videoCount += channel.videos.length;
+    console.error(
+      `  done in ${formatDuration(performance.now() - channelStartedAt)}`,
+    );
   }
 
   await writeFile(path, stringify(config, { lineWidth: 0 }));
+
+  console.error(
+    `summary: ${count(sourceCount, "source")}, ${count(videoCount, "video")}, ${formatDuration(performance.now() - startedAt)} elapsed`,
+  );
 }

package/src/generate-markdown.js

@@ -275,7 +275,7 @@ function cleanBody(text) {
 }
 
 async function generateOnce(
-  { model, retryDelays, session, limiter },
+  { model, retryDelays, session, limiter, priceAt },
   { system, prompt, maxOutputTokens, onRetry },
 ) {
   const totalAttempts = retryDelays.length + 1;
@@ -283,8 +283,10 @@ async function generateOnce(
   for (let attempt = 1;; attempt++) {
     const started = performance.now();
     try {
-      const call = () =>
-        generateText({
+      let at;
+      const call = () => {
+        at = new Date();
+        return generateText({
           model,
           system,
           prompt,
@@ -294,12 +296,15 @@ async function generateOnce(
           maxRetries: 0,
           headers: { "X-OpenCode-Session": session },
         });
+      };
       const { text, usage, finishReason } = await (
         limiter ? limiter(call) : call()
       );
       const elapsed = (performance.now() - started) / 1000;
+      const price = priceAt ? priceAt(at) : null;
+      const cost = estimateCost(price, usage);
 
-      return { text, usage, finishReason, elapsed };
+      return { text, usage, finishReason, elapsed, price, cost };
     } catch (error) {
       if (!isRetryable(error) || attempt > retryDelays.length) {
         throw new Error(describeError(error, attempt), { cause: error });
@@ -400,6 +405,7 @@ async function toMarkdown(
     chunkTargetTokens,
     session,
     limiter,
+    priceAt,
   },
   title,
   transcript,
@@ -423,12 +429,15 @@ async function toMarkdown(
   const usages = [];
   let elapsed = 0;
   let finishReason = "stop";
+  let cost = 0;
+  let costKnown = true;
+  let price = null;
 
   await Promise.all(
     parts.map(async (part, index) => {
       onProgress?.(index + 1, total);
       const result = await generateOnce(
-        { model, retryDelays, session, limiter },
+        { model, retryDelays, session, limiter, priceAt },
         {
           system: SYSTEM,
           prompt: chunkPrompt(title, part, index, total),
@@ -439,6 +448,9 @@ async function toMarkdown(
       bodies[index] = cleanBody(result.text);
       usages.push(result.usage);
       elapsed += result.elapsed;
+      if (result.cost === null) costKnown = false;
+      else cost += result.cost;
+      if (price === null && result.price) price = result.price;
       if (result.finishReason !== "stop" && finishReason === "stop") {
         finishReason = result.finishReason;
       }
@@ -452,7 +464,7 @@ async function toMarkdown(
       if (!previousTail || !nextHead) continue;
 
       const smoothing = await generateOnce(
-        { model, retryDelays, session, limiter },
+        { model, retryDelays, session, limiter, priceAt },
         {
           system: SMOOTH_SYSTEM,
           prompt: smoothPrompt(previousTail, nextHead),
@@ -462,6 +474,9 @@ async function toMarkdown(
       );
       usages.push(smoothing.usage);
       elapsed += smoothing.elapsed;
+      if (smoothing.cost === null) costKnown = false;
+      else cost += smoothing.cost;
+      if (price === null && smoothing.price) price = smoothing.price;
       if (smoothing.finishReason === "stop") {
         const rewritten = cleanBody(smoothing.text);
         if (rewritten) {
@@ -484,6 +499,8 @@ async function toMarkdown(
     finishReason,
     elapsed,
     chunks: total,
+    cost: costKnown ? cost : null,
+    price,
   };
 }
 
@@ -581,6 +598,7 @@ export async function runGenerateMarkdown({
     : isSet(modelOutputLimit)
       ? Number(modelOutputLimit)
       : DEFAULT_CEILING;
+  const priceAt = prices ? (at) => prices.priceAt(modelId, at) : null;
   console.error(
     `output budget: up to ${ceiling} tokens per call, chunk target ${chunkTargetTokens} tokens`,
   );
@@ -660,34 +678,34 @@ export async function runGenerateMarkdown({
         const name = relative(dir, md);
         try {
           const transcript = vttToText(await readFile(job.vtt, "utf8"));
-          const price = prices ? prices.priceAt(modelId) : null;
-          const { body, usage, finishReason, elapsed, chunks } = await toMarkdown(
-            {
-              model,
-              retryDelays: RETRY_DELAYS,
-              maxOutputTokens,
-              ceiling,
-              chunkTargetTokens,
-              session: videoSession(sessionBase, job),
-              limiter,
-            },
-            job.title,
-            transcript,
-            (nextAttempt, totalAttempts, waitMs) => {
-              task.output = `retrying in ${Math.round(waitMs / 1000)}s ` +
-                `(attempt ${nextAttempt}/${totalAttempts})  ${name}`;
-            },
-            (part, totalParts) => {
-              if (totalParts > 1) {
-                task.output = `generating part ${part}/${totalParts}  ${name}`;
-              }
-            },
-          );
+          const { body, usage, finishReason, elapsed, chunks, cost, price } =
+            await toMarkdown(
+              {
+                model,
+                retryDelays: RETRY_DELAYS,
+                maxOutputTokens,
+                ceiling,
+                chunkTargetTokens,
+                session: videoSession(sessionBase, job),
+                limiter,
+                priceAt,
+              },
+              job.title,
+              transcript,
+              (nextAttempt, totalAttempts, waitMs) => {
+                task.output = `retrying in ${Math.round(waitMs / 1000)}s ` +
+                  `(attempt ${nextAttempt}/${totalAttempts})  ${name}`;
+              },
+              (part, totalParts) => {
+                if (totalParts > 1) {
+                  task.output = `generating part ${part}/${totalParts}  ${name}`;
+                }
+              },
+            );
 
           stats.totalSeconds += elapsed;
           stats.totalInput += usage?.inputTokens ?? 0;
           stats.totalOutput += usage?.outputTokens ?? 0;
-          const cost = estimateCost(price, usage);
           if (cost === null) stats.costKnown = false;
           else stats.totalCost += cost;
           const metrics = metricsLine(modelId, usage, elapsed, cost);
@@ -748,6 +766,7 @@ export {
   chunkPrompt,
   cleanBody,
   createLimiter,
+  estimateCost,
   plannedBudget,
   smoothPrompt,
   sumUsage,

package/src/progress.js

@@ -0,0 +1,32 @@
+export function formatDuration(ms) {
+  const value = Number.isFinite(ms) && ms > 0 ? ms : 0;
+  if (value < 1000) return `${Math.round(value)}ms`;
+
+  const seconds = value / 1000;
+  if (seconds < 59.95) return `${seconds.toFixed(1)}s`;
+
+  const rounded = Math.round(seconds);
+  const minutes = Math.floor(rounded / 60);
+  if (minutes < 60) return `${minutes}m ${String(rounded % 60).padStart(2, "0")}s`;
+
+  const hours = Math.floor(minutes / 60);
+  return `${hours}h ${String(minutes % 60).padStart(2, "0")}m`;
+}
+
+export function progressStats({ done, total, startedAt, now = performance.now() }) {
+  const elapsed = Math.max(0, now - startedAt);
+  const completed = Math.min(Math.max(done, 0), Math.max(total, 0));
+
+  if (completed === 0 || total <= 0) return { elapsed, eta: null };
+
+  return { elapsed, eta: (elapsed * (total - completed)) / completed };
+}
+
+export function progressLine({ label, done, total, startedAt, now }) {
+  const { elapsed, eta } = progressStats({ done, total, startedAt, now });
+  const parts = [`  ${label} ${done}/${total}`, `${formatDuration(elapsed)} elapsed`];
+
+  if (eta !== null && eta > 0) parts.push(`~${formatDuration(eta)} left`);
+
+  return parts.join("  ");
+}