headless-youtube-captions 1.4.0 → 2.0.0

package/CHANGELOG.md

@@ -1,5 +1,31 @@
 # Changelog
 
+## [1.4.1] - 2025-10-28
+
+### Fixed
+- **CRITICAL FIX**: Transcript extraction now works reliably with YouTube's current UI
+  - Fixed description expansion to click on container directly instead of "more" button
+  - Added visibility checks before clicking transcript button
+  - Improved scroll behavior to ensure transcript button is in viewport
+  - Increased wait times for transcript panel initialization (3s → 5s)
+  - Increased timeout for segment loading (10s → 15s)
+  - Better error logging for debugging click failures
+
+### Changed
+- Updated test suite to use video with stable, reliable transcripts
+- Improved transcript button click reliability using `page.click(selector)`
+- Added scroll-into-view for transcript button when not visible
+- Enhanced description expansion with more selector options
+
+### Technical Details
+- Updated `src/index.js` with improved UI interaction logic
+- Updated `test/index.test.js` to use `dQw4w9WgXcQ` (stable test video)
+- All 7 tests now passing successfully
+- Added comprehensive documentation in `docs/` folder:
+  - `action-flow-runbook.csv` - Complete action flow with fix notes
+  - `transcript-fix-analysis.md` - Detailed technical analysis
+  - `fix-summary.md` - Summary of changes
+
 ## [1.3.0] - 2025-01-24
 
 ### Added

package/debug-transcript.js

@@ -0,0 +1,124 @@
+import puppeteer from 'puppeteer';
+
+const videoID = 'E_i0SOsMp5I';
+
+async function debugTranscript() {
+  console.log('Launching browser in HEADED mode with cookie fixes...');
+
+  const browser = await puppeteer.launch({
+    headless: false,
+    args: [
+      '--no-sandbox',
+      '--disable-setuid-sandbox',
+      '--window-size=1920,1080',
+      '--disable-dev-shm-usage',
+      '--disable-features=ThirdPartyCookieBlocking,BlockThirdPartyCookies',
+      '--disable-blink-features=BlockCredentialedSubresources',
+      '--disable-site-isolation-trials',
+    ],
+    defaultViewport: { width: 1920, height: 1080 }
+  });
+
+  const page = await browser.newPage();
+
+  await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36');
+
+  console.log(`Navigating to https://www.youtube.com/watch?v=${videoID}`);
+  await page.goto(`https://www.youtube.com/watch?v=${videoID}`, {
+    waitUntil: 'networkidle2',
+    timeout: 60000
+  });
+
+  console.log('Page loaded. Waiting for video player...');
+  await page.waitForSelector('#movie_player, video', { timeout: 30000 });
+  console.log('Video player found.');
+
+  await new Promise(r => setTimeout(r, 5000));
+
+  // Accept cookie consent
+  try {
+    const accepted = await page.evaluate(() => {
+      const btns = document.querySelectorAll('button');
+      for (const b of btns) {
+        const txt = (b.textContent || '').trim().toLowerCase();
+        if (txt === 'accept all' || txt === 'accept the use of cookies and other data for the purposes described') {
+          b.click(); return true;
+        }
+      }
+      return false;
+    });
+    if (accepted) { console.log('Accepted cookie consent'); await new Promise(r => setTimeout(r, 2000)); }
+  } catch(e) {}
+
+  // Skip ads
+  try {
+    const skipBtn = await page.$('.ytp-ad-skip-button, .ytp-skip-ad-button');
+    if (skipBtn) { await skipBtn.click(); console.log('Skipped ad'); await new Promise(r => setTimeout(r, 2000)); }
+  } catch(e) {}
+
+  console.log('Scrolling down...');
+  await page.evaluate(() => window.scrollBy(0, 800));
+  await new Promise(r => setTimeout(r, 2000));
+
+  console.log('Expanding description...');
+  try {
+    const desc = await page.$('#description-inline-expander');
+    if (desc) { await desc.click(); await new Promise(r => setTimeout(r, 2000)); console.log('Description expanded'); }
+  } catch(e) {}
+
+  await page.evaluate(() => window.scrollBy(0, 400));
+  await new Promise(r => setTimeout(r, 1000));
+
+  console.log('Looking for transcript button...');
+  let clicked = false;
+  const btn = await page.$('button[aria-label="Show transcript"]');
+  if (btn) {
+    await btn.evaluate(e => e.scrollIntoView({ behavior: 'smooth', block: 'center' }));
+    await new Promise(r => setTimeout(r, 500));
+    await btn.click();
+    console.log('Clicked transcript button');
+    clicked = true;
+  }
+
+  if (!clicked) {
+    const textClick = await page.evaluate(() => {
+      const btns = Array.from(document.querySelectorAll('button, yt-button-shape'));
+      for (const b of btns) {
+        if ((b.textContent || '').toLowerCase().includes('transcript') ||
+            (b.getAttribute('aria-label') || '').toLowerCase().includes('transcript')) {
+          b.click(); return true;
+        }
+      }
+      return false;
+    });
+    if (textClick) { console.log('Clicked via text search'); clicked = true; }
+  }
+
+  if (clicked) {
+    console.log('Waiting 15s for transcript panel...');
+    await new Promise(r => setTimeout(r, 15000));
+
+    const debug = await page.evaluate(() => ({
+      segmentRenderers: document.querySelectorAll('ytd-transcript-segment-renderer').length,
+      engagementPanels: document.querySelectorAll('ytd-engagement-panel-section-list-renderer').length,
+      transcriptPanel: !!document.querySelector('ytd-engagement-panel-section-list-renderer[target-id*="transcript"]'),
+      expandedPanels: Array.from(document.querySelectorAll('ytd-engagement-panel-section-list-renderer'))
+        .filter(p => p.getAttribute('visibility') === 'ENGAGEMENT_PANEL_VISIBILITY_EXPANDED').length,
+      continuationItems: document.querySelectorAll('ytd-continuation-item-renderer').length,
+      spinners: document.querySelectorAll('tp-yt-paper-spinner, paper-spinner').length
+    }));
+    console.log('DOM state:', JSON.stringify(debug, null, 2));
+
+    if (debug.segmentRenderers > 0) {
+      console.log('SUCCESS! Transcript segments loaded!');
+    } else {
+      console.log('STILL FAILING - segments did not load');
+    }
+  }
+
+  console.log('\n=== Browser open for inspection. Press Ctrl+C to close. ===');
+  await new Promise(r => setTimeout(r, 300000));
+  await browser.close();
+}
+
+debugTranscript().catch(e => { console.error('ERROR:', e); process.exit(1); });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "headless-youtube-captions",
-  "version": "1.4.0",
-  "description": "Extract YouTube video transcripts, channel videos, comments, and comprehensive video metadata using headless browser automation",
+  "version": "2.0.0",
+  "description": "Extract YouTube video transcripts (via yt-dlp), channel videos, comments, and comprehensive video metadata",
   "main": "src/index.js",
   "types": "src/index.d.ts",
   "type": "module",
@@ -26,7 +26,7 @@
   "dependencies": {
     "he": "^1.2.0",
     "lodash": "^4.17.21",
-    "puppeteer": "^24.10.0",
+    "patchright": "^1.51.1",
     "striptags": "^3.2.0"
   },
   "devDependencies": {},
@@ -41,8 +41,8 @@
     "videos",
     "metadata",
     "description",
-    "puppeteer",
-    "headless",
+    "yt-dlp",
+    "patchright",
     "scraper",
     "api",
     "automation",

package/src/channel.js

@@ -23,7 +23,7 @@ export async function getChannelVideos({ channelURL, limit = 30, pageToken = nul
     
     console.error(`Navigating to ${fullURL}`);
     await page.goto(fullURL, {
-      waitUntil: 'networkidle2',
+      waitUntil: 'networkidle',
       timeout: 60000
     });
     
@@ -105,7 +105,7 @@ export async function searchChannelVideos({ channelURL, query, limit = 30 }) {
     
     console.error(`Navigating to ${fullURL}`);
     await page.goto(fullURL, {
-      waitUntil: 'networkidle2',
+      waitUntil: 'networkidle',
       timeout: 60000
     });
     

package/src/comments.js

@@ -11,7 +11,7 @@ export async function getVideoComments({ videoID, limit = 50, sortBy = 'top', pa
     // Navigate to the YouTube video page
     console.error(`Navigating to https://youtube.com/watch?v=${videoID}`);
     await page.goto(`https://youtube.com/watch?v=${videoID}`, {
-      waitUntil: 'networkidle2',
+      waitUntil: 'networkidle',
       timeout: 60000
     });
     
@@ -35,9 +35,8 @@ export async function getVideoComments({ videoID, limit = 50, sortBy = 'top', pa
     }
     
     // Wait for comment threads to load
-    await page.waitForSelector('ytd-comment-thread-renderer', { 
-      timeout: 10000,
-      visible: true 
+    await page.waitForSelector('ytd-comment-thread-renderer', {
+      timeout: 10000
     });
     
     console.error('Comments section loaded');

package/src/index.d.ts

@@ -15,7 +15,8 @@ export interface GetSubtitlesOptions {
 }
 
 /**
- * Extract YouTube video transcripts using headless browser automation
+ * Extract YouTube video transcripts using yt-dlp (json3 subtitle format).
+ * Requires yt-dlp to be installed and available in PATH.
  * @param options - Configuration options
  * @returns Promise that resolves to an array of subtitle segments
  */

package/src/index.js

@@ -1,258 +1,85 @@
-import { createBrowser, createPage, handleCookieConsent, skipAds } from './utils/browser.js';
+import { execFile } from 'child_process';
+import { mkdtemp, readdir, readFile, rm } from 'fs/promises';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { promisify } from 'util';
 
-// Export existing function
-export async function getSubtitles({ videoID, lang = 'en' }) {
-  const browser = await createBrowser();
+const execFileAsync = promisify(execFile);
 
+// Check if yt-dlp is available
+async function ensureYtDlp() {
   try {
-    const page = await createPage(browser);
-    
-    // Navigate to the YouTube video page
-    console.error(`Navigating to https://youtube.com/watch?v=${videoID}`);
-    await page.goto(`https://youtube.com/watch?v=${videoID}`, {
-      waitUntil: 'networkidle2',
-      timeout: 60000
-    });
-
-    // Wait for initial page load
-    await page.waitForSelector('#movie_player, video', { timeout: 30000 });
-    console.error('Video player loaded');
-
-    // Wait a bit more for dynamic content to load
-    await new Promise(resolve => setTimeout(resolve, 5000));
-
-    // Handle cookie consent if present
-    await handleCookieConsent(page);
+    await execFileAsync('which', ['yt-dlp']);
+  } catch {
+    throw new Error(
+      'yt-dlp is not installed or not in PATH. Install it: https://github.com/yt-dlp/yt-dlp#installation'
+    );
+  }
+}
 
-    // Skip ads if present
-    await skipAds(page);
+// Export existing function
+export async function getSubtitles({ videoID, lang = 'en' }) {
+  await ensureYtDlp();
 
-    // Scroll down to load more content
-    await page.evaluate(() => window.scrollBy(0, 800));
-    await new Promise(resolve => setTimeout(resolve, 2000));
+  const tempDir = await mkdtemp(join(tmpdir(), 'yt-subs-'));
 
-    // Click on "more" to expand description if needed
-    try {
-      // Multiple selectors for the more button
-      const moreSelectors = [
-        'tp-yt-paper-button#expand',
-        'tp-yt-paper-button[id="expand"]',
-        '#expand',
-        '#more',
-        'yt-formatted-string:has-text("...more")',
-        '[aria-label*="more"]'
-      ];
-      
-      for (const selector of moreSelectors) {
-        try {
-          const moreButton = await page.$(selector);
-          if (moreButton) {
-            const isVisible = await moreButton.evaluate(el => {
-              const rect = el.getBoundingClientRect();
-              return rect.width > 0 && rect.height > 0;
-            });
-            
-            if (isVisible) {
-              await moreButton.click();
-              console.error('Clicked "more" button');
-              await new Promise(resolve => setTimeout(resolve, 1000));
-              break;
-            }
-          }
-        } catch (e) {
-          // Try next selector
-        }
-      }
-    } catch (e) {
-      console.error('No "more" button found or error clicking it');
+  try {
+    const url = `https://www.youtube.com/watch?v=${videoID}`;
+    const outputTemplate = join(tempDir, '%(id)s');
+
+    // Try manual subs first, fall back to auto-generated
+    await execFileAsync('yt-dlp', [
+      '--write-subs',
+      '--write-auto-subs',
+      '--sub-lang', lang,
+      '--sub-format', 'json3',
+      '--skip-download',
+      '-o', outputTemplate,
+      url,
+    ], { timeout: 30000 });
+
+    // Find the json3 subtitle file
+    const files = await readdir(tempDir);
+    const subFile = files.find(f => f.endsWith('.json3'));
+
+    if (!subFile) {
+      throw new Error(`No subtitles found for video ${videoID} in language '${lang}'`);
     }
 
-    // Look for and click the "Show transcript" button
-    console.error('Looking for "Show transcript" button...');
-    
-    // Multiple strategies to find the transcript button
-    const transcriptButtonSelectors = [
-      'button[aria-label="Show transcript"]',
-      'yt-button-shape button[aria-label="Show transcript"]',
-      'button[title*="transcript" i]',
-      'button[aria-label*="transcript" i]',
-      'yt-button-shape[aria-label*="transcript" i]',
-      '#button[aria-label*="transcript" i]',
-      'ytd-button-renderer[aria-label*="transcript" i]'
-    ];
+    const raw = await readFile(join(tempDir, subFile), 'utf-8');
+    const json3 = JSON.parse(raw);
 
-    let transcriptClicked = false;
-    
-    for (const selector of transcriptButtonSelectors) {
-      try {
-        await page.waitForSelector(selector, { timeout: 3000, visible: true });
-        await page.click(selector);
-        console.error(`Clicked transcript button with selector: ${selector}`);
-        transcriptClicked = true;
-        break;
-      } catch (e) {
-        // Try next selector
-      }
+    if (!json3.events || !Array.isArray(json3.events)) {
+      throw new Error('Invalid json3 subtitle format: missing events array');
     }
 
-    if (!transcriptClicked) {
-      // Try finding by text content
-      console.error('Trying to find transcript button by text...');
-      const clicked = await page.evaluate(() => {
-        const buttons = Array.from(document.querySelectorAll('button, yt-button-shape'));
-        for (const button of buttons) {
-          const text = button.textContent || '';
-          const ariaLabel = button.getAttribute('aria-label') || '';
-          if (text.toLowerCase().includes('transcript') || ariaLabel.toLowerCase().includes('transcript')) {
-            button.click();
-            return true;
-          }
-        }
-        return false;
-      });
-      
-      if (clicked) {
-        console.error('Clicked transcript button by text search');
-        transcriptClicked = true;
-      }
-    }
-
-    if (!transcriptClicked) {
-      throw new Error('Could not find or click "Show transcript" button');
-    }
-
-    // Wait for the transcript panel to load
-    console.error('Waiting for transcript panel...');
-    await new Promise(resolve => setTimeout(resolve, 3000));
-
-    // Wait for transcript segments
-    await page.waitForSelector('ytd-transcript-segment-renderer, ytd-transcript-body-renderer', { 
-      timeout: 10000,
-      visible: true 
-    });
-
-    // Extract transcript data
-    console.error('Extracting transcript content...');
-    const transcriptData = await page.evaluate(() => {
-      // Multiple selectors for transcript segments
-      const segmentSelectors = [
-        'ytd-transcript-segment-renderer',
-        'ytd-transcript-body-renderer ytd-transcript-segment-renderer',
-        'ytd-engagement-panel-section-list-renderer ytd-transcript-segment-renderer',
-        '#segments-container ytd-transcript-segment-renderer',
-        'ytd-transcript-segment-list-renderer ytd-transcript-segment-renderer'
-      ];
-      
-      let segments = [];
-      for (const selector of segmentSelectors) {
-        segments = document.querySelectorAll(selector);
-        if (segments.length > 0) {
-          break;
-        }
-      }
-
-      if (segments.length === 0) {
-        // Try a more general approach
-        segments = document.querySelectorAll('[class*="transcript"][class*="segment"]');
-      }
-
-      if (segments.length === 0) {
-        return [];
-      }
-
-      // Extract data from each segment
-      return Array.from(segments).map((segment) => {
-        // Extract timestamp - multiple strategies
-        let timestampText = '';
-        const timestampSelectors = [
-          '.segment-timestamp',
-          '[class*="timestamp"]',
-          '.ytd-transcript-segment-renderer:first-child',
-          'div:first-child'
-        ];
-        
-        for (const selector of timestampSelectors) {
-          const elem = segment.querySelector(selector);
-          if (elem && elem.textContent && /\d+:\d+/.test(elem.textContent)) {
-            timestampText = elem.textContent.trim();
-            break;
-          }
-        }
-        
-        // Extract text content - multiple strategies
-        let text = '';
-        const textSelectors = [
-          '.segment-text',
-          'yt-formatted-string.segment-text',
-          '[class*="segment-text"]',
-          'yt-formatted-string:last-child',
-          '.ytd-transcript-segment-renderer:last-child'
-        ];
-        
-        for (const selector of textSelectors) {
-          const elem = segment.querySelector(selector);
-          if (elem && elem.textContent) {
-            const content = elem.textContent.trim();
-            // Make sure it's not the timestamp
-            if (content && !(/^\d+:\d+$/.test(content))) {
-              text = content;
-              break;
-            }
-          }
-        }
-
-        // If still no text, get all text and remove timestamp
-        if (!text) {
-          const fullText = segment.textContent || '';
-          text = fullText.replace(timestampText, '').trim();
-        }
-
-        // Convert timestamp to seconds
-        let startSeconds = 0;
-        if (timestampText && timestampText.includes(':')) {
-          const parts = timestampText.split(':').reverse();
-          startSeconds = parts.reduce((acc, part, idx) => {
-            return acc + (parseInt(part) || 0) * Math.pow(60, idx);
-          }, 0);
-        }
-
+    // Filter to entries with segs (skip style/aAppend-only entries)
+    const segments = json3.events
+      .filter(ev => ev.segs && !ev.aAppend)
+      .map(ev => {
+        const text = ev.segs
+          .map(s => s.utf8 || '')
+          .join('')
+          .trim();
+        const startMs = ev.tStartMs || 0;
+        const durationMs = ev.dDurationMs || 0;
         return {
-          start: startSeconds.toString(),
-          dur: "3",
-          text: text,
-          timestamp: timestampText
+          start: (startMs / 1000).toFixed(1),
+          dur: (durationMs / 1000).toFixed(1),
+          text,
         };
-      }).filter(item => item.text && item.text.trim() && item.text.length > 0);
-    });
+      })
+      .filter(seg => seg.text.length > 0);
 
-    if (!transcriptData || transcriptData.length === 0) {
-      throw new Error('No transcript data extracted');
+    if (segments.length === 0) {
+      throw new Error('No transcript segments extracted from subtitle file');
     }
 
-    console.error(`Successfully extracted ${transcriptData.length} transcript segments`);
-    console.error('First segment:', transcriptData[0]);
-
-    // Calculate proper durations
-    const processedCaptions = transcriptData.map((item, index) => {
-      const nextItem = transcriptData[index + 1];
-      const duration = nextItem 
-        ? (parseFloat(nextItem.start) - parseFloat(item.start)).toFixed(1)
-        : "3.0";
-
-      return {
-        start: item.start,
-        dur: duration,
-        text: item.text
-      };
-    });
-
-    return processedCaptions;
+    console.error(`Successfully extracted ${segments.length} transcript segments via yt-dlp`);
+    return segments;
 
-  } catch (error) {
-    console.error('Error extracting subtitles:', error);
-    throw error;
   } finally {
-    await browser.close();
+    await rm(tempDir, { recursive: true, force: true });
   }
 }
 
@@ -260,4 +87,4 @@ export async function getSubtitles({ videoID, lang = 'en' }) {
 export { getChannelVideos, searchChannelVideos } from './channel.js';
 export { getVideoComments } from './comments.js';
 export { searchYouTubeGlobal } from './search.js';
-export { getVideoMetadata } from './metadata.js';
+export { getVideoMetadata } from './metadata.js';

package/src/metadata.js

@@ -16,7 +16,7 @@ export async function getVideoMetadata({ videoID, expandDescription = true }) {
     // Navigate to the YouTube video page
     console.error(`Navigating to https://youtube.com/watch?v=${videoID}`);
     await page.goto(`https://youtube.com/watch?v=${videoID}`, {
-      waitUntil: 'networkidle2',
+      waitUntil: 'networkidle',
       timeout: 60000
     });
 

package/src/search.js

@@ -28,7 +28,7 @@ export async function searchYouTubeGlobal({ query, maxResults = 10, resultTypes
     console.error(`Navigating to ${searchUrl}`);
     
     await page.goto(searchUrl, {
-      waitUntil: 'networkidle2',
+      waitUntil: 'networkidle',
       timeout: 60000
     });
 

package/src/utils/browser.js

@@ -1,29 +1,30 @@
-import puppeteer from 'puppeteer';
+import { chromium } from 'patchright';
+import { mkdtempSync } from 'fs';
+import { join } from 'path';
+import { tmpdir } from 'os';
 
 export async function createBrowser() {
+  const userDataDir = mkdtempSync(join(tmpdir(), 'patchright-'));
+
   const options = {
-    headless: true,
-    args: ['--no-sandbox', '--disable-setuid-sandbox', '--window-size=1920,1080', '--disable-dev-shm-usage']
+    channel: 'chrome',
+    headless: false,
+    viewport: null,
+    args: ['--disable-dev-shm-usage'],
   };
-  
-  // Add executablePath if environment variable is set
-  if (process.env.PUPPETEER_EXECUTABLE_PATH) {
-    options.executablePath = process.env.PUPPETEER_EXECUTABLE_PATH;
-  }
-  
-  return await puppeteer.launch(options);
+
+  // launchPersistentContext returns a BrowserContext directly (not a Browser)
+  const context = await chromium.launchPersistentContext(userDataDir, options);
+  return context;
 }
 
-export async function createPage(browser) {
-  const page = await browser.newPage();
-  
-  // Set viewport to a standard desktop size
-  await page.setViewport({ width: 1920, height: 1080 });
-  
-  // Set a realistic user agent
-  await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
-  
-  return page;
+export async function createPage(browserContext) {
+  // launchPersistentContext already creates a default page
+  const pages = browserContext.pages();
+  if (pages.length > 0) {
+    return pages[0];
+  }
+  return await browserContext.newPage();
 }
 
 export async function handleCookieConsent(page) {
@@ -50,4 +51,4 @@ export async function skipAds(page) {
   } catch (e) {
     // No skip button
   }
-}
+}