channel-worker 1.5.0 → 1.5.2

package/lib/stats-syncer.js

@@ -10,18 +10,12 @@ class StatsSyncer {
     this.api = apiClient;
   }
 
-  /**
-   * Sync stats for a single channel via YouTube Studio
-   * @param {object} channel - { _id, name, nst_profile_id, proxy }
-   * @returns {object} { subscribers, total_views, video_count }
-   */
   async syncYouTubeStats(channel) {
     const profileId = channel.nst_profile_id;
     if (!profileId) throw new Error(`Channel "${channel.name}" has no nst_profile_id`);
 
     console.log(`[stats] Syncing YouTube stats for "${channel.name}" (profile: ${profileId})`);
 
-    // Launch profile (no extension needed for stats)
     const { wsEndpoint } = await this.nst.launchProfile(profileId, {
       proxy: channel.proxy || null,
     });
@@ -29,10 +23,9 @@ class StatsSyncer {
     if (!wsEndpoint) throw new Error('No wsEndpoint returned from Nstbrowser');
 
     try {
-      const stats = await this.scrapeYouTubeStudio(wsEndpoint);
+      const stats = await this.scrapeViaCDP(wsEndpoint);
       console.log(`[stats] "${channel.name}" → subs: ${stats.subscribers}, views: ${stats.total_views}, videos: ${stats.video_count}`);
 
-      // Send to API
       await this.api.request('POST', '/analytics/sync', {
         channel_id: channel._id,
         platform: 'youtube',
@@ -44,7 +37,6 @@ class StatsSyncer {
       console.log(`[stats] "${channel.name}" stats saved to API`);
       return stats;
     } finally {
-      // Close browser after scraping
       try {
         await this.nst.stopProfile(profileId);
       } catch (err) {
@@ -53,32 +45,28 @@ class StatsSyncer {
     }
   }
 
-  /**
-   * Connect to browser via CDP WebSocket and scrape YouTube Studio
-   */
-  async scrapeYouTubeStudio(wsEndpoint) {
+  scrapeViaCDP(wsEndpoint) {
     return new Promise((resolve, reject) => {
       const ws = new WebSocket(wsEndpoint);
       let msgId = 1;
       const pending = new Map();
 
-      const send = (method, params = {}) => {
+      const send = (method, params = {}, sid) => {
         return new Promise((res, rej) => {
           const id = msgId++;
           pending.set(id, { resolve: res, reject: rej });
-          ws.send(JSON.stringify({ id, method, params }));
+          const msg = { id, method, params };
+          if (sid) msg.sessionId = sid;
+          ws.send(JSON.stringify(msg));
         });
       };
 
       const timeout = setTimeout(() => {
         ws.close();
-        reject(new Error('YouTube Studio scrape timed out (60s)'));
-      }, 60000);
+        reject(new Error('Stats scrape timed out (90s)'));
+      }, 90000);
 
-      ws.on('error', (err) => {
-        clearTimeout(timeout);
-        reject(err);
-      });
+      ws.on('error', (err) => { clearTimeout(timeout); reject(err); });
 
       ws.on('message', (data) => {
         try {
@@ -94,50 +82,231 @@ class StatsSyncer {
 
       ws.on('open', async () => {
         try {
-          // Get list of targets (pages)
           const targets = await send('Target.getTargets');
           const pages = targets.targetInfos.filter(t => t.type === 'page');
+          const targetId = pages.length > 0 ? pages[0].targetId : (await send('Target.createTarget', { url: 'about:blank' })).targetId;
 
-          // Find or create a page
-          let targetId;
-          if (pages.length > 0) {
-            targetId = pages[0].targetId;
-          } else {
-            const created = await send('Target.createTarget', { url: 'about:blank' });
-            targetId = created.targetId;
-          }
+          const { sessionId } = await send('Target.attachToTarget', { targetId, flatten: true });
+          const s = (method, params = {}) => send(method, params, sessionId);
 
-          // Attach to target
-          const { sessionId } = await send('Target.attachToTarget', {
-            targetId,
-            flatten: true,
-          });
+          await s('Page.enable');
+          await s('Runtime.enable');
+
+          let stats = { subscribers: 0, total_views: 0, video_count: 0 };
+
+          // Step 1: Go to YouTube Studio → auto-redirects to correct channel
+          console.log('[stats] Navigating to YouTube Studio...');
+          await s('Page.navigate', { url: 'https://studio.youtube.com' });
+          await this.waitForLoad(s);
+          await this.sleep(5000);
+
+          // Get channel ID from URL
+          const urlResult = await s('Runtime.evaluate', { expression: 'window.location.href', returnByValue: true });
+          const studioUrl = urlResult?.result?.value || '';
+          console.log(`[stats] Studio URL: ${studioUrl}`);
+
+          const channelIdMatch = studioUrl.match(/channel\/(UC[\w-]+)/);
+          const channelId = channelIdMatch ? channelIdMatch[1] : null;
 
-          // Helper to send commands to the attached session
-          const sessionSend = (method, params = {}) => {
-            return new Promise((res, rej) => {
-              const id = msgId++;
-              pending.set(id, { resolve: res, reject: rej });
-              ws.send(JSON.stringify({ id, method, params, sessionId }));
+          if (!channelId) {
+            // Try ytcfg
+            const cfgResult = await s('Runtime.evaluate', {
+              expression: '(window.ytcfg && window.ytcfg.get) ? window.ytcfg.get("CHANNEL_ID") : ""',
+              returnByValue: true,
             });
-          };
+            console.log(`[stats] ytcfg CHANNEL_ID: ${cfgResult?.result?.value}`);
+          }
 
-          // Enable page events
-          await sessionSend('Page.enable');
+          const cid = channelId || (await s('Runtime.evaluate', {
+            expression: '(window.ytcfg && window.ytcfg.get) ? window.ytcfg.get("CHANNEL_ID") : ""',
+            returnByValue: true,
+          }))?.result?.value;
+
+          // Step 2: Navigate to Studio Analytics page (English URL works regardless of UI language)
+          if (cid) {
+            const analyticsUrl = `https://studio.youtube.com/channel/${cid}/analytics/tab-overview/period-default`;
+            console.log(`[stats] Navigating to analytics: ${analyticsUrl}`);
+            await s('Page.navigate', { url: analyticsUrl });
+            await this.waitForLoad(s);
+            await this.sleep(6000);
+
+            // Scrape analytics page — look for numbers regardless of language
+            const analyticsResult = await s('Runtime.evaluate', {
+              expression: `
+                (function() {
+                  var result = { subscribers: 0, total_views: 0, video_count: 0, debug_text: '' };
+                  var text = document.body ? document.body.innerText : '';
+                  result.debug_text = text.substring(0, 800);
+
+                  // YouTube Studio Analytics shows metric cards with labels and numbers
+                  // The layout is: Label (in any language) followed by a number
+                  // We look for large numbers near subscriber/view keywords in ANY language
+
+                  // Strategy: find all metric-like elements by looking for number patterns
+                  // Studio analytics typically shows: Views, Watch time, Subscribers
+                  // in cards with the number prominently displayed
+
+                  // Try to find subscriber-related numbers
+                  // Multi-language patterns: subscribers, người đăng ký, abonnés, Abonnenten, etc.
+                  var subKeywords = /subscri|đăng ký|abonn|подписч|구독/i;
+                  var viewKeywords = /views?|lượt xem|vues?|aufrufe|просмотр|조회/i;
+
+                  var lines = text.split('\\n').map(function(l) { return l.trim(); }).filter(function(l) { return l.length > 0; });
+
+                  for (var i = 0; i < lines.length; i++) {
+                    var line = lines[i];
+                    var nextLine = lines[i + 1] || '';
+                    var prevLine = lines[i - 1] || '';
+
+                    // Check if this line or adjacent lines contain keywords
+                    if (subKeywords.test(line) || subKeywords.test(prevLine)) {
+                      // Look for a number in this line or nearby
+                      var numInLine = extractNumber(line);
+                      var numInNext = extractNumber(nextLine);
+                      if (numInLine > 0 && result.subscribers === 0) result.subscribers = numInLine;
+                      else if (numInNext > 0 && result.subscribers === 0) result.subscribers = numInNext;
+                    }
+
+                    if (viewKeywords.test(line) || viewKeywords.test(prevLine)) {
+                      var numInLine2 = extractNumber(line);
+                      var numInNext2 = extractNumber(nextLine);
+                      if (numInLine2 > 0 && result.total_views === 0) result.total_views = numInLine2;
+                      else if (numInNext2 > 0 && result.total_views === 0) result.total_views = numInNext2;
+                    }
+                  }
+
+                  // Also try: Current subscribers count shown in Studio sidebar
+                  // Pattern: "Current subscribers\\n123" or "Người đăng ký hiện tại\\n123"
+                  var currentSubMatch = text.match(/(?:current subscribers|người đăng ký hiện tại|subscriber|đăng ký)[\\s\\n]*([\\d,\\.]+[KMB]?)/i);
+                  if (currentSubMatch && result.subscribers === 0) {
+                    result.subscribers = parseHumanNum(currentSubMatch[1]);
+                  }
+
+                  function extractNumber(str) {
+                    if (!str) return 0;
+                    // Match standalone numbers like "1,234" or "5.2K" or "123"
+                    var m = str.match(/^([\\d,\\.]+[KMB]?)$/i) || str.match(/\\b([\\d,\\.]+[KMB]?)\\b/);
+                    if (m) return parseHumanNum(m[1]);
+                    return 0;
+                  }
+
+                  function parseHumanNum(str) {
+                    if (!str) return 0;
+                    str = str.replace(/,/g, '').trim();
+                    var m = str.match(/([\\d\\.]+)\\s*([KMB])/i);
+                    if (m) {
+                      var n = parseFloat(m[1]);
+                      var s = m[2].toUpperCase();
+                      if (s === 'K') return Math.round(n * 1000);
+                      if (s === 'M') return Math.round(n * 1000000);
+                      if (s === 'B') return Math.round(n * 1000000000);
+                    }
+                    return parseInt(str, 10) || 0;
+                  }
+
+                  return JSON.stringify(result);
+                })()
+              `,
+              returnByValue: true,
+            });
 
-          // Navigate to YouTube Studio dashboard
-          await sessionSend('Page.navigate', {
-            url: 'https://studio.youtube.com',
-          });
+            try {
+              var parsed = JSON.parse(analyticsResult?.result?.value || '{}');
+              console.log('[stats] Analytics debug text (first 300):', (parsed.debug_text || '').substring(0, 300));
+              if (parsed.subscribers > 0) stats.subscribers = parsed.subscribers;
+              if (parsed.total_views > 0) stats.total_views = parsed.total_views;
+            } catch {}
+          }
 
-          // Wait for page to load
-          await this.waitForLoad(sessionSend, 15000);
+          // Step 3: Get video count + subscriber from channel page via ytInitialData
+          var channelPageUrl = cid
+            ? 'https://www.youtube.com/channel/' + cid
+            : 'https://www.youtube.com/@me';
+
+          console.log('[stats] Navigating to channel page:', channelPageUrl);
+          await s('Page.navigate', { url: channelPageUrl });
+          await this.waitForLoad(s);
+          await this.sleep(4000);
+
+          var ytResult = await s('Runtime.evaluate', {
+            expression: `
+              (function() {
+                var result = { subscribers: 0, total_views: 0, video_count: 0 };
+                try {
+                  var data = window.ytInitialData;
+                  if (!data) return JSON.stringify({ error: 'no ytInitialData' });
+
+                  // Try pageHeaderRenderer (new layout)
+                  var pageHeader = data.header && data.header.pageHeaderRenderer;
+                  if (pageHeader) {
+                    var content = pageHeader.content && pageHeader.content.pageHeaderViewModel;
+                    if (content && content.metadata && content.metadata.contentMetadataViewModel) {
+                      var rows = content.metadata.contentMetadataViewModel.metadataRows || [];
+                      for (var r = 0; r < rows.length; r++) {
+                        var parts = rows[r].metadataParts || [];
+                        for (var p = 0; p < parts.length; p++) {
+                          var t = (parts[p].text && parts[p].text.content) || '';
+                          // subscriber text like "123 subscribers" or "0 người đăng ký"
+                          if (/subscri|đăng ký/i.test(t)) {
+                            var m = t.match(/([\\d,\\.]+[KMB]?)/);
+                            if (m) result.subscribers = parseN(m[1]);
+                          }
+                          // video count like "1 video" or "15 video"
+                          if (/video/i.test(t)) {
+                            var m2 = t.match(/([\\d,]+)/);
+                            if (m2) result.video_count = parseInt(m2[1].replace(/,/g, ''), 10);
+                          }
+                        }
+                      }
+                    }
+                  }
+
+                  // Try c4TabbedHeaderRenderer (old layout)
+                  var c4Header = data.header && data.header.c4TabbedHeaderRenderer;
+                  if (c4Header) {
+                    var subText = c4Header.subscriberCountText && c4Header.subscriberCountText.simpleText || '';
+                    if (subText) {
+                      var m3 = subText.match(/([\\d,\\.]+[KMB]?)/);
+                      if (m3 && result.subscribers === 0) result.subscribers = parseN(m3[1]);
+                    }
+                    if (c4Header.videosCountText) {
+                      var vText = c4Header.videosCountText.runs ? c4Header.videosCountText.runs.map(function(r){return r.text}).join('') : (c4Header.videosCountText.simpleText || '');
+                      var m4 = vText.match(/([\\d,]+)/);
+                      if (m4) result.video_count = parseInt(m4[1].replace(/,/g, ''), 10);
+                    }
+                  }
+
+                } catch (e) {
+                  return JSON.stringify({ error: e.message });
+                }
 
-          // Wait a bit for Studio SPA to render
-          await this.sleep(5000);
+                function parseN(str) {
+                  if (!str) return 0;
+                  str = str.replace(/,/g, '').trim();
+                  var m = str.match(/([\\d\\.]+)\\s*([KMB])/i);
+                  if (m) {
+                    var n = parseFloat(m[1]);
+                    var s = m[2].toUpperCase();
+                    if (s === 'K') return Math.round(n * 1000);
+                    if (s === 'M') return Math.round(n * 1000000);
+                    if (s === 'B') return Math.round(n * 1000000000);
+                  }
+                  return parseInt(str, 10) || 0;
+                }
+
+                return JSON.stringify(result);
+              })()
+            `,
+            returnByValue: true,
+          });
 
-          // Try to scrape stats from YouTube Studio dashboard
-          const stats = await this.extractStudioStats(sessionSend);
+          try {
+            var ytData = JSON.parse(ytResult?.result?.value || '{}');
+            console.log('[stats] ytInitialData result:', JSON.stringify(ytData));
+            if (ytData.subscribers > 0 && stats.subscribers === 0) stats.subscribers = ytData.subscribers;
+            if (ytData.video_count > 0) stats.video_count = ytData.video_count;
+            if (ytData.total_views > 0 && stats.total_views === 0) stats.total_views = ytData.total_views;
+          } catch {}
 
           clearTimeout(timeout);
           ws.close();
@@ -151,232 +320,17 @@ class StatsSyncer {
     });
   }
 
-  async waitForLoad(sessionSend, timeoutMs = 15000) {
+  async waitForLoad(s, timeoutMs = 15000) {
     const start = Date.now();
     while (Date.now() - start < timeoutMs) {
       try {
-        const result = await sessionSend('Runtime.evaluate', {
-          expression: 'document.readyState',
-          returnByValue: true,
-        });
-        if (result?.result?.value === 'complete') return;
+        const r = await s('Runtime.evaluate', { expression: 'document.readyState', returnByValue: true });
+        if (r?.result?.value === 'complete') return;
       } catch {}
       await this.sleep(500);
     }
   }
 
-  async extractStudioStats(sessionSend) {
-    // YouTube Studio dashboard shows channel-level stats
-    // We'll try multiple approaches to get the data
-
-    // Approach 1: Navigate to channel analytics overview and extract from the page
-    await sessionSend('Page.navigate', {
-      url: 'https://studio.youtube.com/channel/UC/analytics/tab-overview/period-default',
-    });
-    await this.sleep(5000);
-
-    // Try to get subscriber count from the Studio header/sidebar
-    const statsResult = await sessionSend('Runtime.evaluate', {
-      expression: `
-        (function() {
-          const stats = { subscribers: 0, total_views: 0, video_count: 0 };
-
-          // Try to get subscriber count from channel dashboard
-          // YouTube Studio shows subscriber count in various places
-          const allText = document.body.innerText || '';
-
-          // Method 1: Look for subscriber count in the top bar
-          // YouTube Studio typically shows "X subscribers" somewhere
-          const subMatch = allText.match(/([\\d,\\.]+[KMB]?)\\s*subscribers?/i);
-          if (subMatch) {
-            stats.subscribers = parseHumanNumber(subMatch[1]);
-          }
-
-          // Method 2: Look for specific YouTube Studio elements
-          // The channel dashboard header often has subscriber count
-          const headerEl = document.querySelector('#channel-title .subscriber-count, .subscriber-count, [class*="subscriber"]');
-          if (headerEl) {
-            const text = headerEl.textContent.trim();
-            const num = text.match(/([\\d,\\.]+[KMB]?)/);
-            if (num) stats.subscribers = parseHumanNumber(num[1]);
-          }
-
-          // Try ytInitialData or ytcfg for exact numbers
-          try {
-            if (window.ytcfg) {
-              const data = window.ytcfg.data_ || {};
-              // Look through ytcfg for subscriber info
-            }
-          } catch {}
-
-          function parseHumanNumber(str) {
-            if (!str) return 0;
-            str = str.replace(/,/g, '').trim();
-            const mult = str.match(/([\\d\\.]+)\\s*([KMB])/i);
-            if (mult) {
-              const num = parseFloat(mult[1]);
-              const suffix = mult[2].toUpperCase();
-              if (suffix === 'K') return Math.round(num * 1000);
-              if (suffix === 'M') return Math.round(num * 1000000);
-              if (suffix === 'B') return Math.round(num * 1000000000);
-            }
-            return parseInt(str, 10) || 0;
-          }
-
-          return JSON.stringify(stats);
-        })()
-      `,
-      returnByValue: true,
-    });
-
-    let stats = { subscribers: 0, total_views: 0, video_count: 0 };
-    try {
-      stats = JSON.parse(statsResult?.result?.value || '{}');
-    } catch {}
-
-    // Approach 2: Use YouTube Studio's internal API via fetch in page context
-    // This gives exact numbers from the Studio API
-    const apiResult = await sessionSend('Runtime.evaluate', {
-      expression: `
-        (async function() {
-          try {
-            // YouTube Studio uses internal API calls
-            // We can intercept the channel data from the page's __INITIAL_DATA__ or similar
-
-            // Try getting data from the Studio SPA state
-            const scripts = document.querySelectorAll('script');
-            for (const s of scripts) {
-              const text = s.textContent || '';
-
-              // Look for serialized channel data
-              if (text.includes('subscriberCount') || text.includes('subscriber_count')) {
-                const subMatch = text.match(/"subscriberCount"\\s*:\\s*"?(\\d+)"?/);
-                const viewMatch = text.match(/"viewCount"\\s*:\\s*"?(\\d+)"?/);
-                const videoMatch = text.match(/"videoCount"\\s*:\\s*"?(\\d+)"?/);
-
-                if (subMatch || viewMatch || videoMatch) {
-                  return JSON.stringify({
-                    subscribers: parseInt(subMatch?.[1] || '0', 10),
-                    total_views: parseInt(viewMatch?.[1] || '0', 10),
-                    video_count: parseInt(videoMatch?.[1] || '0', 10),
-                  });
-                }
-              }
-            }
-
-            // Fallback: try YouTube Data endpoint from within the page (same origin, authenticated)
-            const res = await fetch('https://studio.youtube.com/youtubei/v1/creator/get_creator_channels?key=AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrGktHmQ', {
-              method: 'POST',
-              headers: { 'Content-Type': 'application/json' },
-              body: JSON.stringify({
-                context: {
-                  client: { clientName: 62, clientVersion: '1.0' },
-                },
-              }),
-            });
-
-            if (res.ok) {
-              const data = await res.json();
-              const ch = data?.channels?.[0] || {};
-              return JSON.stringify({
-                subscribers: parseInt(ch.subscriberCount || '0', 10),
-                total_views: parseInt(ch.viewCount || '0', 10),
-                video_count: parseInt(ch.videoCount || '0', 10),
-              });
-            }
-          } catch (e) {
-            return JSON.stringify({ error: e.message });
-          }
-          return JSON.stringify({});
-        })()
-      `,
-      awaitPromise: true,
-      returnByValue: true,
-    });
-
-    try {
-      const apiStats = JSON.parse(apiResult?.result?.value || '{}');
-      if (!apiStats.error) {
-        // Merge: prefer API data if available
-        if (apiStats.subscribers > 0) stats.subscribers = apiStats.subscribers;
-        if (apiStats.total_views > 0) stats.total_views = apiStats.total_views;
-        if (apiStats.video_count > 0) stats.video_count = apiStats.video_count;
-      }
-    } catch {}
-
-    // Approach 3: Navigate to the public channel page as fallback
-    if (stats.subscribers === 0 && stats.total_views === 0) {
-      console.log('[stats] Studio scrape returned no data, trying public channel page...');
-      await sessionSend('Page.navigate', {
-        url: 'https://www.youtube.com/@me',
-      });
-      await this.waitForLoad(sessionSend, 10000);
-      await this.sleep(3000);
-
-      const publicResult = await sessionSend('Runtime.evaluate', {
-        expression: `
-          (function() {
-            const stats = { subscribers: 0, total_views: 0, video_count: 0 };
-
-            // ytInitialData contains channel info
-            try {
-              const data = window.ytInitialData;
-              const header = data?.header?.c4TabbedHeaderRenderer || data?.header?.pageHeaderRenderer;
-
-              // Subscriber count
-              const subText = header?.subscriberCountText?.simpleText || '';
-              const subMatch = subText.match(/([\\d,\\.]+[KMB]?)/);
-              if (subMatch) {
-                stats.subscribers = parseHumanNumber(subMatch[1]);
-              }
-
-              // Video count from tabs
-              const tabs = data?.contents?.twoColumnBrowseResultsRenderer?.tabs || [];
-              for (const tab of tabs) {
-                const title = tab?.tabRenderer?.title || '';
-                if (title === 'Videos' || title === 'Shorts') {
-                  // Video tab might show count
-                }
-              }
-
-              // Try metadata
-              const meta = data?.metadata?.channelMetadataRenderer;
-              if (meta) {
-                // Some data available here
-              }
-            } catch {}
-
-            function parseHumanNumber(str) {
-              if (!str) return 0;
-              str = str.replace(/,/g, '').trim();
-              const mult = str.match(/([\\d\\.]+)\\s*([KMB])/i);
-              if (mult) {
-                const num = parseFloat(mult[1]);
-                const suffix = mult[2].toUpperCase();
-                if (suffix === 'K') return Math.round(num * 1000);
-                if (suffix === 'M') return Math.round(num * 1000000);
-                if (suffix === 'B') return Math.round(num * 1000000000);
-              }
-              return parseInt(str, 10) || 0;
-            }
-
-            return JSON.stringify(stats);
-          })()
-        `,
-        returnByValue: true,
-      });
-
-      try {
-        const publicStats = JSON.parse(publicResult?.result?.value || '{}');
-        if (publicStats.subscribers > 0) stats.subscribers = publicStats.subscribers;
-        if (publicStats.total_views > 0) stats.total_views = publicStats.total_views;
-        if (publicStats.video_count > 0) stats.video_count = publicStats.video_count;
-      } catch {}
-    }
-
-    return stats;
-  }
-
   sleep(ms) {
     return new Promise(resolve => setTimeout(resolve, ms));
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "channel-worker",
-  "version": "1.5.0",
+  "version": "1.5.2",
   "description": "Channel Manager worker daemon — runs on remote machines to execute video pipeline jobs",
   "main": "lib/daemon.js",
   "bin": {