channel-worker 1.5.0 → 1.5.1

package/lib/stats-syncer.js

@@ -10,18 +10,12 @@ class StatsSyncer {
     this.api = apiClient;
   }
 
-  /**
-   * Sync stats for a single channel via YouTube Studio
-   * @param {object} channel - { _id, name, nst_profile_id, proxy }
-   * @returns {object} { subscribers, total_views, video_count }
-   */
   async syncYouTubeStats(channel) {
     const profileId = channel.nst_profile_id;
     if (!profileId) throw new Error(`Channel "${channel.name}" has no nst_profile_id`);
 
     console.log(`[stats] Syncing YouTube stats for "${channel.name}" (profile: ${profileId})`);
 
-    // Launch profile (no extension needed for stats)
     const { wsEndpoint } = await this.nst.launchProfile(profileId, {
       proxy: channel.proxy || null,
     });
@@ -29,10 +23,9 @@ class StatsSyncer {
     if (!wsEndpoint) throw new Error('No wsEndpoint returned from Nstbrowser');
 
     try {
-      const stats = await this.scrapeYouTubeStudio(wsEndpoint);
+      const stats = await this.scrapeViaCDP(wsEndpoint);
       console.log(`[stats] "${channel.name}" → subs: ${stats.subscribers}, views: ${stats.total_views}, videos: ${stats.video_count}`);
 
-      // Send to API
       await this.api.request('POST', '/analytics/sync', {
         channel_id: channel._id,
         platform: 'youtube',
@@ -44,7 +37,6 @@ class StatsSyncer {
       console.log(`[stats] "${channel.name}" stats saved to API`);
       return stats;
     } finally {
-      // Close browser after scraping
       try {
         await this.nst.stopProfile(profileId);
       } catch (err) {
@@ -53,32 +45,28 @@ class StatsSyncer {
     }
   }
 
-  /**
-   * Connect to browser via CDP WebSocket and scrape YouTube Studio
-   */
-  async scrapeYouTubeStudio(wsEndpoint) {
+  scrapeViaCDP(wsEndpoint) {
     return new Promise((resolve, reject) => {
       const ws = new WebSocket(wsEndpoint);
       let msgId = 1;
       const pending = new Map();
 
-      const send = (method, params = {}) => {
+      const send = (method, params = {}, sid) => {
         return new Promise((res, rej) => {
           const id = msgId++;
           pending.set(id, { resolve: res, reject: rej });
-          ws.send(JSON.stringify({ id, method, params }));
+          const msg = { id, method, params };
+          if (sid) msg.sessionId = sid;
+          ws.send(JSON.stringify(msg));
         });
       };
 
       const timeout = setTimeout(() => {
         ws.close();
-        reject(new Error('YouTube Studio scrape timed out (60s)'));
-      }, 60000);
+        reject(new Error('Stats scrape timed out (90s)'));
+      }, 90000);
 
-      ws.on('error', (err) => {
-        clearTimeout(timeout);
-        reject(err);
-      });
+      ws.on('error', (err) => { clearTimeout(timeout); reject(err); });
 
       ws.on('message', (data) => {
         try {
@@ -94,50 +82,297 @@ class StatsSyncer {
 
       ws.on('open', async () => {
         try {
-          // Get list of targets (pages)
+          // Get a page target
           const targets = await send('Target.getTargets');
           const pages = targets.targetInfos.filter(t => t.type === 'page');
+          let targetId = pages.length > 0 ? pages[0].targetId : (await send('Target.createTarget', { url: 'about:blank' })).targetId;
+
+          const { sessionId } = await send('Target.attachToTarget', { targetId, flatten: true });
+          const s = (method, params = {}) => send(method, params, sessionId);
+
+          await s('Page.enable');
+          await s('Runtime.enable');
+
+          // Step 1: Go to YouTube Studio — it auto-redirects to the correct channel
+          console.log('[stats] Navigating to YouTube Studio...');
+          await s('Page.navigate', { url: 'https://studio.youtube.com' });
+          await this.waitForLoad(s);
+          await this.sleep(6000); // SPA needs time to render
+
+          // Log current URL for debugging
+          const urlResult = await s('Runtime.evaluate', { expression: 'window.location.href', returnByValue: true });
+          console.log(`[stats] Current URL: ${urlResult?.result?.value}`);
+
+          // Step 2: Try to extract stats
+          let stats = { subscribers: 0, total_views: 0, video_count: 0 };
+
+          // Method A: Dump all visible text and parse (most reliable for YouTube Studio)
+          const dumpResult = await s('Runtime.evaluate', {
+            expression: `
+              (function() {
+                const result = { subscribers: 0, total_views: 0, video_count: 0, debug: '' };
+                const text = document.body?.innerText || '';
+                result.debug = text.substring(0, 500);
+
+                // YouTube Studio shows "X subscribers" in the dashboard
+                // Format varies: "123 subscribers", "1.2K subscribers", "0 subscribers"
+                const subPatterns = [
+                  /Current subscribers\\n([\\d,\\.]+[KMB]?)/i,
+                  /([\\d,\\.]+[KMB]?)\\s*subscriber/i,
+                  /Subscribers?\\n([\\d,\\.]+[KMB]?)/i,
+                ];
+                for (const p of subPatterns) {
+                  const m = text.match(p);
+                  if (m) { result.subscribers = parseNum(m[1]); break; }
+                }
 
-          // Find or create a page
-          let targetId;
-          if (pages.length > 0) {
-            targetId = pages[0].targetId;
-          } else {
-            const created = await send('Target.createTarget', { url: 'about:blank' });
-            targetId = created.targetId;
-          }
+                // Views pattern
+                const viewPatterns = [
+                  /([\\d,\\.]+[KMB]?)\\s*views?\\b/i,
+                  /Views?\\n([\\d,\\.]+[KMB]?)/i,
+                ];
+                for (const p of viewPatterns) {
+                  const m = text.match(p);
+                  if (m) { result.total_views = parseNum(m[1]); break; }
+                }
 
-          // Attach to target
-          const { sessionId } = await send('Target.attachToTarget', {
-            targetId,
-            flatten: true,
-          });
+                function parseNum(str) {
+                  if (!str) return 0;
+                  str = str.replace(/,/g, '').trim();
+                  const m = str.match(/([\\d\\.]+)\\s*([KMB])/i);
+                  if (m) {
+                    const n = parseFloat(m[1]);
+                    const s = m[2].toUpperCase();
+                    if (s === 'K') return Math.round(n * 1000);
+                    if (s === 'M') return Math.round(n * 1000000);
+                    if (s === 'B') return Math.round(n * 1000000000);
+                  }
+                  return parseInt(str, 10) || 0;
+                }
 
-          // Helper to send commands to the attached session
-          const sessionSend = (method, params = {}) => {
-            return new Promise((res, rej) => {
-              const id = msgId++;
-              pending.set(id, { resolve: res, reject: rej });
-              ws.send(JSON.stringify({ id, method, params, sessionId }));
-            });
-          };
+                return JSON.stringify(result);
+              })()
+            `,
+            returnByValue: true,
+          });
 
-          // Enable page events
-          await sessionSend('Page.enable');
+          try {
+            const parsed = JSON.parse(dumpResult?.result?.value || '{}');
+            console.log(`[stats] Studio text dump (first 200): ${(parsed.debug || '').substring(0, 200)}`);
+            if (parsed.subscribers) stats.subscribers = parsed.subscribers;
+            if (parsed.total_views) stats.total_views = parsed.total_views;
+          } catch {}
 
-          // Navigate to YouTube Studio dashboard
-          await sessionSend('Page.navigate', {
-            url: 'https://studio.youtube.com',
+          // Method B: Use YouTube's internal youtubei API (works from studio.youtube.com origin)
+          console.log('[stats] Trying YouTube internal API...');
+          const apiResult = await s('Runtime.evaluate', {
+            expression: `
+              (async function() {
+                try {
+                  // Get SAPISIDHASH for auth (YouTube uses this for internal API calls)
+                  function getSapisidHash(origin) {
+                    const cookies = document.cookie.split(';').reduce((acc, c) => {
+                      const [k, v] = c.trim().split('=');
+                      acc[k] = v;
+                      return acc;
+                    }, {});
+                    const sapisid = cookies['SAPISID'] || cookies['__Secure-3PAPISID'];
+                    if (!sapisid) return null;
+                    const timestamp = Math.floor(Date.now() / 1000);
+                    // Can't compute SHA1 in plain JS easily, but we can use the session cookies directly
+                    return null;
+                  }
+
+                  // Method 1: Use the ytcfg data that Studio loads
+                  if (window.ytcfg && window.ytcfg.get) {
+                    const channelId = window.ytcfg.get('CHANNEL_ID');
+                    if (channelId) {
+                      // Try fetching channel data from YouTube Data API v3 (internal, no key needed from same origin)
+                      const r = await fetch('https://www.youtube.com/youtubei/v1/browse?prettyPrint=false', {
+                        method: 'POST',
+                        headers: {
+                          'Content-Type': 'application/json',
+                          'X-Origin': 'https://www.youtube.com',
+                        },
+                        credentials: 'include',
+                        body: JSON.stringify({
+                          browseId: channelId,
+                          context: {
+                            client: {
+                              clientName: 'WEB',
+                              clientVersion: '2.20240101.00.00',
+                            },
+                          },
+                        }),
+                      });
+                      if (r.ok) {
+                        const d = await r.json();
+                        // Extract from header
+                        const header = d?.header?.c4TabbedHeaderRenderer || {};
+                        const subText = header?.subscriberCountText?.simpleText || '';
+                        const meta = d?.metadata?.channelMetadataRenderer || {};
+
+                        return JSON.stringify({
+                          source: 'youtubei_browse',
+                          channelId,
+                          subscribers_text: subText,
+                        });
+                      }
+                    }
+                  }
+
+                  // Method 2: Parse ytcfg directly for channel info
+                  if (window.ytcfg && window.ytcfg.data_) {
+                    return JSON.stringify({
+                      source: 'ytcfg',
+                      keys: Object.keys(window.ytcfg.data_).filter(k =>
+                        k.toLowerCase().includes('channel') || k.toLowerCase().includes('subscriber')
+                      ),
+                    });
+                  }
+
+                  return JSON.stringify({ source: 'none' });
+                } catch (e) {
+                  return JSON.stringify({ error: e.message });
+                }
+              })()
+            `,
+            awaitPromise: true,
+            returnByValue: true,
           });
 
-          // Wait for page to load
-          await this.waitForLoad(sessionSend, 15000);
+          try {
+            const apiData = JSON.parse(apiResult?.result?.value || '{}');
+            console.log(`[stats] Internal API result: ${JSON.stringify(apiData).substring(0, 300)}`);
+          } catch {}
 
-          // Wait a bit for Studio SPA to render
-          await this.sleep(5000);
+          // Method C: Navigate to channel page to get video count via ytInitialData
+          if (stats.video_count === 0) {
+            console.log('[stats] Getting video count from channel page...');
+
+            // Get the channel URL from Studio
+            const channelUrlResult = await s('Runtime.evaluate', {
+              expression: `
+                (function() {
+                  // Try to find channel URL/ID from Studio page
+                  const url = window.location.href;
+                  const m = url.match(/channel\\/(UC[\\w-]+)/);
+                  if (m) return 'https://www.youtube.com/channel/' + m[1] + '/videos';
+
+                  // Try ytcfg
+                  if (window.ytcfg && window.ytcfg.get) {
+                    const cid = window.ytcfg.get('CHANNEL_ID');
+                    if (cid) return 'https://www.youtube.com/channel/' + cid + '/videos';
+                  }
+
+                  // Fallback: go to own channel
+                  return 'https://www.youtube.com/@me';
+                })()
+              `,
+              returnByValue: true,
+            });
 
-          // Try to scrape stats from YouTube Studio dashboard
-          const stats = await this.extractStudioStats(sessionSend);
+            const channelUrl = channelUrlResult?.result?.value;
+            if (channelUrl) {
+              console.log(`[stats] Navigating to: ${channelUrl}`);
+              await s('Page.navigate', { url: channelUrl });
+              await this.waitForLoad(s);
+              await this.sleep(4000);
+
+              const ytResult = await s('Runtime.evaluate', {
+                expression: `
+                  (function() {
+                    const result = { subscribers: 0, total_views: 0, video_count: 0 };
+                    try {
+                      const data = window.ytInitialData;
+                      if (!data) return JSON.stringify({ error: 'no ytInitialData' });
+
+                      // Subscriber count from header
+                      const header = data?.header?.c4TabbedHeaderRenderer
+                        || data?.header?.pageHeaderRenderer?.content?.pageHeaderViewModel;
+                      if (header) {
+                        // c4TabbedHeaderRenderer path
+                        const subText = header?.subscriberCountText?.simpleText || '';
+                        if (subText) {
+                          const m = subText.match(/([\\d,\\.]+[KMB]?)/);
+                          if (m) result.subscribers = parseNum(m[1]);
+                        }
+                        // pageHeaderViewModel path
+                        const metadata = header?.metadata?.contentMetadataViewModel?.metadataRows;
+                        if (metadata) {
+                          for (const row of metadata) {
+                            for (const part of (row?.metadataParts || [])) {
+                              const t = part?.text?.content || '';
+                              if (t.includes('subscriber')) {
+                                const m = t.match(/([\\d,\\.]+[KMB]?)/);
+                                if (m) result.subscribers = parseNum(m[1]);
+                              }
+                              if (t.includes('video')) {
+                                const m = t.match(/([\\d,\\.]+)/);
+                                if (m) result.video_count = parseInt(m[1].replace(/,/g, ''), 10);
+                              }
+                            }
+                          }
+                        }
+                      }
+
+                      // Video count from tabs
+                      const tabs = data?.contents?.twoColumnBrowseResultsRenderer?.tabs || [];
+                      for (const tab of tabs) {
+                        const tabData = tab?.tabRenderer;
+                        if (tabData?.title === 'Videos') {
+                          // Sometimes the tab content has total count
+                          const content = tabData?.content;
+                          const grid = content?.richGridRenderer || content?.sectionListRenderer;
+                          // Check for video items count
+                        }
+                      }
+
+                      // Total views from about tab data or metadata
+                      const aboutData = data?.metadata?.channelMetadataRenderer;
+                      if (aboutData) {
+                        // metadata doesn't have view count directly, but try
+                      }
+
+                      // Try to get view count from about page data
+                      const microformat = data?.microformat?.microformatDataRenderer;
+                      if (microformat) {
+                        // Some data here
+                      }
+
+                    } catch (e) {
+                      return JSON.stringify({ error: e.message });
+                    }
+
+                    function parseNum(str) {
+                      if (!str) return 0;
+                      str = str.replace(/,/g, '').trim();
+                      const m = str.match(/([\\d\\.]+)\\s*([KMB])/i);
+                      if (m) {
+                        const n = parseFloat(m[1]);
+                        const s = m[2].toUpperCase();
+                        if (s === 'K') return Math.round(n * 1000);
+                        if (s === 'M') return Math.round(n * 1000000);
+                        if (s === 'B') return Math.round(n * 1000000000);
+                      }
+                      return parseInt(str, 10) || 0;
+                    }
+
+                    return JSON.stringify(result);
+                  })()
+                `,
+                returnByValue: true,
+              });
+
+              try {
+                const ytData = JSON.parse(ytResult?.result?.value || '{}');
+                console.log(`[stats] ytInitialData result: ${JSON.stringify(ytData)}`);
+                if (ytData.subscribers > 0 && stats.subscribers === 0) stats.subscribers = ytData.subscribers;
+                if (ytData.video_count > 0) stats.video_count = ytData.video_count;
+                if (ytData.total_views > 0 && stats.total_views === 0) stats.total_views = ytData.total_views;
+              } catch {}
+            }
+          }
 
           clearTimeout(timeout);
           ws.close();
@@ -151,232 +386,17 @@ class StatsSyncer {
     });
   }
 
-  async waitForLoad(sessionSend, timeoutMs = 15000) {
+  async waitForLoad(s, timeoutMs = 15000) {
     const start = Date.now();
     while (Date.now() - start < timeoutMs) {
       try {
-        const result = await sessionSend('Runtime.evaluate', {
-          expression: 'document.readyState',
-          returnByValue: true,
-        });
-        if (result?.result?.value === 'complete') return;
+        const r = await s('Runtime.evaluate', { expression: 'document.readyState', returnByValue: true });
+        if (r?.result?.value === 'complete') return;
       } catch {}
       await this.sleep(500);
     }
   }
 
-  async extractStudioStats(sessionSend) {
-    // YouTube Studio dashboard shows channel-level stats
-    // We'll try multiple approaches to get the data
-
-    // Approach 1: Navigate to channel analytics overview and extract from the page
-    await sessionSend('Page.navigate', {
-      url: 'https://studio.youtube.com/channel/UC/analytics/tab-overview/period-default',
-    });
-    await this.sleep(5000);
-
-    // Try to get subscriber count from the Studio header/sidebar
-    const statsResult = await sessionSend('Runtime.evaluate', {
-      expression: `
-        (function() {
-          const stats = { subscribers: 0, total_views: 0, video_count: 0 };
-
-          // Try to get subscriber count from channel dashboard
-          // YouTube Studio shows subscriber count in various places
-          const allText = document.body.innerText || '';
-
-          // Method 1: Look for subscriber count in the top bar
-          // YouTube Studio typically shows "X subscribers" somewhere
-          const subMatch = allText.match(/([\\d,\\.]+[KMB]?)\\s*subscribers?/i);
-          if (subMatch) {
-            stats.subscribers = parseHumanNumber(subMatch[1]);
-          }
-
-          // Method 2: Look for specific YouTube Studio elements
-          // The channel dashboard header often has subscriber count
-          const headerEl = document.querySelector('#channel-title .subscriber-count, .subscriber-count, [class*="subscriber"]');
-          if (headerEl) {
-            const text = headerEl.textContent.trim();
-            const num = text.match(/([\\d,\\.]+[KMB]?)/);
-            if (num) stats.subscribers = parseHumanNumber(num[1]);
-          }
-
-          // Try ytInitialData or ytcfg for exact numbers
-          try {
-            if (window.ytcfg) {
-              const data = window.ytcfg.data_ || {};
-              // Look through ytcfg for subscriber info
-            }
-          } catch {}
-
-          function parseHumanNumber(str) {
-            if (!str) return 0;
-            str = str.replace(/,/g, '').trim();
-            const mult = str.match(/([\\d\\.]+)\\s*([KMB])/i);
-            if (mult) {
-              const num = parseFloat(mult[1]);
-              const suffix = mult[2].toUpperCase();
-              if (suffix === 'K') return Math.round(num * 1000);
-              if (suffix === 'M') return Math.round(num * 1000000);
-              if (suffix === 'B') return Math.round(num * 1000000000);
-            }
-            return parseInt(str, 10) || 0;
-          }
-
-          return JSON.stringify(stats);
-        })()
-      `,
-      returnByValue: true,
-    });
-
-    let stats = { subscribers: 0, total_views: 0, video_count: 0 };
-    try {
-      stats = JSON.parse(statsResult?.result?.value || '{}');
-    } catch {}
-
-    // Approach 2: Use YouTube Studio's internal API via fetch in page context
-    // This gives exact numbers from the Studio API
-    const apiResult = await sessionSend('Runtime.evaluate', {
-      expression: `
-        (async function() {
-          try {
-            // YouTube Studio uses internal API calls
-            // We can intercept the channel data from the page's __INITIAL_DATA__ or similar
-
-            // Try getting data from the Studio SPA state
-            const scripts = document.querySelectorAll('script');
-            for (const s of scripts) {
-              const text = s.textContent || '';
-
-              // Look for serialized channel data
-              if (text.includes('subscriberCount') || text.includes('subscriber_count')) {
-                const subMatch = text.match(/"subscriberCount"\\s*:\\s*"?(\\d+)"?/);
-                const viewMatch = text.match(/"viewCount"\\s*:\\s*"?(\\d+)"?/);
-                const videoMatch = text.match(/"videoCount"\\s*:\\s*"?(\\d+)"?/);
-
-                if (subMatch || viewMatch || videoMatch) {
-                  return JSON.stringify({
-                    subscribers: parseInt(subMatch?.[1] || '0', 10),
-                    total_views: parseInt(viewMatch?.[1] || '0', 10),
-                    video_count: parseInt(videoMatch?.[1] || '0', 10),
-                  });
-                }
-              }
-            }
-
-            // Fallback: try YouTube Data endpoint from within the page (same origin, authenticated)
-            const res = await fetch('https://studio.youtube.com/youtubei/v1/creator/get_creator_channels?key=AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrGktHmQ', {
-              method: 'POST',
-              headers: { 'Content-Type': 'application/json' },
-              body: JSON.stringify({
-                context: {
-                  client: { clientName: 62, clientVersion: '1.0' },
-                },
-              }),
-            });
-
-            if (res.ok) {
-              const data = await res.json();
-              const ch = data?.channels?.[0] || {};
-              return JSON.stringify({
-                subscribers: parseInt(ch.subscriberCount || '0', 10),
-                total_views: parseInt(ch.viewCount || '0', 10),
-                video_count: parseInt(ch.videoCount || '0', 10),
-              });
-            }
-          } catch (e) {
-            return JSON.stringify({ error: e.message });
-          }
-          return JSON.stringify({});
-        })()
-      `,
-      awaitPromise: true,
-      returnByValue: true,
-    });
-
-    try {
-      const apiStats = JSON.parse(apiResult?.result?.value || '{}');
-      if (!apiStats.error) {
-        // Merge: prefer API data if available
-        if (apiStats.subscribers > 0) stats.subscribers = apiStats.subscribers;
-        if (apiStats.total_views > 0) stats.total_views = apiStats.total_views;
-        if (apiStats.video_count > 0) stats.video_count = apiStats.video_count;
-      }
-    } catch {}
-
-    // Approach 3: Navigate to the public channel page as fallback
-    if (stats.subscribers === 0 && stats.total_views === 0) {
-      console.log('[stats] Studio scrape returned no data, trying public channel page...');
-      await sessionSend('Page.navigate', {
-        url: 'https://www.youtube.com/@me',
-      });
-      await this.waitForLoad(sessionSend, 10000);
-      await this.sleep(3000);
-
-      const publicResult = await sessionSend('Runtime.evaluate', {
-        expression: `
-          (function() {
-            const stats = { subscribers: 0, total_views: 0, video_count: 0 };
-
-            // ytInitialData contains channel info
-            try {
-              const data = window.ytInitialData;
-              const header = data?.header?.c4TabbedHeaderRenderer || data?.header?.pageHeaderRenderer;
-
-              // Subscriber count
-              const subText = header?.subscriberCountText?.simpleText || '';
-              const subMatch = subText.match(/([\\d,\\.]+[KMB]?)/);
-              if (subMatch) {
-                stats.subscribers = parseHumanNumber(subMatch[1]);
-              }
-
-              // Video count from tabs
-              const tabs = data?.contents?.twoColumnBrowseResultsRenderer?.tabs || [];
-              for (const tab of tabs) {
-                const title = tab?.tabRenderer?.title || '';
-                if (title === 'Videos' || title === 'Shorts') {
-                  // Video tab might show count
-                }
-              }
-
-              // Try metadata
-              const meta = data?.metadata?.channelMetadataRenderer;
-              if (meta) {
-                // Some data available here
-              }
-            } catch {}
-
-            function parseHumanNumber(str) {
-              if (!str) return 0;
-              str = str.replace(/,/g, '').trim();
-              const mult = str.match(/([\\d\\.]+)\\s*([KMB])/i);
-              if (mult) {
-                const num = parseFloat(mult[1]);
-                const suffix = mult[2].toUpperCase();
-                if (suffix === 'K') return Math.round(num * 1000);
-                if (suffix === 'M') return Math.round(num * 1000000);
-                if (suffix === 'B') return Math.round(num * 1000000000);
-              }
-              return parseInt(str, 10) || 0;
-            }
-
-            return JSON.stringify(stats);
-          })()
-        `,
-        returnByValue: true,
-      });
-
-      try {
-        const publicStats = JSON.parse(publicResult?.result?.value || '{}');
-        if (publicStats.subscribers > 0) stats.subscribers = publicStats.subscribers;
-        if (publicStats.total_views > 0) stats.total_views = publicStats.total_views;
-        if (publicStats.video_count > 0) stats.video_count = publicStats.video_count;
-      } catch {}
-    }
-
-    return stats;
-  }
-
   sleep(ms) {
     return new Promise(resolve => setTimeout(resolve, ms));
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "channel-worker",
-  "version": "1.5.0",
+  "version": "1.5.1",
   "description": "Channel Manager worker daemon — runs on remote machines to execute video pipeline jobs",
   "main": "lib/daemon.js",
   "bin": {