channel-worker 1.4.0 → 1.5.0

package/lib/api-client.js

@@ -65,7 +65,7 @@ class ApiClient {
 
   // Commands
   async getNextCommand(workerId) {
-    const workerTypes = 'launch_profile,close_profile,save_file,set_thumbnail,set_tags,set_file_input,click_and_upload,type_text,verify_logins,update_extension';
+    const workerTypes = 'launch_profile,close_profile,save_file,set_thumbnail,set_tags,set_file_input,click_and_upload,type_text,verify_logins,update_extension,sync_youtube_stats';
     return this.request('GET', `/workers/commands?worker_id=${workerId}&types=${encodeURIComponent(workerTypes)}`);
   }
 

package/lib/command-poller.js

@@ -1,5 +1,6 @@
 const { NstManager } = require('./nst-manager');
 const { checkAndUpdateExtension } = require('./extension-updater');
+const { StatsSyncer } = require('./stats-syncer');
 
 class CommandPoller {
   constructor(api, config) {
@@ -68,6 +69,9 @@ class CommandPoller {
         case 'update_extension':
           await this.handleUpdateExtension(command);
           break;
+        case 'sync_youtube_stats':
+          await this.handleSyncYoutubeStats(command);
+          break;
         default:
           // Other commands (scan_facebook_pages, etc.) handled by extension
           console.log(`[commands] Skipping ${command.type} — handled by extension`);
@@ -856,6 +860,48 @@ class CommandPoller {
     // TODO: launch headless, check cookies for Google/Facebook/TikTok
     await this.api.updateCommand(command._id, { status: 'done' });
   }
+
+  async handleSyncYoutubeStats(command) {
+    const { channel_id } = command.payload || {};
+    console.log(`[commands] Syncing YouTube stats for channel: ${channel_id}`);
+
+    try {
+      if (!this.nst) {
+        const apiKey = await this.api.getSetting('nst_api_key');
+        if (apiKey) {
+          this.nst = new NstManager(apiKey);
+        } else {
+          throw new Error('Nstbrowser API key not configured.');
+        }
+      }
+
+      // Fetch channel info
+      const channel = await this.api.getChannel(channel_id);
+      if (!channel) throw new Error(`Channel ${channel_id} not found`);
+      if (!channel.nst_profile_id) throw new Error(`Channel "${channel.name}" has no Nstbrowser profile`);
+
+      const syncer = new StatsSyncer(this.nst, this.api);
+      const stats = await syncer.syncYouTubeStats(channel);
+
+      await this.api.updateCommand(command._id, {
+        status: 'done',
+        result: {
+          channel_id,
+          channel_name: channel.name,
+          ...stats,
+          synced_at: new Date().toISOString(),
+        },
+      });
+
+      console.log(`[commands] YouTube stats synced for "${channel.name}"`);
+    } catch (err) {
+      console.error(`[commands] YouTube stats sync failed: ${err.message}`);
+      await this.api.updateCommand(command._id, {
+        status: 'failed',
+        error: err.message,
+      });
+    }
+  }
 }
 
 module.exports = { CommandPoller };

package/lib/stats-syncer.js

@@ -0,0 +1,385 @@
+const WebSocket = require('ws');
+
+/**
+ * Sync YouTube Studio stats via Nstbrowser CDP.
+ * Flow: Launch profile → connect CDP → navigate to Studio → scrape stats → close
+ */
+class StatsSyncer {
+  constructor(nstManager, apiClient) {
+    this.nst = nstManager;
+    this.api = apiClient;
+  }
+
+  /**
+   * Sync stats for a single channel via YouTube Studio
+   * @param {object} channel - { _id, name, nst_profile_id, proxy }
+   * @returns {object} { subscribers, total_views, video_count }
+   */
+  async syncYouTubeStats(channel) {
+    const profileId = channel.nst_profile_id;
+    if (!profileId) throw new Error(`Channel "${channel.name}" has no nst_profile_id`);
+
+    console.log(`[stats] Syncing YouTube stats for "${channel.name}" (profile: ${profileId})`);
+
+    // Launch profile (no extension needed for stats)
+    const { wsEndpoint } = await this.nst.launchProfile(profileId, {
+      proxy: channel.proxy || null,
+    });
+
+    if (!wsEndpoint) throw new Error('No wsEndpoint returned from Nstbrowser');
+
+    try {
+      const stats = await this.scrapeYouTubeStudio(wsEndpoint);
+      console.log(`[stats] "${channel.name}" → subs: ${stats.subscribers}, views: ${stats.total_views}, videos: ${stats.video_count}`);
+
+      // Send to API
+      await this.api.request('POST', '/analytics/sync', {
+        channel_id: channel._id,
+        platform: 'youtube',
+        subscribers: stats.subscribers,
+        total_views: stats.total_views,
+        video_count: stats.video_count,
+      });
+
+      console.log(`[stats] "${channel.name}" stats saved to API`);
+      return stats;
+    } finally {
+      // Close browser after scraping
+      try {
+        await this.nst.stopProfile(profileId);
+      } catch (err) {
+        console.warn(`[stats] Failed to stop profile: ${err.message}`);
+      }
+    }
+  }
+
+  /**
+   * Connect to browser via CDP WebSocket and scrape YouTube Studio
+   */
+  async scrapeYouTubeStudio(wsEndpoint) {
+    return new Promise((resolve, reject) => {
+      const ws = new WebSocket(wsEndpoint);
+      let msgId = 1;
+      const pending = new Map();
+
+      const send = (method, params = {}) => {
+        return new Promise((res, rej) => {
+          const id = msgId++;
+          pending.set(id, { resolve: res, reject: rej });
+          ws.send(JSON.stringify({ id, method, params }));
+        });
+      };
+
+      const timeout = setTimeout(() => {
+        ws.close();
+        reject(new Error('YouTube Studio scrape timed out (60s)'));
+      }, 60000);
+
+      ws.on('error', (err) => {
+        clearTimeout(timeout);
+        reject(err);
+      });
+
+      ws.on('message', (data) => {
+        try {
+          const msg = JSON.parse(data);
+          if (msg.id && pending.has(msg.id)) {
+            const p = pending.get(msg.id);
+            pending.delete(msg.id);
+            if (msg.error) p.reject(new Error(msg.error.message));
+            else p.resolve(msg.result);
+          }
+        } catch {}
+      });
+
+      ws.on('open', async () => {
+        try {
+          // Get list of targets (pages)
+          const targets = await send('Target.getTargets');
+          const pages = targets.targetInfos.filter(t => t.type === 'page');
+
+          // Find or create a page
+          let targetId;
+          if (pages.length > 0) {
+            targetId = pages[0].targetId;
+          } else {
+            const created = await send('Target.createTarget', { url: 'about:blank' });
+            targetId = created.targetId;
+          }
+
+          // Attach to target
+          const { sessionId } = await send('Target.attachToTarget', {
+            targetId,
+            flatten: true,
+          });
+
+          // Helper to send commands to the attached session
+          const sessionSend = (method, params = {}) => {
+            return new Promise((res, rej) => {
+              const id = msgId++;
+              pending.set(id, { resolve: res, reject: rej });
+              ws.send(JSON.stringify({ id, method, params, sessionId }));
+            });
+          };
+
+          // Enable page events
+          await sessionSend('Page.enable');
+
+          // Navigate to YouTube Studio dashboard
+          await sessionSend('Page.navigate', {
+            url: 'https://studio.youtube.com',
+          });
+
+          // Wait for page to load
+          await this.waitForLoad(sessionSend, 15000);
+
+          // Wait a bit for Studio SPA to render
+          await this.sleep(5000);
+
+          // Try to scrape stats from YouTube Studio dashboard
+          const stats = await this.extractStudioStats(sessionSend);
+
+          clearTimeout(timeout);
+          ws.close();
+          resolve(stats);
+        } catch (err) {
+          clearTimeout(timeout);
+          ws.close();
+          reject(err);
+        }
+      });
+    });
+  }
+
+  async waitForLoad(sessionSend, timeoutMs = 15000) {
+    const start = Date.now();
+    while (Date.now() - start < timeoutMs) {
+      try {
+        const result = await sessionSend('Runtime.evaluate', {
+          expression: 'document.readyState',
+          returnByValue: true,
+        });
+        if (result?.result?.value === 'complete') return;
+      } catch {}
+      await this.sleep(500);
+    }
+  }
+
+  async extractStudioStats(sessionSend) {
+    // YouTube Studio dashboard shows channel-level stats
+    // We'll try multiple approaches to get the data
+
+    // Approach 1: Navigate to channel analytics overview and extract from the page
+    await sessionSend('Page.navigate', {
+      url: 'https://studio.youtube.com/channel/UC/analytics/tab-overview/period-default',
+    });
+    await this.sleep(5000);
+
+    // Try to get subscriber count from the Studio header/sidebar
+    const statsResult = await sessionSend('Runtime.evaluate', {
+      expression: `
+        (function() {
+          const stats = { subscribers: 0, total_views: 0, video_count: 0 };
+
+          // Try to get subscriber count from channel dashboard
+          // YouTube Studio shows subscriber count in various places
+          const allText = document.body.innerText || '';
+
+          // Method 1: Look for subscriber count in the top bar
+          // YouTube Studio typically shows "X subscribers" somewhere
+          const subMatch = allText.match(/([\\d,\\.]+[KMB]?)\\s*subscribers?/i);
+          if (subMatch) {
+            stats.subscribers = parseHumanNumber(subMatch[1]);
+          }
+
+          // Method 2: Look for specific YouTube Studio elements
+          // The channel dashboard header often has subscriber count
+          const headerEl = document.querySelector('#channel-title .subscriber-count, .subscriber-count, [class*="subscriber"]');
+          if (headerEl) {
+            const text = headerEl.textContent.trim();
+            const num = text.match(/([\\d,\\.]+[KMB]?)/);
+            if (num) stats.subscribers = parseHumanNumber(num[1]);
+          }
+
+          // Try ytInitialData or ytcfg for exact numbers
+          try {
+            if (window.ytcfg) {
+              const data = window.ytcfg.data_ || {};
+              // Look through ytcfg for subscriber info
+            }
+          } catch {}
+
+          function parseHumanNumber(str) {
+            if (!str) return 0;
+            str = str.replace(/,/g, '').trim();
+            const mult = str.match(/([\\d\\.]+)\\s*([KMB])/i);
+            if (mult) {
+              const num = parseFloat(mult[1]);
+              const suffix = mult[2].toUpperCase();
+              if (suffix === 'K') return Math.round(num * 1000);
+              if (suffix === 'M') return Math.round(num * 1000000);
+              if (suffix === 'B') return Math.round(num * 1000000000);
+            }
+            return parseInt(str, 10) || 0;
+          }
+
+          return JSON.stringify(stats);
+        })()
+      `,
+      returnByValue: true,
+    });
+
+    let stats = { subscribers: 0, total_views: 0, video_count: 0 };
+    try {
+      stats = JSON.parse(statsResult?.result?.value || '{}');
+    } catch {}
+
+    // Approach 2: Use YouTube Studio's internal API via fetch in page context
+    // This gives exact numbers from the Studio API
+    const apiResult = await sessionSend('Runtime.evaluate', {
+      expression: `
+        (async function() {
+          try {
+            // YouTube Studio uses internal API calls
+            // We can intercept the channel data from the page's __INITIAL_DATA__ or similar
+
+            // Try getting data from the Studio SPA state
+            const scripts = document.querySelectorAll('script');
+            for (const s of scripts) {
+              const text = s.textContent || '';
+
+              // Look for serialized channel data
+              if (text.includes('subscriberCount') || text.includes('subscriber_count')) {
+                const subMatch = text.match(/"subscriberCount"\\s*:\\s*"?(\\d+)"?/);
+                const viewMatch = text.match(/"viewCount"\\s*:\\s*"?(\\d+)"?/);
+                const videoMatch = text.match(/"videoCount"\\s*:\\s*"?(\\d+)"?/);
+
+                if (subMatch || viewMatch || videoMatch) {
+                  return JSON.stringify({
+                    subscribers: parseInt(subMatch?.[1] || '0', 10),
+                    total_views: parseInt(viewMatch?.[1] || '0', 10),
+                    video_count: parseInt(videoMatch?.[1] || '0', 10),
+                  });
+                }
+              }
+            }
+
+            // Fallback: try YouTube Data endpoint from within the page (same origin, authenticated)
+            const res = await fetch('https://studio.youtube.com/youtubei/v1/creator/get_creator_channels?key=AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrGktHmQ', {
+              method: 'POST',
+              headers: { 'Content-Type': 'application/json' },
+              body: JSON.stringify({
+                context: {
+                  client: { clientName: 62, clientVersion: '1.0' },
+                },
+              }),
+            });
+
+            if (res.ok) {
+              const data = await res.json();
+              const ch = data?.channels?.[0] || {};
+              return JSON.stringify({
+                subscribers: parseInt(ch.subscriberCount || '0', 10),
+                total_views: parseInt(ch.viewCount || '0', 10),
+                video_count: parseInt(ch.videoCount || '0', 10),
+              });
+            }
+          } catch (e) {
+            return JSON.stringify({ error: e.message });
+          }
+          return JSON.stringify({});
+        })()
+      `,
+      awaitPromise: true,
+      returnByValue: true,
+    });
+
+    try {
+      const apiStats = JSON.parse(apiResult?.result?.value || '{}');
+      if (!apiStats.error) {
+        // Merge: prefer API data if available
+        if (apiStats.subscribers > 0) stats.subscribers = apiStats.subscribers;
+        if (apiStats.total_views > 0) stats.total_views = apiStats.total_views;
+        if (apiStats.video_count > 0) stats.video_count = apiStats.video_count;
+      }
+    } catch {}
+
+    // Approach 3: Navigate to the public channel page as fallback
+    if (stats.subscribers === 0 && stats.total_views === 0) {
+      console.log('[stats] Studio scrape returned no data, trying public channel page...');
+      await sessionSend('Page.navigate', {
+        url: 'https://www.youtube.com/@me',
+      });
+      await this.waitForLoad(sessionSend, 10000);
+      await this.sleep(3000);
+
+      const publicResult = await sessionSend('Runtime.evaluate', {
+        expression: `
+          (function() {
+            const stats = { subscribers: 0, total_views: 0, video_count: 0 };
+
+            // ytInitialData contains channel info
+            try {
+              const data = window.ytInitialData;
+              const header = data?.header?.c4TabbedHeaderRenderer || data?.header?.pageHeaderRenderer;
+
+              // Subscriber count
+              const subText = header?.subscriberCountText?.simpleText || '';
+              const subMatch = subText.match(/([\\d,\\.]+[KMB]?)/);
+              if (subMatch) {
+                stats.subscribers = parseHumanNumber(subMatch[1]);
+              }
+
+              // Video count from tabs
+              const tabs = data?.contents?.twoColumnBrowseResultsRenderer?.tabs || [];
+              for (const tab of tabs) {
+                const title = tab?.tabRenderer?.title || '';
+                if (title === 'Videos' || title === 'Shorts') {
+                  // Video tab might show count
+                }
+              }
+
+              // Try metadata
+              const meta = data?.metadata?.channelMetadataRenderer;
+              if (meta) {
+                // Some data available here
+              }
+            } catch {}
+
+            function parseHumanNumber(str) {
+              if (!str) return 0;
+              str = str.replace(/,/g, '').trim();
+              const mult = str.match(/([\\d\\.]+)\\s*([KMB])/i);
+              if (mult) {
+                const num = parseFloat(mult[1]);
+                const suffix = mult[2].toUpperCase();
+                if (suffix === 'K') return Math.round(num * 1000);
+                if (suffix === 'M') return Math.round(num * 1000000);
+                if (suffix === 'B') return Math.round(num * 1000000000);
+              }
+              return parseInt(str, 10) || 0;
+            }
+
+            return JSON.stringify(stats);
+          })()
+        `,
+        returnByValue: true,
+      });
+
+      try {
+        const publicStats = JSON.parse(publicResult?.result?.value || '{}');
+        if (publicStats.subscribers > 0) stats.subscribers = publicStats.subscribers;
+        if (publicStats.total_views > 0) stats.total_views = publicStats.total_views;
+        if (publicStats.video_count > 0) stats.video_count = publicStats.video_count;
+      } catch {}
+    }
+
+    return stats;
+  }
+
+  sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+}
+
+module.exports = { StatsSyncer };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "channel-worker",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "description": "Channel Manager worker daemon — runs on remote machines to execute video pipeline jobs",
   "main": "lib/daemon.js",
   "bin": {