channel-worker 1.4.0 → 1.5.1

package/lib/api-client.js

@@ -65,7 +65,7 @@ class ApiClient {
 
   // Commands
   async getNextCommand(workerId) {
-    const workerTypes = 'launch_profile,close_profile,save_file,set_thumbnail,set_tags,set_file_input,click_and_upload,type_text,verify_logins,update_extension';
+    const workerTypes = 'launch_profile,close_profile,save_file,set_thumbnail,set_tags,set_file_input,click_and_upload,type_text,verify_logins,update_extension,sync_youtube_stats';
     return this.request('GET', `/workers/commands?worker_id=${workerId}&types=${encodeURIComponent(workerTypes)}`);
   }
 

package/lib/command-poller.js

@@ -1,5 +1,6 @@
 const { NstManager } = require('./nst-manager');
 const { checkAndUpdateExtension } = require('./extension-updater');
+const { StatsSyncer } = require('./stats-syncer');
 
 class CommandPoller {
   constructor(api, config) {
@@ -68,6 +69,9 @@ class CommandPoller {
         case 'update_extension':
           await this.handleUpdateExtension(command);
           break;
+        case 'sync_youtube_stats':
+          await this.handleSyncYoutubeStats(command);
+          break;
         default:
           // Other commands (scan_facebook_pages, etc.) handled by extension
           console.log(`[commands] Skipping ${command.type} — handled by extension`);
@@ -856,6 +860,48 @@ class CommandPoller {
     // TODO: launch headless, check cookies for Google/Facebook/TikTok
     await this.api.updateCommand(command._id, { status: 'done' });
   }
+
+  async handleSyncYoutubeStats(command) {
+    const { channel_id } = command.payload || {};
+    console.log(`[commands] Syncing YouTube stats for channel: ${channel_id}`);
+
+    try {
+      if (!this.nst) {
+        const apiKey = await this.api.getSetting('nst_api_key');
+        if (apiKey) {
+          this.nst = new NstManager(apiKey);
+        } else {
+          throw new Error('Nstbrowser API key not configured.');
+        }
+      }
+
+      // Fetch channel info
+      const channel = await this.api.getChannel(channel_id);
+      if (!channel) throw new Error(`Channel ${channel_id} not found`);
+      if (!channel.nst_profile_id) throw new Error(`Channel "${channel.name}" has no Nstbrowser profile`);
+
+      const syncer = new StatsSyncer(this.nst, this.api);
+      const stats = await syncer.syncYouTubeStats(channel);
+
+      await this.api.updateCommand(command._id, {
+        status: 'done',
+        result: {
+          channel_id,
+          channel_name: channel.name,
+          ...stats,
+          synced_at: new Date().toISOString(),
+        },
+      });
+
+      console.log(`[commands] YouTube stats synced for "${channel.name}"`);
+    } catch (err) {
+      console.error(`[commands] YouTube stats sync failed: ${err.message}`);
+      await this.api.updateCommand(command._id, {
+        status: 'failed',
+        error: err.message,
+      });
+    }
+  }
 }
 
 module.exports = { CommandPoller };

package/lib/stats-syncer.js

@@ -0,0 +1,405 @@
+const WebSocket = require('ws');
+
+/**
+ * Sync YouTube Studio stats via Nstbrowser CDP.
+ * Flow: Launch profile → connect CDP → navigate to Studio → scrape stats → close
+ */
+class StatsSyncer {
+  constructor(nstManager, apiClient) {
+    this.nst = nstManager;
+    this.api = apiClient;
+  }
+
+  async syncYouTubeStats(channel) {
+    const profileId = channel.nst_profile_id;
+    if (!profileId) throw new Error(`Channel "${channel.name}" has no nst_profile_id`);
+
+    console.log(`[stats] Syncing YouTube stats for "${channel.name}" (profile: ${profileId})`);
+
+    const { wsEndpoint } = await this.nst.launchProfile(profileId, {
+      proxy: channel.proxy || null,
+    });
+
+    if (!wsEndpoint) throw new Error('No wsEndpoint returned from Nstbrowser');
+
+    try {
+      const stats = await this.scrapeViaCDP(wsEndpoint);
+      console.log(`[stats] "${channel.name}" → subs: ${stats.subscribers}, views: ${stats.total_views}, videos: ${stats.video_count}`);
+
+      await this.api.request('POST', '/analytics/sync', {
+        channel_id: channel._id,
+        platform: 'youtube',
+        subscribers: stats.subscribers,
+        total_views: stats.total_views,
+        video_count: stats.video_count,
+      });
+
+      console.log(`[stats] "${channel.name}" stats saved to API`);
+      return stats;
+    } finally {
+      try {
+        await this.nst.stopProfile(profileId);
+      } catch (err) {
+        console.warn(`[stats] Failed to stop profile: ${err.message}`);
+      }
+    }
+  }
+
+  scrapeViaCDP(wsEndpoint) {
+    return new Promise((resolve, reject) => {
+      const ws = new WebSocket(wsEndpoint);
+      let msgId = 1;
+      const pending = new Map();
+
+      const send = (method, params = {}, sid) => {
+        return new Promise((res, rej) => {
+          const id = msgId++;
+          pending.set(id, { resolve: res, reject: rej });
+          const msg = { id, method, params };
+          if (sid) msg.sessionId = sid;
+          ws.send(JSON.stringify(msg));
+        });
+      };
+
+      const timeout = setTimeout(() => {
+        ws.close();
+        reject(new Error('Stats scrape timed out (90s)'));
+      }, 90000);
+
+      ws.on('error', (err) => { clearTimeout(timeout); reject(err); });
+
+      ws.on('message', (data) => {
+        try {
+          const msg = JSON.parse(data);
+          if (msg.id && pending.has(msg.id)) {
+            const p = pending.get(msg.id);
+            pending.delete(msg.id);
+            if (msg.error) p.reject(new Error(msg.error.message));
+            else p.resolve(msg.result);
+          }
+        } catch {}
+      });
+
+      ws.on('open', async () => {
+        try {
+          // Get a page target
+          const targets = await send('Target.getTargets');
+          const pages = targets.targetInfos.filter(t => t.type === 'page');
+          let targetId = pages.length > 0 ? pages[0].targetId : (await send('Target.createTarget', { url: 'about:blank' })).targetId;
+
+          const { sessionId } = await send('Target.attachToTarget', { targetId, flatten: true });
+          const s = (method, params = {}) => send(method, params, sessionId);
+
+          await s('Page.enable');
+          await s('Runtime.enable');
+
+          // Step 1: Go to YouTube Studio — it auto-redirects to the correct channel
+          console.log('[stats] Navigating to YouTube Studio...');
+          await s('Page.navigate', { url: 'https://studio.youtube.com' });
+          await this.waitForLoad(s);
+          await this.sleep(6000); // SPA needs time to render
+
+          // Log current URL for debugging
+          const urlResult = await s('Runtime.evaluate', { expression: 'window.location.href', returnByValue: true });
+          console.log(`[stats] Current URL: ${urlResult?.result?.value}`);
+
+          // Step 2: Try to extract stats
+          let stats = { subscribers: 0, total_views: 0, video_count: 0 };
+
+          // Method A: Dump all visible text and parse (most reliable for YouTube Studio)
+          const dumpResult = await s('Runtime.evaluate', {
+            expression: `
+              (function() {
+                const result = { subscribers: 0, total_views: 0, video_count: 0, debug: '' };
+                const text = document.body?.innerText || '';
+                result.debug = text.substring(0, 500);
+
+                // YouTube Studio shows "X subscribers" in the dashboard
+                // Format varies: "123 subscribers", "1.2K subscribers", "0 subscribers"
+                const subPatterns = [
+                  /Current subscribers\\n([\\d,\\.]+[KMB]?)/i,
+                  /([\\d,\\.]+[KMB]?)\\s*subscriber/i,
+                  /Subscribers?\\n([\\d,\\.]+[KMB]?)/i,
+                ];
+                for (const p of subPatterns) {
+                  const m = text.match(p);
+                  if (m) { result.subscribers = parseNum(m[1]); break; }
+                }
+
+                // Views pattern
+                const viewPatterns = [
+                  /([\\d,\\.]+[KMB]?)\\s*views?\\b/i,
+                  /Views?\\n([\\d,\\.]+[KMB]?)/i,
+                ];
+                for (const p of viewPatterns) {
+                  const m = text.match(p);
+                  if (m) { result.total_views = parseNum(m[1]); break; }
+                }
+
+                function parseNum(str) {
+                  if (!str) return 0;
+                  str = str.replace(/,/g, '').trim();
+                  const m = str.match(/([\\d\\.]+)\\s*([KMB])/i);
+                  if (m) {
+                    const n = parseFloat(m[1]);
+                    const s = m[2].toUpperCase();
+                    if (s === 'K') return Math.round(n * 1000);
+                    if (s === 'M') return Math.round(n * 1000000);
+                    if (s === 'B') return Math.round(n * 1000000000);
+                  }
+                  return parseInt(str, 10) || 0;
+                }
+
+                return JSON.stringify(result);
+              })()
+            `,
+            returnByValue: true,
+          });
+
+          try {
+            const parsed = JSON.parse(dumpResult?.result?.value || '{}');
+            console.log(`[stats] Studio text dump (first 200): ${(parsed.debug || '').substring(0, 200)}`);
+            if (parsed.subscribers) stats.subscribers = parsed.subscribers;
+            if (parsed.total_views) stats.total_views = parsed.total_views;
+          } catch {}
+
+          // Method B: Use YouTube's internal youtubei API (works from studio.youtube.com origin)
+          console.log('[stats] Trying YouTube internal API...');
+          const apiResult = await s('Runtime.evaluate', {
+            expression: `
+              (async function() {
+                try {
+                  // Get SAPISIDHASH for auth (YouTube uses this for internal API calls)
+                  function getSapisidHash(origin) {
+                    const cookies = document.cookie.split(';').reduce((acc, c) => {
+                      const [k, v] = c.trim().split('=');
+                      acc[k] = v;
+                      return acc;
+                    }, {});
+                    const sapisid = cookies['SAPISID'] || cookies['__Secure-3PAPISID'];
+                    if (!sapisid) return null;
+                    const timestamp = Math.floor(Date.now() / 1000);
+                    // Can't compute SHA1 in plain JS easily, but we can use the session cookies directly
+                    return null;
+                  }
+
+                  // Method 1: Use the ytcfg data that Studio loads
+                  if (window.ytcfg && window.ytcfg.get) {
+                    const channelId = window.ytcfg.get('CHANNEL_ID');
+                    if (channelId) {
+                      // Try fetching channel data from YouTube Data API v3 (internal, no key needed from same origin)
+                      const r = await fetch('https://www.youtube.com/youtubei/v1/browse?prettyPrint=false', {
+                        method: 'POST',
+                        headers: {
+                          'Content-Type': 'application/json',
+                          'X-Origin': 'https://www.youtube.com',
+                        },
+                        credentials: 'include',
+                        body: JSON.stringify({
+                          browseId: channelId,
+                          context: {
+                            client: {
+                              clientName: 'WEB',
+                              clientVersion: '2.20240101.00.00',
+                            },
+                          },
+                        }),
+                      });
+                      if (r.ok) {
+                        const d = await r.json();
+                        // Extract from header
+                        const header = d?.header?.c4TabbedHeaderRenderer || {};
+                        const subText = header?.subscriberCountText?.simpleText || '';
+                        const meta = d?.metadata?.channelMetadataRenderer || {};
+
+                        return JSON.stringify({
+                          source: 'youtubei_browse',
+                          channelId,
+                          subscribers_text: subText,
+                        });
+                      }
+                    }
+                  }
+
+                  // Method 2: Parse ytcfg directly for channel info
+                  if (window.ytcfg && window.ytcfg.data_) {
+                    return JSON.stringify({
+                      source: 'ytcfg',
+                      keys: Object.keys(window.ytcfg.data_).filter(k =>
+                        k.toLowerCase().includes('channel') || k.toLowerCase().includes('subscriber')
+                      ),
+                    });
+                  }
+
+                  return JSON.stringify({ source: 'none' });
+                } catch (e) {
+                  return JSON.stringify({ error: e.message });
+                }
+              })()
+            `,
+            awaitPromise: true,
+            returnByValue: true,
+          });
+
+          try {
+            const apiData = JSON.parse(apiResult?.result?.value || '{}');
+            console.log(`[stats] Internal API result: ${JSON.stringify(apiData).substring(0, 300)}`);
+          } catch {}
+
+          // Method C: Navigate to channel page to get video count via ytInitialData
+          if (stats.video_count === 0) {
+            console.log('[stats] Getting video count from channel page...');
+
+            // Get the channel URL from Studio
+            const channelUrlResult = await s('Runtime.evaluate', {
+              expression: `
+                (function() {
+                  // Try to find channel URL/ID from Studio page
+                  const url = window.location.href;
+                  const m = url.match(/channel\\/(UC[\\w-]+)/);
+                  if (m) return 'https://www.youtube.com/channel/' + m[1] + '/videos';
+
+                  // Try ytcfg
+                  if (window.ytcfg && window.ytcfg.get) {
+                    const cid = window.ytcfg.get('CHANNEL_ID');
+                    if (cid) return 'https://www.youtube.com/channel/' + cid + '/videos';
+                  }
+
+                  // Fallback: go to own channel
+                  return 'https://www.youtube.com/@me';
+                })()
+              `,
+              returnByValue: true,
+            });
+
+            const channelUrl = channelUrlResult?.result?.value;
+            if (channelUrl) {
+              console.log(`[stats] Navigating to: ${channelUrl}`);
+              await s('Page.navigate', { url: channelUrl });
+              await this.waitForLoad(s);
+              await this.sleep(4000);
+
+              const ytResult = await s('Runtime.evaluate', {
+                expression: `
+                  (function() {
+                    const result = { subscribers: 0, total_views: 0, video_count: 0 };
+                    try {
+                      const data = window.ytInitialData;
+                      if (!data) return JSON.stringify({ error: 'no ytInitialData' });
+
+                      // Subscriber count from header
+                      const header = data?.header?.c4TabbedHeaderRenderer
+                        || data?.header?.pageHeaderRenderer?.content?.pageHeaderViewModel;
+                      if (header) {
+                        // c4TabbedHeaderRenderer path
+                        const subText = header?.subscriberCountText?.simpleText || '';
+                        if (subText) {
+                          const m = subText.match(/([\\d,\\.]+[KMB]?)/);
+                          if (m) result.subscribers = parseNum(m[1]);
+                        }
+                        // pageHeaderViewModel path
+                        const metadata = header?.metadata?.contentMetadataViewModel?.metadataRows;
+                        if (metadata) {
+                          for (const row of metadata) {
+                            for (const part of (row?.metadataParts || [])) {
+                              const t = part?.text?.content || '';
+                              if (t.includes('subscriber')) {
+                                const m = t.match(/([\\d,\\.]+[KMB]?)/);
+                                if (m) result.subscribers = parseNum(m[1]);
+                              }
+                              if (t.includes('video')) {
+                                const m = t.match(/([\\d,\\.]+)/);
+                                if (m) result.video_count = parseInt(m[1].replace(/,/g, ''), 10);
+                              }
+                            }
+                          }
+                        }
+                      }
+
+                      // Video count from tabs
+                      const tabs = data?.contents?.twoColumnBrowseResultsRenderer?.tabs || [];
+                      for (const tab of tabs) {
+                        const tabData = tab?.tabRenderer;
+                        if (tabData?.title === 'Videos') {
+                          // Sometimes the tab content has total count
+                          const content = tabData?.content;
+                          const grid = content?.richGridRenderer || content?.sectionListRenderer;
+                          // Check for video items count
+                        }
+                      }
+
+                      // Total views from about tab data or metadata
+                      const aboutData = data?.metadata?.channelMetadataRenderer;
+                      if (aboutData) {
+                        // metadata doesn't have view count directly, but try
+                      }
+
+                      // Try to get view count from about page data
+                      const microformat = data?.microformat?.microformatDataRenderer;
+                      if (microformat) {
+                        // Some data here
+                      }
+
+                    } catch (e) {
+                      return JSON.stringify({ error: e.message });
+                    }
+
+                    function parseNum(str) {
+                      if (!str) return 0;
+                      str = str.replace(/,/g, '').trim();
+                      const m = str.match(/([\\d\\.]+)\\s*([KMB])/i);
+                      if (m) {
+                        const n = parseFloat(m[1]);
+                        const s = m[2].toUpperCase();
+                        if (s === 'K') return Math.round(n * 1000);
+                        if (s === 'M') return Math.round(n * 1000000);
+                        if (s === 'B') return Math.round(n * 1000000000);
+                      }
+                      return parseInt(str, 10) || 0;
+                    }
+
+                    return JSON.stringify(result);
+                  })()
+                `,
+                returnByValue: true,
+              });
+
+              try {
+                const ytData = JSON.parse(ytResult?.result?.value || '{}');
+                console.log(`[stats] ytInitialData result: ${JSON.stringify(ytData)}`);
+                if (ytData.subscribers > 0 && stats.subscribers === 0) stats.subscribers = ytData.subscribers;
+                if (ytData.video_count > 0) stats.video_count = ytData.video_count;
+                if (ytData.total_views > 0 && stats.total_views === 0) stats.total_views = ytData.total_views;
+              } catch {}
+            }
+          }
+
+          clearTimeout(timeout);
+          ws.close();
+          resolve(stats);
+        } catch (err) {
+          clearTimeout(timeout);
+          ws.close();
+          reject(err);
+        }
+      });
+    });
+  }
+
+  async waitForLoad(s, timeoutMs = 15000) {
+    const start = Date.now();
+    while (Date.now() - start < timeoutMs) {
+      try {
+        const r = await s('Runtime.evaluate', { expression: 'document.readyState', returnByValue: true });
+        if (r?.result?.value === 'complete') return;
+      } catch {}
+      await this.sleep(500);
+    }
+  }
+
+  sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+}
+
+module.exports = { StatsSyncer };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "channel-worker",
-  "version": "1.4.0",
+  "version": "1.5.1",
   "description": "Channel Manager worker daemon — runs on remote machines to execute video pipeline jobs",
   "main": "lib/daemon.js",
   "bin": {