channel-worker 2.2.6 → 2.3.1

package/lib/cache-server.js

@@ -0,0 +1,196 @@
+/**
+ * Local HTTP cache server.
+ *
+ * Extensions running in Nstbrowser profiles share this single daemon-side
+ * cache instead of each downloading the same S3 ref (~1MB master cast)
+ * separately. First request misses, fetches via API; subsequent requests
+ * across ALL profiles on this machine hit the in-memory cache.
+ *
+ * Endpoint: GET /cache/s3?file_key=<key>
+ *   → mirrors the channel-manager API /extension/resolve-s3-url response:
+ *     { success, data: { base64, content_type, size } }
+ */
+
+const http = require('http');
+const url = require('url');
+const { execSync } = require('child_process');
+
+class CacheServer {
+  constructor(api, options = {}) {
+    this.api = api;
+    this.port = options.port || 8849;
+    this.host = options.host || '127.0.0.1';
+    this.maxEntries = options.maxEntries || 32; // ~32 × 1MB ≈ 32MB worst case
+    this.cache = new Map(); // file_key → { base64, contentType, size, ts }
+    this.server = null;
+    this.stats = { hits: 0, misses: 0, errors: 0 };
+  }
+
+  _evictIfNeeded() {
+    while (this.cache.size > this.maxEntries) {
+      // Drop oldest entry (Map preserves insertion order)
+      const oldestKey = this.cache.keys().next().value;
+      this.cache.delete(oldestKey);
+    }
+  }
+
+  async _resolveS3(fileKey) {
+    if (this.cache.has(fileKey)) {
+      this.stats.hits++;
+      // Touch — re-insert to mark as MRU
+      const entry = this.cache.get(fileKey);
+      this.cache.delete(fileKey);
+      this.cache.set(fileKey, entry);
+      return { ok: true, ...entry };
+    }
+    this.stats.misses++;
+    // Fetch via API client (extension/resolve-s3-url is POST)
+    const baseUrl = this.api.baseUrl;
+    const res = await fetch(`${baseUrl}/extension/resolve-s3-url`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', 'x-worker-token': this.api.workerToken },
+      body: JSON.stringify({ file_key: fileKey, as_base64: true }),
+    });
+    const json = await res.json();
+    if (!json.success || !json.data?.base64) {
+      this.stats.errors++;
+      return { ok: false, error: json.message || 'resolve failed' };
+    }
+    const entry = {
+      base64: json.data.base64,
+      contentType: json.data.content_type || 'image/png',
+      size: json.data.size || json.data.base64.length,
+      ts: Date.now(),
+    };
+    this.cache.set(fileKey, entry);
+    this._evictIfNeeded();
+    return { ok: true, ...entry };
+  }
+
+  _handleRequest(req, res) {
+    const parsed = url.parse(req.url, true);
+    if (req.method === 'GET' && parsed.pathname === '/health') {
+      res.writeHead(200, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify({ ok: true, cache: this.cache.size, stats: this.stats }));
+      return;
+    }
+    if (req.method === 'GET' && parsed.pathname === '/cache/s3') {
+      const fileKey = parsed.query.file_key;
+      if (!fileKey) {
+        res.writeHead(400, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({ success: false, message: 'file_key required' }));
+        return;
+      }
+      this._resolveS3(fileKey).then(result => {
+        if (!result.ok) {
+          res.writeHead(502, { 'Content-Type': 'application/json' });
+          res.end(JSON.stringify({ success: false, message: result.error }));
+          return;
+        }
+        // CORS — extensions may treat localhost as cross-origin
+        res.writeHead(200, {
+          'Content-Type': 'application/json',
+          'Access-Control-Allow-Origin': '*',
+          'Cache-Control': 'no-store',
+        });
+        res.end(JSON.stringify({
+          success: true,
+          data: {
+            base64: result.base64,
+            content_type: result.contentType,
+            size: result.size,
+          },
+        }));
+      }).catch(err => {
+        res.writeHead(500, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({ success: false, message: err.message }));
+      });
+      return;
+    }
+    res.writeHead(404, { 'Content-Type': 'application/json' });
+    res.end(JSON.stringify({ success: false, message: 'not found' }));
+  }
+
+  // Find PID(s) holding our port and kill them. Cross-platform best-effort.
+  // Used when bind fails with EADDRINUSE — typically a leaked previous daemon.
+  _killPortHolder() {
+    try {
+      if (process.platform === 'win32') {
+        // netstat columns: Proto Local Foreign State PID
+        const out = execSync(`netstat -ano -p TCP | findstr :${this.port} | findstr LISTENING`,
+          { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] });
+        const pids = new Set();
+        for (const line of out.split('\n')) {
+          const m = line.trim().match(/\s(\d+)\s*$/);
+          if (m) pids.add(m[1]);
+        }
+        for (const pid of pids) {
+          if (Number(pid) === process.pid) continue;
+          console.log(`[cache-server] Killing leaked PID ${pid} holding port ${this.port}`);
+          try { execSync(`taskkill /PID ${pid} /F`, { stdio: 'ignore' }); } catch {}
+        }
+        return pids.size;
+      }
+      // mac/linux
+      const out = execSync(`lsof -ti:${this.port} -sTCP:LISTEN`,
+        { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] });
+      const pids = out.split('\n').map(s => s.trim()).filter(Boolean);
+      for (const pid of pids) {
+        if (Number(pid) === process.pid) continue;
+        console.log(`[cache-server] Killing leaked PID ${pid} holding port ${this.port}`);
+        try { execSync(`kill -9 ${pid}`, { stdio: 'ignore' }); } catch {}
+      }
+      return pids.length;
+    } catch {
+      return 0;
+    }
+  }
+
+  _bindOnce() {
+    return new Promise((resolve, reject) => {
+      const server = http.createServer((req, res) => this._handleRequest(req, res));
+      server.on('error', (err) => reject(err));
+      server.listen(this.port, this.host, () => {
+        this.server = server;
+        resolve();
+      });
+    });
+  }
+
+  async start() {
+    try {
+      await this._bindOnce();
+    } catch (err) {
+      if (err.code !== 'EADDRINUSE') throw err;
+      console.warn(`[cache-server] Port ${this.port} busy — trying to kill leaked holder...`);
+      const killed = this._killPortHolder();
+      if (killed === 0) {
+        console.error(`[cache-server] No leaked process found for port ${this.port}. ` +
+          `Maybe TIME_WAIT — wait 30-60s and retry.`);
+        throw err;
+      }
+      // Wait for OS to release the port after kill, then retry once
+      await new Promise(r => setTimeout(r, 500));
+      try {
+        await this._bindOnce();
+      } catch (err2) {
+        console.error(`[cache-server] Still cannot bind ${this.port} after killing leaked PIDs.`);
+        throw err2;
+      }
+    }
+    console.log(`[cache-server] Listening on http://${this.host}:${this.port} (max ${this.maxEntries} entries)`);
+  }
+
+  stop() {
+    if (this.server) {
+      this.server.close();
+      this.server = null;
+    }
+  }
+
+  url() {
+    return `http://${this.host}:${this.port}`;
+  }
+}
+
+module.exports = { CacheServer };

package/lib/command-poller.js

@@ -146,6 +146,7 @@ class CommandPoller {
             channelManagerApi: 'https://api.channel.tunasm.art',
             profileId: profile_id,
             workerToken: this.config.worker_token || '',
+            daemonUrl: this.config.daemon_url || '',
           }));
           extensionPath = uniqueExtPath;
           console.log(`[commands] Extension dir: ${uniqueExtPath}`);
@@ -222,6 +223,7 @@ class CommandPoller {
             profileId: nst_profile_id,
             workerToken: this.config.worker_token || '',
             workerType: 'veo3',
+            daemonUrl: this.config.daemon_url || '',
           }));
           extensionPath = uniqueExtPath;
           console.log(`[commands] Veo3 ext dir: ${uniqueExtPath}`);

package/lib/daemon.js

@@ -4,6 +4,7 @@ const { JobPoller } = require('./job-poller');
 const { CommandPoller } = require('./command-poller');
 const { UpdateChecker, getLocalVersion } = require('./updater');
 const { checkAndUpdateExtension } = require('./extension-updater');
+const { CacheServer } = require('./cache-server');
 
 class Daemon {
   constructor(config) {
@@ -11,6 +12,9 @@ class Daemon {
     this.api = new ApiClient(config.api_url, config.worker_token);
     this.heartbeat = new Heartbeat(this.api, config.worker_id, 30000, config);
     this.poller = new JobPoller(this.api, config);
+    this.cacheServer = new CacheServer(this.api, { port: config.cache_port || 8849 });
+    // Pass daemon URL down so command-poller can write it into per-profile config
+    config.daemon_url = this.cacheServer.url();
     this.commandPoller = new CommandPoller(this.api, config);
     this.updateChecker = new UpdateChecker(5 * 60 * 1000); // check every 5min
     this.extCheckTimer = null;
@@ -28,6 +32,13 @@ class Daemon {
 ╚══════════════════════════════════════╝
 `);
 
+    // Start local cache server (extension shares this across all profiles)
+    try {
+      await this.cacheServer.start();
+    } catch (err) {
+      console.warn(`[daemon] Cache server failed to start (port in use?): ${err.message}`);
+    }
+
     // Register with dashboard
     try {
       await this.api.register({
@@ -98,6 +109,7 @@ class Daemon {
       this.commandPoller.stop();
       this.updateChecker.stop();
       if (this.extCheckTimer) clearInterval(this.extCheckTimer);
+      try { this.cacheServer.stop(); } catch {}
 
       // Mark offline
       try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "channel-worker",
-  "version": "2.2.6",
+  "version": "2.3.1",
   "description": "Channel Manager worker daemon — runs on remote machines to execute video pipeline jobs",
   "main": "lib/daemon.js",
   "bin": {