channel-worker 2.5.150 → 2.5.151

package/lib/api-client.js

@@ -1,7 +1,21 @@
+// Final-result reports (done / failed / pending) ride out an API restart:
+// one lost PUT left the command 'running' until the stale cleanup marked it
+// "Timed out — có thể bài ĐÃ lên" and the scheduler stopped retrying the
+// idea. ~2.5 minutes covers a pm2 restart with room to spare.
+const REPORT_RETRY_DELAYS_MS = [2000, 4000, 8000, 15000, 30000, 30000, 60000];
+
+// The API never answered: connection refused/reset, a gateway 502/503/504, or
+// a non-JSON page while it restarts. An answer from the API itself — success:
+// false, even a 500 from a handler — is final: repeating it changes nothing.
+class ApiUnreachableError extends Error {}
+const GATEWAY_STATUSES = new Set([502, 503, 504]);
+
 class ApiClient {
   constructor(baseUrl, workerToken) {
     this.baseUrl = baseUrl.replace(/\/$/, '');
     this.workerToken = workerToken || '';
+    this.reportRetryDelaysMs = REPORT_RETRY_DELAYS_MS;
+    this.sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
   }
 
   async request(method, path, body = null) {
@@ -15,8 +29,21 @@ class ApiClient {
     };
     if (body) options.body = JSON.stringify(body);
 
-    const res = await fetch(url, options);
-    const json = await res.json();
+    let res;
+    try {
+      res = await fetch(url, options);
+    } catch (err) {
+      throw new ApiUnreachableError(`API unreachable: ${err.message}`);
+    }
+    let json;
+    try {
+      json = await res.json();
+    } catch {
+      throw new ApiUnreachableError(`API error ${res.status} (no JSON body)`);
+    }
+    if (GATEWAY_STATUSES.has(res.status) && !json?.success) {
+      throw new ApiUnreachableError(json?.message || `API error ${res.status}`);
+    }
 
     if (!json.success) {
       throw new Error(json.message || `API error ${res.status}`);
@@ -24,6 +51,21 @@ class ApiClient {
     return json.data;
   }
 
+  // Same request, retried while the API is unreachable. Only for writes that
+  // are safe to repeat — every report here sets a final state.
+  async reportRequest(method, path, body) {
+    for (let attempt = 0; ; attempt++) {
+      try {
+        return await this.request(method, path, body);
+      } catch (err) {
+        const delay = this.reportRetryDelaysMs[attempt];
+        if (!(err instanceof ApiUnreachableError) || delay === undefined) throw err;
+        console.warn(`[api] ${method} ${path} failed (${err.message}) — retry ${attempt + 1}/${this.reportRetryDelaysMs.length} in ${Math.round(delay / 1000)}s`);
+        await this.sleep(delay);
+      }
+    }
+  }
+
   // Worker
   async register(workerData) {
     return this.request('POST', '/workers/register', workerData);
@@ -79,8 +121,12 @@ class ApiClient {
     return this.request('GET', `/workers/commands?worker_id=${workerId}&types=${encodeURIComponent(workerTypes)}`);
   }
 
+  // A 'running' touch is a liveness beat: NOT retried — a beat landing late
+  // would overwrite the final 'done' that followed it.
   async updateCommand(commandId, data) {
-    return this.request('PUT', `/workers/commands/${commandId}`, data);
+    const path = `/workers/commands/${commandId}`;
+    if (data?.status === 'running') return this.request('PUT', path, data);
+    return this.reportRequest('PUT', path, data);
   }
 
   // Report a command result via the extension-style endpoint, which also
@@ -88,7 +134,7 @@ class ApiClient {
   // idea UI reflects pw-pipeline outcomes the same way it does for extension
   // uploads). Use this for upload_*_pw results.
   async postCommandResult(commandId, { status, result, error } = {}) {
-    return this.request('POST', `/extension/commands/${commandId}/result`, { status, result: result || {}, error: error || null });
+    return this.reportRequest('POST', `/extension/commands/${commandId}/result`, { status, result: result || {}, error: error || null });
   }
 
   // Shopee scraper callbacks — worker ships parsed products, API upserts into
@@ -222,4 +268,4 @@ class ApiClient {
   }
 }
 
-module.exports = { ApiClient };
+module.exports = { ApiClient, ApiUnreachableError, REPORT_RETRY_DELAYS_MS };

package/lib/command-poller.js

@@ -2,6 +2,7 @@ const { NstManager } = require('./nst-manager');
 const { refreshPinnedAgentAddress } = require('./nst-agent-locator');
 const { checkAndUpdateExtension } = require('./extension-updater');
 const { StatsSyncer } = require('./stats-syncer');
+const { ApiUnreachableError } = require('./api-client');
 
 // Quãng RẢNH trước khi đóng profile của luồng đăng bài. Phải đủ dài để lượt
 // upload nền tảng kế tiếp (facebook → youtube → tiktok của cùng một idea) kịp
@@ -284,11 +285,23 @@ class CommandPoller {
       )), ms);
     });
     try {
-      const patch = await Promise.race([job, deadline]);
-      await this.api.updateCommand(command._id, patch);
-    } catch (err) {
-      console.error(`${failLog}: ${err.message}`);
-      await this.api.updateCommand(command._id, { status: 'failed', error: String(err.message || err).slice(0, 500) });
+      let patch;
+      try {
+        patch = await Promise.race([job, deadline]);
+      } catch (err) {
+        console.error(`${failLog}: ${err.message}`);
+        patch = { status: 'failed', error: String(err.message || err).slice(0, 500) };
+      }
+      // Outside the work's try: a report that cannot reach the API must not
+      // turn a finished job into 'failed'.
+      try {
+        await this.api.updateCommand(command._id, patch);
+      } catch (err) {
+        console.error(`${failLog}: ${patch?.status} NOT reported: ${err.message}`);
+        if (!(err instanceof ApiUnreachableError) && patch?.status !== 'failed') {
+          await this.api.updateCommand(command._id, { status: 'failed', error: String(err.message || err).slice(0, 500) }).catch(() => {});
+        }
+      }
     } finally {
       clearTimeout(timer);
       abandoned = true;
@@ -747,23 +760,43 @@ class CommandPoller {
     }, RUN_BEAT_MS);
 
     try {
-      const result = await runPlaywrightScript({
-        nst: this.nst,
-        profileId,
-        scriptName,
-        payload,
-        log,
-        keepOpen: keepOpen.enabled,
-        info: runInfo,
-      });
-      // postCommandResult (not updateCommand) so the existing extension-side
-      // logic mirrors status into ContentJob + ContentIdea.publish_results +
-      // broadcasts the WS update — pw and extension paths share idea-state UX.
-      await this.api.postCommandResult(command._id, { status: 'done', result: result || {} });
-      console.log(`[commands/pw] ${command.type} done`);
-    } catch (err) {
-      console.error(`[commands/pw] ${command.type} failed: ${err.message}`);
-      await this.api.postCommandResult(command._id, { status: 'failed', error: String(err.message || err).slice(0, 500) });
+      let outcome;
+      try {
+        const result = await runPlaywrightScript({
+          nst: this.nst,
+          profileId,
+          scriptName,
+          payload,
+          log,
+          keepOpen: keepOpen.enabled,
+          info: runInfo,
+        });
+        outcome = { status: 'done', result: result || {} };
+        console.log(`[commands/pw] ${command.type} done`);
+      } catch (err) {
+        console.error(`[commands/pw] ${command.type} failed: ${err.message}`);
+        outcome = { status: 'failed', error: String(err.message || err).slice(0, 500) };
+      }
+      // Stop the 'running' beat first: a beat landing after the result would
+      // put the command back to 'running'.
+      clearInterval(beat);
+      // Reported once, outside the script's try: a report that cannot reach
+      // the API used to fall into the catch above and report the posted video
+      // as 'failed'. postCommandResult (not updateCommand) so the existing
+      // extension-side logic mirrors status into ContentJob +
+      // ContentIdea.publish_results + broadcasts the WS update — pw and
+      // extension paths share idea-state UX.
+      try {
+        await this.api.postCommandResult(command._id, outcome);
+      } catch (err) {
+        console.error(`[commands/pw] ${command.type} ${outcome.status} NOT reported: ${err.message}`);
+        // The API answered and refused it: settle the command as failed
+        // rather than leave it 'running'. Unreachable after every retry:
+        // nothing more can be sent.
+        if (!(err instanceof ApiUnreachableError) && outcome.status !== 'failed') {
+          await this.api.postCommandResult(command._id, { status: 'failed', error: `API từ chối kết quả: ${String(err.message).slice(0, 300)}` }).catch(() => {});
+        }
+      }
     } finally {
       clearInterval(beat);
       // Always release the per-profile mutex — even on throw — or sibling

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "channel-worker",
-  "version": "2.5.150",
+  "version": "2.5.151",
   "description": "Channel Manager worker daemon — runs on remote machines to execute video pipeline jobs",
   "main": "lib/daemon.js",
   "bin": {