@remits/remits-cli 0.1.162 → 0.1.164

package/README.md

@@ -139,6 +139,10 @@ remits-cli install --skills --overwrite true
 - The service groups sessions by `baseUrl` so one websocket connection can service multiple authenticated accounts on the same Remits environment.
 - Websocket subscriptions are deduplicated by user topic. If multiple authenticated accounts share the same websocket topic, the listener subscribes once and maps that topic back to all related accounts.
 - The daemon keeps STOMP heartbeats enabled and runs a maintenance watchdog that recreates unhealthy websocket connections.
+- Terminal server authentication/authorization errors suspend WebSocket retries and show
+  `authentication_required`. Reauthenticate to replace the managed client's credentials; the watchdog
+  does not revive rejected credentials. Transient failures use jittered exponential backoff up to
+  60 seconds. Test watchers keep HTTP polling; restart local-AI workers after reauthentication.
 - If the machine sleeps, the network drops, or auth sessions change while the daemon is already running, the service now attempts to reconnect and resubscribe automatically once connectivity returns.
 - Support tickets are the primary unit of dispatched work. A serving agent launches one fresh worker process per routed ticket.
 - Agent sessions register themselves with `remits-cli agent register` or `remits-cli agent serve`; `serve` also starts the local supervisor.

package/index.js

@@ -4,18 +4,18 @@
 ## Table of Contents
 
 - L22  Runtime Bootstrap And Shared State
-- L124  Sessions, Account Resolution, And Production Guards
-- L630  Local State, Workspaces, And Verification Evidence
-- L2668  Account Repos, Guide Sync, And Platform Repo
-- L3508  Component Discovery And HTTP Logging
-- L4276  Skill Delivery And TOC Resolution
-- L4591  Auth And Component Staging
-- L5223  Component Summaries, Status, And Sync Gates
-- L7547  Branches, Promotion, Commit, And Test Runs
-- L9791  Tokens, Tools, Verification, And Config
-- L11308  Service Dashboard And WebSocket Listener
-- L13808  Agent And Ticket Workflows
-- L18069  Help, Auto Update, And Command Dispatch
+- L125  Sessions, Account Resolution, And Production Guards
+- L631  Local State, Workspaces, And Verification Evidence
+- L2669  Account Repos, Guide Sync, And Platform Repo
+- L3509  Component Discovery And HTTP Logging
+- L4277  Skill Delivery And TOC Resolution
+- L4592  Auth And Component Staging
+- L5224  Component Summaries, Status, And Sync Gates
+- L7548  Branches, Promotion, Commit, And Test Runs
+- L9802  Tokens, Tools, Verification, And Config
+- L11319  Service Dashboard And WebSocket Listener
+- L13829  Agent And Ticket Workflows
+- L18090  Help, Auto Update, And Command Dispatch
 */
 
 /*
@@ -32,6 +32,7 @@ const axios = require('axios');
 const openModule = require('open');
 const { Client } = require('@stomp/stompjs');
 const WebSocket = require('ws');
+const { installWebsocketRetry } = require('./lib/websocket-retry');
 const YAML = require('yaml');
 
 Object.assign(global, { WebSocket });
@@ -4160,7 +4161,7 @@ const WORKLOAD_BUSY_MAX_RETRIES = 6;
  * How long to wait before retrying a request the platform refused as `WORKLOAD_BUSY`, or null when the
  * failure is anything else (or the retries are spent).
  *
- * The platform admits only a few heavyweight requests per instance and answers the rest with HTTP 429
+ * The platform admits only a few heavyweight requests at once and answers the rest with HTTP 429
  * BEFORE doing any work, so repeating the identical request is safe — which is not true of any other
  * failure, and is why this matches on the code rather than on the status alone. The wait starts at the
  * server's Retry-After and grows, with jitter so several refused agents do not come back in step.
@@ -4186,7 +4187,7 @@ async function postRidingOutBusy(api, endpoint, payload) {
     } catch (err) {
       const delayMs = workloadBusyRetryDelayMs(err, attempt, maxRetries, Math.random);
       if (delayMs == null) throw err;
-      console.error('Platform instance is busy with ' + (err.response.data.workload || 'other work') +
+      console.error('Platform is busy with ' + (err.response.data.workload || 'other work') +
         '; retrying in ' + Math.round(delayMs / 1000) + 's (' + (attempt + 1) + '/' + maxRetries + ').');
       await new Promise((resolve) => setTimeout(resolve, delayMs));
     }
@@ -9144,7 +9145,16 @@ function watchWebsocket(baseUrl, topicId, taskId, options = {}) {
   const emit = options.stderr ? console.error : console.log;
   const client = new Client({
     brokerURL: webSocketUrl(baseUrl),
-    connectHeaders: { 'X-Auth-Token': options.token || '' },
+    connectHeaders: {
+      'X-Auth-Token': options.token || '',
+      'x-remits-ws-label': 'cli-test',
+      'x-remits-client-kind': 'cli',
+      'x-remits-tab-id': 'cli:' + process.pid + ':test:' + crypto.randomUUID(),
+      'x-remits-origin': 'remits-cli://test',
+      'x-remits-path': 'test/watch',
+      'x-remits-location': normalizeBaseUrl(baseUrl),
+      'x-remits-broker': webSocketUrl(baseUrl)
+    },
     reconnectDelay: 3000,
     heartbeatIncoming: 10000,
     heartbeatOutgoing: 10000,
@@ -9176,6 +9186,7 @@ function watchWebsocket(baseUrl, topicId, taskId, options = {}) {
     });
   };
 
+  installWebsocketRetry(client, { onBlocked: code => emit('[ws] Authentication required (' + code + '); HTTP polling continues.') });
   client.activate();
   return () => {
     client.deactivate();
@@ -9228,7 +9239,7 @@ function testRunOptionsFromFlags(flags = {}) {
 // How long a later shard of a parallel run waits for a free suite slot before the launch gives up.
 const SHARD_ADMISSION_WAIT_MS = 30 * 60 * 1000;
 
-// A start the platform refused because every suite slot on the instance is taken (safe to repeat: no work began).
+// A start the platform refused because every suite slot is taken (safe to repeat: no work began).
 function shardStartBusy(err) {
   const response = err && err.response;
   return !!(response && response.status === 429 && response.data && response.data.code === 'WORKLOAD_BUSY');
@@ -9340,7 +9351,7 @@ async function testCommand(flags) {
   };
   // Shards are started one after another (each start returns at once) and then run side by side on the platform.
   const shardStarts = [];
-  // The platform admits only a few suites per instance at once and refuses the rest as WORKLOAD_BUSY. Once one
+  // The platform admits only a few suites at once and refuses the rest as WORKLOAD_BUSY. Once one
   // of our shards is running it holds a slot it will give back, so a later shard keeps waiting for one (bounded)
   // instead of giving up after the usual retries and leaving a group that can never finish.
   const shardWaitDeadline = Date.now() + SHARD_ADMISSION_WAIT_MS;
@@ -9354,7 +9365,7 @@ async function testCommand(flags) {
         const data = err && err.response && err.response.data;
         if (shardStarts.length && shardStartBusy(err) && Date.now() < shardWaitDeadline) {
           console.error('Shard ' + (shardStarts.length + 1) + ' of ' + runOptions.parallel + ' is waiting for a free suite slot ' +
-            '(this platform instance runs a limited number of suites at once; the shards already running free them).');
+            '(the platform runs a limited number of suites at once; the shards already running free them).');
           await new Promise((resolve) => setTimeout(resolve, 10000));
           continue;
         }
@@ -13538,6 +13549,7 @@ function createManagedWebsocketClient(baseUrl, urlSessions) {
     discardWebsocketOnCommFailure: true,
     connectHeaders: {
       'x-remits-ws-label': 'remits-cli',
+      'x-remits-client-kind': 'cli',
       'x-remits-ws-scope': headerScope,
       'x-remits-user-id': headerUserIds.length === 1 ? headerUserIds[0] : '',
       'x-remits-account-id': headerAccountIds.length === 1 ? headerAccountIds[0] : '',
@@ -13643,7 +13655,7 @@ function createManagedWebsocketClient(baseUrl, urlSessions) {
     ensureRuntimeWebsocketEntry(normalizedBaseUrl);
     runtimeState.websocket[normalizedBaseUrl] = {
       ...runtimeState.websocket[normalizedBaseUrl],
-      status: 'disconnected',
+      status: client.remitsAuthBlocked ? 'authentication_required' : 'disconnected',
       disconnectedAt: new Date().toISOString()
     };
     console.log('[listener] Disconnected from', normalizedBaseUrl, '- will reconnect');
@@ -13673,7 +13685,7 @@ function createManagedWebsocketClient(baseUrl, urlSessions) {
     ensureRuntimeWebsocketEntry(normalizedBaseUrl);
     runtimeState.websocket[normalizedBaseUrl] = {
       ...runtimeState.websocket[normalizedBaseUrl],
-      status: 'disconnected',
+      status: client.remitsAuthBlocked ? 'authentication_required' : 'disconnected',
       disconnectedAt: new Date().toISOString(),
       lastError: detail
     };
@@ -13700,6 +13712,12 @@ function createManagedWebsocketClient(baseUrl, urlSessions) {
   };
 
   managedClient.client = client;
+  installWebsocketRetry(client, { onBlocked: code => {
+    runtimeState.websocket[normalizedBaseUrl] = {
+      ...runtimeState.websocket[normalizedBaseUrl], status: 'authentication_required', lastError: code
+    };
+    updateRuntimeServiceState();
+  } });
   return managedClient;
 }
 
@@ -13745,6 +13763,9 @@ async function runManagedWebsocketWatchdog() {
     if (!managedClient || !managedClient.client) {
       continue;
     }
+    // A session-file change replaces the client. The watchdog must not revive rejected credentials
+    // or defeat the normal transport backoff while it is waiting to reconnect.
+    if (managedClient.client.remitsAuthBlocked || managedClient.client.remitsRetryPending) continue;
     const state = ensureRuntimeWebsocketEntry(managedClient.baseUrl);
     const socket = managedClient.client.webSocket;
     const readyState = socket && typeof socket.readyState === 'number' ? socket.readyState : null;

package/lib/local-ai.js

@@ -6,6 +6,7 @@ const fs = require('node:fs');
 const path = require('node:path');
 const crypto = require('node:crypto');
 const { spawn } = require('node:child_process');
+const { installWebsocketRetry } = require('./websocket-retry');
 
 const MAX_BYTES = 8 * 1024 * 1024;
 const AGENTS = ['manual', 'codex', 'claude', 'gemini'];
@@ -354,6 +355,13 @@ function describe(state) {
   return parts.length ? ` (${parts.join('; ')})` : '';
 }
 
+// HTTP polling recovers what a WebSocket notification missed, renews worker presence (it expires after 90
+// seconds) and notices cancelled requests. With a live socket and nothing in flight, notifications carry
+// delivery and a slow poll only renews presence; otherwise poll fast. Idle `ai serve` processes had been
+// the largest share of all requests on the CLI node (2026-10-23: one request every 2s each, around the clock).
+const FAST_POLL_MS = 2000;
+const IDLE_POLL_MS = 30000;
+
 class Supervisor {
   constructor({ api, dir, worker, agent, model, log = console.error, run = runNative, clock = Date.now, onStop }) {
     Object.assign(this, { api, dir, worker, agent, model, log, run, clock, onStop });
@@ -361,14 +369,23 @@ class Supervisor {
     this.busySessions = new Set();
     this.stopping = false;
     this.polling = false;
+    this.socketReady = false;
     this.lastContact = clock();
   }
 
+  nextPollDelayMs() {
+    return this.socketReady && this.active.size === 0 ? IDLE_POLL_MS : FAST_POLL_MS;
+  }
+
   async register() { return this.api('register', { ...this.worker, agent: this.agent }); }
 
   async tick() {
-    if (this.polling || this.stopping) return;
+    if (this.stopping) return;
+    // A wakeup during a poll gets ONE follow-up poll: that poll's answer may predate the notification,
+    // and the next scheduled poll can be 30 seconds away.
+    if (this.polling) { this.pollAgain = true; return; }
     this.polling = true;
+    this.pollAgain = false;
     try {
       const work = await this.api('work');
       this.lastContact = this.clock();
@@ -402,6 +419,7 @@ class Supervisor {
         this.onStop?.();
       }
     } finally { this.polling = false; }
+    if (this.pollAgain && !this.stopping) await this.tick();
   }
 
   async handle(summary, task) {
@@ -565,7 +583,15 @@ async function command({ subcommand, flags, context, root, axios, Client, WebSoc
     console.error(`Local AI serving ${baseUrl} account ${accountId} ${dataMode}: ${agent}, worker ${worker.workerId}, capacity ${maxConcurrent}. Ctrl-C to stop.`);
     process.once('SIGINT', stop);
     process.once('SIGTERM', stop);
-    timer = setInterval(() => supervisor.tick(), 2000);
+    const schedule = () => {
+      if (supervisor.stopping) return;
+      timer = setTimeout(async () => {
+        try { await supervisor.tick(); } finally { schedule(); }
+      }, supervisor.nextPollDelayMs());
+    };
+    // A socket change re-paces at once, so losing the socket never leaves a 30-second gap.
+    const repace = () => { clearTimeout(timer); schedule(); };
+    schedule();
     const topic = session.websocketTopic || String(session.user.uuid || '').replace(/-/g, '');
     if (topic) {
       socket = new Client({
@@ -577,23 +603,37 @@ async function command({ subcommand, flags, context, root, axios, Client, WebSoc
         connectHeaders: { 'X-Auth-Token': session.token,
           'x-remits-ws-label': 'remits-cli-ai', 'x-remits-user-id': String(session.user.id) },
         onConnect: () => {
+          supervisor.socketReady = true;
+          repace();
           socket.subscribe('/topic/messages/' + topic, (message) => {
             try {
               const event = JSON.parse(message.body);
               if (event.type === 'cli_ai_request' && event.payload?.workerId === worker.workerId &&
-                  String(event.payload.accountId) === String(accountId) && event.payload.dataMode === dataMode) supervisor.tick();
+                  String(event.payload.accountId) === String(accountId) && event.payload.dataMode === dataMode) {
+                // A request that starts here needs the fast poll at once, to notice a cancellation.
+                supervisor.tick().finally(repace);
+              }
             } catch (_) { /* HTTP polling is authoritative. */ }
           });
           supervisor.tick();
         },
-        onStompError: () => console.error('Local AI WebSocket unavailable; authenticated polling continues.')
+        onWebSocketClose: () => { supervisor.socketReady = false; repace(); },
+        onStompError: () => {
+          supervisor.socketReady = false;
+          repace();
+          console.error('Local AI WebSocket unavailable; authenticated polling continues.');
+        }
       });
+      installWebsocketRetry(socket, { onBlocked: code => {
+        console.error(`Local AI WebSocket authentication required (${code}); reauthenticate and restart the worker.`);
+      } });
       socket.activate();
     }
     await supervisor.tick();
     await stopped;
   } finally {
-    clearInterval(timer);
+    if (supervisor) supervisor.stopping = true;
+    clearTimeout(timer);
     process.removeListener('SIGINT', stop);
     process.removeListener('SIGTERM', stop);
     if (socket) await socket.deactivate({ force: true });
@@ -602,5 +642,5 @@ async function command({ subcommand, flags, context, root, axios, Client, WebSoc
   }
 }
 
-module.exports = { command, help, Supervisor, runNative, nativeCommand, consumeEvent, promptFor, parseDecision,
+module.exports = { command, help, Supervisor, FAST_POLL_MS, IDLE_POLL_MS, runNative, nativeCommand, consumeEvent, promptFor, parseDecision,
   validateDecision, decisionSchema, offeredTools, continuation, assertTextRequest, stateDirectory, acquireLock, read, write, identifier };

package/lib/websocket-retry.js

@@ -0,0 +1,67 @@
+'use strict';
+
+// The server sends stable, credential-free codes. A transport close code alone (including 1002)
+// cannot distinguish expired credentials from a transient broker failure.
+const TERMINAL_CODES = new Set(['credentials_missing', 'credentials_rejected', 'credentials_malformed',
+  'session_required', 'topic_denied', 'destination_denied', 'recipient_denied']);
+
+function terminalCode(frame) {
+  const headers = frame?.headers || {};
+  const code = headers['x-remits-error-code'];
+  return TERMINAL_CODES.has(code) ? code : null;
+}
+
+function installWebsocketRetry(client, { onBlocked = () => {}, random = Math.random,
+  setTimer = setTimeout, clearTimer = clearTimeout } = {}) {
+  const base = Math.max(1, Number(client.reconnectDelay) || 5000);
+  let failures = 0, wait = 0, timer, release;
+  const cancelWait = () => {
+    if (timer) clearTimer(timer);
+    timer = null;
+    if (release) release();
+    release = null;
+    client.remitsRetryPending = false;
+  };
+  const before = client.beforeConnect;
+  client.beforeConnect = async () => {
+    if (client.remitsAuthBlocked) { await client.deactivate(); return; }
+    if (wait > 0) {
+      client.remitsRetryPending = true;
+      await new Promise(resolve => { release = resolve; timer = setTimer(cancelWait, wait); });
+      // A stop/replacement may have resolved the wait; never read or send credentials afterwards.
+      if (!client.active || client.remitsAuthBlocked) return;
+    }
+    client.remitsRetryPending = false;
+    if (before) await before();
+  };
+  const connected = client.onConnect;
+  client.onConnect = frame => {
+    failures = 0; wait = 0; client.remitsRetryPending = false;
+    if (connected) connected(frame);
+  };
+  const closed = client.onWebSocketClose;
+  client.onWebSocketClose = event => {
+    const ceiling = Math.min(60000, base * Math.pow(2, Math.min(++failures, 5)));
+    // STOMP already waits base ms. Add jitter/backoff before its next CONNECT using public APIs.
+    wait = Math.max(0, Math.floor(ceiling / 2 + random() * ceiling / 2) - base);
+    client.remitsRetryPending = !client.remitsAuthBlocked;
+    if (closed) closed(event);
+  };
+  const error = client.onStompError;
+  client.onStompError = frame => {
+    const code = terminalCode(frame);
+    if (code) {
+      client.remitsAuthBlocked = code;
+      cancelWait();
+      // deactivate changes active synchronously, before STOMP schedules the close reconnect.
+      Promise.resolve(client.deactivate({ force: true })).catch(() => {});
+    }
+    if (error) error(frame);
+    if (code) onBlocked(code);
+  };
+  const deactivate = client.deactivate.bind(client);
+  client.deactivate = options => { cancelWait(); return deactivate(options); };
+  return client;
+}
+
+module.exports = { installWebsocketRetry, terminalCode };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remits/remits-cli",
-  "version": "0.1.162",
+  "version": "0.1.164",
   "description": "Local CLI for auth, component sync, and live test execution against Remits",
   "license": "MIT",
   "private": false,
@@ -13,6 +13,7 @@
     "index.js",
     "lib/sync-operation.js",
     "lib/local-ai.js",
+    "lib/websocket-retry.js",
     "README.md",
     "scripts/prepare-index-toc.js",
     "skills/remits-cli/SKILL.md",