@agentchatme/openclaw 0.8.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -1
- package/RUNBOOK.md +2 -2
- package/dist/index.cjs +140 -50
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +46 -14
- package/dist/index.d.ts +46 -14
- package/dist/index.js +140 -50
- package/dist/index.js.map +1 -1
- package/dist/setup-entry.cjs +140 -50
- package/dist/setup-entry.cjs.map +1 -1
- package/dist/setup-entry.js +140 -50
- package/dist/setup-entry.js.map +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -98,12 +98,16 @@ type UnixMillis = number;
|
|
|
98
98
|
* DEGRADED — heartbeat missed or send buffer full; still connected but impaired
|
|
99
99
|
* DRAINING — SIGTERM received; flush in-flight sends, then close
|
|
100
100
|
* CLOSED — terminal after clean shutdown
|
|
101
|
-
* AUTH_FAIL —
|
|
101
|
+
* AUTH_FAIL — the server rejected the key; the runtime retries slowly
|
|
102
|
+
* (keys get rotated back, accounts get reinstated)
|
|
102
103
|
*
|
|
103
104
|
* Invariants:
|
|
104
|
-
* -
|
|
105
|
-
*
|
|
106
|
-
*
|
|
105
|
+
* - CLOSED is the only terminal state: it is entered only by DRAIN_REQUESTED
|
|
106
|
+
* / DRAIN_COMPLETED (the caller's own stop), never by anything the
|
|
107
|
+
* network did. Every other state has a way back to CONNECTING.
|
|
108
|
+
* - From AUTH_FAIL, CONNECT is the slow retry and RECONFIGURED is the
|
|
109
|
+
* operator's "retry now"; both lead back to a fresh HELLO.
|
|
110
|
+
* - From CLOSED, every event is a no-op.
|
|
107
111
|
* - DRAINING has a deadline (ms timestamp) after which we force-close.
|
|
108
112
|
*/
|
|
109
113
|
|
|
@@ -162,9 +166,13 @@ type ConnectionState = {
|
|
|
162
166
|
* is what settles delivery. There is no code path in this module that
|
|
163
167
|
* can emit a frame other than HELLO: delivery is judged, never
|
|
164
168
|
* confirmed by the transport.
|
|
165
|
-
* - `
|
|
166
|
-
*
|
|
167
|
-
*
|
|
169
|
+
* - The client never gives up on its own. Only `stop()` ends it. A close
|
|
170
|
+
* the client itself initiated (hello-ack timeout, ping timeout, its own
|
|
171
|
+
* drain) is never read as a verdict from the server; a server-originated
|
|
172
|
+
* auth close enters `AUTH_FAIL` and retries slowly, because keys get
|
|
173
|
+
* rotated back and accounts get reinstated. A long outage widens the
|
|
174
|
+
* backoff cap to `RECONNECT_LONG_OUTAGE_BACKOFF_MS` and logs one error
|
|
175
|
+
* line so an operator sees it without a log flood.
|
|
168
176
|
* - Frame parsing is defensive: malformed JSON, unknown `type`, missing
|
|
169
177
|
* payload — all classified as `validation` errors (non-terminal for
|
|
170
178
|
* the connection, but dropped with a metric + log).
|
|
@@ -186,7 +194,7 @@ interface WsClientEvents {
|
|
|
186
194
|
error: (error: AgentChatChannelError) => void;
|
|
187
195
|
/** Fired once per `hello.ok` — the runtime turns it into the connect knock. */
|
|
188
196
|
authenticated: (at: UnixMillis) => void;
|
|
189
|
-
/** Fired
|
|
197
|
+
/** Fired once the client reached CLOSED — only ever after `stop()`. */
|
|
190
198
|
closed: () => void;
|
|
191
199
|
}
|
|
192
200
|
interface WsClientOptions {
|
|
@@ -236,6 +244,15 @@ declare class AgentchatWsClient {
|
|
|
236
244
|
private ws;
|
|
237
245
|
private attempt;
|
|
238
246
|
private _ackNegotiated;
|
|
247
|
+
/**
|
|
248
|
+
* Set by `closeSocket()` before it closes the current socket, read and
|
|
249
|
+
* cleared by `handleClose()`. A close this client asked for says nothing
|
|
250
|
+
* about the server, whatever code travelled with it.
|
|
251
|
+
*/
|
|
252
|
+
private closeInitiatedLocally;
|
|
253
|
+
/** One error line per long outage / per run of rejected keys, then debug. */
|
|
254
|
+
private longOutageAnnounced;
|
|
255
|
+
private authFailAnnounced;
|
|
239
256
|
private helloAckTimer;
|
|
240
257
|
private pingTimer;
|
|
241
258
|
private pongTimer;
|
|
@@ -252,9 +269,8 @@ declare class AgentchatWsClient {
|
|
|
252
269
|
*/
|
|
253
270
|
get ackNegotiated(): boolean;
|
|
254
271
|
/**
|
|
255
|
-
* Kick off connection. No-op if already connecting, authenticating,
|
|
256
|
-
*
|
|
257
|
-
* after rotating the API key before retrying.
|
|
272
|
+
* Kick off connection. No-op if already connecting, authenticating, ready,
|
|
273
|
+
* or waiting out a backoff / auth retry. Throws only on `CLOSED`.
|
|
258
274
|
*/
|
|
259
275
|
start(): void;
|
|
260
276
|
/**
|
|
@@ -264,9 +280,9 @@ declare class AgentchatWsClient {
|
|
|
264
280
|
*/
|
|
265
281
|
stop(deadline?: UnixMillis): void;
|
|
266
282
|
/**
|
|
267
|
-
* Signal that operator has rotated the API key
|
|
268
|
-
*
|
|
269
|
-
*
|
|
283
|
+
* Signal that the operator has rotated the API key: leave AUTH_FAIL and
|
|
284
|
+
* retry now instead of waiting for the slow retry timer. A no-op in every
|
|
285
|
+
* other state (the client is already connecting or connected).
|
|
270
286
|
*/
|
|
271
287
|
reconfigured(): void;
|
|
272
288
|
on<E extends keyof WsClientEvents>(event: E, handler: Listener<E>): () => void;
|
|
@@ -277,7 +293,14 @@ declare class AgentchatWsClient {
|
|
|
277
293
|
private extractReason;
|
|
278
294
|
private extractPayload;
|
|
279
295
|
private handleHelloOk;
|
|
296
|
+
/** The server answered HELLO with a rejection frame (not a close). */
|
|
280
297
|
private handleAuthRejected;
|
|
298
|
+
/**
|
|
299
|
+
* The server rejected this key. AUTH_FAIL is not the end: retry a fresh
|
|
300
|
+
* HELLO every `AUTH_RETRY_INTERVAL_MS` (jittered). The first rejection in a
|
|
301
|
+
* run logs at error so an operator sees it; the following ones at debug.
|
|
302
|
+
*/
|
|
303
|
+
private enterAuthFail;
|
|
281
304
|
private handleClose;
|
|
282
305
|
private closeSocket;
|
|
283
306
|
private startHeartbeat;
|
|
@@ -285,7 +308,16 @@ declare class AgentchatWsClient {
|
|
|
285
308
|
private sendPing;
|
|
286
309
|
private handlePong;
|
|
287
310
|
private scheduleReconnect;
|
|
311
|
+
/** AUTH_FAIL's own cadence: a fresh HELLO every AUTH_RETRY_INTERVAL_MS, jittered. */
|
|
312
|
+
private scheduleAuthRetry;
|
|
313
|
+
private clearReconnectTimer;
|
|
288
314
|
private computeReconnectDelay;
|
|
315
|
+
/**
|
|
316
|
+
* Symmetric ±`jitterRatio`. Thundering-herd-safe: a fleet that all drops
|
|
317
|
+
* at the same instant re-enters the connect queue spread across
|
|
318
|
+
* ms * [1-jitterRatio, 1+jitterRatio].
|
|
319
|
+
*/
|
|
320
|
+
private jittered;
|
|
289
321
|
private scheduleDrainDeadline;
|
|
290
322
|
/**
|
|
291
323
|
* Called by upper layers when their pending work has fully drained, to
|
package/dist/index.d.ts
CHANGED
|
@@ -98,12 +98,16 @@ type UnixMillis = number;
|
|
|
98
98
|
* DEGRADED — heartbeat missed or send buffer full; still connected but impaired
|
|
99
99
|
* DRAINING — SIGTERM received; flush in-flight sends, then close
|
|
100
100
|
* CLOSED — terminal after clean shutdown
|
|
101
|
-
* AUTH_FAIL —
|
|
101
|
+
* AUTH_FAIL — the server rejected the key; the runtime retries slowly
|
|
102
|
+
* (keys get rotated back, accounts get reinstated)
|
|
102
103
|
*
|
|
103
104
|
* Invariants:
|
|
104
|
-
* -
|
|
105
|
-
*
|
|
106
|
-
*
|
|
105
|
+
* - CLOSED is the only terminal state: it is entered only by DRAIN_REQUESTED
|
|
106
|
+
* / DRAIN_COMPLETED (the caller's own stop), never by anything the
|
|
107
|
+
* network did. Every other state has a way back to CONNECTING.
|
|
108
|
+
* - From AUTH_FAIL, CONNECT is the slow retry and RECONFIGURED is the
|
|
109
|
+
* operator's "retry now"; both lead back to a fresh HELLO.
|
|
110
|
+
* - From CLOSED, every event is a no-op.
|
|
107
111
|
* - DRAINING has a deadline (ms timestamp) after which we force-close.
|
|
108
112
|
*/
|
|
109
113
|
|
|
@@ -162,9 +166,13 @@ type ConnectionState = {
|
|
|
162
166
|
* is what settles delivery. There is no code path in this module that
|
|
163
167
|
* can emit a frame other than HELLO: delivery is judged, never
|
|
164
168
|
* confirmed by the transport.
|
|
165
|
-
* - `
|
|
166
|
-
*
|
|
167
|
-
*
|
|
169
|
+
* - The client never gives up on its own. Only `stop()` ends it. A close
|
|
170
|
+
* the client itself initiated (hello-ack timeout, ping timeout, its own
|
|
171
|
+
* drain) is never read as a verdict from the server; a server-originated
|
|
172
|
+
* auth close enters `AUTH_FAIL` and retries slowly, because keys get
|
|
173
|
+
* rotated back and accounts get reinstated. A long outage widens the
|
|
174
|
+
* backoff cap to `RECONNECT_LONG_OUTAGE_BACKOFF_MS` and logs one error
|
|
175
|
+
* line so an operator sees it without a log flood.
|
|
168
176
|
* - Frame parsing is defensive: malformed JSON, unknown `type`, missing
|
|
169
177
|
* payload — all classified as `validation` errors (non-terminal for
|
|
170
178
|
* the connection, but dropped with a metric + log).
|
|
@@ -186,7 +194,7 @@ interface WsClientEvents {
|
|
|
186
194
|
error: (error: AgentChatChannelError) => void;
|
|
187
195
|
/** Fired once per `hello.ok` — the runtime turns it into the connect knock. */
|
|
188
196
|
authenticated: (at: UnixMillis) => void;
|
|
189
|
-
/** Fired
|
|
197
|
+
/** Fired once the client reached CLOSED — only ever after `stop()`. */
|
|
190
198
|
closed: () => void;
|
|
191
199
|
}
|
|
192
200
|
interface WsClientOptions {
|
|
@@ -236,6 +244,15 @@ declare class AgentchatWsClient {
|
|
|
236
244
|
private ws;
|
|
237
245
|
private attempt;
|
|
238
246
|
private _ackNegotiated;
|
|
247
|
+
/**
|
|
248
|
+
* Set by `closeSocket()` before it closes the current socket, read and
|
|
249
|
+
* cleared by `handleClose()`. A close this client asked for says nothing
|
|
250
|
+
* about the server, whatever code travelled with it.
|
|
251
|
+
*/
|
|
252
|
+
private closeInitiatedLocally;
|
|
253
|
+
/** One error line per long outage / per run of rejected keys, then debug. */
|
|
254
|
+
private longOutageAnnounced;
|
|
255
|
+
private authFailAnnounced;
|
|
239
256
|
private helloAckTimer;
|
|
240
257
|
private pingTimer;
|
|
241
258
|
private pongTimer;
|
|
@@ -252,9 +269,8 @@ declare class AgentchatWsClient {
|
|
|
252
269
|
*/
|
|
253
270
|
get ackNegotiated(): boolean;
|
|
254
271
|
/**
|
|
255
|
-
* Kick off connection. No-op if already connecting, authenticating,
|
|
256
|
-
*
|
|
257
|
-
* after rotating the API key before retrying.
|
|
272
|
+
* Kick off connection. No-op if already connecting, authenticating, ready,
|
|
273
|
+
* or waiting out a backoff / auth retry. Throws only on `CLOSED`.
|
|
258
274
|
*/
|
|
259
275
|
start(): void;
|
|
260
276
|
/**
|
|
@@ -264,9 +280,9 @@ declare class AgentchatWsClient {
|
|
|
264
280
|
*/
|
|
265
281
|
stop(deadline?: UnixMillis): void;
|
|
266
282
|
/**
|
|
267
|
-
* Signal that operator has rotated the API key
|
|
268
|
-
*
|
|
269
|
-
*
|
|
283
|
+
* Signal that the operator has rotated the API key: leave AUTH_FAIL and
|
|
284
|
+
* retry now instead of waiting for the slow retry timer. A no-op in every
|
|
285
|
+
* other state (the client is already connecting or connected).
|
|
270
286
|
*/
|
|
271
287
|
reconfigured(): void;
|
|
272
288
|
on<E extends keyof WsClientEvents>(event: E, handler: Listener<E>): () => void;
|
|
@@ -277,7 +293,14 @@ declare class AgentchatWsClient {
|
|
|
277
293
|
private extractReason;
|
|
278
294
|
private extractPayload;
|
|
279
295
|
private handleHelloOk;
|
|
296
|
+
/** The server answered HELLO with a rejection frame (not a close). */
|
|
280
297
|
private handleAuthRejected;
|
|
298
|
+
/**
|
|
299
|
+
* The server rejected this key. AUTH_FAIL is not the end: retry a fresh
|
|
300
|
+
* HELLO every `AUTH_RETRY_INTERVAL_MS` (jittered). The first rejection in a
|
|
301
|
+
* run logs at error so an operator sees it; the following ones at debug.
|
|
302
|
+
*/
|
|
303
|
+
private enterAuthFail;
|
|
281
304
|
private handleClose;
|
|
282
305
|
private closeSocket;
|
|
283
306
|
private startHeartbeat;
|
|
@@ -285,7 +308,16 @@ declare class AgentchatWsClient {
|
|
|
285
308
|
private sendPing;
|
|
286
309
|
private handlePong;
|
|
287
310
|
private scheduleReconnect;
|
|
311
|
+
/** AUTH_FAIL's own cadence: a fresh HELLO every AUTH_RETRY_INTERVAL_MS, jittered. */
|
|
312
|
+
private scheduleAuthRetry;
|
|
313
|
+
private clearReconnectTimer;
|
|
288
314
|
private computeReconnectDelay;
|
|
315
|
+
/**
|
|
316
|
+
* Symmetric ±`jitterRatio`. Thundering-herd-safe: a fleet that all drops
|
|
317
|
+
* at the same instant re-enters the connect queue spread across
|
|
318
|
+
* ms * [1-jitterRatio, 1+jitterRatio].
|
|
319
|
+
*/
|
|
320
|
+
private jittered;
|
|
289
321
|
private scheduleDrainDeadline;
|
|
290
322
|
/**
|
|
291
323
|
* Called by upper layers when their pending work has fully drained, to
|
package/dist/index.js
CHANGED
|
@@ -131,7 +131,7 @@ function classifyNetworkError(err) {
|
|
|
131
131
|
}
|
|
132
132
|
|
|
133
133
|
// src/version.ts
|
|
134
|
-
var PACKAGE_VERSION = "0.8.
|
|
134
|
+
var PACKAGE_VERSION = "0.8.1";
|
|
135
135
|
|
|
136
136
|
// src/client-identity.ts
|
|
137
137
|
var AGENTCHAT_CLIENT_NAME = "openclaw";
|
|
@@ -1777,7 +1777,6 @@ function transition(state, event) {
|
|
|
1777
1777
|
return { kind: "CONNECTING", attempt: event.attempt, startedAt: event.now };
|
|
1778
1778
|
}
|
|
1779
1779
|
if (event.type === "DRAIN_REQUESTED") return { kind: "CLOSED" };
|
|
1780
|
-
if (event.type === "AUTH_REJECTED") return { kind: "AUTH_FAIL", reason: event.reason };
|
|
1781
1780
|
return state;
|
|
1782
1781
|
}
|
|
1783
1782
|
case "CONNECTING": {
|
|
@@ -1823,6 +1822,9 @@ function transition(state, event) {
|
|
|
1823
1822
|
return state;
|
|
1824
1823
|
}
|
|
1825
1824
|
case "AUTH_FAIL": {
|
|
1825
|
+
if (event.type === "CONNECT") {
|
|
1826
|
+
return { kind: "CONNECTING", attempt: event.attempt, startedAt: event.now };
|
|
1827
|
+
}
|
|
1826
1828
|
if (event.type === "RECONFIGURED") return { kind: "DISCONNECTED" };
|
|
1827
1829
|
if (event.type === "DRAIN_REQUESTED") return { kind: "CLOSED" };
|
|
1828
1830
|
return state;
|
|
@@ -1833,15 +1835,25 @@ function transition(state, event) {
|
|
|
1833
1835
|
}
|
|
1834
1836
|
}
|
|
1835
1837
|
function isTerminal(state) {
|
|
1836
|
-
return state.kind === "CLOSED"
|
|
1838
|
+
return state.kind === "CLOSED";
|
|
1837
1839
|
}
|
|
1838
1840
|
|
|
1839
1841
|
// src/ws-client.ts
|
|
1840
1842
|
var HELLO_ACK_TIMEOUT_MS = 4e3;
|
|
1841
|
-
var
|
|
1843
|
+
var RECONNECT_LONG_OUTAGE_AFTER_ATTEMPTS = 60;
|
|
1844
|
+
var RECONNECT_LONG_OUTAGE_BACKOFF_MS = 5 * 6e4;
|
|
1845
|
+
var AUTH_RETRY_INTERVAL_MS = 75e4;
|
|
1842
1846
|
var MAX_FRAME_BYTES = 2 * 1024 * 1024;
|
|
1843
1847
|
var HELLO_CAPABILITIES = ["ack"];
|
|
1844
|
-
var
|
|
1848
|
+
var UNAMBIGUOUS_AUTH_CLOSE_CODES = /* @__PURE__ */ new Set([4401, 4403]);
|
|
1849
|
+
var AMBIGUOUS_AUTH_CLOSE_CODE = 1008;
|
|
1850
|
+
var SERVER_AUTH_CLOSE_REASONS = ["invalid token"];
|
|
1851
|
+
function isHandshakeRejection(code, reason) {
|
|
1852
|
+
if (UNAMBIGUOUS_AUTH_CLOSE_CODES.has(code)) return true;
|
|
1853
|
+
if (code !== AMBIGUOUS_AUTH_CLOSE_CODE) return false;
|
|
1854
|
+
const lower = reason.toLowerCase();
|
|
1855
|
+
return SERVER_AUTH_CLOSE_REASONS.some((known) => lower.includes(known));
|
|
1856
|
+
}
|
|
1845
1857
|
var AgentchatWsClient = class {
|
|
1846
1858
|
config;
|
|
1847
1859
|
logger;
|
|
@@ -1857,6 +1869,15 @@ var AgentchatWsClient = class {
|
|
|
1857
1869
|
ws = null;
|
|
1858
1870
|
attempt = 0;
|
|
1859
1871
|
_ackNegotiated = false;
|
|
1872
|
+
/**
|
|
1873
|
+
* Set by `closeSocket()` before it closes the current socket, read and
|
|
1874
|
+
* cleared by `handleClose()`. A close this client asked for says nothing
|
|
1875
|
+
* about the server, whatever code travelled with it.
|
|
1876
|
+
*/
|
|
1877
|
+
closeInitiatedLocally = false;
|
|
1878
|
+
/** One error line per long outage / per run of rejected keys, then debug. */
|
|
1879
|
+
longOutageAnnounced = false;
|
|
1880
|
+
authFailAnnounced = false;
|
|
1860
1881
|
helloAckTimer = null;
|
|
1861
1882
|
pingTimer = null;
|
|
1862
1883
|
pongTimer = null;
|
|
@@ -1896,21 +1917,14 @@ var AgentchatWsClient = class {
|
|
|
1896
1917
|
return this._ackNegotiated;
|
|
1897
1918
|
}
|
|
1898
1919
|
/**
|
|
1899
|
-
* Kick off connection. No-op if already connecting, authenticating,
|
|
1900
|
-
*
|
|
1901
|
-
* after rotating the API key before retrying.
|
|
1920
|
+
* Kick off connection. No-op if already connecting, authenticating, ready,
|
|
1921
|
+
* or waiting out a backoff / auth retry. Throws only on `CLOSED`.
|
|
1902
1922
|
*/
|
|
1903
1923
|
start() {
|
|
1904
|
-
if (this.state.kind === "AUTH_FAIL") {
|
|
1905
|
-
throw new AgentChatChannelError(
|
|
1906
|
-
"terminal-auth",
|
|
1907
|
-
"cannot start: channel is in AUTH_FAIL \u2014 rotate api key and call reconfigured()"
|
|
1908
|
-
);
|
|
1909
|
-
}
|
|
1910
1924
|
if (this.state.kind === "CLOSED") {
|
|
1911
1925
|
throw new Error("cannot start: client is closed. Create a new instance.");
|
|
1912
1926
|
}
|
|
1913
|
-
if (this.state.kind !== "DISCONNECTED") {
|
|
1927
|
+
if (this.state.kind !== "DISCONNECTED" || this.reconnectTimer) {
|
|
1914
1928
|
return;
|
|
1915
1929
|
}
|
|
1916
1930
|
this.openSocket(1);
|
|
@@ -1943,13 +1957,15 @@ var AgentchatWsClient = class {
|
|
|
1943
1957
|
}
|
|
1944
1958
|
}
|
|
1945
1959
|
/**
|
|
1946
|
-
* Signal that operator has rotated the API key
|
|
1947
|
-
*
|
|
1948
|
-
*
|
|
1960
|
+
* Signal that the operator has rotated the API key: leave AUTH_FAIL and
|
|
1961
|
+
* retry now instead of waiting for the slow retry timer. A no-op in every
|
|
1962
|
+
* other state (the client is already connecting or connected).
|
|
1949
1963
|
*/
|
|
1950
1964
|
reconfigured() {
|
|
1951
1965
|
if (this.state.kind !== "AUTH_FAIL") return;
|
|
1966
|
+
this.clearReconnectTimer();
|
|
1952
1967
|
this.applyEvent({ type: "RECONFIGURED" });
|
|
1968
|
+
this.openSocket(1);
|
|
1953
1969
|
}
|
|
1954
1970
|
on(event, handler) {
|
|
1955
1971
|
this.listeners[event].add(handler);
|
|
@@ -1959,7 +1975,9 @@ var AgentchatWsClient = class {
|
|
|
1959
1975
|
}
|
|
1960
1976
|
// ─── Internals: socket lifecycle ─────────────────────────────────────
|
|
1961
1977
|
openSocket(attempt) {
|
|
1978
|
+
if (this.state.kind === "CLOSED") return;
|
|
1962
1979
|
this.attempt = attempt;
|
|
1980
|
+
this.closeInitiatedLocally = false;
|
|
1963
1981
|
const now = this.now();
|
|
1964
1982
|
this.applyEvent({ type: "CONNECT", now, attempt });
|
|
1965
1983
|
const url = `${this.config.apiBase.replace(/^http/, "ws")}/v1/ws`;
|
|
@@ -2031,7 +2049,7 @@ var AgentchatWsClient = class {
|
|
|
2031
2049
|
this.emitError(
|
|
2032
2050
|
new AgentChatChannelError("retry-transient", "hello ack timeout")
|
|
2033
2051
|
);
|
|
2034
|
-
this.closeSocket(
|
|
2052
|
+
this.closeSocket(1001, "hello ack timeout");
|
|
2035
2053
|
}, HELLO_ACK_TIMEOUT_MS);
|
|
2036
2054
|
}
|
|
2037
2055
|
handleMessage(data, isBinary) {
|
|
@@ -2118,26 +2136,43 @@ var AgentchatWsClient = class {
|
|
|
2118
2136
|
this.logger.info({ ackNegotiated: this._ackNegotiated }, "hello.ok");
|
|
2119
2137
|
this.applyEvent({ type: "HELLO_OK", now });
|
|
2120
2138
|
this.attempt = 0;
|
|
2139
|
+
this.longOutageAnnounced = false;
|
|
2140
|
+
this.authFailAnnounced = false;
|
|
2121
2141
|
this.startHeartbeat();
|
|
2122
2142
|
this.emit("authenticated", now);
|
|
2123
2143
|
}
|
|
2144
|
+
/** The server answered HELLO with a rejection frame (not a close). */
|
|
2124
2145
|
handleAuthRejected(reason) {
|
|
2125
2146
|
if (this.helloAckTimer) {
|
|
2126
2147
|
this._clearTimeout(this.helloAckTimer);
|
|
2127
2148
|
this.helloAckTimer = null;
|
|
2128
2149
|
}
|
|
2129
|
-
this.
|
|
2150
|
+
this.enterAuthFail(reason);
|
|
2151
|
+
this.closeSocket(1e3, "auth rejected");
|
|
2152
|
+
}
|
|
2153
|
+
/**
|
|
2154
|
+
* The server rejected this key. AUTH_FAIL is not the end: retry a fresh
|
|
2155
|
+
* HELLO every `AUTH_RETRY_INTERVAL_MS` (jittered). The first rejection in a
|
|
2156
|
+
* run logs at error so an operator sees it; the following ones at debug.
|
|
2157
|
+
*/
|
|
2158
|
+
enterAuthFail(reason) {
|
|
2159
|
+
this.logger[this.authFailAnnounced ? "debug" : "error"](
|
|
2160
|
+
{ reason, retryIntervalMs: AUTH_RETRY_INTERVAL_MS, attempt: this.attempt },
|
|
2161
|
+
"auth rejected \u2014 retrying slowly until the key is accepted again"
|
|
2162
|
+
);
|
|
2163
|
+
this.authFailAnnounced = true;
|
|
2130
2164
|
this.applyEvent({ type: "AUTH_REJECTED", reason });
|
|
2131
2165
|
this.emitError(
|
|
2132
2166
|
new AgentChatChannelError("terminal-auth", `auth rejected: ${reason}`)
|
|
2133
2167
|
);
|
|
2134
|
-
this.
|
|
2168
|
+
this.scheduleAuthRetry();
|
|
2135
2169
|
}
|
|
2136
2170
|
handleClose(code, reason) {
|
|
2137
|
-
this.now();
|
|
2138
2171
|
const wasAuthenticating = this.state.kind === "AUTHENTICATING";
|
|
2139
2172
|
const wasDraining = this.state.kind === "DRAINING";
|
|
2140
|
-
const
|
|
2173
|
+
const ours = this.closeInitiatedLocally;
|
|
2174
|
+
this.closeInitiatedLocally = false;
|
|
2175
|
+
const serverAuthClose = !ours && isHandshakeRejection(code, reason);
|
|
2141
2176
|
this.stopHeartbeat();
|
|
2142
2177
|
if (this.helloAckTimer) {
|
|
2143
2178
|
this._clearTimeout(this.helloAckTimer);
|
|
@@ -2145,19 +2180,17 @@ var AgentchatWsClient = class {
|
|
|
2145
2180
|
}
|
|
2146
2181
|
this.ws = null;
|
|
2147
2182
|
this._ackNegotiated = false;
|
|
2148
|
-
|
|
2183
|
+
const kind = this.state.kind;
|
|
2184
|
+
if (kind === "CLOSED") {
|
|
2149
2185
|
this.logger.debug({ code, reason }, "socket closed after CLOSED \u2014 no-op");
|
|
2150
2186
|
return;
|
|
2151
2187
|
}
|
|
2152
|
-
if (
|
|
2153
|
-
this.
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
)
|
|
2159
|
-
);
|
|
2160
|
-
this.emit("closed");
|
|
2188
|
+
if (kind === "AUTH_FAIL") {
|
|
2189
|
+
this.logger.debug({ code, reason }, "socket closed after AUTH_FAIL \u2014 retry already scheduled");
|
|
2190
|
+
return;
|
|
2191
|
+
}
|
|
2192
|
+
if (serverAuthClose && wasAuthenticating) {
|
|
2193
|
+
this.enterAuthFail(`close ${code}: ${reason}`);
|
|
2161
2194
|
return;
|
|
2162
2195
|
}
|
|
2163
2196
|
this.applyEvent({ type: "SOCKET_CLOSED", code, reason });
|
|
@@ -2166,13 +2199,14 @@ var AgentchatWsClient = class {
|
|
|
2166
2199
|
return;
|
|
2167
2200
|
}
|
|
2168
2201
|
this.logger.info(
|
|
2169
|
-
{ code, reason, attempt: this.attempt },
|
|
2202
|
+
{ code, reason, ours, attempt: this.attempt },
|
|
2170
2203
|
"socket closed \u2014 scheduling reconnect"
|
|
2171
2204
|
);
|
|
2172
2205
|
this.scheduleReconnect(`close-${code}`);
|
|
2173
2206
|
}
|
|
2174
2207
|
closeSocket(code, reason) {
|
|
2175
2208
|
if (!this.ws) return;
|
|
2209
|
+
this.closeInitiatedLocally = true;
|
|
2176
2210
|
try {
|
|
2177
2211
|
this.ws.close(code, reason);
|
|
2178
2212
|
} catch {
|
|
@@ -2232,14 +2266,15 @@ var AgentchatWsClient = class {
|
|
|
2232
2266
|
if (this.state.kind === "CLOSED" || this.state.kind === "AUTH_FAIL") return;
|
|
2233
2267
|
if (this.reconnectTimer) return;
|
|
2234
2268
|
const nextAttempt = this.attempt + 1;
|
|
2235
|
-
if (nextAttempt >
|
|
2236
|
-
|
|
2237
|
-
this.logger.error(
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2269
|
+
if (nextAttempt > RECONNECT_LONG_OUTAGE_AFTER_ATTEMPTS && !this.longOutageAnnounced) {
|
|
2270
|
+
this.longOutageAnnounced = true;
|
|
2271
|
+
this.logger.error(
|
|
2272
|
+
{
|
|
2273
|
+
attempt: nextAttempt,
|
|
2274
|
+
maxBackoffMs: Math.max(this.config.reconnect.maxBackoffMs, RECONNECT_LONG_OUTAGE_BACKOFF_MS)
|
|
2275
|
+
},
|
|
2276
|
+
`still reconnecting after ${RECONNECT_LONG_OUTAGE_AFTER_ATTEMPTS} attempts \u2014 backing off further, not giving up`
|
|
2241
2277
|
);
|
|
2242
|
-
return;
|
|
2243
2278
|
}
|
|
2244
2279
|
const delay = this.computeReconnectDelay(nextAttempt);
|
|
2245
2280
|
this.metrics.incReconnect({ reason });
|
|
@@ -2252,13 +2287,41 @@ var AgentchatWsClient = class {
|
|
|
2252
2287
|
this.openSocket(nextAttempt);
|
|
2253
2288
|
}, delay);
|
|
2254
2289
|
}
|
|
2290
|
+
/** AUTH_FAIL's own cadence: a fresh HELLO every AUTH_RETRY_INTERVAL_MS, jittered. */
|
|
2291
|
+
scheduleAuthRetry() {
|
|
2292
|
+
if (this.state.kind !== "AUTH_FAIL") return;
|
|
2293
|
+
this.clearReconnectTimer();
|
|
2294
|
+
const nextAttempt = this.attempt + 1;
|
|
2295
|
+
const delay = this.jittered(AUTH_RETRY_INTERVAL_MS);
|
|
2296
|
+
this.metrics.incReconnect({ reason: "auth-retry" });
|
|
2297
|
+
this.logger.debug({ attempt: nextAttempt, delayMs: delay }, "scheduling auth retry");
|
|
2298
|
+
this.reconnectTimer = this._setTimeout(() => {
|
|
2299
|
+
this.reconnectTimer = null;
|
|
2300
|
+
this.openSocket(nextAttempt);
|
|
2301
|
+
}, delay);
|
|
2302
|
+
}
|
|
2303
|
+
clearReconnectTimer() {
|
|
2304
|
+
if (this.reconnectTimer) {
|
|
2305
|
+
this._clearTimeout(this.reconnectTimer);
|
|
2306
|
+
this.reconnectTimer = null;
|
|
2307
|
+
}
|
|
2308
|
+
}
|
|
2255
2309
|
computeReconnectDelay(attempt) {
|
|
2256
|
-
const { initialBackoffMs, maxBackoffMs
|
|
2310
|
+
const { initialBackoffMs, maxBackoffMs } = this.config.reconnect;
|
|
2257
2311
|
const reconnectCount = Math.max(1, attempt - 1);
|
|
2258
2312
|
const exp = initialBackoffMs * Math.pow(2, Math.min(reconnectCount - 1, 20));
|
|
2259
|
-
const
|
|
2313
|
+
const cap = attempt > RECONNECT_LONG_OUTAGE_AFTER_ATTEMPTS ? Math.max(maxBackoffMs, RECONNECT_LONG_OUTAGE_BACKOFF_MS) : maxBackoffMs;
|
|
2314
|
+
return this.jittered(Math.min(exp, cap));
|
|
2315
|
+
}
|
|
2316
|
+
/**
|
|
2317
|
+
* Symmetric ±`jitterRatio`. Thundering-herd-safe: a fleet that all drops
|
|
2318
|
+
* at the same instant re-enters the connect queue spread across
|
|
2319
|
+
* ms * [1-jitterRatio, 1+jitterRatio].
|
|
2320
|
+
*/
|
|
2321
|
+
jittered(ms) {
|
|
2322
|
+
const { jitterRatio } = this.config.reconnect;
|
|
2260
2323
|
const jitter = 1 - jitterRatio + this.random() * 2 * jitterRatio;
|
|
2261
|
-
return Math.max(0, Math.floor(
|
|
2324
|
+
return Math.max(0, Math.floor(ms * jitter));
|
|
2262
2325
|
}
|
|
2263
2326
|
// ─── Internals: drain ────────────────────────────────────────────────
|
|
2264
2327
|
scheduleDrainDeadline(deadline) {
|
|
@@ -2301,6 +2364,9 @@ var AgentchatWsClient = class {
|
|
|
2301
2364
|
return;
|
|
2302
2365
|
}
|
|
2303
2366
|
this.state = next;
|
|
2367
|
+
if (next.kind === "CLOSED") {
|
|
2368
|
+
this.clearReconnectTimer();
|
|
2369
|
+
}
|
|
2304
2370
|
this.metrics.setConnectionState(next.kind);
|
|
2305
2371
|
this.logger.info(
|
|
2306
2372
|
{ from: prev.kind, to: next.kind, event: event.type },
|
|
@@ -3411,6 +3477,7 @@ var agentchatGatewayAdapter = {
|
|
|
3411
3477
|
gateFailOpen: readReplyGateFailOpenFromEnv(),
|
|
3412
3478
|
...gateTimeoutMs !== void 0 ? { gateTimeoutMs } : {}
|
|
3413
3479
|
});
|
|
3480
|
+
let authBlocked = false;
|
|
3414
3481
|
return {
|
|
3415
3482
|
invoker,
|
|
3416
3483
|
handlers: {
|
|
@@ -3422,16 +3489,39 @@ var agentchatGatewayAdapter = {
|
|
|
3422
3489
|
void catchUpInvites(ctx.accountId, config, invoker, logger);
|
|
3423
3490
|
},
|
|
3424
3491
|
onError: (err) => {
|
|
3425
|
-
|
|
3492
|
+
const level = err.class_ === "terminal-auth" && authBlocked ? "debug" : "warn";
|
|
3493
|
+
ctx.log?.[level]?.(`[agentchat:${ctx.accountId}] runtime error: ${err.message} (class=${err.class_})`);
|
|
3426
3494
|
},
|
|
3427
3495
|
onStateChanged: (next) => {
|
|
3428
|
-
const running = next.kind !== "CLOSED" && next.kind !== "AUTH_FAIL";
|
|
3429
|
-
const connected = next.kind === "READY" || next.kind === "DEGRADED";
|
|
3430
|
-
const lifecycle = next.kind === "READY" || next.kind === "DEGRADED" ? "ready" : next.kind === "AUTH_FAIL" ? "blocked" : next.kind === "CLOSED" ? "stopped" : runtime.getHealth().counters.knocks > 0 ? "recovering" : "starting";
|
|
3431
|
-
ctx.setStatus(withLifecycle({ ...ctx.getStatus(), running, connected, statusState: statusStateFor(next.kind) }, lifecycle));
|
|
3432
3496
|
if (next.kind === "AUTH_FAIL") {
|
|
3433
|
-
|
|
3497
|
+
if (authBlocked) {
|
|
3498
|
+
ctx.log?.debug?.(`[agentchat:${ctx.accountId}] still AUTH_FAIL (${next.reason}) \u2014 next retry in 10\u201315 min`);
|
|
3499
|
+
return;
|
|
3500
|
+
}
|
|
3501
|
+
authBlocked = true;
|
|
3502
|
+
ctx.log?.error?.(
|
|
3503
|
+
`[agentchat:${ctx.accountId}] AUTH_FAIL (${next.reason}) \u2014 the server rejected the key; retrying every 10\u201315 min; rotate the key or re-run openclaw channels add agentchat`
|
|
3504
|
+
);
|
|
3505
|
+
ctx.setStatus(
|
|
3506
|
+
withLifecycle(
|
|
3507
|
+
{ ...ctx.getStatus(), running: false, connected: false, statusState: "not-linked", lastDisconnect: { at: Date.now(), error: `AUTH_FAIL: ${next.reason}` } },
|
|
3508
|
+
"blocked"
|
|
3509
|
+
)
|
|
3510
|
+
);
|
|
3511
|
+
return;
|
|
3512
|
+
}
|
|
3513
|
+
if (authBlocked && next.kind !== "READY" && next.kind !== "CLOSED") {
|
|
3514
|
+
ctx.log?.debug?.(`[agentchat:${ctx.accountId}] AUTH_FAIL retry: ${next.kind.toLowerCase()}`);
|
|
3515
|
+
return;
|
|
3516
|
+
}
|
|
3517
|
+
if (authBlocked) {
|
|
3518
|
+
authBlocked = false;
|
|
3519
|
+
if (next.kind === "READY") ctx.log?.info?.(`[agentchat:${ctx.accountId}] key accepted again \u2014 AUTH_FAIL cleared`);
|
|
3434
3520
|
}
|
|
3521
|
+
const running = next.kind !== "CLOSED";
|
|
3522
|
+
const connected = next.kind === "READY" || next.kind === "DEGRADED";
|
|
3523
|
+
const lifecycle = next.kind === "READY" || next.kind === "DEGRADED" ? "ready" : next.kind === "CLOSED" ? "stopped" : runtime.getHealth().counters.knocks > 0 ? "recovering" : "starting";
|
|
3524
|
+
ctx.setStatus(withLifecycle({ ...ctx.getStatus(), running, connected, statusState: statusStateFor(next.kind) }, lifecycle));
|
|
3435
3525
|
}
|
|
3436
3526
|
}
|
|
3437
3527
|
};
|