mixdog 0.9.16 → 0.9.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -1
- package/scripts/atomic-lock-tryonce-test.mjs +66 -0
- package/scripts/provider-toolcall-test.mjs +79 -2
- package/src/mixdog-session-runtime.mjs +12 -0
- package/src/runtime/agent/orchestrator/providers/anthropic-effort.mjs +33 -1
- package/src/runtime/agent/orchestrator/providers/anthropic-oauth.mjs +9 -0
- package/src/runtime/agent/orchestrator/providers/anthropic-sse.mjs +49 -0
- package/src/runtime/agent/orchestrator/providers/anthropic.mjs +14 -0
- package/src/runtime/agent/orchestrator/session/context-utils.mjs +7 -0
- package/src/runtime/agent/orchestrator/session/loop.mjs +8 -0
- package/src/runtime/agent/orchestrator/session/manager/pending-messages.mjs +32 -18
- package/src/runtime/agent/orchestrator/session/manager/usage-metrics.mjs +142 -3
- package/src/runtime/agent/orchestrator/session/store-summary-index.mjs +108 -30
- package/src/runtime/agent/orchestrator/session/store.mjs +5 -0
- package/src/runtime/agent/orchestrator/stall-policy.mjs +22 -12
- package/src/runtime/agent/orchestrator/tools/builtin/builtin-tools.mjs +3 -1
- package/src/runtime/agent/orchestrator/tools/builtin/read-single-tool.mjs +15 -15
- package/src/runtime/agent/orchestrator/tools/builtin/snapshot-store.mjs +10 -2
- package/src/runtime/agent/orchestrator/tools/code-graph/disk-cache.mjs +8 -2
- package/src/runtime/channels/backends/discord-attachments.mjs +2 -2
- package/src/runtime/channels/backends/discord-gateway.mjs +12 -2
- package/src/runtime/channels/backends/discord.mjs +97 -7
- package/src/runtime/channels/index.mjs +150 -23
- package/src/runtime/channels/lib/crash-log.mjs +21 -3
- package/src/runtime/channels/lib/output-forwarder.mjs +118 -7
- package/src/runtime/channels/lib/runtime-paths.mjs +21 -19
- package/src/runtime/channels/lib/voice-transcription.mjs +4 -2
- package/src/runtime/memory/index.mjs +37 -0
- package/src/runtime/memory/lib/embedding-warmup.mjs +3 -0
- package/src/runtime/memory/lib/ko-morph.mjs +1 -0
- package/src/runtime/shared/atomic-file.mjs +110 -0
- package/src/runtime/shared/transcript-writer.mjs +46 -4
- package/src/session-runtime/provider-models.mjs +47 -8
- package/src/standalone/channel-worker.mjs +14 -2
- package/src/tui/app/transcript-window.mjs +137 -6
- package/src/tui/app/use-transcript-window.mjs +67 -10
- package/src/tui/components/StatusLine.jsx +1 -1
- package/src/tui/dist/index.mjs +436 -93
- package/src/tui/engine/tui-steering-persist.mjs +66 -35
- package/src/tui/engine.mjs +66 -14
- package/src/tui/index.jsx +97 -6
- package/src/ui/statusline-segments.mjs +54 -36
- package/src/ui/statusline.mjs +141 -95
|
@@ -23,6 +23,14 @@ import { normalizeAccess, safeAttName } from "./discord-access.mjs";
|
|
|
23
23
|
import { MAX_ATTACHMENT_BYTES, downloadSingleAttachment } from "./discord-attachments.mjs";
|
|
24
24
|
import * as gateway from "./discord-gateway.mjs";
|
|
25
25
|
const MAX_CHUNK_LIMIT = MAX_DISCORD_MESSAGE;
|
|
26
|
+
// Per-attempt cap for a single discord.js fetch/send. Without it a wedged
|
|
27
|
+
// socket (network blip, "other side closed") leaves the await pending forever,
|
|
28
|
+
// which upstream leaves the forwarder's `sending` flag stuck and silences all
|
|
29
|
+
// outbound. On timeout we treat the client as dead and reset+reconnect it.
|
|
30
|
+
const SEND_ATTEMPT_TIMEOUT_MS = 30_000;
|
|
31
|
+
// After a timed-out send is reset, cap how long we wait for the original
|
|
32
|
+
// (abandoned) promise to actually settle before giving up with unknown outcome.
|
|
33
|
+
const SETTLE_CAP_MS = 15_000;
|
|
26
34
|
const RECENT_SENT_CAP = 200;
|
|
27
35
|
class DiscordBackend {
|
|
28
36
|
name = "discord";
|
|
@@ -74,11 +82,57 @@ class DiscordBackend {
|
|
|
74
82
|
if (this.client) this.client.destroy();
|
|
75
83
|
this._connectPromise = null;
|
|
76
84
|
}
|
|
85
|
+
// ── Self-heal helpers ──────────────────────────────────────────────
|
|
86
|
+
/** Race a discord.js op-promise against a per-attempt timeout so a wedged
|
|
87
|
+
* socket can't hang the outbound path. On timeout the rejection carries
|
|
88
|
+
* `_timeout` so the caller knows the underlying op is still in flight (and
|
|
89
|
+
* must be settled/adopted, not blindly retried → duplicate). */
|
|
90
|
+
_withTimeout(promise, label) {
|
|
91
|
+
let timer;
|
|
92
|
+
const timeout = new Promise((_, reject) => {
|
|
93
|
+
timer = setTimeout(() => {
|
|
94
|
+
const e = new Error(`${label} timed out after ${SEND_ATTEMPT_TIMEOUT_MS}ms`);
|
|
95
|
+
e._timeout = true;
|
|
96
|
+
reject(e);
|
|
97
|
+
}, SEND_ATTEMPT_TIMEOUT_MS);
|
|
98
|
+
});
|
|
99
|
+
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
|
|
100
|
+
}
|
|
101
|
+
/** True when an error means the client is dead (aborted, "other side
|
|
102
|
+
* closed", or our per-attempt timeout) and must be reset+reconnected. */
|
|
103
|
+
_shouldResetClient(err) {
|
|
104
|
+
if (err?.name === "AbortError" || err?.code === "ABORT_ERR") return true;
|
|
105
|
+
const msg = String(err?.message || err || "").toLowerCase();
|
|
106
|
+
return msg.includes("aborted") || msg.includes("other side closed") ||
|
|
107
|
+
msg.includes("socket hang up") || msg.includes("econnreset") || msg.includes("timed out");
|
|
108
|
+
}
|
|
109
|
+
/** Destroy the wedged client and rebuild it via connect() so the next
|
|
110
|
+
* fetch/send (and the forwarder's queue-head-preserving retry) hits a fresh
|
|
111
|
+
* gateway/REST client. Serialized behind a single in-flight reset so
|
|
112
|
+
* concurrent callers await the SAME reconnect instead of racing multiple
|
|
113
|
+
* destroy()/login() cycles. */
|
|
114
|
+
async _resetClient() {
|
|
115
|
+
if (this._resetPromise) return this._resetPromise;
|
|
116
|
+
this._resetPromise = (async () => {
|
|
117
|
+
try { this.client?.destroy?.(); } catch {}
|
|
118
|
+
this.client = null;
|
|
119
|
+
this._connectPromise = null;
|
|
120
|
+
await this.connect();
|
|
121
|
+
})().finally(() => { this._resetPromise = null; });
|
|
122
|
+
return this._resetPromise;
|
|
123
|
+
}
|
|
124
|
+
/** Health-check the gateway; reset+reconnect if the ws is not READY. Called
|
|
125
|
+
* by the owner re-claim self-heal path. (discord.js Status.Ready === 0) */
|
|
126
|
+
async healthCheck() {
|
|
127
|
+
if (this.client && this.client.ws?.status === 0) return;
|
|
128
|
+
await this._resetClient();
|
|
129
|
+
}
|
|
77
130
|
resetSendCount() {
|
|
78
131
|
this.sendCount = 0;
|
|
79
132
|
}
|
|
80
133
|
startTyping(channelId) {
|
|
81
134
|
this.stopTyping(channelId);
|
|
135
|
+
if (!this.client) return;
|
|
82
136
|
const ch = this.client.channels.cache.get(channelId);
|
|
83
137
|
if (ch && "sendTyping" in ch) {
|
|
84
138
|
void ch.sendTyping().catch(() => {
|
|
@@ -172,13 +226,38 @@ class DiscordBackend {
|
|
|
172
226
|
// resumes at this chunk index instead of re-sending 0..i-1.
|
|
173
227
|
let attempt = 0;
|
|
174
228
|
for (;;) {
|
|
229
|
+
const sendPromise = ch.send(payload);
|
|
175
230
|
try {
|
|
176
|
-
const sent = await
|
|
231
|
+
const sent = await this._withTimeout(sendPromise, "discord send");
|
|
177
232
|
this.noteSent(sent.id);
|
|
178
233
|
sentIds.push(sent.id);
|
|
179
234
|
break;
|
|
180
235
|
} catch (err) {
|
|
181
236
|
attempt++;
|
|
237
|
+
if (err?._timeout) {
|
|
238
|
+
// Watchdog abandoned the send but the socket may still deliver.
|
|
239
|
+
// Destroy the client to tear down the in-flight REST socket so the
|
|
240
|
+
// ORIGINAL promise settles, then AWAIT + adopt its real outcome:
|
|
241
|
+
// delivered → done (no retry → no duplicate); else requeue against
|
|
242
|
+
// the fresh client via a resume token.
|
|
243
|
+
try { await this._resetClient(); } catch {}
|
|
244
|
+
// Cap the settlement wait: destroy() should abort the in-flight
|
|
245
|
+
// socket, but never hang here if it doesn't. On unknown outcome
|
|
246
|
+
// throw retryable (duplicate risk accepted over a permanent hang).
|
|
247
|
+
sendPromise.catch(() => {}); // swallow a late rejection
|
|
248
|
+
const settled = await Promise.race([
|
|
249
|
+
sendPromise.then((v) => ({ ok: true, v }), () => ({ ok: false })),
|
|
250
|
+
new Promise((r) => setTimeout(() => r({ ok: false }), SETTLE_CAP_MS))
|
|
251
|
+
]);
|
|
252
|
+
if (settled.ok) {
|
|
253
|
+
this.noteSent(settled.v.id);
|
|
254
|
+
sentIds.push(settled.v.id);
|
|
255
|
+
break;
|
|
256
|
+
}
|
|
257
|
+
const e = err instanceof Error ? err : new Error(String(err));
|
|
258
|
+
e.resumeToken = { hash: contentHash, nextChunkIdx: i, sentIds: [...sentIds], prefixed: applyPrefix, limit };
|
|
259
|
+
throw e;
|
|
260
|
+
}
|
|
182
261
|
const status = err?.status ?? err?.code ?? err?.httpStatus;
|
|
183
262
|
// Classify: PERMANENT = a 4xx client error (unknown channel/message,
|
|
184
263
|
// missing access/permissions, 404 …) that will never succeed on
|
|
@@ -192,7 +271,15 @@ class DiscordBackend {
|
|
|
192
271
|
e.permanent = true;
|
|
193
272
|
throw e;
|
|
194
273
|
}
|
|
195
|
-
|
|
274
|
+
// Dead client (abort / "other side closed" / per-attempt timeout):
|
|
275
|
+
// destroy + reconnect so the forwarder's transient-retry path hits a
|
|
276
|
+
// fresh client, and throw a resume token now instead of burning the
|
|
277
|
+
// remaining attempts on a stale channel handle bound to the old client.
|
|
278
|
+
const dead = this._shouldResetClient(err);
|
|
279
|
+
if (dead) {
|
|
280
|
+
try { await this._resetClient(); } catch {}
|
|
281
|
+
}
|
|
282
|
+
if (dead || attempt >= 3) {
|
|
196
283
|
// Attach an opaque resume token pointing at the chunk that failed
|
|
197
284
|
// (i). sentIds holds every chunk already delivered (including any
|
|
198
285
|
// seeded from a prior token).
|
|
@@ -301,13 +388,13 @@ class DiscordBackend {
|
|
|
301
388
|
const msg = await ch.messages.fetch(messageId);
|
|
302
389
|
await msg.delete();
|
|
303
390
|
}
|
|
304
|
-
async downloadAttachment(chatId, messageId) {
|
|
391
|
+
async downloadAttachment(chatId, messageId, opts = {}) {
|
|
305
392
|
const ch = await this.fetchAllowedChannel(chatId);
|
|
306
393
|
const msg = await ch.messages.fetch(messageId);
|
|
307
394
|
if (msg.attachments.size === 0) return [];
|
|
308
395
|
const results = [];
|
|
309
396
|
for (const att of msg.attachments.values()) {
|
|
310
|
-
const path = await this.downloadSingleAttachment(att);
|
|
397
|
+
const path = await this.downloadSingleAttachment(att, opts);
|
|
311
398
|
results.push({
|
|
312
399
|
id: att.id,
|
|
313
400
|
path,
|
|
@@ -526,7 +613,10 @@ class DiscordBackend {
|
|
|
526
613
|
}
|
|
527
614
|
// ── Channel helpers ────────────────────────────────────────────────
|
|
528
615
|
async fetchTextChannel(id) {
|
|
529
|
-
|
|
616
|
+
if (!this.client) await this.connect();
|
|
617
|
+
const p = this.client.channels.fetch(id);
|
|
618
|
+
p.catch(() => {}); // if the watchdog wins, don't leak an unhandled rejection
|
|
619
|
+
const ch = await this._withTimeout(p, "discord channels.fetch");
|
|
530
620
|
if (!ch || !ch.isTextBased()) {
|
|
531
621
|
throw new Error(`channel ${id} not found or not text-based`);
|
|
532
622
|
}
|
|
@@ -571,8 +661,8 @@ class DiscordBackend {
|
|
|
571
661
|
throw new Error(`refusing to send channel state: ${f}`);
|
|
572
662
|
}
|
|
573
663
|
}
|
|
574
|
-
async downloadSingleAttachment(att) {
|
|
575
|
-
return downloadSingleAttachment(att, this.inboxDir);
|
|
664
|
+
async downloadSingleAttachment(att, opts = {}) {
|
|
665
|
+
return downloadSingleAttachment(att, this.inboxDir, opts);
|
|
576
666
|
}
|
|
577
667
|
}
|
|
578
668
|
export {
|
|
@@ -305,6 +305,11 @@ const forwarder = new OutputForwarder({
|
|
|
305
305
|
removeReaction: (ch, mid, emoji) => {
|
|
306
306
|
if (!channelBridgeActive) return Promise.resolve();
|
|
307
307
|
return backend.removeReaction(ch, mid, emoji);
|
|
308
|
+
},
|
|
309
|
+
// Watchdog backstop: force the backend to tear down + rebuild a wedged
|
|
310
|
+
// client so an over-budget send fails fast and the queue releases.
|
|
311
|
+
resetBackend: async () => {
|
|
312
|
+
if (typeof backend?._resetClient === "function") await backend._resetClient();
|
|
308
313
|
}
|
|
309
314
|
}, statusState);
|
|
310
315
|
forwarder.setOnIdle(() => {
|
|
@@ -323,7 +328,11 @@ forwarder.setOnIdle(() => {
|
|
|
323
328
|
forwarder.setOwnerGetter(() => bridgeRuntimeConnected && currentOwnerState().owned);
|
|
324
329
|
function applyTranscriptBinding(channelId, transcriptPath, options = {}) {
|
|
325
330
|
if (!transcriptPath) return;
|
|
326
|
-
forwarder.setContext(channelId, transcriptPath, {
|
|
331
|
+
forwarder.setContext(channelId, transcriptPath, {
|
|
332
|
+
replayFromStart: options.replayFromStart,
|
|
333
|
+
catchUpFromPersisted: options.catchUpFromPersisted,
|
|
334
|
+
recoverUnsyncedTail: options.recoverUnsyncedTail,
|
|
335
|
+
});
|
|
327
336
|
const boundTranscriptPath = forwarder.transcriptPath || transcriptPath;
|
|
328
337
|
forwarder.startWatch();
|
|
329
338
|
void memoryIngestTranscript(boundTranscriptPath, { cwd: options.cwd });
|
|
@@ -336,14 +345,31 @@ function applyTranscriptBinding(channelId, transcriptPath, options = {}) {
|
|
|
336
345
|
refreshActiveInstance(INSTANCE_ID, { channelId, transcriptPath: boundTranscriptPath }, { onlyIfOwned: true });
|
|
337
346
|
if (options.persistStatus !== false) {
|
|
338
347
|
statusState.update((state) => {
|
|
348
|
+
const wasSameTranscript = typeof state.transcriptPath === "string"
|
|
349
|
+
&& state.transcriptPath
|
|
350
|
+
&& sameResolvedPath(state.transcriptPath, boundTranscriptPath);
|
|
351
|
+
const prevSentCount = Number(state.sentCount ?? 0);
|
|
352
|
+
const nextSentCount = Number(forwarder.sentCount ?? 0);
|
|
339
353
|
state.channelId = channelId;
|
|
340
354
|
state.transcriptPath = boundTranscriptPath;
|
|
341
355
|
state.lastFileSize = forwarder.lastFileSize;
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
356
|
+
if (wasSameTranscript) {
|
|
357
|
+
state.sentCount = Math.max(
|
|
358
|
+
Number.isFinite(prevSentCount) ? prevSentCount : 0,
|
|
359
|
+
Number.isFinite(nextSentCount) ? nextSentCount : 0,
|
|
360
|
+
);
|
|
361
|
+
if (forwarder.lastHash) state.lastSentHash = forwarder.lastHash;
|
|
362
|
+
else if (typeof state.lastSentHash !== "string") state.lastSentHash = "";
|
|
363
|
+
if (!Number.isFinite(Number(state.lastSentTime))) state.lastSentTime = 0;
|
|
364
|
+
if (typeof state.sessionIdle !== "boolean") state.sessionIdle = false;
|
|
365
|
+
} else {
|
|
366
|
+
state.sentCount = forwarder.sentCount;
|
|
367
|
+
state.lastSentHash = forwarder.lastHash;
|
|
368
|
+
state.lastSentTime = 0;
|
|
369
|
+
state.sessionIdle = false;
|
|
370
|
+
}
|
|
371
|
+
if (options.cwd !== undefined) state.sessionCwd = options.cwd ?? null;
|
|
372
|
+
else if (!wasSameTranscript) state.sessionCwd = null;
|
|
347
373
|
});
|
|
348
374
|
}
|
|
349
375
|
}
|
|
@@ -421,7 +447,8 @@ async function rebindTranscriptContext(channelId, options = {}) {
|
|
|
421
447
|
applyTranscriptBinding(channelId, explicitTranscriptPath, {
|
|
422
448
|
replayFromStart: Boolean(options.catchUp),
|
|
423
449
|
catchUpFromPersisted: options.catchUpFromPersisted,
|
|
424
|
-
persistStatus: options.persistStatus
|
|
450
|
+
persistStatus: options.persistStatus,
|
|
451
|
+
recoverUnsyncedTail: options.recoverUnsyncedTail,
|
|
425
452
|
});
|
|
426
453
|
if (options.catchUp || options.catchUpFromPersisted) {
|
|
427
454
|
await forwarder.forwardNewText();
|
|
@@ -452,6 +479,7 @@ async function rebindTranscriptContext(channelId, options = {}) {
|
|
|
452
479
|
applyTranscriptBinding(channelId, bound.transcriptPath, {
|
|
453
480
|
replayFromStart,
|
|
454
481
|
catchUpFromPersisted: options.catchUpFromPersisted,
|
|
482
|
+
recoverUnsyncedTail: options.recoverUnsyncedTail,
|
|
455
483
|
persistStatus: options.persistStatus,
|
|
456
484
|
cwd: bound.sessionCwd,
|
|
457
485
|
});
|
|
@@ -497,6 +525,7 @@ async function rebindTranscriptContext(channelId, options = {}) {
|
|
|
497
525
|
applyTranscriptBinding(channelId, pendingTranscriptPath, {
|
|
498
526
|
replayFromStart: !samePersisted,
|
|
499
527
|
catchUpFromPersisted: samePersisted,
|
|
528
|
+
recoverUnsyncedTail: options.recoverUnsyncedTail,
|
|
500
529
|
cwd: pendingTranscriptCwd,
|
|
501
530
|
persistStatus: options.persistStatus,
|
|
502
531
|
});
|
|
@@ -512,7 +541,11 @@ async function rebindTranscriptContext(channelId, options = {}) {
|
|
|
512
541
|
}
|
|
513
542
|
}
|
|
514
543
|
if (previousPath && options.fallbackPrevious !== false) {
|
|
515
|
-
applyTranscriptBinding(channelId, previousPath, {
|
|
544
|
+
applyTranscriptBinding(channelId, previousPath, {
|
|
545
|
+
catchUpFromPersisted: true,
|
|
546
|
+
recoverUnsyncedTail: options.recoverUnsyncedTail,
|
|
547
|
+
cwd: statusState.read().sessionCwd
|
|
548
|
+
});
|
|
516
549
|
if (fs.existsSync(previousPath)) {
|
|
517
550
|
await forwarder.forwardNewText();
|
|
518
551
|
} else {
|
|
@@ -631,7 +664,8 @@ async function bindPersistedTranscriptIfAny() {
|
|
|
631
664
|
const bound = await rebindTranscriptContext(currentStatus.channelId, {
|
|
632
665
|
previousPath: getPersistedTranscriptPath(),
|
|
633
666
|
persistStatus: true,
|
|
634
|
-
catchUpFromPersisted: true
|
|
667
|
+
catchUpFromPersisted: true,
|
|
668
|
+
recoverUnsyncedTail: true,
|
|
635
669
|
});
|
|
636
670
|
if (bound) {
|
|
637
671
|
process.stderr.write(`mixdog: initial transcript bind: ${bound}
|
|
@@ -710,8 +744,40 @@ function syncOwnedWebhookAndEventRuntime({ reload = false } = {}) {
|
|
|
710
744
|
webhookServer = null;
|
|
711
745
|
}
|
|
712
746
|
}
|
|
747
|
+
let _ownedRuntimeSelfHealing = false;
|
|
748
|
+
// Explicit restore path for an already-connected owner. The 3s ownership tick
|
|
749
|
+
// must not run this implicitly: re-binding status on every tick can move the
|
|
750
|
+
// transcript cursor to EOF, and probing the gateway during Discord's own
|
|
751
|
+
// reconnect window can reset a healthy reconnect loop. Keep this for explicit
|
|
752
|
+
// activation/reload recovery only.
|
|
753
|
+
async function selfHealOwnedRuntime(options = {}) {
|
|
754
|
+
const shouldRestoreBinding = options.restoreBinding === true;
|
|
755
|
+
const shouldResetBackend = options.resetBackend === true;
|
|
756
|
+
if (!shouldRestoreBinding && !shouldResetBackend) return;
|
|
757
|
+
if (_ownedRuntimeSelfHealing) return;
|
|
758
|
+
_ownedRuntimeSelfHealing = true;
|
|
759
|
+
try {
|
|
760
|
+
if (shouldRestoreBinding) {
|
|
761
|
+
await bindPersistedTranscriptIfAny().catch((e) => {
|
|
762
|
+
process.stderr.write(`mixdog: self-heal bindPersistedTranscriptIfAny failed (non-fatal): ${e instanceof Error ? e.message : String(e)}\n`);
|
|
763
|
+
});
|
|
764
|
+
}
|
|
765
|
+
if (shouldResetBackend && typeof backend?._resetClient === "function") {
|
|
766
|
+
await backend._resetClient().catch((e) => {
|
|
767
|
+
process.stderr.write(`mixdog: self-heal backend reset failed (non-fatal): ${e instanceof Error ? e.message : String(e)}\n`);
|
|
768
|
+
});
|
|
769
|
+
}
|
|
770
|
+
// Nudge the forwarder to drain anything the rebind/reconnect surfaced.
|
|
771
|
+
void forwarder.forwardNewText().catch(() => {});
|
|
772
|
+
} finally {
|
|
773
|
+
_ownedRuntimeSelfHealing = false;
|
|
774
|
+
}
|
|
775
|
+
}
|
|
713
776
|
async function startOwnedRuntime(options = {}) {
|
|
714
|
-
if (bridgeRuntimeConnected)
|
|
777
|
+
if (bridgeRuntimeConnected) {
|
|
778
|
+
if (!bridgeRuntimeStarting) await selfHealOwnedRuntime(options);
|
|
779
|
+
return;
|
|
780
|
+
}
|
|
715
781
|
if (bridgeRuntimeStarting) return;
|
|
716
782
|
if (!channelBridgeActive) return;
|
|
717
783
|
bridgeRuntimeStarting = true;
|
|
@@ -928,8 +994,13 @@ async function refreshBridgeOwnership(options = {}) {
|
|
|
928
994
|
if (active && active.instanceId && active.instanceId !== INSTANCE_ID) {
|
|
929
995
|
if (bridgeRuntimeConnected) {
|
|
930
996
|
await stopOwnedRuntime("ownership lost (newer remote session)");
|
|
931
|
-
notifyRemoteSuperseded();
|
|
932
997
|
}
|
|
998
|
+
// Notify the parent UNCONDITIONALLY (not only when we were connected):
|
|
999
|
+
// a loser whose backend never connected (seat stolen mid-connect,
|
|
1000
|
+
// connect failed, or already in standby) must still drop its Remote
|
|
1001
|
+
// indicator. The parent handler is idempotent — it only acts while
|
|
1002
|
+
// remoteEnabled is still true — so repeat ticks are harmless.
|
|
1003
|
+
notifyRemoteSuperseded();
|
|
933
1004
|
return;
|
|
934
1005
|
}
|
|
935
1006
|
// No live owner — claim the empty seat and start.
|
|
@@ -1658,6 +1729,26 @@ backend.onMessage = (msg) => {
|
|
|
1658
1729
|
});
|
|
1659
1730
|
})();
|
|
1660
1731
|
};
|
|
1732
|
+
const NETWORK_ERR_RE = /fetch failed|ECONNRESET|ETIMEDOUT|ENOTFOUND|EAI_AGAIN|socket hang up|network|timeout|aborted|TimeoutError/i;
|
|
1733
|
+
function isNetworkError(err) {
|
|
1734
|
+
const s = `${err?.code ?? ""} ${err?.name ?? ""} ${err?.message ?? err ?? ""}`;
|
|
1735
|
+
return NETWORK_ERR_RE.test(s);
|
|
1736
|
+
}
|
|
1737
|
+
async function retryOnNetwork(fn, { attempts = 3, baseDelayMs = 300, label = "op" } = {}) {
|
|
1738
|
+
let lastErr;
|
|
1739
|
+
for (let i = 0; i < attempts; i++) {
|
|
1740
|
+
try {
|
|
1741
|
+
return await fn();
|
|
1742
|
+
} catch (err) {
|
|
1743
|
+
lastErr = err;
|
|
1744
|
+
if (i === attempts - 1 || !isNetworkError(err)) throw err;
|
|
1745
|
+
const delay = baseDelayMs * (i + 1);
|
|
1746
|
+
process.stderr.write(`mixdog: ${label} network failure (attempt ${i + 1}/${attempts}), retrying in ${delay}ms: ${err?.message || err}\n`);
|
|
1747
|
+
await new Promise((r) => setTimeout(r, delay));
|
|
1748
|
+
}
|
|
1749
|
+
}
|
|
1750
|
+
throw lastErr;
|
|
1751
|
+
}
|
|
1661
1752
|
async function handleInbound(msg, route, options = {}) {
|
|
1662
1753
|
const handleStartMs = Date.now();
|
|
1663
1754
|
let text = msg.text;
|
|
@@ -1668,11 +1759,19 @@ async function handleInbound(msg, route, options = {}) {
|
|
|
1668
1759
|
text = text || "[voice message]";
|
|
1669
1760
|
} else {
|
|
1670
1761
|
try {
|
|
1671
|
-
const files = await
|
|
1762
|
+
const files = await retryOnNetwork(
|
|
1763
|
+
// Short per-attempt timeout (vs the 180s default) bounds worst-case
|
|
1764
|
+
// blockage of the serial inboundQueue on a bad voice message.
|
|
1765
|
+
() => backend.downloadAttachment(msg.chatId, msg.messageId, { timeoutMs: 20_000 }),
|
|
1766
|
+
{ label: "voice.download" }
|
|
1767
|
+
);
|
|
1672
1768
|
// concurrency handled inside transcribeVoice queue; loop is sequential so last att wins
|
|
1673
1769
|
for (const f of voiceAtts.map(a => files.find(df => df.id === a.id) ?? null).filter(Boolean)) {
|
|
1674
1770
|
const _t0 = Date.now();
|
|
1675
|
-
const transcript = await
|
|
1771
|
+
const transcript = await retryOnNetwork(
|
|
1772
|
+
() => transcribeVoice(f.path, { attachmentId: f.id }),
|
|
1773
|
+
{ label: "voice.transcription" }
|
|
1774
|
+
);
|
|
1676
1775
|
const _elapsed = Date.now() - _t0;
|
|
1677
1776
|
if (transcript) {
|
|
1678
1777
|
text = transcript;
|
|
@@ -1683,8 +1782,12 @@ async function handleInbound(msg, route, options = {}) {
|
|
|
1683
1782
|
}
|
|
1684
1783
|
}
|
|
1685
1784
|
} catch (err) {
|
|
1686
|
-
|
|
1687
|
-
|
|
1785
|
+
const netFail = isNetworkError(err);
|
|
1786
|
+
process.stderr.write(`mixdog: voice.transcription error${netFail ? " (network, retries exhausted)" : ""}: ${err}\n`);
|
|
1787
|
+
const marker = netFail
|
|
1788
|
+
? "[attachment: voice transcription failed (network)]"
|
|
1789
|
+
: `[voice message \u2014 transcription error: ${err?.message || err}]`;
|
|
1790
|
+
text = text ? `${text} ${marker}` : marker;
|
|
1688
1791
|
}
|
|
1689
1792
|
}
|
|
1690
1793
|
}
|
|
@@ -1953,15 +2056,39 @@ if (_isWorkerMode && process.send) {
|
|
|
1953
2056
|
})
|
|
1954
2057
|
void (async () => {
|
|
1955
2058
|
const startedAt = performance.now()
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
2059
|
+
const MAX_START_ATTEMPTS = 3
|
|
2060
|
+
const BASE_BACKOFF_MS = 250
|
|
2061
|
+
const isTransientStartErr = (err) =>
|
|
2062
|
+
err?.code === 'ELOCKTIMEOUT' || /atomic lock timeout/i.test(err?.message || '')
|
|
2063
|
+
let lastErr
|
|
2064
|
+
for (let attempt = 1; attempt <= MAX_START_ATTEMPTS; attempt++) {
|
|
2065
|
+
if (_channelsStopInFlight) return
|
|
2066
|
+
try {
|
|
2067
|
+
await start()
|
|
2068
|
+
bootProfile("worker:ready", { ms: (performance.now() - startedAt).toFixed(1), attempt })
|
|
2069
|
+
process.send({ type: 'ready' })
|
|
2070
|
+
return
|
|
2071
|
+
} catch (e) {
|
|
2072
|
+
lastErr = e
|
|
2073
|
+
const transient = isTransientStartErr(e)
|
|
2074
|
+
bootProfile("worker:start-failed", { attempt, transient, error: e?.message || String(e) })
|
|
2075
|
+
process.stderr.write(`[channels-worker] start() failed (attempt ${attempt}/${MAX_START_ATTEMPTS}, transient=${transient}): ${e && (e.message || e)}\n`)
|
|
2076
|
+
if (!transient || attempt >= MAX_START_ATTEMPTS) break
|
|
2077
|
+
const backoff = BASE_BACKOFF_MS * attempt + Math.floor(Math.random() * BASE_BACKOFF_MS)
|
|
2078
|
+
await new Promise((r) => setTimeout(r, backoff))
|
|
2079
|
+
if (_channelsStopInFlight) return
|
|
2080
|
+
}
|
|
1964
2081
|
}
|
|
2082
|
+
// A stop landed while we were failing — let clean shutdown proceed, never exit over it.
|
|
2083
|
+
if (_channelsStopInFlight) return
|
|
2084
|
+
// Terminal failure: do NOT mask as a (degraded) ready. Exit non-zero so the
|
|
2085
|
+
// parent's exit-before-ready path respawns or rejects startRemote instead of
|
|
2086
|
+
// silently losing remote output forwarding.
|
|
2087
|
+
bootProfile("worker:failed", { ms: (performance.now() - startedAt).toFixed(1), error: lastErr?.message || String(lastErr) })
|
|
2088
|
+
process.stderr.write(`[channels-worker] start() giving up after ${MAX_START_ATTEMPTS} attempts: ${lastErr && (lastErr.message || lastErr)}\n`)
|
|
2089
|
+
// Exit 2 = terminal (non-transient) start failure: parent must reject, not respawn.
|
|
2090
|
+
// Exit 1 = exhausted transient retries: parent may respawn.
|
|
2091
|
+
process.exit(isTransientStartErr(lastErr) ? 1 : 2)
|
|
1965
2092
|
})()
|
|
1966
2093
|
}
|
|
1967
2094
|
|
|
@@ -86,15 +86,33 @@ ${err instanceof Error ? err.stack : ""}
|
|
|
86
86
|
// already retried elsewhere (atomic-file.mjs RETRY_CODES) — a single
|
|
87
87
|
// occurrence must NOT be fatal, only a run of 3+ in a row without an
|
|
88
88
|
// intervening distinct/successful event.
|
|
89
|
-
const BENIGN_CRASH_CODES = new Set([
|
|
89
|
+
const BENIGN_CRASH_CODES = new Set([
|
|
90
|
+
"EPERM", "EACCES", "EBUSY",
|
|
91
|
+
// Transient transport blips: a raw http/https ClientRequest (or an
|
|
92
|
+
// undici socket) can emit these when a keep-alive connection is torn
|
|
93
|
+
// down mid-flight (Discord gateway flapping, proxy resets). One blip
|
|
94
|
+
// must NOT kill the worker — it is benign; only a repeat storm within
|
|
95
|
+
// the streak window force-exits (see _fatalCrash).
|
|
96
|
+
"ECONNRESET", "ETIMEDOUT", "ECONNREFUSED", "EPIPE", "EHOSTUNREACH",
|
|
97
|
+
"ENETUNREACH", "ENETDOWN", "EAI_AGAIN", "ERR_STREAM_PREMATURE_CLOSE",
|
|
98
|
+
"UND_ERR_SOCKET", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_HEADERS_TIMEOUT",
|
|
99
|
+
"UND_ERR_BODY_TIMEOUT", "UND_ERR_ABORTED",
|
|
100
|
+
]);
|
|
101
|
+
// Some transient socket errors arrive with no `.code` (notably the classic
|
|
102
|
+
// "socket hang up" thrown by node:_http_client socketOnEnd — the exact crash
|
|
103
|
+
// this guard exists for). Match those by message so they classify benign too.
|
|
104
|
+
const BENIGN_CRASH_MESSAGE_RE = /socket hang up|ECONNRESET|ETIMEDOUT|EPIPE|UND_ERR_/i;
|
|
90
105
|
const BENIGN_CRASH_FATAL_THRESHOLD = 3;
|
|
91
106
|
// "In a row" needs a time dimension: benign errors minutes/hours apart are
|
|
92
107
|
// independent contention events, not a corrupted-state run. Only count a
|
|
93
108
|
// streak when hits land within this window of the previous one.
|
|
94
109
|
const BENIGN_CRASH_STREAK_WINDOW_MS = 60_000;
|
|
95
110
|
function _isBenignCrash(err) {
|
|
96
|
-
const
|
|
97
|
-
|
|
111
|
+
const msg = String(err?.message || err);
|
|
112
|
+
const code = err?.code
|
|
113
|
+
|| (/\b(EPERM|EACCES|EBUSY|ECONNRESET|ETIMEDOUT|ECONNREFUSED|EPIPE|EHOSTUNREACH|ENETUNREACH|ENETDOWN|EAI_AGAIN|UND_ERR_[A-Z_]+)\b/.exec(msg) || [])[0];
|
|
114
|
+
if (BENIGN_CRASH_CODES.has(code)) return true;
|
|
115
|
+
return BENIGN_CRASH_MESSAGE_RE.test(msg);
|
|
98
116
|
}
|
|
99
117
|
|
|
100
118
|
export {
|