mixdog 0.9.16 → 0.9.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/package.json +2 -1
  2. package/scripts/atomic-lock-tryonce-test.mjs +66 -0
  3. package/scripts/provider-toolcall-test.mjs +79 -2
  4. package/src/mixdog-session-runtime.mjs +12 -0
  5. package/src/runtime/agent/orchestrator/providers/anthropic-effort.mjs +33 -1
  6. package/src/runtime/agent/orchestrator/providers/anthropic-oauth.mjs +9 -0
  7. package/src/runtime/agent/orchestrator/providers/anthropic-sse.mjs +49 -0
  8. package/src/runtime/agent/orchestrator/providers/anthropic.mjs +14 -0
  9. package/src/runtime/agent/orchestrator/session/context-utils.mjs +7 -0
  10. package/src/runtime/agent/orchestrator/session/loop.mjs +8 -0
  11. package/src/runtime/agent/orchestrator/session/manager/pending-messages.mjs +32 -18
  12. package/src/runtime/agent/orchestrator/session/manager/usage-metrics.mjs +142 -3
  13. package/src/runtime/agent/orchestrator/session/store-summary-index.mjs +108 -30
  14. package/src/runtime/agent/orchestrator/session/store.mjs +5 -0
  15. package/src/runtime/agent/orchestrator/stall-policy.mjs +22 -12
  16. package/src/runtime/agent/orchestrator/tools/builtin/builtin-tools.mjs +3 -1
  17. package/src/runtime/agent/orchestrator/tools/builtin/read-single-tool.mjs +15 -15
  18. package/src/runtime/agent/orchestrator/tools/builtin/snapshot-store.mjs +10 -2
  19. package/src/runtime/agent/orchestrator/tools/code-graph/disk-cache.mjs +8 -2
  20. package/src/runtime/channels/backends/discord-attachments.mjs +2 -2
  21. package/src/runtime/channels/backends/discord-gateway.mjs +12 -2
  22. package/src/runtime/channels/backends/discord.mjs +97 -7
  23. package/src/runtime/channels/index.mjs +150 -23
  24. package/src/runtime/channels/lib/crash-log.mjs +21 -3
  25. package/src/runtime/channels/lib/output-forwarder.mjs +118 -7
  26. package/src/runtime/channels/lib/runtime-paths.mjs +21 -19
  27. package/src/runtime/channels/lib/voice-transcription.mjs +4 -2
  28. package/src/runtime/memory/index.mjs +37 -0
  29. package/src/runtime/memory/lib/embedding-warmup.mjs +3 -0
  30. package/src/runtime/memory/lib/ko-morph.mjs +1 -0
  31. package/src/runtime/shared/atomic-file.mjs +110 -0
  32. package/src/runtime/shared/transcript-writer.mjs +46 -4
  33. package/src/session-runtime/provider-models.mjs +47 -8
  34. package/src/standalone/channel-worker.mjs +14 -2
  35. package/src/tui/app/transcript-window.mjs +137 -6
  36. package/src/tui/app/use-transcript-window.mjs +67 -10
  37. package/src/tui/components/StatusLine.jsx +1 -1
  38. package/src/tui/dist/index.mjs +436 -93
  39. package/src/tui/engine/tui-steering-persist.mjs +66 -35
  40. package/src/tui/engine.mjs +66 -14
  41. package/src/tui/index.jsx +97 -6
  42. package/src/ui/statusline-segments.mjs +54 -36
  43. package/src/ui/statusline.mjs +141 -95
@@ -23,6 +23,14 @@ import { normalizeAccess, safeAttName } from "./discord-access.mjs";
23
23
  import { MAX_ATTACHMENT_BYTES, downloadSingleAttachment } from "./discord-attachments.mjs";
24
24
  import * as gateway from "./discord-gateway.mjs";
25
25
  const MAX_CHUNK_LIMIT = MAX_DISCORD_MESSAGE;
26
+ // Per-attempt cap for a single discord.js fetch/send. Without it a wedged
27
+ // socket (network blip, "other side closed") leaves the await pending forever,
28
+ // which upstream leaves the forwarder's `sending` flag stuck and silences all
29
+ // outbound. On timeout we treat the client as dead and reset+reconnect it.
30
+ const SEND_ATTEMPT_TIMEOUT_MS = 30_000;
31
+ // After a timed-out send is reset, cap how long we wait for the original
32
+ // (abandoned) promise to actually settle before giving up with unknown outcome.
33
+ const SETTLE_CAP_MS = 15_000;
26
34
  const RECENT_SENT_CAP = 200;
27
35
  class DiscordBackend {
28
36
  name = "discord";
@@ -74,11 +82,57 @@ class DiscordBackend {
74
82
  if (this.client) this.client.destroy();
75
83
  this._connectPromise = null;
76
84
  }
85
+ // ── Self-heal helpers ──────────────────────────────────────────────
86
+ /** Race a discord.js op-promise against a per-attempt timeout so a wedged
87
+ * socket can't hang the outbound path. On timeout the rejection carries
88
+ * `_timeout` so the caller knows the underlying op is still in flight (and
89
+ * must be settled/adopted, not blindly retried → duplicate). */
90
+ _withTimeout(promise, label) {
91
+ let timer;
92
+ const timeout = new Promise((_, reject) => {
93
+ timer = setTimeout(() => {
94
+ const e = new Error(`${label} timed out after ${SEND_ATTEMPT_TIMEOUT_MS}ms`);
95
+ e._timeout = true;
96
+ reject(e);
97
+ }, SEND_ATTEMPT_TIMEOUT_MS);
98
+ });
99
+ return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
100
+ }
101
+ /** True when an error means the client is dead (aborted, "other side
102
+ * closed", or our per-attempt timeout) and must be reset+reconnected. */
103
+ _shouldResetClient(err) {
104
+ if (err?.name === "AbortError" || err?.code === "ABORT_ERR") return true;
105
+ const msg = String(err?.message || err || "").toLowerCase();
106
+ return msg.includes("aborted") || msg.includes("other side closed") ||
107
+ msg.includes("socket hang up") || msg.includes("econnreset") || msg.includes("timed out");
108
+ }
109
+ /** Destroy the wedged client and rebuild it via connect() so the next
110
+ * fetch/send (and the forwarder's queue-head-preserving retry) hits a fresh
111
+ * gateway/REST client. Serialized behind a single in-flight reset so
112
+ * concurrent callers await the SAME reconnect instead of racing multiple
113
+ * destroy()/login() cycles. */
114
+ async _resetClient() {
115
+ if (this._resetPromise) return this._resetPromise;
116
+ this._resetPromise = (async () => {
117
+ try { this.client?.destroy?.(); } catch {}
118
+ this.client = null;
119
+ this._connectPromise = null;
120
+ await this.connect();
121
+ })().finally(() => { this._resetPromise = null; });
122
+ return this._resetPromise;
123
+ }
124
+ /** Health-check the gateway; reset+reconnect if the ws is not READY. Called
125
+ * by the owner re-claim self-heal path. (discord.js Status.Ready === 0) */
126
+ async healthCheck() {
127
+ if (this.client && this.client.ws?.status === 0) return;
128
+ await this._resetClient();
129
+ }
77
130
  resetSendCount() {
78
131
  this.sendCount = 0;
79
132
  }
80
133
  startTyping(channelId) {
81
134
  this.stopTyping(channelId);
135
+ if (!this.client) return;
82
136
  const ch = this.client.channels.cache.get(channelId);
83
137
  if (ch && "sendTyping" in ch) {
84
138
  void ch.sendTyping().catch(() => {
@@ -172,13 +226,38 @@ class DiscordBackend {
172
226
  // resumes at this chunk index instead of re-sending 0..i-1.
173
227
  let attempt = 0;
174
228
  for (;;) {
229
+ const sendPromise = ch.send(payload);
175
230
  try {
176
- const sent = await ch.send(payload);
231
+ const sent = await this._withTimeout(sendPromise, "discord send");
177
232
  this.noteSent(sent.id);
178
233
  sentIds.push(sent.id);
179
234
  break;
180
235
  } catch (err) {
181
236
  attempt++;
237
+ if (err?._timeout) {
238
+ // Watchdog abandoned the send but the socket may still deliver.
239
+ // Destroy the client to tear down the in-flight REST socket so the
240
+ // ORIGINAL promise settles, then AWAIT + adopt its real outcome:
241
+ // delivered → done (no retry → no duplicate); else requeue against
242
+ // the fresh client via a resume token.
243
+ try { await this._resetClient(); } catch {}
244
+ // Cap the settlement wait: destroy() should abort the in-flight
245
+ // socket, but never hang here if it doesn't. On unknown outcome
246
+ // throw retryable (duplicate risk accepted over a permanent hang).
247
+ sendPromise.catch(() => {}); // swallow a late rejection
248
+ const settled = await Promise.race([
249
+ sendPromise.then((v) => ({ ok: true, v }), () => ({ ok: false })),
250
+ new Promise((r) => setTimeout(() => r({ ok: false }), SETTLE_CAP_MS))
251
+ ]);
252
+ if (settled.ok) {
253
+ this.noteSent(settled.v.id);
254
+ sentIds.push(settled.v.id);
255
+ break;
256
+ }
257
+ const e = err instanceof Error ? err : new Error(String(err));
258
+ e.resumeToken = { hash: contentHash, nextChunkIdx: i, sentIds: [...sentIds], prefixed: applyPrefix, limit };
259
+ throw e;
260
+ }
182
261
  const status = err?.status ?? err?.code ?? err?.httpStatus;
183
262
  // Classify: PERMANENT = a 4xx client error (unknown channel/message,
184
263
  // missing access/permissions, 404 …) that will never succeed on
@@ -192,7 +271,15 @@ class DiscordBackend {
192
271
  e.permanent = true;
193
272
  throw e;
194
273
  }
195
- if (attempt >= 3) {
274
+ // Dead client (abort / "other side closed" / per-attempt timeout):
275
+ // destroy + reconnect so the forwarder's transient-retry path hits a
276
+ // fresh client, and throw a resume token now instead of burning the
277
+ // remaining attempts on a stale channel handle bound to the old client.
278
+ const dead = this._shouldResetClient(err);
279
+ if (dead) {
280
+ try { await this._resetClient(); } catch {}
281
+ }
282
+ if (dead || attempt >= 3) {
196
283
  // Attach an opaque resume token pointing at the chunk that failed
197
284
  // (i). sentIds holds every chunk already delivered (including any
198
285
  // seeded from a prior token).
@@ -301,13 +388,13 @@ class DiscordBackend {
301
388
  const msg = await ch.messages.fetch(messageId);
302
389
  await msg.delete();
303
390
  }
304
- async downloadAttachment(chatId, messageId) {
391
+ async downloadAttachment(chatId, messageId, opts = {}) {
305
392
  const ch = await this.fetchAllowedChannel(chatId);
306
393
  const msg = await ch.messages.fetch(messageId);
307
394
  if (msg.attachments.size === 0) return [];
308
395
  const results = [];
309
396
  for (const att of msg.attachments.values()) {
310
- const path = await this.downloadSingleAttachment(att);
397
+ const path = await this.downloadSingleAttachment(att, opts);
311
398
  results.push({
312
399
  id: att.id,
313
400
  path,
@@ -526,7 +613,10 @@ class DiscordBackend {
526
613
  }
527
614
  // ── Channel helpers ────────────────────────────────────────────────
528
615
  async fetchTextChannel(id) {
529
- const ch = await this.client.channels.fetch(id);
616
+ if (!this.client) await this.connect();
617
+ const p = this.client.channels.fetch(id);
618
+ p.catch(() => {}); // if the watchdog wins, don't leak an unhandled rejection
619
+ const ch = await this._withTimeout(p, "discord channels.fetch");
530
620
  if (!ch || !ch.isTextBased()) {
531
621
  throw new Error(`channel ${id} not found or not text-based`);
532
622
  }
@@ -571,8 +661,8 @@ class DiscordBackend {
571
661
  throw new Error(`refusing to send channel state: ${f}`);
572
662
  }
573
663
  }
574
- async downloadSingleAttachment(att) {
575
- return downloadSingleAttachment(att, this.inboxDir);
664
+ async downloadSingleAttachment(att, opts = {}) {
665
+ return downloadSingleAttachment(att, this.inboxDir, opts);
576
666
  }
577
667
  }
578
668
  export {
@@ -305,6 +305,11 @@ const forwarder = new OutputForwarder({
305
305
  removeReaction: (ch, mid, emoji) => {
306
306
  if (!channelBridgeActive) return Promise.resolve();
307
307
  return backend.removeReaction(ch, mid, emoji);
308
+ },
309
+ // Watchdog backstop: force the backend to tear down + rebuild a wedged
310
+ // client so an over-budget send fails fast and the queue releases.
311
+ resetBackend: async () => {
312
+ if (typeof backend?._resetClient === "function") await backend._resetClient();
308
313
  }
309
314
  }, statusState);
310
315
  forwarder.setOnIdle(() => {
@@ -323,7 +328,11 @@ forwarder.setOnIdle(() => {
323
328
  forwarder.setOwnerGetter(() => bridgeRuntimeConnected && currentOwnerState().owned);
324
329
  function applyTranscriptBinding(channelId, transcriptPath, options = {}) {
325
330
  if (!transcriptPath) return;
326
- forwarder.setContext(channelId, transcriptPath, { replayFromStart: options.replayFromStart, catchUpFromPersisted: options.catchUpFromPersisted });
331
+ forwarder.setContext(channelId, transcriptPath, {
332
+ replayFromStart: options.replayFromStart,
333
+ catchUpFromPersisted: options.catchUpFromPersisted,
334
+ recoverUnsyncedTail: options.recoverUnsyncedTail,
335
+ });
327
336
  const boundTranscriptPath = forwarder.transcriptPath || transcriptPath;
328
337
  forwarder.startWatch();
329
338
  void memoryIngestTranscript(boundTranscriptPath, { cwd: options.cwd });
@@ -336,14 +345,31 @@ function applyTranscriptBinding(channelId, transcriptPath, options = {}) {
336
345
  refreshActiveInstance(INSTANCE_ID, { channelId, transcriptPath: boundTranscriptPath }, { onlyIfOwned: true });
337
346
  if (options.persistStatus !== false) {
338
347
  statusState.update((state) => {
348
+ const wasSameTranscript = typeof state.transcriptPath === "string"
349
+ && state.transcriptPath
350
+ && sameResolvedPath(state.transcriptPath, boundTranscriptPath);
351
+ const prevSentCount = Number(state.sentCount ?? 0);
352
+ const nextSentCount = Number(forwarder.sentCount ?? 0);
339
353
  state.channelId = channelId;
340
354
  state.transcriptPath = boundTranscriptPath;
341
355
  state.lastFileSize = forwarder.lastFileSize;
342
- state.sentCount = forwarder.sentCount;
343
- state.lastSentHash = forwarder.lastHash;
344
- state.lastSentTime = 0;
345
- state.sessionIdle = false;
346
- state.sessionCwd = options.cwd ?? null;
356
+ if (wasSameTranscript) {
357
+ state.sentCount = Math.max(
358
+ Number.isFinite(prevSentCount) ? prevSentCount : 0,
359
+ Number.isFinite(nextSentCount) ? nextSentCount : 0,
360
+ );
361
+ if (forwarder.lastHash) state.lastSentHash = forwarder.lastHash;
362
+ else if (typeof state.lastSentHash !== "string") state.lastSentHash = "";
363
+ if (!Number.isFinite(Number(state.lastSentTime))) state.lastSentTime = 0;
364
+ if (typeof state.sessionIdle !== "boolean") state.sessionIdle = false;
365
+ } else {
366
+ state.sentCount = forwarder.sentCount;
367
+ state.lastSentHash = forwarder.lastHash;
368
+ state.lastSentTime = 0;
369
+ state.sessionIdle = false;
370
+ }
371
+ if (options.cwd !== undefined) state.sessionCwd = options.cwd ?? null;
372
+ else if (!wasSameTranscript) state.sessionCwd = null;
347
373
  });
348
374
  }
349
375
  }
@@ -421,7 +447,8 @@ async function rebindTranscriptContext(channelId, options = {}) {
421
447
  applyTranscriptBinding(channelId, explicitTranscriptPath, {
422
448
  replayFromStart: Boolean(options.catchUp),
423
449
  catchUpFromPersisted: options.catchUpFromPersisted,
424
- persistStatus: options.persistStatus
450
+ persistStatus: options.persistStatus,
451
+ recoverUnsyncedTail: options.recoverUnsyncedTail,
425
452
  });
426
453
  if (options.catchUp || options.catchUpFromPersisted) {
427
454
  await forwarder.forwardNewText();
@@ -452,6 +479,7 @@ async function rebindTranscriptContext(channelId, options = {}) {
452
479
  applyTranscriptBinding(channelId, bound.transcriptPath, {
453
480
  replayFromStart,
454
481
  catchUpFromPersisted: options.catchUpFromPersisted,
482
+ recoverUnsyncedTail: options.recoverUnsyncedTail,
455
483
  persistStatus: options.persistStatus,
456
484
  cwd: bound.sessionCwd,
457
485
  });
@@ -497,6 +525,7 @@ async function rebindTranscriptContext(channelId, options = {}) {
497
525
  applyTranscriptBinding(channelId, pendingTranscriptPath, {
498
526
  replayFromStart: !samePersisted,
499
527
  catchUpFromPersisted: samePersisted,
528
+ recoverUnsyncedTail: options.recoverUnsyncedTail,
500
529
  cwd: pendingTranscriptCwd,
501
530
  persistStatus: options.persistStatus,
502
531
  });
@@ -512,7 +541,11 @@ async function rebindTranscriptContext(channelId, options = {}) {
512
541
  }
513
542
  }
514
543
  if (previousPath && options.fallbackPrevious !== false) {
515
- applyTranscriptBinding(channelId, previousPath, { catchUpFromPersisted: true, cwd: statusState.read().sessionCwd });
544
+ applyTranscriptBinding(channelId, previousPath, {
545
+ catchUpFromPersisted: true,
546
+ recoverUnsyncedTail: options.recoverUnsyncedTail,
547
+ cwd: statusState.read().sessionCwd
548
+ });
516
549
  if (fs.existsSync(previousPath)) {
517
550
  await forwarder.forwardNewText();
518
551
  } else {
@@ -631,7 +664,8 @@ async function bindPersistedTranscriptIfAny() {
631
664
  const bound = await rebindTranscriptContext(currentStatus.channelId, {
632
665
  previousPath: getPersistedTranscriptPath(),
633
666
  persistStatus: true,
634
- catchUpFromPersisted: true
667
+ catchUpFromPersisted: true,
668
+ recoverUnsyncedTail: true,
635
669
  });
636
670
  if (bound) {
637
671
  process.stderr.write(`mixdog: initial transcript bind: ${bound}
@@ -710,8 +744,40 @@ function syncOwnedWebhookAndEventRuntime({ reload = false } = {}) {
710
744
  webhookServer = null;
711
745
  }
712
746
  }
747
+ let _ownedRuntimeSelfHealing = false;
748
+ // Explicit restore path for an already-connected owner. The 3s ownership tick
749
+ // must not run this implicitly: re-binding status on every tick can move the
750
+ // transcript cursor to EOF, and probing the gateway during Discord's own
751
+ // reconnect window can reset a healthy reconnect loop. Keep this for explicit
752
+ // activation/reload recovery only.
753
+ async function selfHealOwnedRuntime(options = {}) {
754
+ const shouldRestoreBinding = options.restoreBinding === true;
755
+ const shouldResetBackend = options.resetBackend === true;
756
+ if (!shouldRestoreBinding && !shouldResetBackend) return;
757
+ if (_ownedRuntimeSelfHealing) return;
758
+ _ownedRuntimeSelfHealing = true;
759
+ try {
760
+ if (shouldRestoreBinding) {
761
+ await bindPersistedTranscriptIfAny().catch((e) => {
762
+ process.stderr.write(`mixdog: self-heal bindPersistedTranscriptIfAny failed (non-fatal): ${e instanceof Error ? e.message : String(e)}\n`);
763
+ });
764
+ }
765
+ if (shouldResetBackend && typeof backend?._resetClient === "function") {
766
+ await backend._resetClient().catch((e) => {
767
+ process.stderr.write(`mixdog: self-heal backend reset failed (non-fatal): ${e instanceof Error ? e.message : String(e)}\n`);
768
+ });
769
+ }
770
+ // Nudge the forwarder to drain anything the rebind/reconnect surfaced.
771
+ void forwarder.forwardNewText().catch(() => {});
772
+ } finally {
773
+ _ownedRuntimeSelfHealing = false;
774
+ }
775
+ }
713
776
  async function startOwnedRuntime(options = {}) {
714
- if (bridgeRuntimeConnected) return;
777
+ if (bridgeRuntimeConnected) {
778
+ if (!bridgeRuntimeStarting) await selfHealOwnedRuntime(options);
779
+ return;
780
+ }
715
781
  if (bridgeRuntimeStarting) return;
716
782
  if (!channelBridgeActive) return;
717
783
  bridgeRuntimeStarting = true;
@@ -928,8 +994,13 @@ async function refreshBridgeOwnership(options = {}) {
928
994
  if (active && active.instanceId && active.instanceId !== INSTANCE_ID) {
929
995
  if (bridgeRuntimeConnected) {
930
996
  await stopOwnedRuntime("ownership lost (newer remote session)");
931
- notifyRemoteSuperseded();
932
997
  }
998
+ // Notify the parent UNCONDITIONALLY (not only when we were connected):
999
+ // a loser whose backend never connected (seat stolen mid-connect,
1000
+ // connect failed, or already in standby) must still drop its Remote
1001
+ // indicator. The parent handler is idempotent — it only acts while
1002
+ // remoteEnabled is still true — so repeat ticks are harmless.
1003
+ notifyRemoteSuperseded();
933
1004
  return;
934
1005
  }
935
1006
  // No live owner — claim the empty seat and start.
@@ -1658,6 +1729,26 @@ backend.onMessage = (msg) => {
1658
1729
  });
1659
1730
  })();
1660
1731
  };
1732
+ const NETWORK_ERR_RE = /fetch failed|ECONNRESET|ETIMEDOUT|ENOTFOUND|EAI_AGAIN|socket hang up|network|timeout|aborted|TimeoutError/i;
1733
+ function isNetworkError(err) {
1734
+ const s = `${err?.code ?? ""} ${err?.name ?? ""} ${err?.message ?? err ?? ""}`;
1735
+ return NETWORK_ERR_RE.test(s);
1736
+ }
1737
+ async function retryOnNetwork(fn, { attempts = 3, baseDelayMs = 300, label = "op" } = {}) {
1738
+ let lastErr;
1739
+ for (let i = 0; i < attempts; i++) {
1740
+ try {
1741
+ return await fn();
1742
+ } catch (err) {
1743
+ lastErr = err;
1744
+ if (i === attempts - 1 || !isNetworkError(err)) throw err;
1745
+ const delay = baseDelayMs * (i + 1);
1746
+ process.stderr.write(`mixdog: ${label} network failure (attempt ${i + 1}/${attempts}), retrying in ${delay}ms: ${err?.message || err}\n`);
1747
+ await new Promise((r) => setTimeout(r, delay));
1748
+ }
1749
+ }
1750
+ throw lastErr;
1751
+ }
1661
1752
  async function handleInbound(msg, route, options = {}) {
1662
1753
  const handleStartMs = Date.now();
1663
1754
  let text = msg.text;
@@ -1668,11 +1759,19 @@ async function handleInbound(msg, route, options = {}) {
1668
1759
  text = text || "[voice message]";
1669
1760
  } else {
1670
1761
  try {
1671
- const files = await backend.downloadAttachment(msg.chatId, msg.messageId);
1762
+ const files = await retryOnNetwork(
1763
+ // Short per-attempt timeout (vs the 180s default) bounds worst-case
1764
+ // blockage of the serial inboundQueue on a bad voice message.
1765
+ () => backend.downloadAttachment(msg.chatId, msg.messageId, { timeoutMs: 20_000 }),
1766
+ { label: "voice.download" }
1767
+ );
1672
1768
  // concurrency handled inside transcribeVoice queue; loop is sequential so last att wins
1673
1769
  for (const f of voiceAtts.map(a => files.find(df => df.id === a.id) ?? null).filter(Boolean)) {
1674
1770
  const _t0 = Date.now();
1675
- const transcript = await transcribeVoice(f.path, { attachmentId: f.id });
1771
+ const transcript = await retryOnNetwork(
1772
+ () => transcribeVoice(f.path, { attachmentId: f.id }),
1773
+ { label: "voice.transcription" }
1774
+ );
1676
1775
  const _elapsed = Date.now() - _t0;
1677
1776
  if (transcript) {
1678
1777
  text = transcript;
@@ -1683,8 +1782,12 @@ async function handleInbound(msg, route, options = {}) {
1683
1782
  }
1684
1783
  }
1685
1784
  } catch (err) {
1686
- process.stderr.write(`mixdog: voice.transcription error: ${err}\n`);
1687
- text = text || `[voice message \u2014 transcription error: ${err?.message || err}]`;
1785
+ const netFail = isNetworkError(err);
1786
+ process.stderr.write(`mixdog: voice.transcription error${netFail ? " (network, retries exhausted)" : ""}: ${err}\n`);
1787
+ const marker = netFail
1788
+ ? "[attachment: voice transcription failed (network)]"
1789
+ : `[voice message \u2014 transcription error: ${err?.message || err}]`;
1790
+ text = text ? `${text} ${marker}` : marker;
1688
1791
  }
1689
1792
  }
1690
1793
  }
@@ -1953,15 +2056,39 @@ if (_isWorkerMode && process.send) {
1953
2056
  })
1954
2057
  void (async () => {
1955
2058
  const startedAt = performance.now()
1956
- try {
1957
- await start()
1958
- bootProfile("worker:ready", { ms: (performance.now() - startedAt).toFixed(1) })
1959
- process.send({ type: 'ready' })
1960
- } catch (e) {
1961
- bootProfile("worker:failed", { ms: (performance.now() - startedAt).toFixed(1), error: e?.message || String(e) })
1962
- process.stderr.write(`[channels-worker] start() failed: ${e && (e.message || e)}\n`)
1963
- process.send({ type: 'ready', degraded: true, error: e?.message || String(e) })
2059
+ const MAX_START_ATTEMPTS = 3
2060
+ const BASE_BACKOFF_MS = 250
2061
+ const isTransientStartErr = (err) =>
2062
+ err?.code === 'ELOCKTIMEOUT' || /atomic lock timeout/i.test(err?.message || '')
2063
+ let lastErr
2064
+ for (let attempt = 1; attempt <= MAX_START_ATTEMPTS; attempt++) {
2065
+ if (_channelsStopInFlight) return
2066
+ try {
2067
+ await start()
2068
+ bootProfile("worker:ready", { ms: (performance.now() - startedAt).toFixed(1), attempt })
2069
+ process.send({ type: 'ready' })
2070
+ return
2071
+ } catch (e) {
2072
+ lastErr = e
2073
+ const transient = isTransientStartErr(e)
2074
+ bootProfile("worker:start-failed", { attempt, transient, error: e?.message || String(e) })
2075
+ process.stderr.write(`[channels-worker] start() failed (attempt ${attempt}/${MAX_START_ATTEMPTS}, transient=${transient}): ${e && (e.message || e)}\n`)
2076
+ if (!transient || attempt >= MAX_START_ATTEMPTS) break
2077
+ const backoff = BASE_BACKOFF_MS * attempt + Math.floor(Math.random() * BASE_BACKOFF_MS)
2078
+ await new Promise((r) => setTimeout(r, backoff))
2079
+ if (_channelsStopInFlight) return
2080
+ }
1964
2081
  }
2082
+ // A stop landed while we were failing — let clean shutdown proceed, never exit over it.
2083
+ if (_channelsStopInFlight) return
2084
+ // Terminal failure: do NOT mask as a (degraded) ready. Exit non-zero so the
2085
+ // parent's exit-before-ready path respawns or rejects startRemote instead of
2086
+ // silently losing remote output forwarding.
2087
+ bootProfile("worker:failed", { ms: (performance.now() - startedAt).toFixed(1), error: lastErr?.message || String(lastErr) })
2088
+ process.stderr.write(`[channels-worker] start() giving up after ${MAX_START_ATTEMPTS} attempts: ${lastErr && (lastErr.message || lastErr)}\n`)
2089
+ // Exit 2 = terminal (non-transient) start failure: parent must reject, not respawn.
2090
+ // Exit 1 = exhausted transient retries: parent may respawn.
2091
+ process.exit(isTransientStartErr(lastErr) ? 1 : 2)
1965
2092
  })()
1966
2093
  }
1967
2094
 
@@ -86,15 +86,33 @@ ${err instanceof Error ? err.stack : ""}
86
86
  // already retried elsewhere (atomic-file.mjs RETRY_CODES) — a single
87
87
  // occurrence must NOT be fatal, only a run of 3+ in a row without an
88
88
  // intervening distinct/successful event.
89
- const BENIGN_CRASH_CODES = new Set(["EPERM", "EACCES", "EBUSY"]);
89
+ const BENIGN_CRASH_CODES = new Set([
90
+ "EPERM", "EACCES", "EBUSY",
91
+ // Transient transport blips: a raw http/https ClientRequest (or an
92
+ // undici socket) can emit these when a keep-alive connection is torn
93
+ // down mid-flight (Discord gateway flapping, proxy resets). One blip
94
+ // must NOT kill the worker — it is benign; only a repeat storm within
95
+ // the streak window force-exits (see _fatalCrash).
96
+ "ECONNRESET", "ETIMEDOUT", "ECONNREFUSED", "EPIPE", "EHOSTUNREACH",
97
+ "ENETUNREACH", "ENETDOWN", "EAI_AGAIN", "ERR_STREAM_PREMATURE_CLOSE",
98
+ "UND_ERR_SOCKET", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_HEADERS_TIMEOUT",
99
+ "UND_ERR_BODY_TIMEOUT", "UND_ERR_ABORTED",
100
+ ]);
101
+ // Some transient socket errors arrive with no `.code` (notably the classic
102
+ // "socket hang up" thrown by node:_http_client socketOnEnd — the exact crash
103
+ // this guard exists for). Match those by message so they classify benign too.
104
+ const BENIGN_CRASH_MESSAGE_RE = /socket hang up|ECONNRESET|ETIMEDOUT|EPIPE|UND_ERR_/i;
90
105
  const BENIGN_CRASH_FATAL_THRESHOLD = 3;
91
106
  // "In a row" needs a time dimension: benign errors minutes/hours apart are
92
107
  // independent contention events, not a corrupted-state run. Only count a
93
108
  // streak when hits land within this window of the previous one.
94
109
  const BENIGN_CRASH_STREAK_WINDOW_MS = 60_000;
95
110
  function _isBenignCrash(err) {
96
- const code = err?.code || (/\b(EPERM|EACCES|EBUSY)\b/.exec(String(err?.message || err)) || [])[0];
97
- return BENIGN_CRASH_CODES.has(code);
111
+ const msg = String(err?.message || err);
112
+ const code = err?.code
113
+ || (/\b(EPERM|EACCES|EBUSY|ECONNRESET|ETIMEDOUT|ECONNREFUSED|EPIPE|EHOSTUNREACH|ENETUNREACH|ENETDOWN|EAI_AGAIN|UND_ERR_[A-Z_]+)\b/.exec(msg) || [])[0];
114
+ if (BENIGN_CRASH_CODES.has(code)) return true;
115
+ return BENIGN_CRASH_MESSAGE_RE.test(msg);
98
116
  }
99
117
 
100
118
  export {