mixdog 0.9.17 → 0.9.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -305,6 +305,11 @@ const forwarder = new OutputForwarder({
305
305
  removeReaction: (ch, mid, emoji) => {
306
306
  if (!channelBridgeActive) return Promise.resolve();
307
307
  return backend.removeReaction(ch, mid, emoji);
308
+ },
309
+ // Watchdog backstop: force the backend to tear down + rebuild a wedged
310
+ // client so an over-budget send fails fast and the queue releases.
311
+ resetBackend: async () => {
312
+ if (typeof backend?._resetClient === "function") await backend._resetClient();
308
313
  }
309
314
  }, statusState);
310
315
  forwarder.setOnIdle(() => {
@@ -323,7 +328,11 @@ forwarder.setOnIdle(() => {
323
328
  forwarder.setOwnerGetter(() => bridgeRuntimeConnected && currentOwnerState().owned);
324
329
  function applyTranscriptBinding(channelId, transcriptPath, options = {}) {
325
330
  if (!transcriptPath) return;
326
- forwarder.setContext(channelId, transcriptPath, { replayFromStart: options.replayFromStart, catchUpFromPersisted: options.catchUpFromPersisted });
331
+ forwarder.setContext(channelId, transcriptPath, {
332
+ replayFromStart: options.replayFromStart,
333
+ catchUpFromPersisted: options.catchUpFromPersisted,
334
+ recoverUnsyncedTail: options.recoverUnsyncedTail,
335
+ });
327
336
  const boundTranscriptPath = forwarder.transcriptPath || transcriptPath;
328
337
  forwarder.startWatch();
329
338
  void memoryIngestTranscript(boundTranscriptPath, { cwd: options.cwd });
@@ -336,14 +345,31 @@ function applyTranscriptBinding(channelId, transcriptPath, options = {}) {
336
345
  refreshActiveInstance(INSTANCE_ID, { channelId, transcriptPath: boundTranscriptPath }, { onlyIfOwned: true });
337
346
  if (options.persistStatus !== false) {
338
347
  statusState.update((state) => {
348
+ const wasSameTranscript = typeof state.transcriptPath === "string"
349
+ && state.transcriptPath
350
+ && sameResolvedPath(state.transcriptPath, boundTranscriptPath);
351
+ const prevSentCount = Number(state.sentCount ?? 0);
352
+ const nextSentCount = Number(forwarder.sentCount ?? 0);
339
353
  state.channelId = channelId;
340
354
  state.transcriptPath = boundTranscriptPath;
341
355
  state.lastFileSize = forwarder.lastFileSize;
342
- state.sentCount = forwarder.sentCount;
343
- state.lastSentHash = forwarder.lastHash;
344
- state.lastSentTime = 0;
345
- state.sessionIdle = false;
346
- state.sessionCwd = options.cwd ?? null;
356
+ if (wasSameTranscript) {
357
+ state.sentCount = Math.max(
358
+ Number.isFinite(prevSentCount) ? prevSentCount : 0,
359
+ Number.isFinite(nextSentCount) ? nextSentCount : 0,
360
+ );
361
+ if (forwarder.lastHash) state.lastSentHash = forwarder.lastHash;
362
+ else if (typeof state.lastSentHash !== "string") state.lastSentHash = "";
363
+ if (!Number.isFinite(Number(state.lastSentTime))) state.lastSentTime = 0;
364
+ if (typeof state.sessionIdle !== "boolean") state.sessionIdle = false;
365
+ } else {
366
+ state.sentCount = forwarder.sentCount;
367
+ state.lastSentHash = forwarder.lastHash;
368
+ state.lastSentTime = 0;
369
+ state.sessionIdle = false;
370
+ }
371
+ if (options.cwd !== undefined) state.sessionCwd = options.cwd ?? null;
372
+ else if (!wasSameTranscript) state.sessionCwd = null;
347
373
  });
348
374
  }
349
375
  }
@@ -421,7 +447,8 @@ async function rebindTranscriptContext(channelId, options = {}) {
421
447
  applyTranscriptBinding(channelId, explicitTranscriptPath, {
422
448
  replayFromStart: Boolean(options.catchUp),
423
449
  catchUpFromPersisted: options.catchUpFromPersisted,
424
- persistStatus: options.persistStatus
450
+ persistStatus: options.persistStatus,
451
+ recoverUnsyncedTail: options.recoverUnsyncedTail,
425
452
  });
426
453
  if (options.catchUp || options.catchUpFromPersisted) {
427
454
  await forwarder.forwardNewText();
@@ -452,6 +479,7 @@ async function rebindTranscriptContext(channelId, options = {}) {
452
479
  applyTranscriptBinding(channelId, bound.transcriptPath, {
453
480
  replayFromStart,
454
481
  catchUpFromPersisted: options.catchUpFromPersisted,
482
+ recoverUnsyncedTail: options.recoverUnsyncedTail,
455
483
  persistStatus: options.persistStatus,
456
484
  cwd: bound.sessionCwd,
457
485
  });
@@ -497,6 +525,7 @@ async function rebindTranscriptContext(channelId, options = {}) {
497
525
  applyTranscriptBinding(channelId, pendingTranscriptPath, {
498
526
  replayFromStart: !samePersisted,
499
527
  catchUpFromPersisted: samePersisted,
528
+ recoverUnsyncedTail: options.recoverUnsyncedTail,
500
529
  cwd: pendingTranscriptCwd,
501
530
  persistStatus: options.persistStatus,
502
531
  });
@@ -512,7 +541,11 @@ async function rebindTranscriptContext(channelId, options = {}) {
512
541
  }
513
542
  }
514
543
  if (previousPath && options.fallbackPrevious !== false) {
515
- applyTranscriptBinding(channelId, previousPath, { catchUpFromPersisted: true, cwd: statusState.read().sessionCwd });
544
+ applyTranscriptBinding(channelId, previousPath, {
545
+ catchUpFromPersisted: true,
546
+ recoverUnsyncedTail: options.recoverUnsyncedTail,
547
+ cwd: statusState.read().sessionCwd
548
+ });
516
549
  if (fs.existsSync(previousPath)) {
517
550
  await forwarder.forwardNewText();
518
551
  } else {
@@ -631,7 +664,8 @@ async function bindPersistedTranscriptIfAny() {
631
664
  const bound = await rebindTranscriptContext(currentStatus.channelId, {
632
665
  previousPath: getPersistedTranscriptPath(),
633
666
  persistStatus: true,
634
- catchUpFromPersisted: true
667
+ catchUpFromPersisted: true,
668
+ recoverUnsyncedTail: true,
635
669
  });
636
670
  if (bound) {
637
671
  process.stderr.write(`mixdog: initial transcript bind: ${bound}
@@ -710,8 +744,40 @@ function syncOwnedWebhookAndEventRuntime({ reload = false } = {}) {
710
744
  webhookServer = null;
711
745
  }
712
746
  }
747
+ let _ownedRuntimeSelfHealing = false;
748
+ // Explicit restore path for an already-connected owner. The 3s ownership tick
749
+ // must not run this implicitly: re-binding status on every tick can move the
750
+ // transcript cursor to EOF, and probing the gateway during Discord's own
751
+ // reconnect window can reset a healthy reconnect loop. Keep this for explicit
752
+ // activation/reload recovery only.
753
+ async function selfHealOwnedRuntime(options = {}) {
754
+ const shouldRestoreBinding = options.restoreBinding === true;
755
+ const shouldResetBackend = options.resetBackend === true;
756
+ if (!shouldRestoreBinding && !shouldResetBackend) return;
757
+ if (_ownedRuntimeSelfHealing) return;
758
+ _ownedRuntimeSelfHealing = true;
759
+ try {
760
+ if (shouldRestoreBinding) {
761
+ await bindPersistedTranscriptIfAny().catch((e) => {
762
+ process.stderr.write(`mixdog: self-heal bindPersistedTranscriptIfAny failed (non-fatal): ${e instanceof Error ? e.message : String(e)}\n`);
763
+ });
764
+ }
765
+ if (shouldResetBackend && typeof backend?._resetClient === "function") {
766
+ await backend._resetClient().catch((e) => {
767
+ process.stderr.write(`mixdog: self-heal backend reset failed (non-fatal): ${e instanceof Error ? e.message : String(e)}\n`);
768
+ });
769
+ }
770
+ // Nudge the forwarder to drain anything the rebind/reconnect surfaced.
771
+ void forwarder.forwardNewText().catch(() => {});
772
+ } finally {
773
+ _ownedRuntimeSelfHealing = false;
774
+ }
775
+ }
713
776
  async function startOwnedRuntime(options = {}) {
714
- if (bridgeRuntimeConnected) return;
777
+ if (bridgeRuntimeConnected) {
778
+ if (!bridgeRuntimeStarting) await selfHealOwnedRuntime(options);
779
+ return;
780
+ }
715
781
  if (bridgeRuntimeStarting) return;
716
782
  if (!channelBridgeActive) return;
717
783
  bridgeRuntimeStarting = true;
@@ -928,8 +994,13 @@ async function refreshBridgeOwnership(options = {}) {
928
994
  if (active && active.instanceId && active.instanceId !== INSTANCE_ID) {
929
995
  if (bridgeRuntimeConnected) {
930
996
  await stopOwnedRuntime("ownership lost (newer remote session)");
931
- notifyRemoteSuperseded();
932
997
  }
998
+ // Notify the parent UNCONDITIONALLY (not only when we were connected):
999
+ // a loser whose backend never connected (seat stolen mid-connect,
1000
+ // connect failed, or already in standby) must still drop its Remote
1001
+ // indicator. The parent handler is idempotent — it only acts while
1002
+ // remoteEnabled is still true — so repeat ticks are harmless.
1003
+ notifyRemoteSuperseded();
933
1004
  return;
934
1005
  }
935
1006
  // No live owner — claim the empty seat and start.
@@ -1658,6 +1729,26 @@ backend.onMessage = (msg) => {
1658
1729
  });
1659
1730
  })();
1660
1731
  };
1732
+ const NETWORK_ERR_RE = /fetch failed|ECONNRESET|ETIMEDOUT|ENOTFOUND|EAI_AGAIN|socket hang up|network|timeout|aborted|TimeoutError/i;
1733
+ function isNetworkError(err) {
1734
+ const s = `${err?.code ?? ""} ${err?.name ?? ""} ${err?.message ?? err ?? ""}`;
1735
+ return NETWORK_ERR_RE.test(s);
1736
+ }
1737
+ async function retryOnNetwork(fn, { attempts = 3, baseDelayMs = 300, label = "op" } = {}) {
1738
+ let lastErr;
1739
+ for (let i = 0; i < attempts; i++) {
1740
+ try {
1741
+ return await fn();
1742
+ } catch (err) {
1743
+ lastErr = err;
1744
+ if (i === attempts - 1 || !isNetworkError(err)) throw err;
1745
+ const delay = baseDelayMs * (i + 1);
1746
+ process.stderr.write(`mixdog: ${label} network failure (attempt ${i + 1}/${attempts}), retrying in ${delay}ms: ${err?.message || err}\n`);
1747
+ await new Promise((r) => setTimeout(r, delay));
1748
+ }
1749
+ }
1750
+ throw lastErr;
1751
+ }
1661
1752
  async function handleInbound(msg, route, options = {}) {
1662
1753
  const handleStartMs = Date.now();
1663
1754
  let text = msg.text;
@@ -1668,11 +1759,19 @@ async function handleInbound(msg, route, options = {}) {
1668
1759
  text = text || "[voice message]";
1669
1760
  } else {
1670
1761
  try {
1671
- const files = await backend.downloadAttachment(msg.chatId, msg.messageId);
1762
+ const files = await retryOnNetwork(
1763
+ // Short per-attempt timeout (vs the 180s default) bounds worst-case
1764
+ // blockage of the serial inboundQueue on a bad voice message.
1765
+ () => backend.downloadAttachment(msg.chatId, msg.messageId, { timeoutMs: 20_000 }),
1766
+ { label: "voice.download" }
1767
+ );
1672
1768
  // concurrency handled inside transcribeVoice queue; loop is sequential so last att wins
1673
1769
  for (const f of voiceAtts.map(a => files.find(df => df.id === a.id) ?? null).filter(Boolean)) {
1674
1770
  const _t0 = Date.now();
1675
- const transcript = await transcribeVoice(f.path, { attachmentId: f.id });
1771
+ const transcript = await retryOnNetwork(
1772
+ () => transcribeVoice(f.path, { attachmentId: f.id }),
1773
+ { label: "voice.transcription" }
1774
+ );
1676
1775
  const _elapsed = Date.now() - _t0;
1677
1776
  if (transcript) {
1678
1777
  text = transcript;
@@ -1683,8 +1782,12 @@ async function handleInbound(msg, route, options = {}) {
1683
1782
  }
1684
1783
  }
1685
1784
  } catch (err) {
1686
- process.stderr.write(`mixdog: voice.transcription error: ${err}\n`);
1687
- text = text || `[voice message \u2014 transcription error: ${err?.message || err}]`;
1785
+ const netFail = isNetworkError(err);
1786
+ process.stderr.write(`mixdog: voice.transcription error${netFail ? " (network, retries exhausted)" : ""}: ${err}\n`);
1787
+ const marker = netFail
1788
+ ? "[attachment: voice transcription failed (network)]"
1789
+ : `[voice message \u2014 transcription error: ${err?.message || err}]`;
1790
+ text = text ? `${text} ${marker}` : marker;
1688
1791
  }
1689
1792
  }
1690
1793
  }
@@ -1953,15 +2056,39 @@ if (_isWorkerMode && process.send) {
1953
2056
  })
1954
2057
  void (async () => {
1955
2058
  const startedAt = performance.now()
1956
- try {
1957
- await start()
1958
- bootProfile("worker:ready", { ms: (performance.now() - startedAt).toFixed(1) })
1959
- process.send({ type: 'ready' })
1960
- } catch (e) {
1961
- bootProfile("worker:failed", { ms: (performance.now() - startedAt).toFixed(1), error: e?.message || String(e) })
1962
- process.stderr.write(`[channels-worker] start() failed: ${e && (e.message || e)}\n`)
1963
- process.send({ type: 'ready', degraded: true, error: e?.message || String(e) })
2059
+ const MAX_START_ATTEMPTS = 3
2060
+ const BASE_BACKOFF_MS = 250
2061
+ const isTransientStartErr = (err) =>
2062
+ err?.code === 'ELOCKTIMEOUT' || /atomic lock timeout/i.test(err?.message || '')
2063
+ let lastErr
2064
+ for (let attempt = 1; attempt <= MAX_START_ATTEMPTS; attempt++) {
2065
+ if (_channelsStopInFlight) return
2066
+ try {
2067
+ await start()
2068
+ bootProfile("worker:ready", { ms: (performance.now() - startedAt).toFixed(1), attempt })
2069
+ process.send({ type: 'ready' })
2070
+ return
2071
+ } catch (e) {
2072
+ lastErr = e
2073
+ const transient = isTransientStartErr(e)
2074
+ bootProfile("worker:start-failed", { attempt, transient, error: e?.message || String(e) })
2075
+ process.stderr.write(`[channels-worker] start() failed (attempt ${attempt}/${MAX_START_ATTEMPTS}, transient=${transient}): ${e && (e.message || e)}\n`)
2076
+ if (!transient || attempt >= MAX_START_ATTEMPTS) break
2077
+ const backoff = BASE_BACKOFF_MS * attempt + Math.floor(Math.random() * BASE_BACKOFF_MS)
2078
+ await new Promise((r) => setTimeout(r, backoff))
2079
+ if (_channelsStopInFlight) return
2080
+ }
1964
2081
  }
2082
+ // A stop landed while we were failing — let clean shutdown proceed, never exit over it.
2083
+ if (_channelsStopInFlight) return
2084
+ // Terminal failure: do NOT mask as a (degraded) ready. Exit non-zero so the
2085
+ // parent's exit-before-ready path respawns or rejects startRemote instead of
2086
+ // silently losing remote output forwarding.
2087
+ bootProfile("worker:failed", { ms: (performance.now() - startedAt).toFixed(1), error: lastErr?.message || String(lastErr) })
2088
+ process.stderr.write(`[channels-worker] start() giving up after ${MAX_START_ATTEMPTS} attempts: ${lastErr && (lastErr.message || lastErr)}\n`)
2089
+ // Exit 2 = terminal (non-transient) start failure: parent must reject, not respawn.
2090
+ // Exit 1 = exhausted transient retries: parent may respawn.
2091
+ process.exit(isTransientStartErr(lastErr) ? 1 : 2)
1965
2092
  })()
1966
2093
  }
1967
2094
 
@@ -86,15 +86,33 @@ ${err instanceof Error ? err.stack : ""}
86
86
  // already retried elsewhere (atomic-file.mjs RETRY_CODES) — a single
87
87
  // occurrence must NOT be fatal, only a run of 3+ in a row without an
88
88
  // intervening distinct/successful event.
89
- const BENIGN_CRASH_CODES = new Set(["EPERM", "EACCES", "EBUSY"]);
89
+ const BENIGN_CRASH_CODES = new Set([
90
+ "EPERM", "EACCES", "EBUSY",
91
+ // Transient transport blips: a raw http/https ClientRequest (or an
92
+ // undici socket) can emit these when a keep-alive connection is torn
93
+ // down mid-flight (Discord gateway flapping, proxy resets). One blip
94
+ // must NOT kill the worker — it is benign; only a repeat storm within
95
+ // the streak window force-exits (see _fatalCrash).
96
+ "ECONNRESET", "ETIMEDOUT", "ECONNREFUSED", "EPIPE", "EHOSTUNREACH",
97
+ "ENETUNREACH", "ENETDOWN", "EAI_AGAIN", "ERR_STREAM_PREMATURE_CLOSE",
98
+ "UND_ERR_SOCKET", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_HEADERS_TIMEOUT",
99
+ "UND_ERR_BODY_TIMEOUT", "UND_ERR_ABORTED",
100
+ ]);
101
+ // Some transient socket errors arrive with no `.code` (notably the classic
102
+ // "socket hang up" thrown by node:_http_client socketOnEnd — the exact crash
103
+ // this guard exists for). Match those by message so they classify benign too.
104
+ const BENIGN_CRASH_MESSAGE_RE = /socket hang up|ECONNRESET|ETIMEDOUT|EPIPE|UND_ERR_/i;
90
105
  const BENIGN_CRASH_FATAL_THRESHOLD = 3;
91
106
  // "In a row" needs a time dimension: benign errors minutes/hours apart are
92
107
  // independent contention events, not a corrupted-state run. Only count a
93
108
  // streak when hits land within this window of the previous one.
94
109
  const BENIGN_CRASH_STREAK_WINDOW_MS = 60_000;
95
110
  function _isBenignCrash(err) {
96
- const code = err?.code || (/\b(EPERM|EACCES|EBUSY)\b/.exec(String(err?.message || err)) || [])[0];
97
- return BENIGN_CRASH_CODES.has(code);
111
+ const msg = String(err?.message || err);
112
+ const code = err?.code
113
+ || (/\b(EPERM|EACCES|EBUSY|ECONNRESET|ETIMEDOUT|ECONNREFUSED|EPIPE|EHOSTUNREACH|ENETUNREACH|ENETDOWN|EAI_AGAIN|UND_ERR_[A-Z_]+)\b/.exec(msg) || [])[0];
114
+ if (BENIGN_CRASH_CODES.has(code)) return true;
115
+ return BENIGN_CRASH_MESSAGE_RE.test(msg);
98
116
  }
99
117
 
100
118
  export {
@@ -28,8 +28,24 @@ import {
28
28
  buildToolLine
29
29
  } from "./tool-format.mjs";
30
30
 
31
+ // Per-item send watchdog. drainQueue awaits ONE backend send at a time; a hung
32
+ // send would otherwise never settle, so drainQueue never returns and `sending`
33
+ // stays true forever — wedging ALL outbound behind the queue head. The watchdog
34
+ // does NOT abandon the send (racing a still-running send against a retry would
35
+ // double-deliver): it only pokes the backend to reset its wedged client so the
36
+ // AWAITED send fails fast and `sending` releases on real settlement. Sized above
37
+ // the backend's worst-case per-attempt retries.
38
+ const SEND_ITEM_TIMEOUT_MS = 120_000;
39
+ // After the watchdog resets the backend, wait at most this long for the
40
+ // abandoned send to actually settle before releasing the queue.
41
+ const SEND_SETTLE_CAP_MS = 15_000;
42
+ const TAIL_RECOVERY_BYTES = 256 * 1024;
43
+
31
44
  class OutputForwarder {
32
45
  ownerGetter = null;
46
+ // Generation of the in-flight supervised send. Bumped when a send is
47
+ // abandoned (settlement cap) so a late/zombie completion can't double-advance.
48
+ _activeSendGen = 0;
33
49
  setOwnerGetter(fn) {
34
50
  this.ownerGetter = fn;
35
51
  }
@@ -111,20 +127,35 @@ class OutputForwarder {
111
127
  try {
112
128
  const stat = existsSync(this.transcriptPath) ? statSync(this.transcriptPath) : null;
113
129
  const currentSize = stat?.size ?? 0;
130
+ const persisted = this.statusState?.read?.();
131
+ const sameTranscript = persisted?.transcriptPath &&
132
+ sameResolvedPath(persisted.transcriptPath, this.transcriptPath);
114
133
  let fileSize;
115
134
  if (options.replayFromStart) {
116
135
  fileSize = 0;
117
136
  } else if (options.catchUpFromPersisted) {
118
- const persisted = this.statusState?.read?.();
119
137
  const persistedSize = typeof persisted?.lastFileSize === "number" ? persisted.lastFileSize : -1;
120
- const sameTranscript = persisted?.transcriptPath &&
121
- sameResolvedPath(persisted.transcriptPath, this.transcriptPath);
122
138
  fileSize = (sameTranscript && persistedSize >= 0)
123
139
  ? Math.min(Math.max(persistedSize, 0), currentSize)
124
140
  : currentSize;
125
141
  } else {
126
142
  fileSize = currentSize;
127
143
  }
144
+ if (options.recoverUnsyncedTail && fileSize === currentSize && currentSize > 0) {
145
+ const sentCount = Number(persisted?.sentCount ?? 0);
146
+ const lastSentTime = Number(persisted?.lastSentTime ?? 0);
147
+ const hasSentHash = typeof persisted?.lastSentHash === "string" && persisted.lastSentHash.length > 0;
148
+ const noSendEvidence = (!Number.isFinite(sentCount) || sentCount <= 0)
149
+ && (!Number.isFinite(lastSentTime) || lastSentTime <= 0)
150
+ && !hasSentHash;
151
+ if (!sameTranscript || noSendEvidence) {
152
+ const tailOffset = this.findLastAssistantTextOffset(currentSize);
153
+ if (tailOffset >= 0 && tailOffset < currentSize) {
154
+ fileSize = tailOffset;
155
+ dropTrace("context.recoverUnsyncedTail", { path: this.transcriptPath, offset: tailOffset, currentSize, sameTranscript: !!sameTranscript });
156
+ }
157
+ }
158
+ }
128
159
  this.lastFileSize = fileSize;
129
160
  this.readFileSize = fileSize;
130
161
  } catch {
@@ -132,6 +163,46 @@ class OutputForwarder {
132
163
  this.readFileSize = 0;
133
164
  }
134
165
  }
166
+ findLastAssistantTextOffset(fileSize) {
167
+ if (!this.transcriptPath || fileSize <= 0) return -1;
168
+ const bytesToRead = Math.min(fileSize, TAIL_RECOVERY_BYTES);
169
+ const startOffset = fileSize - bytesToRead;
170
+ let fd = null;
171
+ try {
172
+ fd = openSync(this.transcriptPath, "r");
173
+ const buf = Buffer.alloc(bytesToRead);
174
+ readSync(fd, buf, 0, bytesToRead, startOffset);
175
+ const raw = buf.toString("utf8");
176
+ let chunkStart = 0;
177
+ if (startOffset > 0) {
178
+ const firstNewline = raw.indexOf("\n");
179
+ if (firstNewline < 0) return -1;
180
+ chunkStart = firstNewline + 1;
181
+ }
182
+ let cursor = startOffset + chunkStart;
183
+ let lastOffset = -1;
184
+ for (const line of raw.slice(chunkStart).split("\n")) {
185
+ const lineOffset = cursor;
186
+ cursor += Buffer.byteLength(line, "utf8") + 1;
187
+ if (!line.trim()) continue;
188
+ try {
189
+ const entry = JSON.parse(line);
190
+ if (entry.isSidechain || entry.type !== "assistant") continue;
191
+ const parts = entry.message?.content;
192
+ if (!Array.isArray(parts)) continue;
193
+ if (parts.some((c) => c?.type === "text" && typeof c.text === "string" && c.text.trim())) {
194
+ lastOffset = lineOffset;
195
+ }
196
+ } catch {
197
+ }
198
+ }
199
+ return lastOffset;
200
+ } catch {
201
+ return -1;
202
+ } finally {
203
+ if (fd != null) closeSync(fd);
204
+ }
205
+ }
135
206
  /** Reset counters for new turn */
136
207
  reset() {
137
208
  this.sentCount = 0;
@@ -282,6 +353,7 @@ class OutputForwarder {
282
353
  }
283
354
  async deliverQueueItem(item) {
284
355
  const targetChannelId = item.channelId ?? this.channelId;
356
+ const gen = ++this._activeSendGen;
285
357
  if (!item.text || !targetChannelId) {
286
358
  this.commitReadProgress(item.nextFileSize);
287
359
  return;
@@ -314,10 +386,13 @@ class OutputForwarder {
314
386
  // re-sending chunks that already landed. The token is opaque here —
315
387
  // the forwarder only stores/passes/clears it, never interprets it.
316
388
  await this.cb.send(targetChannelId, formatted, item._resumeToken ? { resumeToken: item._resumeToken } : undefined);
389
+ dropTrace("discord.send.ok", null);
390
+ // A zombie completion (send abandoned past the settlement cap, gen bumped)
391
+ // must not mutate ANY item/queue state — including the resume token.
392
+ if (gen !== this._activeSendGen) return;
317
393
  // Full success — drop any stale token so a later reuse of this item
318
394
  // object can't carry a dead token.
319
395
  item._resumeToken = undefined;
320
- dropTrace("discord.send.ok", null);
321
396
  } catch (err) {
322
397
  // Persist the opaque resume token (if any) on the item so the requeued
323
398
  // retry resumes from the failed chunk.
@@ -349,7 +424,18 @@ ${_bt.trim()}` : _bt.trim();
349
424
  }
350
425
  /** Forward new assistant text to Discord. Returns true if text was sent. */
351
426
  async forwardNewText() {
352
- if (!this._isOwner()) return false;
427
+ if (!this._isOwner()) {
428
+ // Silent no-send is invisible and has masked real ownership-gate bugs
429
+ // (a live seat that never forwards). Surface it, rate-limited to one
430
+ // line per 60s so a persistent non-owner state stays visible without
431
+ // flooding stderr on every poll tick.
432
+ const now = Date.now();
433
+ if (!this._lastOwnerGateLogAt || now - this._lastOwnerGateLogAt >= 60_000) {
434
+ this._lastOwnerGateLogAt = now;
435
+ process.stderr.write(`[output-forwarder] forwardNewText skipped: not seat owner (channelId=${this.channelId ?? "none"})\n`);
436
+ }
437
+ return false;
438
+ }
353
439
  if (!this.channelId) return false;
354
440
  const { text: newText, nextFileSize } = this.extractNewText();
355
441
  if (!newText) {
@@ -428,6 +514,31 @@ ${_bt.trim()}` : _bt.trim();
428
514
  _kickDrain() {
429
515
  this.drainQueue().catch(err => process.stderr.write(`[output-forwarder] drainQueue rejected: ${err?.message || err}\n`));
430
516
  }
517
+ /** Supervise a per-item send. The underlying promise is AWAITED to real
518
+ * settlement so `sending` is never released early (no unsupervised send
519
+ * racing a retry → duplicate). The watchdog's only job is to reset the
520
+ * backend's wedged client so the awaited send fails fast. */
521
+ _sendSupervised(promise, label) {
522
+ const settled = Promise.resolve(promise);
523
+ settled.catch(() => {}); // never leak an unhandled rejection on late settle
524
+ return new Promise((resolve, reject) => {
525
+ let capTimer = null;
526
+ const watchdog = setTimeout(() => {
527
+ process.stderr.write(`[output-forwarder] ${label} watchdog fired after ${SEND_ITEM_TIMEOUT_MS}ms; resetting backend\n`);
528
+ Promise.resolve(this.cb?.resetBackend?.()).catch(() => {});
529
+ // Bound the post-reset settlement wait; past the cap, invalidate the
530
+ // in-flight send (gen bump → zombie can't double-advance) and release.
531
+ capTimer = setTimeout(() => {
532
+ this._activeSendGen++;
533
+ reject(new Error(`${label} abandoned after ${SEND_SETTLE_CAP_MS}ms settlement cap`));
534
+ }, SEND_SETTLE_CAP_MS);
535
+ }, SEND_ITEM_TIMEOUT_MS);
536
+ settled.then(
537
+ (v) => { clearTimeout(watchdog); if (capTimer) clearTimeout(capTimer); resolve(v); },
538
+ (e) => { clearTimeout(watchdog); if (capTimer) clearTimeout(capTimer); reject(e); }
539
+ );
540
+ });
541
+ }
431
542
  /** Drain both priority lanes sequentially. finalLane drains first when non-empty. */
432
543
  async drainQueue() {
433
544
  if (this.sending) return;
@@ -440,9 +551,9 @@ ${_bt.trim()}` : _bt.trim();
440
551
  const item = lane[0];
441
552
  try {
442
553
  if (item.type === "text") {
443
- await this.deliverQueueItem(item);
554
+ await this._sendSupervised(this.deliverQueueItem(item), "deliverQueueItem");
444
555
  } else if (item.type === "toolLog") {
445
- await this.processToolLog(item);
556
+ await this._sendSupervised(this.processToolLog(item), "processToolLog");
446
557
  }
447
558
  lane.shift();
448
559
  } catch (err) {
@@ -360,6 +360,19 @@ function refreshActiveInstance(instanceId, meta, options) {
360
360
  }
361
361
  }
362
362
  }
363
+ // Worker-owned seat: when THIS process is the channels worker and it is
364
+ // the seat owner (no distinct TUI/supervisor Lead — ownerLeadPid resolves
365
+ // to our own pid), keep ui_heartbeat_at fresh. A headless worker never
366
+ // runs the TUI render loop, so an inherited heartbeat (written by a prior
367
+ // TUI session and carried forward in prevRest) would otherwise go stale at
368
+ // UI_HEARTBEAT_STALE_MS and falsely flag a live worker-owned seat. A
369
+ // TUI-owned seat has a distinct supervisor (ownerLeadPid !== our pid), so
370
+ // this branch is a no-op there and TUI staleness is preserved.
371
+ const workerOwnsSeat = process.env.MIXDOG_WORKER_MODE === "1"
372
+ && identity.ownerLeadPid === process.pid;
373
+ if (workerOwnsSeat) {
374
+ next.ui_heartbeat_at = Date.now();
375
+ }
363
376
  return { ...preservedExtra, ...next };
364
377
  }, { compact: true, fsync: false, fsyncDir: false, renameFallback: 'truncate' });
365
378
  }
@@ -167,9 +167,11 @@ function createVoiceTranscription({ getConfig, dataDir }) {
167
167
  if (attachmentId && result) _voiceTranscriptCache.set(attachmentId, result);
168
168
  return result;
169
169
  } catch (err) {
170
- if (err?.message?.startsWith('voice runtime not installed')) throw err; // propagate setup errors; caller posts user-visible failure
170
+ // Propagate ALL real failures so the caller's retry (network-class) and
171
+ // failure marker (index.mjs catch) fire; null is reserved strictly for a
172
+ // legit empty transcript.
171
173
  process.stderr.write(`mixdog: voice.transcription failed: ${err}\n`);
172
- return null;
174
+ throw err;
173
175
  }
174
176
  }
175
177
 
@@ -75,6 +75,8 @@ export function createStandaloneChannelWorker({
75
75
  let readyResolve = null;
76
76
  let readyReject = null;
77
77
  let stopPromise = null;
78
+ let stopRequested = false;
79
+ let bootGeneration = 0;
78
80
  let inProcessMod = null;
79
81
  let inProcessStartPromise = null;
80
82
  let nextCallId = 1;
@@ -153,6 +155,7 @@ export function createStandaloneChannelWorker({
153
155
  return stopPromise.then(() => start());
154
156
  }
155
157
  if (child && child.exitCode == null && !child.killed) return readyPromise || Promise.resolve(status());
158
+ stopRequested = false;
156
159
  readyPromise = new Promise((resolve, reject) => {
157
160
  readyResolve = resolve;
158
161
  readyReject = reject;
@@ -181,6 +184,9 @@ export function createStandaloneChannelWorker({
181
184
  windowsHide: true,
182
185
  });
183
186
  const spawnedPid = child.pid;
187
+ // Per-boot generation: a later start()/respawn bumps this, so a stale old
188
+ // child's late exit is ignored instead of respawning/rejecting the current one.
189
+ const myGeneration = ++bootGeneration;
184
190
  startChildGuardian({ childPid: spawnedPid, label: 'channel-worker', orphanGraceMs: 8000, forceGraceMs: 3000 });
185
191
  if (spawnedPid) ownedChildPids.add(spawnedPid);
186
192
  installParentExitHook();
@@ -215,13 +221,17 @@ export function createStandaloneChannelWorker({
215
221
  });
216
222
 
217
223
  child.on('exit', (code, signal) => {
224
+ if (myGeneration !== bootGeneration) return;
218
225
  if (spawnedPid) ownedChildPids.delete(spawnedPid);
219
226
  const error = new Error(`channels runtime exited (${signal || (code ?? 'unknown')})`);
220
- if (!becameReady && readyReject && !stopPromise && attempt < WORKER_BOOT_MAX_ATTEMPTS) {
227
+ // Exit code 2 = terminal (non-transient) worker start failure: reject, never respawn.
228
+ // A requested stop (stopRequested) also suppresses respawn even if stopPromise's
229
+ // settle timer already cleared it before the child exited.
230
+ if (!becameReady && readyReject && !stopPromise && !stopRequested && code !== 2 && attempt < WORKER_BOOT_MAX_ATTEMPTS) {
221
231
  logLine(logPath, `worker exited before ready (${signal || (code ?? 'unknown')}), attempt ${attempt}/${WORKER_BOOT_MAX_ATTEMPTS}; retrying in ${WORKER_BOOT_RETRY_DELAY_MS}ms`);
222
232
  child = null;
223
233
  setTimeout(() => {
224
- if (stopPromise || !readyReject) return;
234
+ if (stopPromise || stopRequested || !readyReject) return;
225
235
  spawnWorkerChild(attempt + 1);
226
236
  }, WORKER_BOOT_RETRY_DELAY_MS);
227
237
  return;
@@ -235,6 +245,7 @@ export function createStandaloneChannelWorker({
235
245
  });
236
246
 
237
247
  child.on('error', (error) => {
248
+ if (myGeneration !== bootGeneration) return;
238
249
  logLine(logPath, `runtime error: ${error?.message || error}`);
239
250
  if (readyReject) readyReject(error);
240
251
  readyResolve = null;
@@ -347,6 +358,7 @@ export function createStandaloneChannelWorker({
347
358
 
348
359
  function stop(reason = 'standalone shutdown', options = {}) {
349
360
  const waitForExit = options?.waitForExit !== false;
361
+ stopRequested = true;
350
362
  stopClientHeartbeat();
351
363
  if (stopPromise) return stopPromise;
352
364
  if (!useProcessWorker) {
@@ -562,6 +562,17 @@ export function measuredTranscriptRows(item, columns, toolOutputExpanded) {
562
562
 
563
563
  const STREAMING_ROW_QUANTUM = 1;
564
564
 
565
+ // Bottom-pinned flag for the streaming-tail row estimate. Set once per hook
566
+ // render (use-transcript-window.mjs) BEFORE any tail resolution so every path
567
+ // that resolves the streaming tail height (tailSig, buildTranscriptRowIndex,
568
+ // incremental tail, transcriptStructureSignature) reads the same value and
569
+ // cannot diverge. Bottom-pinned → fold the live estimate in (geometry right on
570
+ // the first commit); away from bottom → d19cad1e defer-growth.
571
+ let streamingBottomPinned = false;
572
+ export function setStreamingBottomPinned(pinned) {
573
+ streamingBottomPinned = !!pinned;
574
+ }
575
+
565
576
  function assistantTextForStreamingRowEstimate(text) {
566
577
  return streamingLayoutText(text);
567
578
  }
@@ -580,6 +591,17 @@ export function estimateTranscriptItemRowsCached(item, columns, toolOutputExpand
580
591
  const toolExpanded = toolOutputExpanded ? 1 : 0;
581
592
  const idEntry = streamingMeasuredRowsById.get(item.id);
582
593
  if (idEntry && idEntry.rows > 0 && idEntry.columns === columns && idEntry.toolExpanded === toolExpanded) {
594
+ if (streamingBottomPinned) {
595
+ // Bottom-pinned: the view follows the tail, so its geometry must be
596
+ // correct on the FIRST commit. Fold the freshly-grown live estimate in
597
+ // — max(measuredFloor, liveEstimate) — so totalRows/scrollOffset are
598
+ // right this frame instead of committing the stale measured floor and
599
+ // growing a frame later when the harvest bumps measuredRowsVersion
600
+ // (the judder). The measured floor still guards against an estimate
601
+ // that undercounts the real wrap. Away from bottom the defer-growth
602
+ // path below is kept (its anchor-stability / perf intent).
603
+ return Math.max(idEntry.rows, streamingEstimateRows(item, columns, toolOutputExpanded));
604
+ }
583
605
  // Defer-growth: while a confirmed (post-commit Yoga) measurement exists
584
606
  // for this streaming id, THIS render keeps that value instead of
585
607
  // folding in the freshly-grown estimate. Combining