mixdog 0.9.17 → 0.9.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/bench/cache-probe-tasks.json +1 -1
- package/scripts/bench/lead-review-tasks-r3.json +1 -1
- package/scripts/bench/r4-mixed-tasks.json +1 -1
- package/scripts/bench/review-tasks.json +1 -1
- package/scripts/build-runtime-windows.ps1 +242 -242
- package/scripts/recall-usecase-cases.json +1 -1
- package/scripts/smoke-runtime-negative.ps1 +106 -106
- package/scripts/tool-efficiency-diag.mjs +1 -1
- package/src/rules/lead/02-channels.md +3 -3
- package/src/runtime/channels/backends/discord-attachments.mjs +2 -2
- package/src/runtime/channels/backends/discord-gateway.mjs +12 -2
- package/src/runtime/channels/backends/discord.mjs +97 -7
- package/src/runtime/channels/index.mjs +150 -23
- package/src/runtime/channels/lib/crash-log.mjs +21 -3
- package/src/runtime/channels/lib/output-forwarder.mjs +118 -7
- package/src/runtime/channels/lib/runtime-paths.mjs +13 -0
- package/src/runtime/channels/lib/voice-transcription.mjs +4 -2
- package/src/standalone/channel-worker.mjs +14 -2
- package/src/tui/app/transcript-window.mjs +22 -0
- package/src/tui/app/use-transcript-window.mjs +9 -3
- package/src/tui/dist/index.mjs +63 -12
- package/src/tui/engine.mjs +60 -10
|
@@ -305,6 +305,11 @@ const forwarder = new OutputForwarder({
|
|
|
305
305
|
removeReaction: (ch, mid, emoji) => {
|
|
306
306
|
if (!channelBridgeActive) return Promise.resolve();
|
|
307
307
|
return backend.removeReaction(ch, mid, emoji);
|
|
308
|
+
},
|
|
309
|
+
// Watchdog backstop: force the backend to tear down + rebuild a wedged
|
|
310
|
+
// client so an over-budget send fails fast and the queue releases.
|
|
311
|
+
resetBackend: async () => {
|
|
312
|
+
if (typeof backend?._resetClient === "function") await backend._resetClient();
|
|
308
313
|
}
|
|
309
314
|
}, statusState);
|
|
310
315
|
forwarder.setOnIdle(() => {
|
|
@@ -323,7 +328,11 @@ forwarder.setOnIdle(() => {
|
|
|
323
328
|
forwarder.setOwnerGetter(() => bridgeRuntimeConnected && currentOwnerState().owned);
|
|
324
329
|
function applyTranscriptBinding(channelId, transcriptPath, options = {}) {
|
|
325
330
|
if (!transcriptPath) return;
|
|
326
|
-
forwarder.setContext(channelId, transcriptPath, {
|
|
331
|
+
forwarder.setContext(channelId, transcriptPath, {
|
|
332
|
+
replayFromStart: options.replayFromStart,
|
|
333
|
+
catchUpFromPersisted: options.catchUpFromPersisted,
|
|
334
|
+
recoverUnsyncedTail: options.recoverUnsyncedTail,
|
|
335
|
+
});
|
|
327
336
|
const boundTranscriptPath = forwarder.transcriptPath || transcriptPath;
|
|
328
337
|
forwarder.startWatch();
|
|
329
338
|
void memoryIngestTranscript(boundTranscriptPath, { cwd: options.cwd });
|
|
@@ -336,14 +345,31 @@ function applyTranscriptBinding(channelId, transcriptPath, options = {}) {
|
|
|
336
345
|
refreshActiveInstance(INSTANCE_ID, { channelId, transcriptPath: boundTranscriptPath }, { onlyIfOwned: true });
|
|
337
346
|
if (options.persistStatus !== false) {
|
|
338
347
|
statusState.update((state) => {
|
|
348
|
+
const wasSameTranscript = typeof state.transcriptPath === "string"
|
|
349
|
+
&& state.transcriptPath
|
|
350
|
+
&& sameResolvedPath(state.transcriptPath, boundTranscriptPath);
|
|
351
|
+
const prevSentCount = Number(state.sentCount ?? 0);
|
|
352
|
+
const nextSentCount = Number(forwarder.sentCount ?? 0);
|
|
339
353
|
state.channelId = channelId;
|
|
340
354
|
state.transcriptPath = boundTranscriptPath;
|
|
341
355
|
state.lastFileSize = forwarder.lastFileSize;
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
356
|
+
if (wasSameTranscript) {
|
|
357
|
+
state.sentCount = Math.max(
|
|
358
|
+
Number.isFinite(prevSentCount) ? prevSentCount : 0,
|
|
359
|
+
Number.isFinite(nextSentCount) ? nextSentCount : 0,
|
|
360
|
+
);
|
|
361
|
+
if (forwarder.lastHash) state.lastSentHash = forwarder.lastHash;
|
|
362
|
+
else if (typeof state.lastSentHash !== "string") state.lastSentHash = "";
|
|
363
|
+
if (!Number.isFinite(Number(state.lastSentTime))) state.lastSentTime = 0;
|
|
364
|
+
if (typeof state.sessionIdle !== "boolean") state.sessionIdle = false;
|
|
365
|
+
} else {
|
|
366
|
+
state.sentCount = forwarder.sentCount;
|
|
367
|
+
state.lastSentHash = forwarder.lastHash;
|
|
368
|
+
state.lastSentTime = 0;
|
|
369
|
+
state.sessionIdle = false;
|
|
370
|
+
}
|
|
371
|
+
if (options.cwd !== undefined) state.sessionCwd = options.cwd ?? null;
|
|
372
|
+
else if (!wasSameTranscript) state.sessionCwd = null;
|
|
347
373
|
});
|
|
348
374
|
}
|
|
349
375
|
}
|
|
@@ -421,7 +447,8 @@ async function rebindTranscriptContext(channelId, options = {}) {
|
|
|
421
447
|
applyTranscriptBinding(channelId, explicitTranscriptPath, {
|
|
422
448
|
replayFromStart: Boolean(options.catchUp),
|
|
423
449
|
catchUpFromPersisted: options.catchUpFromPersisted,
|
|
424
|
-
persistStatus: options.persistStatus
|
|
450
|
+
persistStatus: options.persistStatus,
|
|
451
|
+
recoverUnsyncedTail: options.recoverUnsyncedTail,
|
|
425
452
|
});
|
|
426
453
|
if (options.catchUp || options.catchUpFromPersisted) {
|
|
427
454
|
await forwarder.forwardNewText();
|
|
@@ -452,6 +479,7 @@ async function rebindTranscriptContext(channelId, options = {}) {
|
|
|
452
479
|
applyTranscriptBinding(channelId, bound.transcriptPath, {
|
|
453
480
|
replayFromStart,
|
|
454
481
|
catchUpFromPersisted: options.catchUpFromPersisted,
|
|
482
|
+
recoverUnsyncedTail: options.recoverUnsyncedTail,
|
|
455
483
|
persistStatus: options.persistStatus,
|
|
456
484
|
cwd: bound.sessionCwd,
|
|
457
485
|
});
|
|
@@ -497,6 +525,7 @@ async function rebindTranscriptContext(channelId, options = {}) {
|
|
|
497
525
|
applyTranscriptBinding(channelId, pendingTranscriptPath, {
|
|
498
526
|
replayFromStart: !samePersisted,
|
|
499
527
|
catchUpFromPersisted: samePersisted,
|
|
528
|
+
recoverUnsyncedTail: options.recoverUnsyncedTail,
|
|
500
529
|
cwd: pendingTranscriptCwd,
|
|
501
530
|
persistStatus: options.persistStatus,
|
|
502
531
|
});
|
|
@@ -512,7 +541,11 @@ async function rebindTranscriptContext(channelId, options = {}) {
|
|
|
512
541
|
}
|
|
513
542
|
}
|
|
514
543
|
if (previousPath && options.fallbackPrevious !== false) {
|
|
515
|
-
applyTranscriptBinding(channelId, previousPath, {
|
|
544
|
+
applyTranscriptBinding(channelId, previousPath, {
|
|
545
|
+
catchUpFromPersisted: true,
|
|
546
|
+
recoverUnsyncedTail: options.recoverUnsyncedTail,
|
|
547
|
+
cwd: statusState.read().sessionCwd
|
|
548
|
+
});
|
|
516
549
|
if (fs.existsSync(previousPath)) {
|
|
517
550
|
await forwarder.forwardNewText();
|
|
518
551
|
} else {
|
|
@@ -631,7 +664,8 @@ async function bindPersistedTranscriptIfAny() {
|
|
|
631
664
|
const bound = await rebindTranscriptContext(currentStatus.channelId, {
|
|
632
665
|
previousPath: getPersistedTranscriptPath(),
|
|
633
666
|
persistStatus: true,
|
|
634
|
-
catchUpFromPersisted: true
|
|
667
|
+
catchUpFromPersisted: true,
|
|
668
|
+
recoverUnsyncedTail: true,
|
|
635
669
|
});
|
|
636
670
|
if (bound) {
|
|
637
671
|
process.stderr.write(`mixdog: initial transcript bind: ${bound}
|
|
@@ -710,8 +744,40 @@ function syncOwnedWebhookAndEventRuntime({ reload = false } = {}) {
|
|
|
710
744
|
webhookServer = null;
|
|
711
745
|
}
|
|
712
746
|
}
|
|
747
|
+
let _ownedRuntimeSelfHealing = false;
|
|
748
|
+
// Explicit restore path for an already-connected owner. The 3s ownership tick
|
|
749
|
+
// must not run this implicitly: re-binding status on every tick can move the
|
|
750
|
+
// transcript cursor to EOF, and probing the gateway during Discord's own
|
|
751
|
+
// reconnect window can reset a healthy reconnect loop. Keep this for explicit
|
|
752
|
+
// activation/reload recovery only.
|
|
753
|
+
async function selfHealOwnedRuntime(options = {}) {
|
|
754
|
+
const shouldRestoreBinding = options.restoreBinding === true;
|
|
755
|
+
const shouldResetBackend = options.resetBackend === true;
|
|
756
|
+
if (!shouldRestoreBinding && !shouldResetBackend) return;
|
|
757
|
+
if (_ownedRuntimeSelfHealing) return;
|
|
758
|
+
_ownedRuntimeSelfHealing = true;
|
|
759
|
+
try {
|
|
760
|
+
if (shouldRestoreBinding) {
|
|
761
|
+
await bindPersistedTranscriptIfAny().catch((e) => {
|
|
762
|
+
process.stderr.write(`mixdog: self-heal bindPersistedTranscriptIfAny failed (non-fatal): ${e instanceof Error ? e.message : String(e)}\n`);
|
|
763
|
+
});
|
|
764
|
+
}
|
|
765
|
+
if (shouldResetBackend && typeof backend?._resetClient === "function") {
|
|
766
|
+
await backend._resetClient().catch((e) => {
|
|
767
|
+
process.stderr.write(`mixdog: self-heal backend reset failed (non-fatal): ${e instanceof Error ? e.message : String(e)}\n`);
|
|
768
|
+
});
|
|
769
|
+
}
|
|
770
|
+
// Nudge the forwarder to drain anything the rebind/reconnect surfaced.
|
|
771
|
+
void forwarder.forwardNewText().catch(() => {});
|
|
772
|
+
} finally {
|
|
773
|
+
_ownedRuntimeSelfHealing = false;
|
|
774
|
+
}
|
|
775
|
+
}
|
|
713
776
|
async function startOwnedRuntime(options = {}) {
|
|
714
|
-
if (bridgeRuntimeConnected)
|
|
777
|
+
if (bridgeRuntimeConnected) {
|
|
778
|
+
if (!bridgeRuntimeStarting) await selfHealOwnedRuntime(options);
|
|
779
|
+
return;
|
|
780
|
+
}
|
|
715
781
|
if (bridgeRuntimeStarting) return;
|
|
716
782
|
if (!channelBridgeActive) return;
|
|
717
783
|
bridgeRuntimeStarting = true;
|
|
@@ -928,8 +994,13 @@ async function refreshBridgeOwnership(options = {}) {
|
|
|
928
994
|
if (active && active.instanceId && active.instanceId !== INSTANCE_ID) {
|
|
929
995
|
if (bridgeRuntimeConnected) {
|
|
930
996
|
await stopOwnedRuntime("ownership lost (newer remote session)");
|
|
931
|
-
notifyRemoteSuperseded();
|
|
932
997
|
}
|
|
998
|
+
// Notify the parent UNCONDITIONALLY (not only when we were connected):
|
|
999
|
+
// a loser whose backend never connected (seat stolen mid-connect,
|
|
1000
|
+
// connect failed, or already in standby) must still drop its Remote
|
|
1001
|
+
// indicator. The parent handler is idempotent — it only acts while
|
|
1002
|
+
// remoteEnabled is still true — so repeat ticks are harmless.
|
|
1003
|
+
notifyRemoteSuperseded();
|
|
933
1004
|
return;
|
|
934
1005
|
}
|
|
935
1006
|
// No live owner — claim the empty seat and start.
|
|
@@ -1658,6 +1729,26 @@ backend.onMessage = (msg) => {
|
|
|
1658
1729
|
});
|
|
1659
1730
|
})();
|
|
1660
1731
|
};
|
|
1732
|
+
const NETWORK_ERR_RE = /fetch failed|ECONNRESET|ETIMEDOUT|ENOTFOUND|EAI_AGAIN|socket hang up|network|timeout|aborted|TimeoutError/i;
|
|
1733
|
+
function isNetworkError(err) {
|
|
1734
|
+
const s = `${err?.code ?? ""} ${err?.name ?? ""} ${err?.message ?? err ?? ""}`;
|
|
1735
|
+
return NETWORK_ERR_RE.test(s);
|
|
1736
|
+
}
|
|
1737
|
+
async function retryOnNetwork(fn, { attempts = 3, baseDelayMs = 300, label = "op" } = {}) {
|
|
1738
|
+
let lastErr;
|
|
1739
|
+
for (let i = 0; i < attempts; i++) {
|
|
1740
|
+
try {
|
|
1741
|
+
return await fn();
|
|
1742
|
+
} catch (err) {
|
|
1743
|
+
lastErr = err;
|
|
1744
|
+
if (i === attempts - 1 || !isNetworkError(err)) throw err;
|
|
1745
|
+
const delay = baseDelayMs * (i + 1);
|
|
1746
|
+
process.stderr.write(`mixdog: ${label} network failure (attempt ${i + 1}/${attempts}), retrying in ${delay}ms: ${err?.message || err}\n`);
|
|
1747
|
+
await new Promise((r) => setTimeout(r, delay));
|
|
1748
|
+
}
|
|
1749
|
+
}
|
|
1750
|
+
throw lastErr;
|
|
1751
|
+
}
|
|
1661
1752
|
async function handleInbound(msg, route, options = {}) {
|
|
1662
1753
|
const handleStartMs = Date.now();
|
|
1663
1754
|
let text = msg.text;
|
|
@@ -1668,11 +1759,19 @@ async function handleInbound(msg, route, options = {}) {
|
|
|
1668
1759
|
text = text || "[voice message]";
|
|
1669
1760
|
} else {
|
|
1670
1761
|
try {
|
|
1671
|
-
const files = await
|
|
1762
|
+
const files = await retryOnNetwork(
|
|
1763
|
+
// Short per-attempt timeout (vs the 180s default) bounds worst-case
|
|
1764
|
+
// blockage of the serial inboundQueue on a bad voice message.
|
|
1765
|
+
() => backend.downloadAttachment(msg.chatId, msg.messageId, { timeoutMs: 20_000 }),
|
|
1766
|
+
{ label: "voice.download" }
|
|
1767
|
+
);
|
|
1672
1768
|
// concurrency handled inside transcribeVoice queue; loop is sequential so last att wins
|
|
1673
1769
|
for (const f of voiceAtts.map(a => files.find(df => df.id === a.id) ?? null).filter(Boolean)) {
|
|
1674
1770
|
const _t0 = Date.now();
|
|
1675
|
-
const transcript = await
|
|
1771
|
+
const transcript = await retryOnNetwork(
|
|
1772
|
+
() => transcribeVoice(f.path, { attachmentId: f.id }),
|
|
1773
|
+
{ label: "voice.transcription" }
|
|
1774
|
+
);
|
|
1676
1775
|
const _elapsed = Date.now() - _t0;
|
|
1677
1776
|
if (transcript) {
|
|
1678
1777
|
text = transcript;
|
|
@@ -1683,8 +1782,12 @@ async function handleInbound(msg, route, options = {}) {
|
|
|
1683
1782
|
}
|
|
1684
1783
|
}
|
|
1685
1784
|
} catch (err) {
|
|
1686
|
-
|
|
1687
|
-
|
|
1785
|
+
const netFail = isNetworkError(err);
|
|
1786
|
+
process.stderr.write(`mixdog: voice.transcription error${netFail ? " (network, retries exhausted)" : ""}: ${err}\n`);
|
|
1787
|
+
const marker = netFail
|
|
1788
|
+
? "[attachment: voice transcription failed (network)]"
|
|
1789
|
+
: `[voice message \u2014 transcription error: ${err?.message || err}]`;
|
|
1790
|
+
text = text ? `${text} ${marker}` : marker;
|
|
1688
1791
|
}
|
|
1689
1792
|
}
|
|
1690
1793
|
}
|
|
@@ -1953,15 +2056,39 @@ if (_isWorkerMode && process.send) {
|
|
|
1953
2056
|
})
|
|
1954
2057
|
void (async () => {
|
|
1955
2058
|
const startedAt = performance.now()
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
2059
|
+
const MAX_START_ATTEMPTS = 3
|
|
2060
|
+
const BASE_BACKOFF_MS = 250
|
|
2061
|
+
const isTransientStartErr = (err) =>
|
|
2062
|
+
err?.code === 'ELOCKTIMEOUT' || /atomic lock timeout/i.test(err?.message || '')
|
|
2063
|
+
let lastErr
|
|
2064
|
+
for (let attempt = 1; attempt <= MAX_START_ATTEMPTS; attempt++) {
|
|
2065
|
+
if (_channelsStopInFlight) return
|
|
2066
|
+
try {
|
|
2067
|
+
await start()
|
|
2068
|
+
bootProfile("worker:ready", { ms: (performance.now() - startedAt).toFixed(1), attempt })
|
|
2069
|
+
process.send({ type: 'ready' })
|
|
2070
|
+
return
|
|
2071
|
+
} catch (e) {
|
|
2072
|
+
lastErr = e
|
|
2073
|
+
const transient = isTransientStartErr(e)
|
|
2074
|
+
bootProfile("worker:start-failed", { attempt, transient, error: e?.message || String(e) })
|
|
2075
|
+
process.stderr.write(`[channels-worker] start() failed (attempt ${attempt}/${MAX_START_ATTEMPTS}, transient=${transient}): ${e && (e.message || e)}\n`)
|
|
2076
|
+
if (!transient || attempt >= MAX_START_ATTEMPTS) break
|
|
2077
|
+
const backoff = BASE_BACKOFF_MS * attempt + Math.floor(Math.random() * BASE_BACKOFF_MS)
|
|
2078
|
+
await new Promise((r) => setTimeout(r, backoff))
|
|
2079
|
+
if (_channelsStopInFlight) return
|
|
2080
|
+
}
|
|
1964
2081
|
}
|
|
2082
|
+
// A stop landed while we were failing — let clean shutdown proceed, never exit over it.
|
|
2083
|
+
if (_channelsStopInFlight) return
|
|
2084
|
+
// Terminal failure: do NOT mask as a (degraded) ready. Exit non-zero so the
|
|
2085
|
+
// parent's exit-before-ready path respawns or rejects startRemote instead of
|
|
2086
|
+
// silently losing remote output forwarding.
|
|
2087
|
+
bootProfile("worker:failed", { ms: (performance.now() - startedAt).toFixed(1), error: lastErr?.message || String(lastErr) })
|
|
2088
|
+
process.stderr.write(`[channels-worker] start() giving up after ${MAX_START_ATTEMPTS} attempts: ${lastErr && (lastErr.message || lastErr)}\n`)
|
|
2089
|
+
// Exit 2 = terminal (non-transient) start failure: parent must reject, not respawn.
|
|
2090
|
+
// Exit 1 = exhausted transient retries: parent may respawn.
|
|
2091
|
+
process.exit(isTransientStartErr(lastErr) ? 1 : 2)
|
|
1965
2092
|
})()
|
|
1966
2093
|
}
|
|
1967
2094
|
|
|
@@ -86,15 +86,33 @@ ${err instanceof Error ? err.stack : ""}
|
|
|
86
86
|
// already retried elsewhere (atomic-file.mjs RETRY_CODES) — a single
|
|
87
87
|
// occurrence must NOT be fatal, only a run of 3+ in a row without an
|
|
88
88
|
// intervening distinct/successful event.
|
|
89
|
-
const BENIGN_CRASH_CODES = new Set([
|
|
89
|
+
const BENIGN_CRASH_CODES = new Set([
|
|
90
|
+
"EPERM", "EACCES", "EBUSY",
|
|
91
|
+
// Transient transport blips: a raw http/https ClientRequest (or an
|
|
92
|
+
// undici socket) can emit these when a keep-alive connection is torn
|
|
93
|
+
// down mid-flight (Discord gateway flapping, proxy resets). One blip
|
|
94
|
+
// must NOT kill the worker — it is benign; only a repeat storm within
|
|
95
|
+
// the streak window force-exits (see _fatalCrash).
|
|
96
|
+
"ECONNRESET", "ETIMEDOUT", "ECONNREFUSED", "EPIPE", "EHOSTUNREACH",
|
|
97
|
+
"ENETUNREACH", "ENETDOWN", "EAI_AGAIN", "ERR_STREAM_PREMATURE_CLOSE",
|
|
98
|
+
"UND_ERR_SOCKET", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_HEADERS_TIMEOUT",
|
|
99
|
+
"UND_ERR_BODY_TIMEOUT", "UND_ERR_ABORTED",
|
|
100
|
+
]);
|
|
101
|
+
// Some transient socket errors arrive with no `.code` (notably the classic
|
|
102
|
+
// "socket hang up" thrown by node:_http_client socketOnEnd — the exact crash
|
|
103
|
+
// this guard exists for). Match those by message so they classify benign too.
|
|
104
|
+
const BENIGN_CRASH_MESSAGE_RE = /socket hang up|ECONNRESET|ETIMEDOUT|EPIPE|UND_ERR_/i;
|
|
90
105
|
const BENIGN_CRASH_FATAL_THRESHOLD = 3;
|
|
91
106
|
// "In a row" needs a time dimension: benign errors minutes/hours apart are
|
|
92
107
|
// independent contention events, not a corrupted-state run. Only count a
|
|
93
108
|
// streak when hits land within this window of the previous one.
|
|
94
109
|
const BENIGN_CRASH_STREAK_WINDOW_MS = 60_000;
|
|
95
110
|
function _isBenignCrash(err) {
|
|
96
|
-
const
|
|
97
|
-
|
|
111
|
+
const msg = String(err?.message || err);
|
|
112
|
+
const code = err?.code
|
|
113
|
+
|| (/\b(EPERM|EACCES|EBUSY|ECONNRESET|ETIMEDOUT|ECONNREFUSED|EPIPE|EHOSTUNREACH|ENETUNREACH|ENETDOWN|EAI_AGAIN|UND_ERR_[A-Z_]+)\b/.exec(msg) || [])[0];
|
|
114
|
+
if (BENIGN_CRASH_CODES.has(code)) return true;
|
|
115
|
+
return BENIGN_CRASH_MESSAGE_RE.test(msg);
|
|
98
116
|
}
|
|
99
117
|
|
|
100
118
|
export {
|
|
@@ -28,8 +28,24 @@ import {
|
|
|
28
28
|
buildToolLine
|
|
29
29
|
} from "./tool-format.mjs";
|
|
30
30
|
|
|
31
|
+
// Per-item send watchdog. drainQueue awaits ONE backend send at a time; a hung
|
|
32
|
+
// send would otherwise never settle, so drainQueue never returns and `sending`
|
|
33
|
+
// stays true forever — wedging ALL outbound behind the queue head. The watchdog
|
|
34
|
+
// does NOT abandon the send (racing a still-running send against a retry would
|
|
35
|
+
// double-deliver): it only pokes the backend to reset its wedged client so the
|
|
36
|
+
// AWAITED send fails fast and `sending` releases on real settlement. Sized above
|
|
37
|
+
// the backend's worst-case per-attempt retries.
|
|
38
|
+
const SEND_ITEM_TIMEOUT_MS = 120_000;
|
|
39
|
+
// After the watchdog resets the backend, wait at most this long for the
|
|
40
|
+
// abandoned send to actually settle before releasing the queue.
|
|
41
|
+
const SEND_SETTLE_CAP_MS = 15_000;
|
|
42
|
+
const TAIL_RECOVERY_BYTES = 256 * 1024;
|
|
43
|
+
|
|
31
44
|
class OutputForwarder {
|
|
32
45
|
ownerGetter = null;
|
|
46
|
+
// Generation of the in-flight supervised send. Bumped when a send is
|
|
47
|
+
// abandoned (settlement cap) so a late/zombie completion can't double-advance.
|
|
48
|
+
_activeSendGen = 0;
|
|
33
49
|
setOwnerGetter(fn) {
|
|
34
50
|
this.ownerGetter = fn;
|
|
35
51
|
}
|
|
@@ -111,20 +127,35 @@ class OutputForwarder {
|
|
|
111
127
|
try {
|
|
112
128
|
const stat = existsSync(this.transcriptPath) ? statSync(this.transcriptPath) : null;
|
|
113
129
|
const currentSize = stat?.size ?? 0;
|
|
130
|
+
const persisted = this.statusState?.read?.();
|
|
131
|
+
const sameTranscript = persisted?.transcriptPath &&
|
|
132
|
+
sameResolvedPath(persisted.transcriptPath, this.transcriptPath);
|
|
114
133
|
let fileSize;
|
|
115
134
|
if (options.replayFromStart) {
|
|
116
135
|
fileSize = 0;
|
|
117
136
|
} else if (options.catchUpFromPersisted) {
|
|
118
|
-
const persisted = this.statusState?.read?.();
|
|
119
137
|
const persistedSize = typeof persisted?.lastFileSize === "number" ? persisted.lastFileSize : -1;
|
|
120
|
-
const sameTranscript = persisted?.transcriptPath &&
|
|
121
|
-
sameResolvedPath(persisted.transcriptPath, this.transcriptPath);
|
|
122
138
|
fileSize = (sameTranscript && persistedSize >= 0)
|
|
123
139
|
? Math.min(Math.max(persistedSize, 0), currentSize)
|
|
124
140
|
: currentSize;
|
|
125
141
|
} else {
|
|
126
142
|
fileSize = currentSize;
|
|
127
143
|
}
|
|
144
|
+
if (options.recoverUnsyncedTail && fileSize === currentSize && currentSize > 0) {
|
|
145
|
+
const sentCount = Number(persisted?.sentCount ?? 0);
|
|
146
|
+
const lastSentTime = Number(persisted?.lastSentTime ?? 0);
|
|
147
|
+
const hasSentHash = typeof persisted?.lastSentHash === "string" && persisted.lastSentHash.length > 0;
|
|
148
|
+
const noSendEvidence = (!Number.isFinite(sentCount) || sentCount <= 0)
|
|
149
|
+
&& (!Number.isFinite(lastSentTime) || lastSentTime <= 0)
|
|
150
|
+
&& !hasSentHash;
|
|
151
|
+
if (!sameTranscript || noSendEvidence) {
|
|
152
|
+
const tailOffset = this.findLastAssistantTextOffset(currentSize);
|
|
153
|
+
if (tailOffset >= 0 && tailOffset < currentSize) {
|
|
154
|
+
fileSize = tailOffset;
|
|
155
|
+
dropTrace("context.recoverUnsyncedTail", { path: this.transcriptPath, offset: tailOffset, currentSize, sameTranscript: !!sameTranscript });
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
128
159
|
this.lastFileSize = fileSize;
|
|
129
160
|
this.readFileSize = fileSize;
|
|
130
161
|
} catch {
|
|
@@ -132,6 +163,46 @@ class OutputForwarder {
|
|
|
132
163
|
this.readFileSize = 0;
|
|
133
164
|
}
|
|
134
165
|
}
|
|
166
|
+
findLastAssistantTextOffset(fileSize) {
|
|
167
|
+
if (!this.transcriptPath || fileSize <= 0) return -1;
|
|
168
|
+
const bytesToRead = Math.min(fileSize, TAIL_RECOVERY_BYTES);
|
|
169
|
+
const startOffset = fileSize - bytesToRead;
|
|
170
|
+
let fd = null;
|
|
171
|
+
try {
|
|
172
|
+
fd = openSync(this.transcriptPath, "r");
|
|
173
|
+
const buf = Buffer.alloc(bytesToRead);
|
|
174
|
+
readSync(fd, buf, 0, bytesToRead, startOffset);
|
|
175
|
+
const raw = buf.toString("utf8");
|
|
176
|
+
let chunkStart = 0;
|
|
177
|
+
if (startOffset > 0) {
|
|
178
|
+
const firstNewline = raw.indexOf("\n");
|
|
179
|
+
if (firstNewline < 0) return -1;
|
|
180
|
+
chunkStart = firstNewline + 1;
|
|
181
|
+
}
|
|
182
|
+
let cursor = startOffset + chunkStart;
|
|
183
|
+
let lastOffset = -1;
|
|
184
|
+
for (const line of raw.slice(chunkStart).split("\n")) {
|
|
185
|
+
const lineOffset = cursor;
|
|
186
|
+
cursor += Buffer.byteLength(line, "utf8") + 1;
|
|
187
|
+
if (!line.trim()) continue;
|
|
188
|
+
try {
|
|
189
|
+
const entry = JSON.parse(line);
|
|
190
|
+
if (entry.isSidechain || entry.type !== "assistant") continue;
|
|
191
|
+
const parts = entry.message?.content;
|
|
192
|
+
if (!Array.isArray(parts)) continue;
|
|
193
|
+
if (parts.some((c) => c?.type === "text" && typeof c.text === "string" && c.text.trim())) {
|
|
194
|
+
lastOffset = lineOffset;
|
|
195
|
+
}
|
|
196
|
+
} catch {
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
return lastOffset;
|
|
200
|
+
} catch {
|
|
201
|
+
return -1;
|
|
202
|
+
} finally {
|
|
203
|
+
if (fd != null) closeSync(fd);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
135
206
|
/** Reset counters for new turn */
|
|
136
207
|
reset() {
|
|
137
208
|
this.sentCount = 0;
|
|
@@ -282,6 +353,7 @@ class OutputForwarder {
|
|
|
282
353
|
}
|
|
283
354
|
async deliverQueueItem(item) {
|
|
284
355
|
const targetChannelId = item.channelId ?? this.channelId;
|
|
356
|
+
const gen = ++this._activeSendGen;
|
|
285
357
|
if (!item.text || !targetChannelId) {
|
|
286
358
|
this.commitReadProgress(item.nextFileSize);
|
|
287
359
|
return;
|
|
@@ -314,10 +386,13 @@ class OutputForwarder {
|
|
|
314
386
|
// re-sending chunks that already landed. The token is opaque here —
|
|
315
387
|
// the forwarder only stores/passes/clears it, never interprets it.
|
|
316
388
|
await this.cb.send(targetChannelId, formatted, item._resumeToken ? { resumeToken: item._resumeToken } : undefined);
|
|
389
|
+
dropTrace("discord.send.ok", null);
|
|
390
|
+
// A zombie completion (send abandoned past the settlement cap, gen bumped)
|
|
391
|
+
// must not mutate ANY item/queue state — including the resume token.
|
|
392
|
+
if (gen !== this._activeSendGen) return;
|
|
317
393
|
// Full success — drop any stale token so a later reuse of this item
|
|
318
394
|
// object can't carry a dead token.
|
|
319
395
|
item._resumeToken = undefined;
|
|
320
|
-
dropTrace("discord.send.ok", null);
|
|
321
396
|
} catch (err) {
|
|
322
397
|
// Persist the opaque resume token (if any) on the item so the requeued
|
|
323
398
|
// retry resumes from the failed chunk.
|
|
@@ -349,7 +424,18 @@ ${_bt.trim()}` : _bt.trim();
|
|
|
349
424
|
}
|
|
350
425
|
/** Forward new assistant text to Discord. Returns true if text was sent. */
|
|
351
426
|
async forwardNewText() {
|
|
352
|
-
if (!this._isOwner())
|
|
427
|
+
if (!this._isOwner()) {
|
|
428
|
+
// Silent no-send is invisible and has masked real ownership-gate bugs
|
|
429
|
+
// (a live seat that never forwards). Surface it, rate-limited to one
|
|
430
|
+
// line per 60s so a persistent non-owner state stays visible without
|
|
431
|
+
// flooding stderr on every poll tick.
|
|
432
|
+
const now = Date.now();
|
|
433
|
+
if (!this._lastOwnerGateLogAt || now - this._lastOwnerGateLogAt >= 60_000) {
|
|
434
|
+
this._lastOwnerGateLogAt = now;
|
|
435
|
+
process.stderr.write(`[output-forwarder] forwardNewText skipped: not seat owner (channelId=${this.channelId ?? "none"})\n`);
|
|
436
|
+
}
|
|
437
|
+
return false;
|
|
438
|
+
}
|
|
353
439
|
if (!this.channelId) return false;
|
|
354
440
|
const { text: newText, nextFileSize } = this.extractNewText();
|
|
355
441
|
if (!newText) {
|
|
@@ -428,6 +514,31 @@ ${_bt.trim()}` : _bt.trim();
|
|
|
428
514
|
_kickDrain() {
|
|
429
515
|
this.drainQueue().catch(err => process.stderr.write(`[output-forwarder] drainQueue rejected: ${err?.message || err}\n`));
|
|
430
516
|
}
|
|
517
|
+
/** Supervise a per-item send. The underlying promise is AWAITED to real
|
|
518
|
+
* settlement so `sending` is never released early (no unsupervised send
|
|
519
|
+
* racing a retry → duplicate). The watchdog's only job is to reset the
|
|
520
|
+
* backend's wedged client so the awaited send fails fast. */
|
|
521
|
+
_sendSupervised(promise, label) {
|
|
522
|
+
const settled = Promise.resolve(promise);
|
|
523
|
+
settled.catch(() => {}); // never leak an unhandled rejection on late settle
|
|
524
|
+
return new Promise((resolve, reject) => {
|
|
525
|
+
let capTimer = null;
|
|
526
|
+
const watchdog = setTimeout(() => {
|
|
527
|
+
process.stderr.write(`[output-forwarder] ${label} watchdog fired after ${SEND_ITEM_TIMEOUT_MS}ms; resetting backend\n`);
|
|
528
|
+
Promise.resolve(this.cb?.resetBackend?.()).catch(() => {});
|
|
529
|
+
// Bound the post-reset settlement wait; past the cap, invalidate the
|
|
530
|
+
// in-flight send (gen bump → zombie can't double-advance) and release.
|
|
531
|
+
capTimer = setTimeout(() => {
|
|
532
|
+
this._activeSendGen++;
|
|
533
|
+
reject(new Error(`${label} abandoned after ${SEND_SETTLE_CAP_MS}ms settlement cap`));
|
|
534
|
+
}, SEND_SETTLE_CAP_MS);
|
|
535
|
+
}, SEND_ITEM_TIMEOUT_MS);
|
|
536
|
+
settled.then(
|
|
537
|
+
(v) => { clearTimeout(watchdog); if (capTimer) clearTimeout(capTimer); resolve(v); },
|
|
538
|
+
(e) => { clearTimeout(watchdog); if (capTimer) clearTimeout(capTimer); reject(e); }
|
|
539
|
+
);
|
|
540
|
+
});
|
|
541
|
+
}
|
|
431
542
|
/** Drain both priority lanes sequentially. finalLane drains first when non-empty. */
|
|
432
543
|
async drainQueue() {
|
|
433
544
|
if (this.sending) return;
|
|
@@ -440,9 +551,9 @@ ${_bt.trim()}` : _bt.trim();
|
|
|
440
551
|
const item = lane[0];
|
|
441
552
|
try {
|
|
442
553
|
if (item.type === "text") {
|
|
443
|
-
await this.deliverQueueItem(item);
|
|
554
|
+
await this._sendSupervised(this.deliverQueueItem(item), "deliverQueueItem");
|
|
444
555
|
} else if (item.type === "toolLog") {
|
|
445
|
-
await this.processToolLog(item);
|
|
556
|
+
await this._sendSupervised(this.processToolLog(item), "processToolLog");
|
|
446
557
|
}
|
|
447
558
|
lane.shift();
|
|
448
559
|
} catch (err) {
|
|
@@ -360,6 +360,19 @@ function refreshActiveInstance(instanceId, meta, options) {
|
|
|
360
360
|
}
|
|
361
361
|
}
|
|
362
362
|
}
|
|
363
|
+
// Worker-owned seat: when THIS process is the channels worker and it is
|
|
364
|
+
// the seat owner (no distinct TUI/supervisor Lead — ownerLeadPid resolves
|
|
365
|
+
// to our own pid), keep ui_heartbeat_at fresh. A headless worker never
|
|
366
|
+
// runs the TUI render loop, so an inherited heartbeat (written by a prior
|
|
367
|
+
// TUI session and carried forward in prevRest) would otherwise go stale at
|
|
368
|
+
// UI_HEARTBEAT_STALE_MS and falsely flag a live worker-owned seat. A
|
|
369
|
+
// TUI-owned seat has a distinct supervisor (ownerLeadPid !== our pid), so
|
|
370
|
+
// this branch is a no-op there and TUI staleness is preserved.
|
|
371
|
+
const workerOwnsSeat = process.env.MIXDOG_WORKER_MODE === "1"
|
|
372
|
+
&& identity.ownerLeadPid === process.pid;
|
|
373
|
+
if (workerOwnsSeat) {
|
|
374
|
+
next.ui_heartbeat_at = Date.now();
|
|
375
|
+
}
|
|
363
376
|
return { ...preservedExtra, ...next };
|
|
364
377
|
}, { compact: true, fsync: false, fsyncDir: false, renameFallback: 'truncate' });
|
|
365
378
|
}
|
|
@@ -167,9 +167,11 @@ function createVoiceTranscription({ getConfig, dataDir }) {
|
|
|
167
167
|
if (attachmentId && result) _voiceTranscriptCache.set(attachmentId, result);
|
|
168
168
|
return result;
|
|
169
169
|
} catch (err) {
|
|
170
|
-
|
|
170
|
+
// Propagate ALL real failures so the caller's retry (network-class) and
|
|
171
|
+
// failure marker (index.mjs catch) fire; null is reserved strictly for a
|
|
172
|
+
// legit empty transcript.
|
|
171
173
|
process.stderr.write(`mixdog: voice.transcription failed: ${err}\n`);
|
|
172
|
-
|
|
174
|
+
throw err;
|
|
173
175
|
}
|
|
174
176
|
}
|
|
175
177
|
|
|
@@ -75,6 +75,8 @@ export function createStandaloneChannelWorker({
|
|
|
75
75
|
let readyResolve = null;
|
|
76
76
|
let readyReject = null;
|
|
77
77
|
let stopPromise = null;
|
|
78
|
+
let stopRequested = false;
|
|
79
|
+
let bootGeneration = 0;
|
|
78
80
|
let inProcessMod = null;
|
|
79
81
|
let inProcessStartPromise = null;
|
|
80
82
|
let nextCallId = 1;
|
|
@@ -153,6 +155,7 @@ export function createStandaloneChannelWorker({
|
|
|
153
155
|
return stopPromise.then(() => start());
|
|
154
156
|
}
|
|
155
157
|
if (child && child.exitCode == null && !child.killed) return readyPromise || Promise.resolve(status());
|
|
158
|
+
stopRequested = false;
|
|
156
159
|
readyPromise = new Promise((resolve, reject) => {
|
|
157
160
|
readyResolve = resolve;
|
|
158
161
|
readyReject = reject;
|
|
@@ -181,6 +184,9 @@ export function createStandaloneChannelWorker({
|
|
|
181
184
|
windowsHide: true,
|
|
182
185
|
});
|
|
183
186
|
const spawnedPid = child.pid;
|
|
187
|
+
// Per-boot generation: a later start()/respawn bumps this, so a stale old
|
|
188
|
+
// child's late exit is ignored instead of respawning/rejecting the current one.
|
|
189
|
+
const myGeneration = ++bootGeneration;
|
|
184
190
|
startChildGuardian({ childPid: spawnedPid, label: 'channel-worker', orphanGraceMs: 8000, forceGraceMs: 3000 });
|
|
185
191
|
if (spawnedPid) ownedChildPids.add(spawnedPid);
|
|
186
192
|
installParentExitHook();
|
|
@@ -215,13 +221,17 @@ export function createStandaloneChannelWorker({
|
|
|
215
221
|
});
|
|
216
222
|
|
|
217
223
|
child.on('exit', (code, signal) => {
|
|
224
|
+
if (myGeneration !== bootGeneration) return;
|
|
218
225
|
if (spawnedPid) ownedChildPids.delete(spawnedPid);
|
|
219
226
|
const error = new Error(`channels runtime exited (${signal || (code ?? 'unknown')})`);
|
|
220
|
-
|
|
227
|
+
// Exit code 2 = terminal (non-transient) worker start failure: reject, never respawn.
|
|
228
|
+
// A requested stop (stopRequested) also suppresses respawn even if stopPromise's
|
|
229
|
+
// settle timer already cleared it before the child exited.
|
|
230
|
+
if (!becameReady && readyReject && !stopPromise && !stopRequested && code !== 2 && attempt < WORKER_BOOT_MAX_ATTEMPTS) {
|
|
221
231
|
logLine(logPath, `worker exited before ready (${signal || (code ?? 'unknown')}), attempt ${attempt}/${WORKER_BOOT_MAX_ATTEMPTS}; retrying in ${WORKER_BOOT_RETRY_DELAY_MS}ms`);
|
|
222
232
|
child = null;
|
|
223
233
|
setTimeout(() => {
|
|
224
|
-
if (stopPromise || !readyReject) return;
|
|
234
|
+
if (stopPromise || stopRequested || !readyReject) return;
|
|
225
235
|
spawnWorkerChild(attempt + 1);
|
|
226
236
|
}, WORKER_BOOT_RETRY_DELAY_MS);
|
|
227
237
|
return;
|
|
@@ -235,6 +245,7 @@ export function createStandaloneChannelWorker({
|
|
|
235
245
|
});
|
|
236
246
|
|
|
237
247
|
child.on('error', (error) => {
|
|
248
|
+
if (myGeneration !== bootGeneration) return;
|
|
238
249
|
logLine(logPath, `runtime error: ${error?.message || error}`);
|
|
239
250
|
if (readyReject) readyReject(error);
|
|
240
251
|
readyResolve = null;
|
|
@@ -347,6 +358,7 @@ export function createStandaloneChannelWorker({
|
|
|
347
358
|
|
|
348
359
|
function stop(reason = 'standalone shutdown', options = {}) {
|
|
349
360
|
const waitForExit = options?.waitForExit !== false;
|
|
361
|
+
stopRequested = true;
|
|
350
362
|
stopClientHeartbeat();
|
|
351
363
|
if (stopPromise) return stopPromise;
|
|
352
364
|
if (!useProcessWorker) {
|
|
@@ -562,6 +562,17 @@ export function measuredTranscriptRows(item, columns, toolOutputExpanded) {
|
|
|
562
562
|
|
|
563
563
|
const STREAMING_ROW_QUANTUM = 1;
|
|
564
564
|
|
|
565
|
+
// Bottom-pinned flag for the streaming-tail row estimate. Set once per hook
|
|
566
|
+
// render (use-transcript-window.mjs) BEFORE any tail resolution so every path
|
|
567
|
+
// that resolves the streaming tail height (tailSig, buildTranscriptRowIndex,
|
|
568
|
+
// incremental tail, transcriptStructureSignature) reads the same value and
|
|
569
|
+
// cannot diverge. Bottom-pinned → fold the live estimate in (geometry right on
|
|
570
|
+
// the first commit); away from bottom → d19cad1e defer-growth.
|
|
571
|
+
let streamingBottomPinned = false;
|
|
572
|
+
export function setStreamingBottomPinned(pinned) {
|
|
573
|
+
streamingBottomPinned = !!pinned;
|
|
574
|
+
}
|
|
575
|
+
|
|
565
576
|
function assistantTextForStreamingRowEstimate(text) {
|
|
566
577
|
return streamingLayoutText(text);
|
|
567
578
|
}
|
|
@@ -580,6 +591,17 @@ export function estimateTranscriptItemRowsCached(item, columns, toolOutputExpand
|
|
|
580
591
|
const toolExpanded = toolOutputExpanded ? 1 : 0;
|
|
581
592
|
const idEntry = streamingMeasuredRowsById.get(item.id);
|
|
582
593
|
if (idEntry && idEntry.rows > 0 && idEntry.columns === columns && idEntry.toolExpanded === toolExpanded) {
|
|
594
|
+
if (streamingBottomPinned) {
|
|
595
|
+
// Bottom-pinned: the view follows the tail, so its geometry must be
|
|
596
|
+
// correct on the FIRST commit. Fold the freshly-grown live estimate in
|
|
597
|
+
// — max(measuredFloor, liveEstimate) — so totalRows/scrollOffset are
|
|
598
|
+
// right this frame instead of committing the stale measured floor and
|
|
599
|
+
// growing a frame later when the harvest bumps measuredRowsVersion
|
|
600
|
+
// (the judder). The measured floor still guards against an estimate
|
|
601
|
+
// that undercounts the real wrap. Away from bottom the defer-growth
|
|
602
|
+
// path below is kept (its anchor-stability / perf intent).
|
|
603
|
+
return Math.max(idEntry.rows, streamingEstimateRows(item, columns, toolOutputExpanded));
|
|
604
|
+
}
|
|
583
605
|
// Defer-growth: while a confirmed (post-commit Yoga) measurement exists
|
|
584
606
|
// for this streaming id, THIS render keeps that value instead of
|
|
585
607
|
// folding in the freshly-grown estimate. Combining
|