mixdog 0.9.20 → 0.9.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -12
- package/package.json +1 -1
- package/scripts/build-runtime-windows.ps1 +242 -242
- package/scripts/recall-usecase-cases.json +1 -1
- package/scripts/smoke-runtime-negative.ps1 +106 -106
- package/scripts/tool-efficiency-diag.mjs +1 -1
- package/src/defaults/skills/setup/SKILL.md +327 -0
- package/src/rules/lead/02-channels.md +3 -3
- package/src/runtime/agent/orchestrator/providers/anthropic.mjs +34 -2
- package/src/runtime/agent/orchestrator/providers/gemini.mjs +3 -0
- package/src/runtime/agent/orchestrator/providers/grok-oauth.mjs +67 -10
- package/src/runtime/agent/orchestrator/providers/openai-compat.mjs +3 -0
- package/src/runtime/agent/orchestrator/providers/openai-oauth-http-sse.mjs +40 -3
- package/src/runtime/agent/orchestrator/providers/openai-ws.mjs +73 -3
- package/src/runtime/channels/lib/owned-runtime.mjs +95 -15
- package/src/runtime/channels/lib/runtime-paths.mjs +20 -9
- package/src/runtime/channels/lib/worker-main.mjs +7 -1
- package/src/runtime/memory/index.mjs +90 -0
- package/src/runtime/memory/lib/http-router.mjs +39 -0
- package/src/runtime/memory/lib/memory-cycle2-gate.mjs +6 -0
- package/src/runtime/memory/lib/memory-cycle2-mutations.mjs +46 -9
- package/src/runtime/memory/lib/memory-cycle2.mjs +87 -11
- package/src/runtime/memory/lib/memory.mjs +39 -0
- package/src/runtime/shared/atomic-file.mjs +138 -80
- package/src/runtime/shared/child-guardian.mjs +61 -3
- package/src/session-runtime/runtime-core.mjs +75 -17
- package/src/standalone/channel-worker.mjs +63 -7
- package/src/standalone/folder-dialog.mjs +4 -1
- package/src/standalone/memory-runtime-proxy.mjs +98 -11
- package/src/standalone/seeds.mjs +28 -1
- package/src/tui/App.jsx +1 -0
- package/src/tui/app/doctor.mjs +175 -0
- package/src/tui/app/slash-commands.mjs +1 -0
- package/src/tui/app/slash-dispatch.mjs +9 -0
- package/src/tui/dist/index.mjs +297 -71
- package/src/tui/engine/session-api.mjs +19 -0
|
@@ -308,10 +308,7 @@ export function forgetGrokOAuthCredentials() {
|
|
|
308
308
|
}
|
|
309
309
|
|
|
310
310
|
let _refreshInFlight = null;
|
|
311
|
-
async function
|
|
312
|
-
if (!tokens?.refresh_token) {
|
|
313
|
-
throw new Error('[grok-oauth] refresh token not available — open /providers in mixdog to sign in again');
|
|
314
|
-
}
|
|
311
|
+
async function _postRefresh(tokens) {
|
|
315
312
|
const tokenEndpoint = tokens.token_endpoint
|
|
316
313
|
? assertTrustedXaiEndpoint(tokens.token_endpoint, 'token endpoint')
|
|
317
314
|
: (await fetchDiscovery()).token_endpoint;
|
|
@@ -365,6 +362,34 @@ async function refreshTokens(tokens) {
|
|
|
365
362
|
}
|
|
366
363
|
}
|
|
367
364
|
|
|
365
|
+
// 16 mixdog processes share one grok-oauth.json and xAI rotates refresh
|
|
366
|
+
// tokens single-use. Re-read the store immediately before the network POST so
|
|
367
|
+
// a peer's just-completed rotation is adopted instead of spending our
|
|
368
|
+
// (now-stale) refresh_token; on invalid_grant, re-read once and retry with a
|
|
369
|
+
// peer-rotated refresh_token before propagating.
|
|
370
|
+
async function refreshTokens(tokens, { force = false } = {}) {
|
|
371
|
+
if (!tokens?.refresh_token) {
|
|
372
|
+
throw new Error('[grok-oauth] refresh token not available — open /providers in mixdog to sign in again');
|
|
373
|
+
}
|
|
374
|
+
const disk = _loadOwnTokens();
|
|
375
|
+
const validAfter = Date.now() + (force ? 0 : TOKEN_REFRESH_SKEW_MS);
|
|
376
|
+
if (disk?.access_token && disk.access_token !== tokens.access_token
|
|
377
|
+
&& (!disk.expires_at || disk.expires_at >= validAfter)) {
|
|
378
|
+
return disk;
|
|
379
|
+
}
|
|
380
|
+
try {
|
|
381
|
+
return await _postRefresh(tokens);
|
|
382
|
+
} catch (err) {
|
|
383
|
+
if (err?.isInvalidGrant) {
|
|
384
|
+
const rotated = _loadOwnTokens();
|
|
385
|
+
if (rotated?.refresh_token && rotated.refresh_token !== tokens.refresh_token) {
|
|
386
|
+
return await _postRefresh(rotated);
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
throw err;
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
|
|
368
393
|
// --- Model catalog cache (24h disk TTL) ---
|
|
369
394
|
const _modelCache = makeModelCache({
|
|
370
395
|
fileName: 'grok-oauth-models.json',
|
|
@@ -530,6 +555,9 @@ export class GrokOAuthProvider {
|
|
|
530
555
|
tokens = null;
|
|
531
556
|
_inner = null;
|
|
532
557
|
_innerKey = null;
|
|
558
|
+
// Grace window after a non-force refresh failure: keep serving the current
|
|
559
|
+
// still-valid access_token instead of thrashing the shared refresh_token.
|
|
560
|
+
_refreshFallbackUntil = 0;
|
|
533
561
|
|
|
534
562
|
constructor(config) {
|
|
535
563
|
this.config = config || {};
|
|
@@ -553,15 +581,44 @@ export class GrokOAuthProvider {
|
|
|
553
581
|
if (disk?.access_token) this.tokens = disk;
|
|
554
582
|
this._lastDiskScan = ownM;
|
|
555
583
|
}
|
|
584
|
+
if (!forceRefresh && this._refreshFallbackUntil > Date.now() && this.tokens?.access_token
|
|
585
|
+
&& (!this.tokens.expires_at || this.tokens.expires_at > Date.now())) {
|
|
586
|
+
return this.tokens;
|
|
587
|
+
}
|
|
556
588
|
const expiring = this.tokens.expires_at
|
|
557
589
|
&& this.tokens.expires_at < Date.now() + TOKEN_REFRESH_SKEW_MS;
|
|
558
590
|
if (forceRefresh || expiring) {
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
.
|
|
564
|
-
|
|
591
|
+
const currentToken = this.tokens?.access_token || null;
|
|
592
|
+
try {
|
|
593
|
+
if (_refreshInFlight) {
|
|
594
|
+
const shared = await _refreshInFlight;
|
|
595
|
+
this.tokens = shared;
|
|
596
|
+
// A forced caller must not accept a shared non-force result
|
|
597
|
+
// that merely handed back its own prior token: start a fresh
|
|
598
|
+
// forced refresh instead (mirror openai-oauth).
|
|
599
|
+
if (forceRefresh && shared?.access_token === currentToken) {
|
|
600
|
+
if (!_refreshInFlight) {
|
|
601
|
+
_refreshInFlight = refreshTokens(this.tokens, { force: true })
|
|
602
|
+
.finally(() => { _refreshInFlight = null; });
|
|
603
|
+
}
|
|
604
|
+
this.tokens = await _refreshInFlight;
|
|
605
|
+
}
|
|
606
|
+
} else {
|
|
607
|
+
_refreshInFlight = refreshTokens(this.tokens, { force: forceRefresh })
|
|
608
|
+
.finally(() => { _refreshInFlight = null; });
|
|
609
|
+
this.tokens = await _refreshInFlight;
|
|
610
|
+
}
|
|
611
|
+
this._refreshFallbackUntil = 0;
|
|
612
|
+
} catch (err) {
|
|
613
|
+
// Non-force failure while the current token is still valid: serve
|
|
614
|
+
// it under a grace window rather than throwing (mirror openai-oauth).
|
|
615
|
+
if (!forceRefresh && currentToken
|
|
616
|
+
&& (!this.tokens?.expires_at || this.tokens.expires_at > Date.now())) {
|
|
617
|
+
this._refreshFallbackUntil = Date.now() + TOKEN_REFRESH_SKEW_MS;
|
|
618
|
+
process.stderr.write(`[grok-oauth] Refresh failed (${String(err?.message || err).slice(0, 120)}); using still-valid current token\n`);
|
|
619
|
+
return this.tokens;
|
|
620
|
+
}
|
|
621
|
+
throw err;
|
|
565
622
|
}
|
|
566
623
|
}
|
|
567
624
|
return this.tokens;
|
|
@@ -173,6 +173,9 @@ export class OpenAICompatProvider {
|
|
|
173
173
|
return await this._doSend(messages, model, tools, sendOpts);
|
|
174
174
|
} catch (err) {
|
|
175
175
|
if (err.message && (err.message.includes('401') || err.message.includes('403'))) {
|
|
176
|
+
if (err.liveTextEmitted === true || err.emittedToolCall === true || err.unsafeToRetry === true) {
|
|
177
|
+
throw err;
|
|
178
|
+
}
|
|
176
179
|
process.stderr.write(`[provider] Auth error, re-reading config...\n`);
|
|
177
180
|
this.reloadApiKey();
|
|
178
181
|
return await this._doSend(messages, model, tools, sendOpts);
|
|
@@ -27,6 +27,13 @@ import { createLeakGuard, createToolCallDedupe, dedupeToolCallList } from './ant
|
|
|
27
27
|
import { customToolCallFromResponseItem } from './custom-tool-wire.mjs';
|
|
28
28
|
import { CODEX_OAUTH_ORIGINATOR, CODEX_RESPONSES_URL, _displayCodexModel } from './openai-oauth.mjs';
|
|
29
29
|
|
|
30
|
+
// Public OpenAI Responses API endpoint for the api-key `openai` provider.
|
|
31
|
+
// The openai-direct WS transport hits the same origin (openai-ws-pool
|
|
32
|
+
// OPENAI_WS_URL = wss://api.openai.com/v1/responses); this HTTP/SSE fallback
|
|
33
|
+
// mirrors it so OpenAIDirectProvider can fall back off WebSocket like
|
|
34
|
+
// openai-oauth. Same Responses SSE wire format, only endpoint + auth differ.
|
|
35
|
+
const OPENAI_DIRECT_RESPONSES_URL = 'https://api.openai.com/v1/responses';
|
|
36
|
+
|
|
30
37
|
export function _envFlag(name, fallback = true) {
|
|
31
38
|
const raw = process.env[name];
|
|
32
39
|
if (raw == null || raw === '') return fallback;
|
|
@@ -116,6 +123,18 @@ function _pushOutputTextAnnotations(part, citations, citationKeys) {
|
|
|
116
123
|
}
|
|
117
124
|
|
|
118
125
|
function _buildOpenAIHttpFallbackHeaders({ auth, cacheKey }) {
|
|
126
|
+
if (auth?.type === 'openai-direct') {
|
|
127
|
+
// Public API-key auth: Bearer <OPENAI_API_KEY>, no chatgpt-account-id /
|
|
128
|
+
// originator (mirrors openai-ws-pool _buildHandshakeHeaders' direct
|
|
129
|
+
// branch). session_id anchors are an OAuth-backend behavior, so omit
|
|
130
|
+
// them — the public API keys its prefix cache off body.prompt_cache_key.
|
|
131
|
+
return {
|
|
132
|
+
Authorization: `Bearer ${auth.apiKey}`,
|
|
133
|
+
'Content-Type': 'application/json',
|
|
134
|
+
Accept: 'text/event-stream',
|
|
135
|
+
'x-client-request-id': randomBytes(16).toString('hex'),
|
|
136
|
+
};
|
|
137
|
+
}
|
|
119
138
|
const headers = {
|
|
120
139
|
Authorization: `Bearer ${auth.access_token}`,
|
|
121
140
|
'Content-Type': 'application/json',
|
|
@@ -189,10 +208,13 @@ export async function sendViaHttpSse({
|
|
|
189
208
|
);
|
|
190
209
|
const headers = _buildOpenAIHttpFallbackHeaders({ auth, cacheKey });
|
|
191
210
|
const fetchStartedAt = Date.now();
|
|
211
|
+
const responsesUrl = auth?.type === 'openai-direct'
|
|
212
|
+
? OPENAI_DIRECT_RESPONSES_URL
|
|
213
|
+
: CODEX_RESPONSES_URL;
|
|
192
214
|
let response;
|
|
193
215
|
try {
|
|
194
216
|
try { onStageChange?.('requesting'); } catch {}
|
|
195
|
-
response = await fetchFn(
|
|
217
|
+
response = await fetchFn(responsesUrl, {
|
|
196
218
|
method: 'POST',
|
|
197
219
|
headers,
|
|
198
220
|
body: JSON.stringify(body),
|
|
@@ -304,6 +326,17 @@ export async function sendViaHttpSse({
|
|
|
304
326
|
// a second attempt.
|
|
305
327
|
let emittedText = false;
|
|
306
328
|
|
|
329
|
+
// Tool-emit invariant (mirrors emittedText, WS path's emittedToolCall): set
|
|
330
|
+
// once onToolCall has actually dispatched a call. A failure afterwards is
|
|
331
|
+
// non-retryable — the side-effecting tool already ran, and any upstream
|
|
332
|
+
// retry/fallback would double-execute it. Stamped onto errors below so
|
|
333
|
+
// shouldFallbackTransport / the WS auth-retry gate refuse to reissue.
|
|
334
|
+
let emittedToolCall = false;
|
|
335
|
+
const _stampToolSafety = (err) => {
|
|
336
|
+
if (emittedToolCall && err) { try { err.emittedToolCall = true; err.unsafeToRetry = true; } catch {} }
|
|
337
|
+
return err;
|
|
338
|
+
};
|
|
339
|
+
|
|
307
340
|
// Single-emit guard for tool calls (matches the WS path's
|
|
308
341
|
// emittedToolCall intent). The HTTP/SSE event stream can surface the
|
|
309
342
|
// same function_call across multiple frames — response.function_call_arguments.done,
|
|
@@ -324,6 +357,7 @@ export async function sendViaHttpSse({
|
|
|
324
357
|
if (emittedToolCallIds.has(call.id)) return;
|
|
325
358
|
emittedToolCallIds.add(call.id);
|
|
326
359
|
if (!_toolDedupe.shouldDispatch(call.name, call.arguments)) return;
|
|
360
|
+
emittedToolCall = true;
|
|
327
361
|
try { onToolCall?.(call); } catch {}
|
|
328
362
|
};
|
|
329
363
|
|
|
@@ -676,6 +710,9 @@ export async function sendViaHttpSse({
|
|
|
676
710
|
// Live-text invariant: once a non-empty chunk has been relayed it
|
|
677
711
|
// cannot be withdrawn — flag the error so no upstream layer retries.
|
|
678
712
|
if (emittedText && err) { try { err.liveTextEmitted = true; err.unsafeToRetry = true; } catch {} }
|
|
713
|
+
// Tool-emit invariant: an error after a dispatched tool call must not
|
|
714
|
+
// reissue the turn (double-execution). Stamp emittedToolCall too.
|
|
715
|
+
_stampToolSafety(err);
|
|
679
716
|
throw err;
|
|
680
717
|
} finally {
|
|
681
718
|
_clearSemanticIdle();
|
|
@@ -688,10 +725,10 @@ export async function sendViaHttpSse({
|
|
|
688
725
|
|
|
689
726
|
const unresolved = toolCalls.find(t => t._pendingItemId);
|
|
690
727
|
if (unresolved) {
|
|
691
|
-
throw new Error(`OpenAI OAuth HTTP fallback function_call salvage failed: missing call_id/name for item_id=${unresolved._pendingItemId || '?'}`);
|
|
728
|
+
throw _stampToolSafety(new Error(`OpenAI OAuth HTTP fallback function_call salvage failed: missing call_id/name for item_id=${unresolved._pendingItemId || '?'}`));
|
|
692
729
|
}
|
|
693
730
|
if (!completed && !content && !toolCalls.length) {
|
|
694
|
-
throw new Error('OpenAI OAuth HTTP fallback ended before response.completed');
|
|
731
|
+
throw _stampToolSafety(new Error('OpenAI OAuth HTTP fallback ended before response.completed'));
|
|
695
732
|
}
|
|
696
733
|
|
|
697
734
|
const liveModel = model || useModel;
|
|
@@ -16,6 +16,9 @@ import { sendViaWebSocket } from './openai-oauth-ws.mjs';
|
|
|
16
16
|
import { buildRequestBody } from './openai-oauth.mjs';
|
|
17
17
|
import { enrichModels } from './model-catalog.mjs';
|
|
18
18
|
import { sanitizeModelList } from './model-list-sanitize.mjs';
|
|
19
|
+
import { sendViaHttpSse, _envFlag } from './openai-oauth-http-sse.mjs';
|
|
20
|
+
import { shouldFallbackTransport } from './retry-classifier.mjs';
|
|
21
|
+
import { loadConfig } from '../config.mjs';
|
|
19
22
|
import {
|
|
20
23
|
resolveProviderCacheKey,
|
|
21
24
|
resolveProviderPromptCacheLane,
|
|
@@ -52,6 +55,23 @@ export class OpenAIDirectProvider {
|
|
|
52
55
|
if (!k) throw new Error('OPENAI_API_KEY not configured (providers.openai.apiKey)');
|
|
53
56
|
return k;
|
|
54
57
|
}
|
|
58
|
+
// Auth-recovery mirror of openai-compat.reloadApiKey: on a 401/403 the key
|
|
59
|
+
// was likely rotated in config after this provider instance was built, so
|
|
60
|
+
// re-read providers.openai.apiKey from disk before the single retry.
|
|
61
|
+
// Returns the fresh key (or null if none) — no client to rebuild here since
|
|
62
|
+
// the WS/HTTP transports take the key per-call via the `auth` object.
|
|
63
|
+
reloadApiKey() {
|
|
64
|
+
try {
|
|
65
|
+
const freshConfig = loadConfig();
|
|
66
|
+
const cfg = freshConfig.providers?.openai;
|
|
67
|
+
const newKey = cfg?.apiKey || this.config.apiKey;
|
|
68
|
+
if (newKey) {
|
|
69
|
+
this.config = { ...(this.config || {}), ...(cfg || {}), apiKey: newKey };
|
|
70
|
+
return newKey;
|
|
71
|
+
}
|
|
72
|
+
} catch { /* best effort */ }
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
55
75
|
async send(messages, model, tools, sendOpts) {
|
|
56
76
|
const opts = sendOpts || {};
|
|
57
77
|
const onStageChange = typeof opts.onStageChange === 'function' ? opts.onStageChange : null;
|
|
@@ -109,10 +129,8 @@ export class OpenAIDirectProvider {
|
|
|
109
129
|
const cacheKey = body.prompt_cache_key || resolveProviderCacheKey(opts, 'openai');
|
|
110
130
|
const iteration = Number.isFinite(Number(opts.iteration)) ? Number(opts.iteration) : null;
|
|
111
131
|
const auth = { type: 'openai-direct', apiKey };
|
|
112
|
-
|
|
113
|
-
auth,
|
|
132
|
+
const common = {
|
|
114
133
|
body,
|
|
115
|
-
sendOpts: opts,
|
|
116
134
|
onStreamDelta,
|
|
117
135
|
onToolCall,
|
|
118
136
|
onTextDelta,
|
|
@@ -122,8 +140,60 @@ export class OpenAIDirectProvider {
|
|
|
122
140
|
cacheKey,
|
|
123
141
|
iteration,
|
|
124
142
|
useModel,
|
|
143
|
+
};
|
|
144
|
+
const dispatchWs = (a) => sendViaWebSocket({
|
|
145
|
+
...common,
|
|
146
|
+
auth: a,
|
|
147
|
+
sendOpts: opts,
|
|
125
148
|
displayModel: (id) => id,
|
|
126
149
|
});
|
|
150
|
+
// WS→HTTP/SSE fallback mirrors the openai-oauth wrapper: the shared
|
|
151
|
+
// HTTP transport now accepts auth.type==='openai-direct' (public
|
|
152
|
+
// Responses endpoint + Bearer <apiKey>), so the api-key provider gets
|
|
153
|
+
// the same envelope. Gate via shouldFallbackTransport (denies
|
|
154
|
+
// 401/403/404/429 + liveTextEmitted/emittedToolCall/unsafeToRetry).
|
|
155
|
+
const httpFallbackEnabled = _envFlag('MIXDOG_OPENAI_HTTP_FALLBACK', true);
|
|
156
|
+
const dispatchHttp = (a) => {
|
|
157
|
+
if (!process.env.MIXDOG_QUIET_PROVIDER_LOG) {
|
|
158
|
+
process.stderr.write('[openai-ws] WebSocket unhealthy; falling back to HTTP/SSE\n');
|
|
159
|
+
}
|
|
160
|
+
return sendViaHttpSse({ ...common, auth: a, opts, fetchFn: opts._fetchFn });
|
|
161
|
+
};
|
|
162
|
+
try {
|
|
163
|
+
return await dispatchWs(auth);
|
|
164
|
+
} catch (err) {
|
|
165
|
+
const status = err?.httpStatus;
|
|
166
|
+
// Live-text/tool invariant: never reissue a turn that already
|
|
167
|
+
// relayed visible output or dispatched a tool call.
|
|
168
|
+
const unsafeToRetry = err?.liveTextEmitted === true
|
|
169
|
+
|| err?.emittedToolCall === true
|
|
170
|
+
|| err?.unsafeToRetry === true;
|
|
171
|
+
// (1) 401/403 → reload apiKey from config and retry once over WS.
|
|
172
|
+
// shouldFallbackTransport denies 401/403, so this branch owns
|
|
173
|
+
// its own guard.
|
|
174
|
+
if ((status === 401 || status === 403) && !unsafeToRetry) {
|
|
175
|
+
process.stderr.write(`[openai-ws] ${status} — reloading apiKey and retrying once\n`);
|
|
176
|
+
const freshKey = this.reloadApiKey();
|
|
177
|
+
if (freshKey) {
|
|
178
|
+
const retryAuth = { type: 'openai-direct', apiKey: freshKey };
|
|
179
|
+
try {
|
|
180
|
+
return await dispatchWs(retryAuth);
|
|
181
|
+
} catch (retryErr) {
|
|
182
|
+
if (shouldFallbackTransport(retryErr, { signal: externalSignal, enabled: httpFallbackEnabled })) {
|
|
183
|
+
return await dispatchHttp(retryAuth);
|
|
184
|
+
}
|
|
185
|
+
throw retryErr;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
throw err;
|
|
189
|
+
}
|
|
190
|
+
// (2) WS transport failure → HTTP/SSE fallback (predicate handles
|
|
191
|
+
// the safety denies).
|
|
192
|
+
if (shouldFallbackTransport(err, { signal: externalSignal, enabled: httpFallbackEnabled })) {
|
|
193
|
+
return await dispatchHttp(auth);
|
|
194
|
+
}
|
|
195
|
+
throw err;
|
|
196
|
+
}
|
|
127
197
|
}
|
|
128
198
|
async listModels() {
|
|
129
199
|
try {
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
releaseOwnedChannelLocks,
|
|
11
11
|
clearActiveInstance,
|
|
12
12
|
probeActiveOwner,
|
|
13
|
+
RUNTIME_ROOT,
|
|
13
14
|
} from "./runtime-paths.mjs";
|
|
14
15
|
// Owned-runtime lifecycle extracted from channels/index.mjs (behavior-
|
|
15
16
|
// preserving): bridge-ownership claim/refresh/loss, backend connect/disconnect,
|
|
@@ -52,11 +53,49 @@ export function createOwnedRuntime({
|
|
|
52
53
|
let bridgeOwnershipRefreshInFlight = null;
|
|
53
54
|
let bridgeOwnershipTimer = null;
|
|
54
55
|
let _memoryDrainTimer = null;
|
|
56
|
+
// Event-driven ownership signal: an fs.watch on the runtime dir fires the
|
|
57
|
+
// ownership refresh the instant active-instance.json changes (a newer owner
|
|
58
|
+
// claims, or the owner releases/clears it), instead of waiting up to 3s for
|
|
59
|
+
// the poll tick. This shrinks the double-owner window on takeover (the old
|
|
60
|
+
// owner observes owned=false and tears down in ms) and signals ownership
|
|
61
|
+
// loss to contenders immediately on release. The 3s timer stays as fallback.
|
|
62
|
+
let activeInstanceWatcher = null;
|
|
63
|
+
let _activeInstanceWatchDebounce = null;
|
|
64
|
+
function armActiveInstanceWatcher() {
|
|
65
|
+
if (activeInstanceWatcher) return;
|
|
66
|
+
try {
|
|
67
|
+
activeInstanceWatcher = fs.watch(RUNTIME_ROOT, { persistent: false }, (_event, filename) => {
|
|
68
|
+
if (filename && filename !== 'active-instance.json') return;
|
|
69
|
+
// Coalesce the burst of events an atomic rename/truncate emits.
|
|
70
|
+
if (_activeInstanceWatchDebounce) return;
|
|
71
|
+
_activeInstanceWatchDebounce = setTimeout(() => {
|
|
72
|
+
_activeInstanceWatchDebounce = null;
|
|
73
|
+
refreshBridgeOwnershipSafe();
|
|
74
|
+
}, 50);
|
|
75
|
+
_activeInstanceWatchDebounce.unref?.();
|
|
76
|
+
});
|
|
77
|
+
// fs.watch emits 'error' (and an unhandled one CRASHES the worker) when
|
|
78
|
+
// the watched dir is removed or the handle is invalidated (common on
|
|
79
|
+
// Windows). Close the dead handle and fall back to the 3s poll, which is
|
|
80
|
+
// still armed — the event signal is a latency optimization, never the
|
|
81
|
+
// sole ownership mechanism.
|
|
82
|
+
activeInstanceWatcher.on('error', (err) => {
|
|
83
|
+
process.stderr.write(`[ownership] active-instance watch error, falling back to poll: ${err?.message || err}\n`);
|
|
84
|
+
clearActiveInstanceWatcher();
|
|
85
|
+
});
|
|
86
|
+
activeInstanceWatcher.unref?.();
|
|
87
|
+
} catch { activeInstanceWatcher = null; }
|
|
88
|
+
}
|
|
89
|
+
function clearActiveInstanceWatcher() {
|
|
90
|
+
if (_activeInstanceWatchDebounce) { clearTimeout(_activeInstanceWatchDebounce); _activeInstanceWatchDebounce = null; }
|
|
91
|
+
if (activeInstanceWatcher) { try { activeInstanceWatcher.close(); } catch {} activeInstanceWatcher = null; }
|
|
92
|
+
}
|
|
55
93
|
function clearBridgeOwnershipTimer() {
|
|
56
94
|
if (bridgeOwnershipTimer) {
|
|
57
95
|
clearInterval(bridgeOwnershipTimer);
|
|
58
96
|
bridgeOwnershipTimer = null;
|
|
59
97
|
}
|
|
98
|
+
clearActiveInstanceWatcher();
|
|
60
99
|
}
|
|
61
100
|
function shouldStartEventPipelineRuntime() {
|
|
62
101
|
return getConfig().webhook?.enabled === true || (Array.isArray(getConfig().events?.rules) && getConfig().events.rules.length > 0);
|
|
@@ -176,6 +215,9 @@ async function startOwnedRuntime(options = {}) {
|
|
|
176
215
|
// both-backends-live window.
|
|
177
216
|
const startingBackend = getBackend();
|
|
178
217
|
const claimAfterReady = options.claimAfterReady === true;
|
|
218
|
+
// Auto-start intent: claim the seat ONLY if it is vacant/stale (never steal a
|
|
219
|
+
// live owner). Threaded from worker start() (MIXDOG_REMOTE_INTENT=auto).
|
|
220
|
+
const claimIfVacant = options.claimIfVacant === true;
|
|
179
221
|
// Single-holder correctness: the seat is claimed BEFORE getBackend().connect(),
|
|
180
222
|
// never after. The old make-before-break (claim-after-ready) boot left two
|
|
181
223
|
// gateways connected and contending during the multi-second connect window;
|
|
@@ -190,8 +232,29 @@ async function startOwnedRuntime(options = {}) {
|
|
|
190
232
|
// leave it stuck true and permanently block every future ownership attempt.
|
|
191
233
|
try {
|
|
192
234
|
if (claimAfterReady) {
|
|
193
|
-
|
|
194
|
-
|
|
235
|
+
if (claimIfVacant) {
|
|
236
|
+
// Auto-start claim-if-vacant. Probe first: a live owner other than us
|
|
237
|
+
// holds the seat -> back off SILENTLY (no claim, no acquire notify) so
|
|
238
|
+
// this session stays non-remote. Then claim atomically via the
|
|
239
|
+
// onlyIfVacant CAS (guards the probe->write TOCTOU: a live owner landing
|
|
240
|
+
// in that gap aborts the write, leaving the seat untouched).
|
|
241
|
+
const probe = probeActiveOwner();
|
|
242
|
+
if (probe.status === 'live' && probe.state?.instanceId && probe.state.instanceId !== instanceId) {
|
|
243
|
+
bridgeRuntimeStarting = false;
|
|
244
|
+
logOwnership("autostart backoff (live owner holds seat)");
|
|
245
|
+
return;
|
|
246
|
+
}
|
|
247
|
+
const casResult = refreshActiveInstance(instanceId, { backendReady: false }, { onlyIfVacant: true, timeoutMs: 0 });
|
|
248
|
+
if (casResult?.instanceId !== instanceId) {
|
|
249
|
+
bridgeRuntimeStarting = false;
|
|
250
|
+
logOwnership("autostart backoff (seat claimed by newer owner)");
|
|
251
|
+
return;
|
|
252
|
+
}
|
|
253
|
+
logOwnership("boot claim (pre-connect, claim-if-vacant)");
|
|
254
|
+
} else {
|
|
255
|
+
refreshActiveInstance(instanceId, { backendReady: false });
|
|
256
|
+
logOwnership("boot claim (pre-connect, last-wins)");
|
|
257
|
+
}
|
|
195
258
|
} else {
|
|
196
259
|
// Try-once (timeoutMs:0): a refresh-path re-acquire must never block on
|
|
197
260
|
// the active-instance lock. Contention throws → caught below and treated
|
|
@@ -289,11 +352,18 @@ async function startOwnedRuntime(options = {}) {
|
|
|
289
352
|
return;
|
|
290
353
|
}
|
|
291
354
|
setBridgeRuntimeConnected(true);
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
355
|
+
// Fresh confirmed ownership — tell the parent it holds the seat so it flips
|
|
356
|
+
// remote ON. Reached ONLY on a not-connected -> connected transition (the
|
|
357
|
+
// top-of-fn early-return skips already-connected re-ticks), so this fires
|
|
358
|
+
// exactly once per acquire and covers EVERY win path, not just boot:
|
|
359
|
+
// - explicit/auto boot claim (claimAfterReady),
|
|
360
|
+
// - the deferred claim when a bridge timer's refreshBridgeOwnership()
|
|
361
|
+
// claims a seat vacated by a departing owner (finding 1).
|
|
362
|
+
// The parent's acquired handler is idempotent (no-op when already remote),
|
|
363
|
+
// so notifying post-connect — never pre-connect — means a connect FAILURE
|
|
364
|
+
// below leaves the parent non-remote instead of stuck remote-with-no-bridge
|
|
365
|
+
// (finding 2).
|
|
366
|
+
notifyRemoteAcquired();
|
|
297
367
|
// initProviders must complete before scheduler.start() — otherwise the
|
|
298
368
|
// scheduler's first fire can land before the registry is populated and
|
|
299
369
|
// return `Provider "<name>" not found or not enabled`. The previous
|
|
@@ -402,15 +472,28 @@ function armBridgeOwnershipTimer() {
|
|
|
402
472
|
refreshBridgeOwnershipSafe();
|
|
403
473
|
}, 3e3);
|
|
404
474
|
bridgeOwnershipTimer.unref?.();
|
|
475
|
+
// Arm the event-driven signal alongside the poll fallback.
|
|
476
|
+
armActiveInstanceWatcher();
|
|
477
|
+
}
|
|
478
|
+
// Guarded IPC send to the parent: no-ops when there is no channel or it is
|
|
479
|
+
// already disconnected, and swallows both the synchronous throw and the async
|
|
480
|
+
// error-callback path of ERR_IPC_CHANNEL_CLOSED (channel closing between the
|
|
481
|
+
// connected check and delivery). Log-and-continue — never crash the worker.
|
|
482
|
+
function sendToParent(message) {
|
|
483
|
+
if (!process.send || process.connected === false) return;
|
|
484
|
+
try {
|
|
485
|
+
process.send(message, undefined, undefined, err => {
|
|
486
|
+
if (err) process.stderr.write(`[channels] parent IPC send failed: ${err?.message || err}\n`);
|
|
487
|
+
});
|
|
488
|
+
} catch (err) {
|
|
489
|
+
process.stderr.write(`[channels] parent IPC send threw: ${err?.message || err}\n`);
|
|
490
|
+
}
|
|
405
491
|
}
|
|
406
492
|
// Tell the parent session that this worker LOST the bridge seat to a newer
|
|
407
493
|
// remote session (last-wins). The parent flips its remote mode OFF entirely —
|
|
408
494
|
// exactly one session holds remote; losers fully release, no handover.
|
|
409
495
|
function notifyRemoteSuperseded() {
|
|
410
|
-
|
|
411
|
-
try {
|
|
412
|
-
process.send({ type: 'notify', method: 'notifications/mixdog/remote', params: { state: 'superseded' } });
|
|
413
|
-
} catch {}
|
|
496
|
+
sendToParent({ type: 'notify', method: 'notifications/mixdog/remote', params: { state: 'superseded' } });
|
|
414
497
|
}
|
|
415
498
|
// Symmetric to notifyRemoteSuperseded: tell the parent session this worker
|
|
416
499
|
// ACQUIRED the bridge seat so it flips remote mode ON (badge/transcript writer).
|
|
@@ -419,10 +502,7 @@ function notifyRemoteSuperseded() {
|
|
|
419
502
|
// transition (boot make-before-break, activate when not already owned) — never
|
|
420
503
|
// on a heartbeat refresh — so the parent's idempotent handler sees it once.
|
|
421
504
|
function notifyRemoteAcquired() {
|
|
422
|
-
|
|
423
|
-
try {
|
|
424
|
-
process.send({ type: 'notify', method: 'notifications/mixdog/remote', params: { state: 'acquired' } });
|
|
425
|
-
} catch {}
|
|
505
|
+
sendToParent({ type: 'notify', method: 'notifications/mixdog/remote', params: { state: 'acquired' } });
|
|
426
506
|
}
|
|
427
507
|
async function refreshBridgeOwnership(options = {}) {
|
|
428
508
|
// Coalesce concurrent callers onto the in-flight refresh so getBackend() tool
|
|
@@ -279,6 +279,14 @@ function refreshActiveInstance(instanceId, meta, options) {
|
|
|
279
279
|
if (options?.onlyIfOwned && (!prev?.instanceId || prev.instanceId !== instanceId)) {
|
|
280
280
|
return undefined;
|
|
281
281
|
}
|
|
282
|
+
// CAS guard (opt-in via options.onlyIfVacant): auto-start claim-if-vacant.
|
|
283
|
+
// Abort the write when a live (non-stale) owner OTHER than us already holds
|
|
284
|
+
// the seat — an auto-start must never steal from a live owner. A stale/dead
|
|
285
|
+
// prior owner leaves prev=null above, so it does NOT block the claim; an
|
|
286
|
+
// absent seat (prev=null) is claimable too.
|
|
287
|
+
if (options?.onlyIfVacant && prev?.instanceId && prev.instanceId !== instanceId) {
|
|
288
|
+
return undefined;
|
|
289
|
+
}
|
|
282
290
|
// Drop stale fields (pid/startedAt) written by older server versions.
|
|
283
291
|
const { pid: _legacyPid, startedAt: _legacyStartedAt, ...prevRest } = prev ?? {};
|
|
284
292
|
const identity = buildRuntimeIdentity();
|
|
@@ -393,18 +401,21 @@ function refreshActiveInstance(instanceId, meta, options) {
|
|
|
393
401
|
}
|
|
394
402
|
}
|
|
395
403
|
}
|
|
396
|
-
//
|
|
397
|
-
// the seat
|
|
398
|
-
//
|
|
399
|
-
//
|
|
400
|
-
//
|
|
401
|
-
//
|
|
402
|
-
//
|
|
403
|
-
//
|
|
404
|
+
// Headless worker-owned seat: THIS process is the channels worker AND no
|
|
405
|
+
// distinct terminal lead owns the seat (ownerLeadPid resolves to our own
|
|
406
|
+
// pid). A worker never runs the TUI render loop, so it owns no heartbeat —
|
|
407
|
+
// it must NOT keep refreshing a ui_heartbeat_at it does not own. An
|
|
408
|
+
// inherited value (written by a prior TUI session, carried forward in
|
|
409
|
+
// prevRest) would otherwise be perpetually renewed here and falsely mask a
|
|
410
|
+
// dead render loop, so DROP it and let pid-only judgment stand.
|
|
411
|
+
// When a distinct terminal lead owns the seat (ownerLeadPid !== our pid),
|
|
412
|
+
// this branch is a no-op: the TUI's own heartbeat is carried forward
|
|
413
|
+
// untouched, so a stale (zombie) render loop still evicts the seat even
|
|
414
|
+
// while the worker pid is alive.
|
|
404
415
|
const workerOwnsSeat = process.env.MIXDOG_WORKER_MODE === "1"
|
|
405
416
|
&& identity.ownerLeadPid === process.pid;
|
|
406
417
|
if (workerOwnsSeat) {
|
|
407
|
-
next.ui_heartbeat_at
|
|
418
|
+
delete next.ui_heartbeat_at;
|
|
408
419
|
}
|
|
409
420
|
return { ...preservedExtra, ...next };
|
|
410
421
|
}, writeOpts);
|
|
@@ -687,7 +687,13 @@ async function start() {
|
|
|
687
687
|
// takeover" second claim (double reconnect) is gone.
|
|
688
688
|
const _bindingReadyStart = Date.now();
|
|
689
689
|
try {
|
|
690
|
-
|
|
690
|
+
// Boot claim intent (published by the parent via env before this fork):
|
|
691
|
+
// explicit (/remote, --remote) -> last-wins force-takeover.
|
|
692
|
+
// auto (config/delayed autoStart) -> claim-if-vacant: take the seat only
|
|
693
|
+
// when no live owner holds it, otherwise back off silently (no claim, no
|
|
694
|
+
// acquire notify) so a live owner is never stolen.
|
|
695
|
+
const autoStartClaim = process.env.MIXDOG_REMOTE_INTENT === 'auto';
|
|
696
|
+
await startOwnedRuntime({ restoreBinding: true, claimAfterReady: true, claimIfVacant: autoStartClaim });
|
|
691
697
|
bindingReadyStatus = "resolved";
|
|
692
698
|
dropTrace("bindingReady.resolve", { elapsedMs: Date.now() - _bindingReadyStart, status: bindingReadyStatus });
|
|
693
699
|
_bindingReadyResolve(true);
|
|
@@ -164,6 +164,88 @@ function touchDaemonIdleTimer(reason = 'activity') {
|
|
|
164
164
|
_idleShutdownTimer.unref?.()
|
|
165
165
|
}
|
|
166
166
|
|
|
167
|
+
// ── Connected-client tracking + prompt shutdown ───────────────────────────
|
|
168
|
+
// The daemon is shared by multiple proxy clients (TUI host + channels worker,
|
|
169
|
+
// potentially several sessions). Each client registers/deregisters over HTTP
|
|
170
|
+
// (see /client/register, /client/deregister in lib/http-router.mjs). When the
|
|
171
|
+
// last client goes away we arm a short grace timer (default 10s) so a quick
|
|
172
|
+
// reconnect keeps the daemon warm, but an actually-closed CLI reaps the daemon
|
|
173
|
+
// in seconds instead of waiting out the 10-minute idle TTL (kept as backstop).
|
|
174
|
+
const MEMORY_CLIENT_GRACE_MS = Math.max(0, Number(process.env.MIXDOG_MEMORY_CLIENT_GRACE_MS) || 10_000)
|
|
175
|
+
const _connectedClients = new Map() // clientPid -> lastSeenMs
|
|
176
|
+
let _everHadClient = false
|
|
177
|
+
let _clientGraceTimer = null
|
|
178
|
+
let _clientSweepTimer = null
|
|
179
|
+
|
|
180
|
+
function _clientShutdownEnabled() {
|
|
181
|
+
return MEMORY_DAEMON_MODE && MEMORY_CLIENT_GRACE_MS > 0
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function pruneDeadClients() {
|
|
185
|
+
for (const pid of [..._connectedClients.keys()]) {
|
|
186
|
+
if (!_isPidAliveLocal(pid)) _connectedClients.delete(pid)
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function cancelClientGrace() {
|
|
191
|
+
if (_clientGraceTimer) {
|
|
192
|
+
try { clearTimeout(_clientGraceTimer) } catch {}
|
|
193
|
+
_clientGraceTimer = null
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function armClientGrace(reason = 'last client gone') {
|
|
198
|
+
if (!_clientShutdownEnabled() || _clientGraceTimer) return
|
|
199
|
+
_clientGraceTimer = setTimeout(() => {
|
|
200
|
+
_clientGraceTimer = null
|
|
201
|
+
pruneDeadClients()
|
|
202
|
+
if (_connectedClients.size > 0) return
|
|
203
|
+
__mixdogMemoryLog(`[memory-service] daemon client grace elapsed (${reason}); shutting down\n`)
|
|
204
|
+
stop()
|
|
205
|
+
.then(() => process.exit(0))
|
|
206
|
+
.catch((e) => {
|
|
207
|
+
__mixdogMemoryLog(`[memory-service] daemon client-grace shutdown failed: ${e?.message || e}\n`)
|
|
208
|
+
process.exit(1)
|
|
209
|
+
})
|
|
210
|
+
}, MEMORY_CLIENT_GRACE_MS)
|
|
211
|
+
_clientGraceTimer.unref?.()
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function startClientSweep() {
|
|
215
|
+
if (_clientSweepTimer || !_clientShutdownEnabled()) return
|
|
216
|
+
// Reap clients that died without deregistering so a crashed CLI still frees
|
|
217
|
+
// the daemon in grace-scale time rather than waiting for the idle TTL.
|
|
218
|
+
const interval = Math.max(1000, Math.min(MEMORY_CLIENT_GRACE_MS, 5000))
|
|
219
|
+
_clientSweepTimer = setInterval(() => {
|
|
220
|
+
pruneDeadClients()
|
|
221
|
+
if (_everHadClient && _connectedClients.size === 0) armClientGrace('all clients gone (sweep)')
|
|
222
|
+
}, interval)
|
|
223
|
+
_clientSweepTimer.unref?.()
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function registerClient(clientPid) {
|
|
227
|
+
const pid = parsePositivePid(clientPid)
|
|
228
|
+
if (!pid) return true
|
|
229
|
+
// Reject registration once shutdown has begun. stop() sets _stopPromise
|
|
230
|
+
// synchronously (before its first await) and, in daemon mode, always ends
|
|
231
|
+
// in process.exit — so a daemon that is draining will never revive. Signal
|
|
232
|
+
// the proxy (via a distinct 503) to respawn a fresh daemon instead of
|
|
233
|
+
// binding to this dying one, which would fail the subsequent /api/tool.
|
|
234
|
+
if (_stopPromise) return false
|
|
235
|
+
_connectedClients.set(pid, Date.now())
|
|
236
|
+
_everHadClient = true
|
|
237
|
+
cancelClientGrace()
|
|
238
|
+
startClientSweep()
|
|
239
|
+
return true
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function deregisterClient(clientPid) {
|
|
243
|
+
const pid = parsePositivePid(clientPid)
|
|
244
|
+
if (pid) _connectedClients.delete(pid)
|
|
245
|
+
pruneDeadClients()
|
|
246
|
+
if (_everHadClient && _connectedClients.size === 0) armClientGrace('last client deregistered')
|
|
247
|
+
}
|
|
248
|
+
|
|
167
249
|
function advertiseMemoryPort(boundPort, attempt = 0) {
|
|
168
250
|
if (!Number.isFinite(boundPort) || boundPort <= 0) return
|
|
169
251
|
_currentAdvertisedPort = boundPort
|
|
@@ -660,6 +742,9 @@ const _httpRouter = createHttpRouter({
|
|
|
660
742
|
handleMemoryAction,
|
|
661
743
|
handleToolCall,
|
|
662
744
|
stop,
|
|
745
|
+
registerClient,
|
|
746
|
+
deregisterClient,
|
|
747
|
+
getDraining: () => _stopPromise != null,
|
|
663
748
|
getCycle1CallLlm,
|
|
664
749
|
getTraceDb: () => _traceDb,
|
|
665
750
|
setTraceDb: (v) => { _traceDb = v },
|
|
@@ -720,6 +805,11 @@ export async function stop() {
|
|
|
720
805
|
try { clearTimeout(_idleShutdownTimer) } catch {}
|
|
721
806
|
_idleShutdownTimer = null
|
|
722
807
|
}
|
|
808
|
+
cancelClientGrace()
|
|
809
|
+
if (_clientSweepTimer) {
|
|
810
|
+
try { clearInterval(_clientSweepTimer) } catch {}
|
|
811
|
+
_clientSweepTimer = null
|
|
812
|
+
}
|
|
723
813
|
await stopLlmWorker()
|
|
724
814
|
resetHttpListenErrorHandler()
|
|
725
815
|
if (_httpBoundPort != null || _httpReadyPromise) {
|