mixdog 0.9.142 → 0.9.143
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/agent-lead-e2e-probe.mjs +1 -6
- package/scripts/agent-turn-trace-probe.mjs +3 -5
- package/src/runtime/agent/orchestrator/agent-runtime/agent-dispatch.mjs +0 -39
- package/src/runtime/agent/orchestrator/providers/admission-scheduler.mjs +28 -22
- package/src/runtime/agent/orchestrator/session/manager/turn-checkpoint.mjs +1 -16
- package/src/runtime/agent/orchestrator/session/token-native.mjs +40 -40
- package/src/runtime/agent/orchestrator/tools/builtin/list-tool-integrity.test.mjs +10 -10
- package/src/runtime/agent/orchestrator/tools/builtin/list-tool.mjs +7 -7
- package/src/runtime/agent/orchestrator/tools/builtin/native-search-client.mjs +6 -6
- package/src/runtime/shared/child-spawn-remote.mjs +7 -8
- package/src/runtime/shared/resource-admission.mjs +20 -0
- package/src/runtime/shared/resource-admission.test.mjs +43 -0
- package/src/runtime/shared/{session-shard-health.mjs → session-runtime-health.mjs} +14 -14
- package/src/runtime/shared/session-runtime-health.test.mjs +30 -0
- package/src/session-runtime/runtime-core.mjs +21 -41
- package/src/session-runtime/session-lifecycle.mjs +0 -32
- package/src/session-runtime/session-turn-api.mjs +1 -1
- package/src/standalone/agent-tool/helpers.mjs +1 -1
- package/src/standalone/agent-tool/spawn-flow.mjs +1 -30
- package/src/standalone/agent-tool.mjs +1 -7
- package/src/standalone/channel-worker.mjs +3 -3
- package/src/standalone/daemon.mjs +14 -18
- package/src/standalone/session-client.mjs +1 -1
- package/src/standalone/{session-runtime-pool-health.test.mjs → session-runtime-host-health.test.mjs} +22 -23
- package/src/standalone/{session-runtime-pool.mjs → session-runtime-host.mjs} +69 -291
- package/src/standalone/session-runtime-record.mjs +2 -2
- package/src/standalone/session-runtime-worker.mjs +20 -21
- package/src/standalone/session-service.mjs +3 -28
- package/src/standalone/session-state-patch.mjs +1 -1
- package/src/standalone/session-transport.mjs +1 -1
- package/src/tui/session-local.mjs +1 -3
- package/scripts/agent-long-prompt-repro.mjs +0 -82
- package/scripts/agent-shard-spread-perf.mjs +0 -245
- package/scripts/fixtures/session-shard-fixture-worker.mjs +0 -13
- package/src/runtime/shared/session-shard-health.test.mjs +0 -30
- package/src/standalone/agent-tool/shard-spread.mjs +0 -709
- package/src/standalone/agent-tool/shard-spread.test.mjs +0 -68
package/package.json
CHANGED
|
@@ -15,12 +15,7 @@ import { homedir, tmpdir } from 'node:os';
|
|
|
15
15
|
import { join } from 'node:path';
|
|
16
16
|
|
|
17
17
|
const ROOT = mkdtempSync(join(tmpdir(), 'mixdog-lead-e2e-'));
|
|
18
|
-
|
|
19
|
-
// discovery) and opt into agent shard spread, so the spawned worker runs on
|
|
20
|
-
// the INSTALLED daemon's shard pool — the deployed remote-completion path.
|
|
21
|
-
const LIVE_DAEMON = process.env.MIXDOG_LEAD_E2E_LIVE_DAEMON === '1';
|
|
22
|
-
if (LIVE_DAEMON) process.env.MIXDOG_AGENT_SHARD_SPREAD = '1';
|
|
23
|
-
else process.env.MIXDOG_RUNTIME_ROOT = ROOT;
|
|
18
|
+
process.env.MIXDOG_RUNTIME_ROOT = ROOT;
|
|
24
19
|
process.env.MIXDOG_BOOT_CORE_MEMORY = '0';
|
|
25
20
|
process.env.MIXDOG_DAEMON_SKIP_MEMORY = '1';
|
|
26
21
|
process.env.MIXDOG_FEATURE_MEMORY = '0';
|
|
@@ -17,9 +17,7 @@ const ROOT = mkdtempSync(join(tmpdir(), 'mixdog-turn-trace-'));
|
|
|
17
17
|
process.env.MIXDOG_RUNTIME_ROOT = ROOT;
|
|
18
18
|
process.env.MIXDOG_DAEMON_SKIP_MEMORY = '1';
|
|
19
19
|
process.env.MIXDOG_BOOT_CORE_MEMORY = '0';
|
|
20
|
-
|
|
21
|
-
// The probe's whole point is the trace: explicit path (shared by daemon and
|
|
22
|
-
// shard children via env inheritance) + timing rows.
|
|
20
|
+
// The probe's whole point is the trace: explicit path plus timing rows.
|
|
23
21
|
const TRACE_PATH = join(ROOT, 'agent-trace.jsonl');
|
|
24
22
|
delete process.env.MIXDOG_AGENT_TRACE_DISABLE;
|
|
25
23
|
process.env.MIXDOG_AGENT_TRACE_PATH = TRACE_PATH;
|
|
@@ -75,7 +73,7 @@ try {
|
|
|
75
73
|
'2) read README.md (first 40 lines)',
|
|
76
74
|
'3) run the shell command: node -v',
|
|
77
75
|
'4) read apps/desktop/package.json',
|
|
78
|
-
'5) grep the string "
|
|
76
|
+
'5) grep the string "createSessionRuntimeHost" under src/standalone (files list only)',
|
|
79
77
|
'Then reply with one line: DONE <package name> <node version>. Do not edit anything.',
|
|
80
78
|
].join('\n'),
|
|
81
79
|
}, { invocationSource: 'model-tool', cwd: REPO });
|
|
@@ -90,7 +88,7 @@ try {
|
|
|
90
88
|
process.stdout.write(`wall=${((Date.now() - t0) / 1000).toFixed(1)}s\n--- result ---\n${last.slice(0, 400)}\n`);
|
|
91
89
|
try { agent.closeAll('turn-trace probe end'); } catch { /* teardown */ }
|
|
92
90
|
|
|
93
|
-
//
|
|
91
|
+
// The runtime worker flushes its local trace buffer on a short timer.
|
|
94
92
|
await sleep(9_000);
|
|
95
93
|
const rows = existsSync(TRACE_PATH)
|
|
96
94
|
? readFileSync(TRACE_PATH, 'utf8').split('\n').filter(Boolean).flatMap((line) => {
|
|
@@ -334,45 +334,6 @@ export function makeAgentDispatch(opts = {}) {
|
|
|
334
334
|
// pooled or resumed. Cache prefix matching happens at the provider
|
|
335
335
|
// layer (account-level), not the session level.
|
|
336
336
|
const finalPrompt = prompt;
|
|
337
|
-
// Agent shard spread: inside a shard child the ephemeral hidden-role
|
|
338
|
-
// session runs on a peer shard through the daemon session protocol —
|
|
339
|
-
// the same single spawn path the agent tool uses. ONE path per mode:
|
|
340
|
-
// a remote failure fails the dispatch (no in-process fallback).
|
|
341
|
-
const spreadMod = await import('../../../../standalone/agent-tool/shard-spread.mjs')
|
|
342
|
-
.catch(() => null);
|
|
343
|
-
if (spreadMod?.agentShardSpreadEnabled?.()) {
|
|
344
|
-
const raw = await spreadMod.dispatchHiddenAgentRemote({
|
|
345
|
-
spec: {
|
|
346
|
-
agent,
|
|
347
|
-
presetName,
|
|
348
|
-
preset,
|
|
349
|
-
runtimeSpec,
|
|
350
|
-
permission,
|
|
351
|
-
cwd,
|
|
352
|
-
sourceType: opts.sourceType,
|
|
353
|
-
sourceName: sourceNameArg || opts.sourceName,
|
|
354
|
-
parentSessionId: opts.parentSessionId || null,
|
|
355
|
-
ownerSessionId: opts.ownerSessionId === undefined ? (opts.parentSessionId || null) : opts.ownerSessionId,
|
|
356
|
-
clientHostPid: opts.clientHostPid,
|
|
357
|
-
skipRoleReminder: isPoolC,
|
|
358
|
-
schemaAllowedTools: resolveHiddenRoleSchemaAllowedTools(hidden),
|
|
359
|
-
taskType: opts.taskType,
|
|
360
|
-
maxLoopIterations: opts.maxLoopIterations,
|
|
361
|
-
},
|
|
362
|
-
provider: preset.provider,
|
|
363
|
-
model: preset.model,
|
|
364
|
-
cwd,
|
|
365
|
-
prompt: finalPrompt,
|
|
366
|
-
parentSignals: [opts.parentSignal, callParentSignal],
|
|
367
|
-
watchdogPolicy: resolveAgentWatchdogPolicy(agent, {
|
|
368
|
-
idleTimeoutMs: Number.isFinite(callIdleTimeoutMs)
|
|
369
|
-
? callIdleTimeoutMs
|
|
370
|
-
: opts.idleTimeoutMs,
|
|
371
|
-
firstResponseTimeoutMs: opts.firstResponseTimeoutMs,
|
|
372
|
-
}),
|
|
373
|
-
});
|
|
374
|
-
return opts.brief === false ? raw : applyBriefCap(raw);
|
|
375
|
-
}
|
|
376
337
|
const { session } = prepareAgentSession({
|
|
377
338
|
agent,
|
|
378
339
|
presetName,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
3
|
+
import { resourceAdmission } from '../../../shared/resource-admission.mjs';
|
|
3
4
|
|
|
4
5
|
// Normal provider traffic is not concurrency-gated. Independent sessions and
|
|
5
6
|
// accounts start immediately; a finite limit exists only when an operator
|
|
@@ -577,28 +578,33 @@ export function wrapProviderAdmission(provider, providerName, scheduler = provid
|
|
|
577
578
|
const opts = sendOpts || {};
|
|
578
579
|
const signal = opts.signal || null;
|
|
579
580
|
const key = providerAdmissionKey(providerName, this);
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
581
|
+
// Provider queueing and network wait consume no local CPU slot. The
|
|
582
|
+
// resource controller reacquires the agent lease before model output
|
|
583
|
+
// continues through local context/tool processing.
|
|
584
|
+
return resourceAdmission.runYielded(() =>
|
|
585
|
+
scheduler.run(key, (admissionSignal) => {
|
|
586
|
+
// Admission is the common request-clock boundary for WS/SSE/HTTP.
|
|
587
|
+
// Queue wait therefore cannot consume first-byte or agent-watchdog
|
|
588
|
+
// time. Provider-local retry remains the sole retry owner.
|
|
589
|
+
try { opts.onStageChange?.('requesting'); } catch {}
|
|
590
|
+
return originalSend.call(this, messages, model, tools, {
|
|
591
|
+
...opts,
|
|
592
|
+
signal: admissionSignal,
|
|
593
|
+
});
|
|
594
|
+
}, {
|
|
595
|
+
signal,
|
|
596
|
+
ownerKey: opts.admissionOwner || opts.sessionId || null,
|
|
597
|
+
priority: opts.admissionPriority || 'user-visible',
|
|
598
|
+
onCooldownWait: (waitMs) => {
|
|
599
|
+
// Display-only: the TUI/desktop 'reconnecting' stage already
|
|
600
|
+
// renders a custom verb, so no new stage vocabulary is needed.
|
|
601
|
+
const secs = Math.max(1, Math.ceil(waitMs / 1000));
|
|
602
|
+
try {
|
|
603
|
+
opts.onStageChange?.('reconnecting', { message: `Rate-limited — waiting ~${secs}s for the provider window` });
|
|
604
|
+
} catch { /* display-only */ }
|
|
605
|
+
},
|
|
606
|
+
})
|
|
607
|
+
);
|
|
602
608
|
};
|
|
603
609
|
return provider;
|
|
604
610
|
}
|
|
@@ -64,27 +64,12 @@ export function projectTurnCheckpointMessages(session, checkpoint) {
|
|
|
64
64
|
|| !Array.isArray(checkpoint?.turnMessages)) return source;
|
|
65
65
|
const sessionGeneration = Number(session.generation) || 0;
|
|
66
66
|
const checkpointGeneration = Number(checkpoint.generation) || 0;
|
|
67
|
-
const owner = String(session.owner || '').trim().toLowerCase();
|
|
68
|
-
const agent = String(session.agent || '').trim().toLowerCase();
|
|
69
|
-
const detachedLiveAgent = session.closed === true
|
|
70
|
-
&& (owner === 'agent' || (agent && agent !== 'lead'))
|
|
71
|
-
&& Boolean(String(session.ownerSessionId || session.parentSessionId || '').trim());
|
|
72
67
|
const markerToken = typeof session.activeTurnCheckpoint?.turnToken === 'string'
|
|
73
68
|
? session.activeTurnCheckpoint.turnToken
|
|
74
69
|
: null;
|
|
75
70
|
const markerMatches = markerToken === checkpoint.turnToken;
|
|
76
71
|
const checkpointIsNewer = Number(checkpoint.updatedAt) > Number(session.updatedAt || 0);
|
|
77
|
-
|
|
78
|
-
// shard-owned turn finishes. closeSession writes the lifecycle fence by
|
|
79
|
-
// advancing the session generation once, while that already-running turn
|
|
80
|
-
// keeps publishing under its original generation. Accept exactly that
|
|
81
|
-
// one-generation handoff only when the durable marker token still proves
|
|
82
|
-
// identity and the checkpoint was written after the detach.
|
|
83
|
-
const detachedGenerationHandoff = detachedLiveAgent
|
|
84
|
-
&& sessionGeneration === checkpointGeneration + 1
|
|
85
|
-
&& markerMatches
|
|
86
|
-
&& checkpointIsNewer;
|
|
87
|
-
if ((checkpointGeneration !== sessionGeneration && !detachedGenerationHandoff)
|
|
72
|
+
if (checkpointGeneration !== sessionGeneration
|
|
88
73
|
|| (markerToken && !markerMatches)
|
|
89
74
|
|| (!markerToken && !checkpointIsNewer)) return source;
|
|
90
75
|
let start = -1;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Native mixdog-token client: a Node Worker owns the in-process Node-API addon
|
|
2
|
-
// so CPU-heavy counting never blocks the daemon event loop.
|
|
3
|
-
//
|
|
2
|
+
// so CPU-heavy counting never blocks the daemon event loop. The session
|
|
3
|
+
// runtime worker relays to this single owner over its existing IPC channel.
|
|
4
4
|
//
|
|
5
5
|
// Unavailable addon / dead worker / timeout all resolve `null`, and callers
|
|
6
6
|
// fall back to the WASM worker path. Kill switch:
|
|
@@ -23,16 +23,16 @@ const LOCAL_ADDON = pathJoin(
|
|
|
23
23
|
);
|
|
24
24
|
const READY_TIMEOUT_MS = 10_000;
|
|
25
25
|
const REQUEST_TIMEOUT_MS = 30_000;
|
|
26
|
-
const
|
|
27
|
-
const
|
|
28
|
-
const
|
|
26
|
+
const RUNTIME_PREWARM_MESSAGE = 'token-native-prewarm';
|
|
27
|
+
const RUNTIME_COUNT_MESSAGE = 'token-native-count';
|
|
28
|
+
const RUNTIME_RESULT_MESSAGE = 'token-native-result';
|
|
29
29
|
|
|
30
|
-
function
|
|
31
|
-
return process.env.
|
|
30
|
+
function isSessionRuntimeWorkerProcess() {
|
|
31
|
+
return process.env.MIXDOG_SESSION_RUNTIME_WORKER_PID === String(process.pid);
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
-
function
|
|
35
|
-
return
|
|
34
|
+
function sessionRuntimeClientEnabled() {
|
|
35
|
+
return isSessionRuntimeWorkerProcess()
|
|
36
36
|
&& typeof process.send === 'function'
|
|
37
37
|
&& process.connected === true;
|
|
38
38
|
}
|
|
@@ -66,61 +66,61 @@ function _resolveAddon() {
|
|
|
66
66
|
|
|
67
67
|
let _workerOwner = null; // { worker, pending, seq, ready, resolveReady, readyTimer }
|
|
68
68
|
let _workerFailed = false;
|
|
69
|
-
let
|
|
70
|
-
let
|
|
71
|
-
const
|
|
69
|
+
let _runtimeSequence = 0;
|
|
70
|
+
let _runtimeListenerInstalled = false;
|
|
71
|
+
const _runtimePending = new Map();
|
|
72
72
|
|
|
73
|
-
function
|
|
74
|
-
for (const entry of
|
|
73
|
+
function _dropRuntimeRequests() {
|
|
74
|
+
for (const entry of _runtimePending.values()) {
|
|
75
75
|
clearTimeout(entry.timer);
|
|
76
76
|
entry.resolve(null);
|
|
77
77
|
}
|
|
78
|
-
|
|
78
|
+
_runtimePending.clear();
|
|
79
79
|
}
|
|
80
80
|
|
|
81
|
-
function
|
|
82
|
-
if (
|
|
83
|
-
|
|
81
|
+
function _ensureRuntimeResponseListener() {
|
|
82
|
+
if (_runtimeListenerInstalled) return;
|
|
83
|
+
_runtimeListenerInstalled = true;
|
|
84
84
|
process.on('message', (message) => {
|
|
85
|
-
if (!message || message.type !==
|
|
85
|
+
if (!message || message.type !== RUNTIME_RESULT_MESSAGE) return;
|
|
86
86
|
const tokenRequestId = String(message.tokenRequestId || '');
|
|
87
|
-
const entry =
|
|
87
|
+
const entry = _runtimePending.get(tokenRequestId);
|
|
88
88
|
if (!entry) return;
|
|
89
|
-
|
|
89
|
+
_runtimePending.delete(tokenRequestId);
|
|
90
90
|
clearTimeout(entry.timer);
|
|
91
91
|
const count = Number(message.count);
|
|
92
92
|
entry.resolve(Number.isFinite(count) && count >= 0 ? count : null);
|
|
93
93
|
});
|
|
94
|
-
process.once('disconnect',
|
|
94
|
+
process.once('disconnect', _dropRuntimeRequests);
|
|
95
95
|
}
|
|
96
96
|
|
|
97
|
-
function
|
|
98
|
-
if (!
|
|
99
|
-
|
|
100
|
-
const tokenRequestId = `token-${process.pid}-${++
|
|
97
|
+
function _requestRuntimeTokenCount(text, timeoutMs = REQUEST_TIMEOUT_MS) {
|
|
98
|
+
if (!sessionRuntimeClientEnabled()) return Promise.resolve(null);
|
|
99
|
+
_ensureRuntimeResponseListener();
|
|
100
|
+
const tokenRequestId = `token-${process.pid}-${++_runtimeSequence}`;
|
|
101
101
|
return new Promise((resolve) => {
|
|
102
102
|
const timer = setTimeout(() => {
|
|
103
|
-
if (!
|
|
103
|
+
if (!_runtimePending.delete(tokenRequestId)) return;
|
|
104
104
|
resolve(null);
|
|
105
105
|
}, timeoutMs);
|
|
106
106
|
timer.unref?.();
|
|
107
|
-
|
|
107
|
+
_runtimePending.set(tokenRequestId, { resolve, timer });
|
|
108
108
|
if (!safeIpcSend(process, {
|
|
109
|
-
type:
|
|
109
|
+
type: RUNTIME_COUNT_MESSAGE,
|
|
110
110
|
tokenRequestId,
|
|
111
111
|
text: String(text ?? ''),
|
|
112
112
|
}, {
|
|
113
113
|
onError: () => {
|
|
114
|
-
const entry =
|
|
114
|
+
const entry = _runtimePending.get(tokenRequestId);
|
|
115
115
|
if (!entry) return;
|
|
116
|
-
|
|
116
|
+
_runtimePending.delete(tokenRequestId);
|
|
117
117
|
clearTimeout(entry.timer);
|
|
118
118
|
entry.resolve(null);
|
|
119
119
|
},
|
|
120
120
|
})) {
|
|
121
|
-
const entry =
|
|
121
|
+
const entry = _runtimePending.get(tokenRequestId);
|
|
122
122
|
if (entry) {
|
|
123
|
-
|
|
123
|
+
_runtimePending.delete(tokenRequestId);
|
|
124
124
|
clearTimeout(entry.timer);
|
|
125
125
|
entry.resolve(null);
|
|
126
126
|
}
|
|
@@ -241,17 +241,17 @@ function _requestWorker(owner, text, timeoutMs = REQUEST_TIMEOUT_MS) {
|
|
|
241
241
|
|
|
242
242
|
/** True when the native counter can be attempted (mode on + addon present). */
|
|
243
243
|
export function nativeTokenCounterEnabled() {
|
|
244
|
-
if (
|
|
245
|
-
return nativeTokenModeEnabled() &&
|
|
244
|
+
if (isSessionRuntimeWorkerProcess()) {
|
|
245
|
+
return nativeTokenModeEnabled() && sessionRuntimeClientEnabled();
|
|
246
246
|
}
|
|
247
247
|
return nativeTokenModeEnabled() && !_workerFailed && Boolean(_resolveAddon());
|
|
248
248
|
}
|
|
249
249
|
|
|
250
250
|
/** Precise o200k count via the daemon-owned addon worker. */
|
|
251
251
|
export async function countTokensNative(text) {
|
|
252
|
-
if (
|
|
252
|
+
if (isSessionRuntimeWorkerProcess()) {
|
|
253
253
|
if (!nativeTokenModeEnabled()) return null;
|
|
254
|
-
return
|
|
254
|
+
return _requestRuntimeTokenCount(text);
|
|
255
255
|
}
|
|
256
256
|
const owner = _ensureWorker();
|
|
257
257
|
if (!owner) return null;
|
|
@@ -267,9 +267,9 @@ let _fetchAttempted = false;
|
|
|
267
267
|
* With no local/env/cached addon, kick ONE background manifest fetch; on
|
|
268
268
|
* success the resolution cache resets so the next estimate adopts it. */
|
|
269
269
|
export function prewarmNativeTokenCounter() {
|
|
270
|
-
if (
|
|
271
|
-
if (!nativeTokenModeEnabled() || !
|
|
272
|
-
return safeIpcSend(process, { type:
|
|
270
|
+
if (isSessionRuntimeWorkerProcess()) {
|
|
271
|
+
if (!nativeTokenModeEnabled() || !sessionRuntimeClientEnabled()) return false;
|
|
272
|
+
return safeIpcSend(process, { type: RUNTIME_PREWARM_MESSAGE }, { onError: () => {} });
|
|
273
273
|
}
|
|
274
274
|
const owner = _ensureWorker();
|
|
275
275
|
if (!owner) {
|
|
@@ -4,7 +4,7 @@ import { tmpdir } from 'node:os';
|
|
|
4
4
|
import { join, resolve } from 'node:path';
|
|
5
5
|
import test from 'node:test';
|
|
6
6
|
|
|
7
|
-
import {
|
|
7
|
+
import { recordRuntimeDirectoryReadSuccess } from '../../../../shared/session-runtime-health.mjs';
|
|
8
8
|
import { executeListTool, executeTreeTool } from './list-tool.mjs';
|
|
9
9
|
|
|
10
10
|
function readdirError(code, message = 'injected readdir failure') {
|
|
@@ -32,14 +32,14 @@ test('aborted list work cannot poison the result cache', async () => {
|
|
|
32
32
|
}
|
|
33
33
|
});
|
|
34
34
|
|
|
35
|
-
test('root readdir failures are errors, stay uncached, and mark a repeatedly failing
|
|
36
|
-
const
|
|
35
|
+
test('root readdir failures are errors, stay uncached, and mark a repeatedly failing runtime worker unhealthy', async () => {
|
|
36
|
+
const originalRuntimeWorkerPid = process.env.MIXDOG_SESSION_RUNTIME_WORKER_PID;
|
|
37
37
|
const roots = [];
|
|
38
38
|
let unhealthy = null;
|
|
39
39
|
const onUnhealthy = (detail) => { unhealthy = detail; };
|
|
40
|
-
process.env.
|
|
41
|
-
process.on('mixdog:session-
|
|
42
|
-
|
|
40
|
+
process.env.MIXDOG_SESSION_RUNTIME_WORKER_PID = String(process.pid);
|
|
41
|
+
process.on('mixdog:session-runtime-worker-unhealthy', onUnhealthy);
|
|
42
|
+
recordRuntimeDirectoryReadSuccess();
|
|
43
43
|
try {
|
|
44
44
|
for (let index = 0; index < 3; index += 1) {
|
|
45
45
|
const root = await mkdtemp(join(tmpdir(), `mixdog-list-root-fail-${index}-`));
|
|
@@ -58,10 +58,10 @@ test('root readdir failures are errors, stay uncached, and mark a repeatedly fai
|
|
|
58
58
|
/entry-0\.txt\tfile/,
|
|
59
59
|
);
|
|
60
60
|
} finally {
|
|
61
|
-
|
|
62
|
-
process.off('mixdog:session-
|
|
63
|
-
if (
|
|
64
|
-
else process.env.
|
|
61
|
+
recordRuntimeDirectoryReadSuccess();
|
|
62
|
+
process.off('mixdog:session-runtime-worker-unhealthy', onUnhealthy);
|
|
63
|
+
if (originalRuntimeWorkerPid === undefined) delete process.env.MIXDOG_SESSION_RUNTIME_WORKER_PID;
|
|
64
|
+
else process.env.MIXDOG_SESSION_RUNTIME_WORKER_PID = originalRuntimeWorkerPid;
|
|
65
65
|
await Promise.all(roots.map((root) => rm(root, { recursive: true, force: true })));
|
|
66
66
|
}
|
|
67
67
|
});
|
|
@@ -48,9 +48,9 @@ import {
|
|
|
48
48
|
import { assertPathReachable } from './fs-reachability.mjs';
|
|
49
49
|
import { listGuardPath, normalizeListHeadLimit, readFamilyPathEnoentOrError } from './lib/list-helpers.mjs';
|
|
50
50
|
import {
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
} from '../../../../shared/session-
|
|
51
|
+
recordRuntimeDirectoryReadSuccess,
|
|
52
|
+
reportRuntimeDirectoryReadFailure,
|
|
53
|
+
} from '../../../../shared/session-runtime-health.mjs';
|
|
54
54
|
|
|
55
55
|
const FIND_WALK_TIMEOUT_MS = 20_000;
|
|
56
56
|
const LIST_WALK_TIMEOUT_MS = 20_000;
|
|
@@ -358,12 +358,12 @@ export async function executeListTool(args, workDir, options = {}) {
|
|
|
358
358
|
throwIfDirectoryWalkAborted(options.signal, walkResult, 'list walk');
|
|
359
359
|
const rootFailure = walkWarnings.find((warning) => warning.root);
|
|
360
360
|
if (rootFailure) {
|
|
361
|
-
|
|
361
|
+
reportRuntimeDirectoryReadFailure(fullPath, rootFailure.error);
|
|
362
362
|
recordDirectoryWalkTelemetry(options, 'failed', walkResult, walkWarnings.length);
|
|
363
363
|
if (options?.scopedCacheOutcome) markScopedCacheIncomplete(options.scopedCacheOutcome);
|
|
364
364
|
return directoryReadFailureLine(rootFailure);
|
|
365
365
|
}
|
|
366
|
-
|
|
366
|
+
recordRuntimeDirectoryReadSuccess();
|
|
367
367
|
|
|
368
368
|
if (!nativeDeep && needsGlobalStat && rows.length > 0) {
|
|
369
369
|
// lstat: symlinks should report own metadata, not the target's.
|
|
@@ -523,12 +523,12 @@ export async function executeTreeTool(args, workDir, options = {}) {
|
|
|
523
523
|
throwIfDirectoryWalkAborted(options.signal, walkResult, 'tree walk');
|
|
524
524
|
const rootFailure = walkWarnings.find((warning) => warning.root);
|
|
525
525
|
if (rootFailure) {
|
|
526
|
-
|
|
526
|
+
reportRuntimeDirectoryReadFailure(fullPath, rootFailure.error);
|
|
527
527
|
recordDirectoryWalkTelemetry(options, 'failed', walkResult, walkWarnings.length);
|
|
528
528
|
if (options?.scopedCacheOutcome) markScopedCacheIncomplete(options.scopedCacheOutcome);
|
|
529
529
|
return directoryReadFailureLine(rootFailure);
|
|
530
530
|
}
|
|
531
|
-
|
|
531
|
+
recordRuntimeDirectoryReadSuccess();
|
|
532
532
|
const root = lines[0];
|
|
533
533
|
const body = lines.slice(1);
|
|
534
534
|
const windowed = offset > 0 ? body.slice(offset) : body;
|
|
@@ -8,7 +8,7 @@ import { spawn } from 'node:child_process';
|
|
|
8
8
|
import { existsSync } from 'node:fs';
|
|
9
9
|
import { createInterface } from 'node:readline';
|
|
10
10
|
import { hiddenSpawnOpts } from '../../../../shared/spawn-flags.mjs';
|
|
11
|
-
import {
|
|
11
|
+
import { reportRuntimeWorkerUnhealthy } from '../../../../shared/session-runtime-health.mjs';
|
|
12
12
|
import { invalidateBuiltinResultCache } from './cache-layers.mjs';
|
|
13
13
|
import { getPluginData } from '../../config.mjs';
|
|
14
14
|
import { ensureGraphBinary } from '../graph-binary-fetcher.mjs';
|
|
@@ -103,8 +103,8 @@ function noteSearchTimeout(server, now = Date.now()) {
|
|
|
103
103
|
if (sincePrevious <= SEARCH_TIMEOUT_BURST_MS) return 'none';
|
|
104
104
|
// The server is no longer recycled on a first timeout, so a recurrence
|
|
105
105
|
// within the window — on this server or its replacement — is what marks the
|
|
106
|
-
//
|
|
107
|
-
return sincePrevious <= SEARCH_TIMEOUT_RECYCLE_WINDOW_MS ? '
|
|
106
|
+
// runtime worker unhealthy.
|
|
107
|
+
return sincePrevious <= SEARCH_TIMEOUT_RECYCLE_WINDOW_MS ? 'runtime' : 'server';
|
|
108
108
|
}
|
|
109
109
|
|
|
110
110
|
function subscribeAbortSignal(signal, callback) {
|
|
@@ -226,7 +226,7 @@ export function _bindNativeSearchServerLifecycle(child, { onError, onExit } = {}
|
|
|
226
226
|
child.on('exit', onExit);
|
|
227
227
|
// ChildProcess stdin emits write failures asynchronously. A sync try/catch
|
|
228
228
|
// around stdin.write cannot catch EPIPE; without this listener the session
|
|
229
|
-
//
|
|
229
|
+
// runtime worker itself terminates and every active/queued turn is recovered.
|
|
230
230
|
child.stdin?.on?.('error', onError);
|
|
231
231
|
}
|
|
232
232
|
|
|
@@ -480,8 +480,8 @@ async function requestNative(server, request, execOptions, deadlineMs) {
|
|
|
480
480
|
// already told it to stop, and the cancellation watchdog tears it down
|
|
481
481
|
// if it does not comply. Killing it here also destroyed every warm
|
|
482
482
|
// inventory, so the next call re-walked from cold and timed out again.
|
|
483
|
-
if (action !== '
|
|
484
|
-
|
|
483
|
+
if (action !== 'runtime') return;
|
|
484
|
+
reportRuntimeWorkerUnhealthy({
|
|
485
485
|
reason: 'native search timed out again after server recycle',
|
|
486
486
|
code: 'NATIVE_SEARCH_TIMEOUT_STREAK',
|
|
487
487
|
subsystem: 'native-search',
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
//
|
|
3
|
+
// Runtime-worker client for the MACHINE-WIDE child-spawn budget.
|
|
4
4
|
//
|
|
5
5
|
// child-spawn-gate's semaphore was written for the single-process daemon; the
|
|
6
|
-
// session-
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
// daemon-side pool grants leases from its OWN child-spawn-gate instance,
|
|
6
|
+
// session-runtime split creates another process-local copy, which would
|
|
7
|
+
// multiply the intended machine-wide cap. In the runtime worker the gate
|
|
8
|
+
// therefore forwards each
|
|
9
|
+
// acquire as a lease request over the existing runtime IPC channel; the
|
|
10
|
+
// daemon-side host grants leases from its OWN child-spawn-gate instance,
|
|
12
11
|
// which becomes the single machine-wide authority (daemon-hosted maintenance
|
|
13
12
|
// agents already use that instance locally). Transport loss degrades to the
|
|
14
13
|
// local per-process lane — bounded, never unbounded.
|
|
@@ -53,7 +52,7 @@ function _ensureListener() {
|
|
|
53
52
|
|
|
54
53
|
export function remoteSpawnLeasesEnabled(env = process.env) {
|
|
55
54
|
return !_broken
|
|
56
|
-
&& env.
|
|
55
|
+
&& env.MIXDOG_SESSION_RUNTIME_WORKER === '1'
|
|
57
56
|
&& env.MIXDOG_DISABLE_MACHINE_SPAWN_BUDGET !== '1'
|
|
58
57
|
&& typeof process.send === 'function'
|
|
59
58
|
&& process.connected === true;
|
|
@@ -255,6 +255,26 @@ export class ResourceAdmissionController {
|
|
|
255
255
|
return this.context.run(lease, task);
|
|
256
256
|
}
|
|
257
257
|
|
|
258
|
+
/**
|
|
259
|
+
* Yield the current agent lease while waiting on remote I/O, then reacquire
|
|
260
|
+
* it before the caller continues local processing. Non-agent callers are a
|
|
261
|
+
* no-op. Nested yields share the same dependency depth and restore once.
|
|
262
|
+
*/
|
|
263
|
+
async runYielded(task) {
|
|
264
|
+
if (typeof task !== 'function') return Promise.resolve().then(task);
|
|
265
|
+
const lease = this.context.getStore();
|
|
266
|
+
if (!lease || lease.controller !== this || lease.kind !== 'agent' || lease.released) {
|
|
267
|
+
return Promise.resolve().then(task);
|
|
268
|
+
}
|
|
269
|
+
const suspended = this._suspendParent(lease);
|
|
270
|
+
this._drain();
|
|
271
|
+
try {
|
|
272
|
+
return await task();
|
|
273
|
+
} finally {
|
|
274
|
+
await this._resumeParent(suspended);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
|
|
258
278
|
acquire(kind, {
|
|
259
279
|
signal = null, label = null, dependency = 'scoped', ownerKey = null,
|
|
260
280
|
priority = 'user-visible',
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import test from 'node:test';
|
|
3
|
+
|
|
4
|
+
import { ResourceAdmissionController } from './resource-admission.mjs';
|
|
5
|
+
|
|
6
|
+
test('provider-style waits yield and reacquire the current agent lease', async () => {
|
|
7
|
+
const admission = new ResourceAdmissionController({
|
|
8
|
+
limits: {
|
|
9
|
+
maxAgents: 1,
|
|
10
|
+
maxShells: 1,
|
|
11
|
+
maxHighLoad: 1,
|
|
12
|
+
maxQueue: 8,
|
|
13
|
+
},
|
|
14
|
+
});
|
|
15
|
+
const first = await admission.acquire('agent', { ownerKey: 'first' });
|
|
16
|
+
const waitStarted = Promise.withResolvers();
|
|
17
|
+
const finishWait = Promise.withResolvers();
|
|
18
|
+
let yieldedFinished = false;
|
|
19
|
+
|
|
20
|
+
const yielding = admission.runWithLease(first, () => admission.runYielded(async () => {
|
|
21
|
+
waitStarted.resolve();
|
|
22
|
+
await finishWait.promise;
|
|
23
|
+
return 'provider-result';
|
|
24
|
+
})).then((value) => {
|
|
25
|
+
yieldedFinished = true;
|
|
26
|
+
return value;
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
await waitStarted.promise;
|
|
30
|
+
assert.equal(admission.snapshot().active.agent, 0);
|
|
31
|
+
const second = await admission.acquire('agent', { ownerKey: 'second' });
|
|
32
|
+
assert.equal(admission.snapshot().active.agent, 1);
|
|
33
|
+
|
|
34
|
+
finishWait.resolve();
|
|
35
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
36
|
+
assert.equal(yieldedFinished, false, 'the first continuation waits to reacquire its slot');
|
|
37
|
+
|
|
38
|
+
await second.release();
|
|
39
|
+
assert.equal(await yielding, 'provider-result');
|
|
40
|
+
assert.equal(admission.snapshot().active.agent, 1);
|
|
41
|
+
await first.release();
|
|
42
|
+
assert.equal(admission.snapshot().active.agent, 0);
|
|
43
|
+
});
|