agent-dealer 1.2.9 → 1.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle/server/dist/adapters/github.js +13 -0
- package/bundle/server/dist/adapters/github.test.js +13 -0
- package/bundle/server/dist/coordinator/commands.js +147 -0
- package/bundle/server/dist/coordinator/developer-effect.js +93 -1
- package/bundle/server/dist/coordinator/developer-effect.test.js +172 -0
- package/bundle/server/dist/coordinator/merge-conflict-sync.js +100 -12
- package/bundle/server/dist/coordinator/merge-conflict-sync.test.js +190 -0
- package/bundle/server/dist/coordinator/muse-spawn.js +22 -4
- package/bundle/server/dist/coordinator/muse-spawn.test.js +166 -10
- package/bundle/server/dist/coordinator/routing.js +10 -0
- package/bundle/server/dist/coordinator/session-timeouts.js +22 -0
- package/bundle/server/dist/runners/muse-code-jsonl.js +61 -0
- package/bundle/server/dist/runners/muse-code.test.js +13 -1
- package/bundle/server/dist/runners/spawn-cli.js +93 -20
- package/bundle/server/dist/runners/spawn-cli.test.js +38 -0
- package/bundle/server/package.json +2 -2
- package/bundle/shared/package.json +1 -1
- package/package.json +1 -1
|
@@ -2,30 +2,36 @@
|
|
|
2
2
|
// arrival-stamped normalized events, and the five-field stall metadata object.
|
|
3
3
|
// The full `runMuseDeveloperSession` needs a Muse binary + credentials, so these
|
|
4
4
|
// test the pure pieces it wires together (plus the parser contract it relies on).
|
|
5
|
-
import { test } from "node:test";
|
|
5
|
+
import { test, after } from "node:test";
|
|
6
6
|
import assert from "node:assert/strict";
|
|
7
|
+
import { execFileSync } from "node:child_process";
|
|
7
8
|
import fs from "node:fs";
|
|
8
9
|
import os from "node:os";
|
|
9
10
|
import path from "node:path";
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
11
|
+
import { fileURLToPath } from "node:url";
|
|
12
|
+
import { MUSE_CODE_CONTRIBUTOR_MODEL } from "@agent-dealer/shared";
|
|
13
|
+
import { parseMuseRun, isMuseTerminalStdoutLine, createMusePrimaryTerminalMatcher, } from "../runners/muse-code-jsonl.js";
|
|
14
|
+
import { museIdleTimeoutMs, museTerminalGraceMs, DEFAULT_MUSE_IDLE_TIMEOUT_MS, DEFAULT_MUSE_TERMINAL_GRACE_MS, } from "./session-timeouts.js";
|
|
15
|
+
import { museIdleMinutes, museStallMetadata, runMuseDeveloperSession, sessionLogActivityMs, writeMuseNormalizedLog, } from "./muse-spawn.js";
|
|
13
16
|
const MODEL = "muse-spark-1.3-contributor";
|
|
14
17
|
// ── MUSE_IDLE_TIMEOUT_MS parsing ─────────────────────────────────────────────
|
|
15
18
|
function withIdleEnv(raw, fn) {
|
|
16
|
-
|
|
19
|
+
withEnv("MUSE_IDLE_TIMEOUT_MS", raw, fn);
|
|
20
|
+
}
|
|
21
|
+
function withEnv(name, raw, fn) {
|
|
22
|
+
const prev = process.env[name];
|
|
17
23
|
try {
|
|
18
24
|
if (raw === undefined)
|
|
19
|
-
delete process.env
|
|
25
|
+
delete process.env[name];
|
|
20
26
|
else
|
|
21
|
-
process.env
|
|
27
|
+
process.env[name] = raw;
|
|
22
28
|
fn();
|
|
23
29
|
}
|
|
24
30
|
finally {
|
|
25
31
|
if (prev === undefined)
|
|
26
|
-
delete process.env
|
|
32
|
+
delete process.env[name];
|
|
27
33
|
else
|
|
28
|
-
process.env
|
|
34
|
+
process.env[name] = prev;
|
|
29
35
|
}
|
|
30
36
|
}
|
|
31
37
|
test("museIdleTimeoutMs defaults to 30 minutes when unset or blank", () => {
|
|
@@ -97,19 +103,21 @@ test("museIdleMinutes measures silence from last activity, else the bound", () =
|
|
|
97
103
|
assert.equal(museIdleMinutes({ lastActivityAt: null, idleTimeoutMs: undefined }), null, "disabled watchdog has no idle kill to explain");
|
|
98
104
|
});
|
|
99
105
|
// ── stall metadata object ────────────────────────────────────────────────────
|
|
100
|
-
test("museStallMetadata carries
|
|
106
|
+
test("museStallMetadata carries stall-evidence fields including lingeredAfterTerminal", () => {
|
|
101
107
|
assert.deepEqual(museStallMetadata({
|
|
102
108
|
lastActivityAt: "2026-10-01T19:00:00.000Z",
|
|
103
109
|
toolCallCount: 24,
|
|
104
110
|
lastToolName: "npm_test",
|
|
105
111
|
firstOutputMs: 1234,
|
|
106
112
|
idleTimedOut: true,
|
|
113
|
+
lingeredAfterTerminal: false,
|
|
107
114
|
}), {
|
|
108
115
|
lastActivityAt: "2026-10-01T19:00:00.000Z",
|
|
109
116
|
toolCallCount: 24,
|
|
110
117
|
lastToolName: "npm_test",
|
|
111
118
|
firstOutputMs: 1234,
|
|
112
119
|
idleTimedOut: true,
|
|
120
|
+
lingeredAfterTerminal: false,
|
|
113
121
|
});
|
|
114
122
|
});
|
|
115
123
|
// ── arrival-stamped normalized events ────────────────────────────────────────
|
|
@@ -185,6 +193,7 @@ test("the written ndjson carries ts on every event and durationMs on completed t
|
|
|
185
193
|
firstOutputMs: 900,
|
|
186
194
|
lastActivityAt: new Date(t0 + 63_000).toISOString(),
|
|
187
195
|
idleTimedOut: false,
|
|
196
|
+
lingeredAfterTerminal: false,
|
|
188
197
|
toolCallCount: r.tools.length,
|
|
189
198
|
lastToolName: "npm_test",
|
|
190
199
|
};
|
|
@@ -206,5 +215,152 @@ test("the written ndjson carries ts on every event and durationMs on completed t
|
|
|
206
215
|
assert.equal(muse.toolCallCount, 2);
|
|
207
216
|
assert.equal(muse.lastToolName, "npm_test");
|
|
208
217
|
assert.equal(muse.idleTimedOut, false);
|
|
218
|
+
assert.equal(muse.lingeredAfterTerminal, false);
|
|
209
219
|
assert.equal(muse.lastActivityAt, new Date(t0 + 63_000).toISOString());
|
|
210
220
|
});
|
|
221
|
+
// ── MUSE_TERMINAL_GRACE_MS parsing ───────────────────────────────────────────
|
|
222
|
+
test("museTerminalGraceMs defaults to 30 seconds when unset or blank", () => {
|
|
223
|
+
withEnv("MUSE_TERMINAL_GRACE_MS", undefined, () => assert.equal(museTerminalGraceMs(), DEFAULT_MUSE_TERMINAL_GRACE_MS));
|
|
224
|
+
withEnv("MUSE_TERMINAL_GRACE_MS", "", () => assert.equal(museTerminalGraceMs(), DEFAULT_MUSE_TERMINAL_GRACE_MS));
|
|
225
|
+
withEnv("MUSE_TERMINAL_GRACE_MS", " ", () => assert.equal(museTerminalGraceMs(), DEFAULT_MUSE_TERMINAL_GRACE_MS));
|
|
226
|
+
assert.equal(DEFAULT_MUSE_TERMINAL_GRACE_MS, 30_000);
|
|
227
|
+
});
|
|
228
|
+
test("MUSE_TERMINAL_GRACE_MS=0 disables the post-terminal kill", () => {
|
|
229
|
+
withEnv("MUSE_TERMINAL_GRACE_MS", "0", () => assert.equal(museTerminalGraceMs(), undefined));
|
|
230
|
+
});
|
|
231
|
+
test("isMuseTerminalStdoutLine matches completed and failed envelopes only", () => {
|
|
232
|
+
assert.equal(isMuseTerminalStdoutLine(JSON.stringify({ payload_type: "run.terminal.failed", payload: { terminal: "failed", reason: "x" } })), true);
|
|
233
|
+
assert.equal(isMuseTerminalStdoutLine(JSON.stringify({ payload_type: "run.terminal.completed", payload: {} })), true);
|
|
234
|
+
assert.equal(isMuseTerminalStdoutLine(JSON.stringify({ payload_type: "tool.result", payload: {} })), false);
|
|
235
|
+
assert.equal(isMuseTerminalStdoutLine("not json"), false);
|
|
236
|
+
});
|
|
237
|
+
// ── NOT-342: linger after terminal, through the Muse spawn ───────────────────
|
|
238
|
+
const FAKE_MUSE = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures", "fake-muse.mjs");
|
|
239
|
+
const LINGER_HOME = fs.mkdtempSync(path.join(os.homedir(), ".dealer-muse-linger-"));
|
|
240
|
+
after(() => {
|
|
241
|
+
fs.rmSync(LINGER_HOME, { recursive: true, force: true });
|
|
242
|
+
});
|
|
243
|
+
function lingerScratchWorktree() {
|
|
244
|
+
const wt = fs.mkdtempSync(path.join(LINGER_HOME, "wt-"));
|
|
245
|
+
execFileSync("git", ["init", "-q", "-b", "main"], { cwd: wt });
|
|
246
|
+
execFileSync("git", ["config", "user.email", "t@example.com"], { cwd: wt });
|
|
247
|
+
execFileSync("git", ["config", "user.name", "T"], { cwd: wt });
|
|
248
|
+
fs.writeFileSync(path.join(wt, "a.txt"), "a\n");
|
|
249
|
+
execFileSync("git", ["add", "."], { cwd: wt });
|
|
250
|
+
execFileSync("git", ["-c", "user.email=t@example.com", "-c", "user.name=T", "commit", "-q", "-m", "init"], {
|
|
251
|
+
cwd: wt,
|
|
252
|
+
});
|
|
253
|
+
return wt;
|
|
254
|
+
}
|
|
255
|
+
async function runLingeringMuse(opts) {
|
|
256
|
+
const keys = [
|
|
257
|
+
"MUSE_CLI",
|
|
258
|
+
"FAKE_MUSE_SCENARIO",
|
|
259
|
+
"META_API_KEY",
|
|
260
|
+
"AGENT_DEALER_HOME",
|
|
261
|
+
"MUSE_TERMINAL_GRACE_MS",
|
|
262
|
+
"MUSE_IDLE_TIMEOUT_MS",
|
|
263
|
+
];
|
|
264
|
+
const prev = Object.fromEntries(keys.map((k) => [k, process.env[k]]));
|
|
265
|
+
const wt = lingerScratchWorktree();
|
|
266
|
+
const dealerHome = fs.mkdtempSync(path.join(LINGER_HOME, "home-"));
|
|
267
|
+
const startedAt = Date.now();
|
|
268
|
+
try {
|
|
269
|
+
process.env.MUSE_CLI = FAKE_MUSE;
|
|
270
|
+
process.env.FAKE_MUSE_SCENARIO = opts.scenario;
|
|
271
|
+
process.env.META_API_KEY = "mk-test-linger-key";
|
|
272
|
+
process.env.AGENT_DEALER_HOME = dealerHome;
|
|
273
|
+
process.env.MUSE_TERMINAL_GRACE_MS = opts.graceMs;
|
|
274
|
+
process.env.MUSE_IDLE_TIMEOUT_MS = "0";
|
|
275
|
+
const result = await runMuseDeveloperSession({
|
|
276
|
+
sessionId: opts.sessionId,
|
|
277
|
+
runtime: "muse_code",
|
|
278
|
+
policy: { worktreeWrite: true },
|
|
279
|
+
model: MUSE_CODE_CONTRIBUTOR_MODEL,
|
|
280
|
+
deckId: "00000000-0000-4000-a000-000000000099",
|
|
281
|
+
agentDeckUrl: "http://127.0.0.1:1110/mcp",
|
|
282
|
+
prompt: "Implement it",
|
|
283
|
+
cwd: wt,
|
|
284
|
+
timeoutMs: opts.timeoutMs,
|
|
285
|
+
logPath: path.join(dealerHome, `muse-${opts.sessionId}.ndjson`),
|
|
286
|
+
});
|
|
287
|
+
return {
|
|
288
|
+
timedOut: result.timedOut,
|
|
289
|
+
// Assert top-level and muse summary separately — a regression that drops
|
|
290
|
+
// either field must fail (do not coalesce with ??).
|
|
291
|
+
lingeredAfterTerminal: result.lingeredAfterTerminal,
|
|
292
|
+
museLingeredAfterTerminal: result.muse?.lingeredAfterTerminal,
|
|
293
|
+
failureMessage: result.muse?.failure?.message ?? null,
|
|
294
|
+
failureKind: result.muse?.failure?.kind ?? null,
|
|
295
|
+
elapsedMs: Date.now() - startedAt,
|
|
296
|
+
logPath: result.logPath,
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
finally {
|
|
300
|
+
for (const k of keys) {
|
|
301
|
+
if (prev[k] === undefined)
|
|
302
|
+
delete process.env[k];
|
|
303
|
+
else
|
|
304
|
+
process.env[k] = prev[k];
|
|
305
|
+
}
|
|
306
|
+
fs.rmSync(wt, { recursive: true, force: true });
|
|
307
|
+
fs.rmSync(dealerHome, { recursive: true, force: true });
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
test("a Muse transport-error that lingers is recorded as a failure, not a timeout", { timeout: 15_000 }, async () => {
|
|
311
|
+
const out = await runLingeringMuse({
|
|
312
|
+
scenario: "transport-linger",
|
|
313
|
+
graceMs: "200",
|
|
314
|
+
timeoutMs: 20_000,
|
|
315
|
+
sessionId: "00000000-0000-4000-8000-000000000342",
|
|
316
|
+
});
|
|
317
|
+
assert.ok(out.elapsedMs < 5_000, `settled on the grace, took ${out.elapsedMs}ms`);
|
|
318
|
+
assert.equal(out.timedOut, false);
|
|
319
|
+
assert.equal(out.lingeredAfterTerminal, true);
|
|
320
|
+
assert.equal(out.museLingeredAfterTerminal, true);
|
|
321
|
+
assert.match(out.failureMessage ?? "", /transport error/);
|
|
322
|
+
});
|
|
323
|
+
test("a completed Muse run that lingers is still a completion, not a timeout", { timeout: 15_000 }, async () => {
|
|
324
|
+
const out = await runLingeringMuse({
|
|
325
|
+
scenario: "completed-linger",
|
|
326
|
+
graceMs: "200",
|
|
327
|
+
timeoutMs: 20_000,
|
|
328
|
+
sessionId: "00000000-0000-4000-8000-000000000344",
|
|
329
|
+
});
|
|
330
|
+
assert.equal(out.timedOut, false);
|
|
331
|
+
assert.equal(out.lingeredAfterTerminal, true);
|
|
332
|
+
assert.equal(out.museLingeredAfterTerminal, true);
|
|
333
|
+
assert.equal(out.failureMessage, null);
|
|
334
|
+
});
|
|
335
|
+
test("MUSE_TERMINAL_GRACE_MS=0 leaves a lingering Muse child to the wall clock", { timeout: 15_000 }, async () => {
|
|
336
|
+
const out = await runLingeringMuse({
|
|
337
|
+
scenario: "transport-linger",
|
|
338
|
+
graceMs: "0",
|
|
339
|
+
timeoutMs: 500,
|
|
340
|
+
sessionId: "00000000-0000-4000-8000-000000000343",
|
|
341
|
+
});
|
|
342
|
+
assert.equal(out.lingeredAfterTerminal, false);
|
|
343
|
+
assert.equal(out.museLingeredAfterTerminal, false);
|
|
344
|
+
assert.equal(out.timedOut, true, "without the grace, the wall clock still kills");
|
|
345
|
+
});
|
|
346
|
+
// NOT-342 repair: a cron run's terminal must not arm the 30s grace and SIGTERM the
|
|
347
|
+
// still-working primary. The child has no primary terminal — only the wall clock kills.
|
|
348
|
+
test("a cron terminal before the primary does not arm terminal grace", { timeout: 15_000 }, async () => {
|
|
349
|
+
const out = await runLingeringMuse({
|
|
350
|
+
scenario: "cron-terminal-then-primary-linger",
|
|
351
|
+
graceMs: "200",
|
|
352
|
+
timeoutMs: 800,
|
|
353
|
+
sessionId: "00000000-0000-4000-8000-000000000345",
|
|
354
|
+
});
|
|
355
|
+
assert.equal(out.lingeredAfterTerminal, false, "cron terminal must not linger-kill");
|
|
356
|
+
assert.equal(out.museLingeredAfterTerminal, false);
|
|
357
|
+
assert.equal(out.timedOut, true, "wall clock still owns the kill when primary never terminals");
|
|
358
|
+
assert.notEqual(out.failureKind, "malformed_stream");
|
|
359
|
+
});
|
|
360
|
+
test("createMusePrimaryTerminalMatcher is what Muse wires for terminalGrace", () => {
|
|
361
|
+
// Sanity: the factory exists and matches the shape spawnCli expects.
|
|
362
|
+
const m = createMusePrimaryTerminalMatcher();
|
|
363
|
+
assert.equal(typeof m.isTerminalLine, "function");
|
|
364
|
+
assert.equal(typeof m.sawPrimaryTerminal, "function");
|
|
365
|
+
assert.equal(m.sawPrimaryTerminal(), false);
|
|
366
|
+
});
|
|
@@ -176,6 +176,16 @@ export function routeDeveloperOutcome(outcome, limits) {
|
|
|
176
176
|
// session ended cleanly with a pushed branch and an open PR. Wait on backoff
|
|
177
177
|
// and re-poll; the deferral path (not this route) enforces the wait ceiling.
|
|
178
178
|
return { next: "defer_work", reason: outcome.reason };
|
|
179
|
+
case "checks_conflicting":
|
|
180
|
+
// NOT-354: applyCompletion handles this before routing (it needs the lease to
|
|
181
|
+
// queue the repair round). Reaching here is a bug — fail safe to a human.
|
|
182
|
+
return {
|
|
183
|
+
next: "human_action",
|
|
184
|
+
actionType: "policy_escalation",
|
|
185
|
+
reason: outcome.sync.result === "escalate"
|
|
186
|
+
? outcome.sync.reason
|
|
187
|
+
: `PR #${outcome.prNumber} conflicts with ${outcome.baseBranch}; CI cannot start.`,
|
|
188
|
+
};
|
|
179
189
|
}
|
|
180
190
|
}
|
|
181
191
|
function infraFailureReason(outcome) {
|
|
@@ -35,6 +35,28 @@ export function museIdleTimeoutMs() {
|
|
|
35
35
|
}
|
|
36
36
|
return parsed;
|
|
37
37
|
}
|
|
38
|
+
/**
|
|
39
|
+
* NOT-342: how long Muse may linger after a `run.terminal` envelope before Dealer
|
|
40
|
+
* SIGTERMs it. Ships ENABLED at 30 seconds: long enough for a clean self-exit,
|
|
41
|
+
* short enough that a provider `transport error` is not held until the 1-hour
|
|
42
|
+
* wall (and then recorded as `timed_out`). `0` disables. A non-numeric or
|
|
43
|
+
* negative value falls back to the default with a logged warning, matching
|
|
44
|
+
* `museIdleTimeoutMs`.
|
|
45
|
+
*/
|
|
46
|
+
export const DEFAULT_MUSE_TERMINAL_GRACE_MS = 30_000;
|
|
47
|
+
export function museTerminalGraceMs() {
|
|
48
|
+
const raw = process.env.MUSE_TERMINAL_GRACE_MS;
|
|
49
|
+
if (raw === undefined || raw.trim() === "")
|
|
50
|
+
return DEFAULT_MUSE_TERMINAL_GRACE_MS;
|
|
51
|
+
const parsed = Number(raw);
|
|
52
|
+
if (Number.isFinite(parsed) && parsed === 0)
|
|
53
|
+
return undefined;
|
|
54
|
+
if (!Number.isFinite(parsed) || parsed < 0) {
|
|
55
|
+
console.warn(`[dealer] ignoring invalid MUSE_TERMINAL_GRACE_MS=${JSON.stringify(raw)} — using default ${DEFAULT_MUSE_TERMINAL_GRACE_MS}`);
|
|
56
|
+
return DEFAULT_MUSE_TERMINAL_GRACE_MS;
|
|
57
|
+
}
|
|
58
|
+
return parsed;
|
|
59
|
+
}
|
|
38
60
|
export function sessionTimeoutMsFor(role) {
|
|
39
61
|
return role === "reviewer" ? reviewerSessionTimeoutMs() : developerSessionTimeoutMs();
|
|
40
62
|
}
|
|
@@ -22,6 +22,25 @@ function num(v) {
|
|
|
22
22
|
function cap(s) {
|
|
23
23
|
return s.length > ERROR_TEXT_MAX ? `${s.slice(0, ERROR_TEXT_MAX)}…` : s;
|
|
24
24
|
}
|
|
25
|
+
/**
|
|
26
|
+
* NOT-342: true when a stdout line is a Muse `run.terminal.completed` or
|
|
27
|
+
* `run.terminal.failed` envelope (any run). Prefer
|
|
28
|
+
* {@link createMusePrimaryTerminalMatcher} for `spawnCli` terminal grace — a
|
|
29
|
+
* cron run's terminal must not arm the kill. Garbage lines are not terminal
|
|
30
|
+
* (never throw).
|
|
31
|
+
*/
|
|
32
|
+
export function isMuseTerminalStdoutLine(line) {
|
|
33
|
+
const trimmed = line.trim();
|
|
34
|
+
if (!trimmed.startsWith("{"))
|
|
35
|
+
return false;
|
|
36
|
+
try {
|
|
37
|
+
const env = JSON.parse(trimmed);
|
|
38
|
+
return env.payload_type === "run.terminal.completed" || env.payload_type === "run.terminal.failed";
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
25
44
|
function firstLine(s) {
|
|
26
45
|
return s.trim().split("\n")[0] ?? "";
|
|
27
46
|
}
|
|
@@ -143,6 +162,48 @@ function runIdOf(payload) {
|
|
|
143
162
|
const rs = asRecord(payload.run_stream);
|
|
144
163
|
return rs?.kind === "run" ? str(rs.id) : undefined;
|
|
145
164
|
}
|
|
165
|
+
/**
|
|
166
|
+
* NOT-342: stateful predicate for Muse `terminalGrace`. Tracks the first-seen
|
|
167
|
+
* run id with the same rule as {@link parseMuseRun}, and returns true only when
|
|
168
|
+
* that primary run's terminal envelope arrives. A cron (or other secondary) run
|
|
169
|
+
* completing first must not start the grace — otherwise Dealer SIGTERMs the
|
|
170
|
+
* still-working primary and the session is mis-recorded as `malformed_stream`.
|
|
171
|
+
*/
|
|
172
|
+
export function createMusePrimaryTerminalMatcher() {
|
|
173
|
+
let primaryRunId = null;
|
|
174
|
+
let sawPrimary = false;
|
|
175
|
+
return {
|
|
176
|
+
sawPrimaryTerminal: () => sawPrimary,
|
|
177
|
+
isTerminalLine(line) {
|
|
178
|
+
const trimmed = line.trim();
|
|
179
|
+
if (!trimmed.startsWith("{"))
|
|
180
|
+
return false;
|
|
181
|
+
let parsed;
|
|
182
|
+
try {
|
|
183
|
+
parsed = JSON.parse(trimmed);
|
|
184
|
+
}
|
|
185
|
+
catch {
|
|
186
|
+
return false;
|
|
187
|
+
}
|
|
188
|
+
const env = toEnvelope(parsed);
|
|
189
|
+
if (!env)
|
|
190
|
+
return false;
|
|
191
|
+
const runId = runIdOf(env.payload);
|
|
192
|
+
// Same first-run-id rule as parseMuseRun: the first envelope that names a
|
|
193
|
+
// run wins; envelopes with no run id leave primary unset (any later terminal
|
|
194
|
+
// with no correlating id is accepted, matching parseMuseRun).
|
|
195
|
+
if (primaryRunId === null)
|
|
196
|
+
primaryRunId = runId ?? null;
|
|
197
|
+
if (env.payloadType !== "run.terminal.completed" && env.payloadType !== "run.terminal.failed") {
|
|
198
|
+
return false;
|
|
199
|
+
}
|
|
200
|
+
if (primaryRunId !== null && runId !== primaryRunId)
|
|
201
|
+
return false;
|
|
202
|
+
sawPrimary = true;
|
|
203
|
+
return true;
|
|
204
|
+
},
|
|
205
|
+
};
|
|
206
|
+
}
|
|
146
207
|
function hasRateLimitFacet(payload) {
|
|
147
208
|
const details = asRecord(asRecord(payload.event)?.details);
|
|
148
209
|
const facets = details?.facets;
|
|
@@ -4,7 +4,7 @@ import { readFileSync } from "node:fs";
|
|
|
4
4
|
import { dirname, join } from "node:path";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
6
|
import { buildMuseDeveloperInvocation } from "./muse-code-args.js";
|
|
7
|
-
import { parseMuseRun } from "./muse-code-jsonl.js";
|
|
7
|
+
import { parseMuseRun, createMusePrimaryTerminalMatcher } from "./muse-code-jsonl.js";
|
|
8
8
|
import { extractResultText, extractSessionId, extractUsage } from "./stream-json.js";
|
|
9
9
|
// NOT-179: developer argv + JSONL parser, table-driven over the sanitized NOT-177 fixtures.
|
|
10
10
|
const DIR = join(dirname(fileURLToPath(import.meta.url)), "fixtures", "muse-code");
|
|
@@ -157,6 +157,18 @@ test("terminals are selected by the primary run id, not by arrival order", () =>
|
|
|
157
157
|
assert.equal(r.usage.modelDurationMs, 3);
|
|
158
158
|
assert.equal(r.failure, null);
|
|
159
159
|
});
|
|
160
|
+
// NOT-342: terminalGrace must follow the same primary-run rule — a cron terminal
|
|
161
|
+
// arriving first must not arm the kill while the primary is still working.
|
|
162
|
+
test("createMusePrimaryTerminalMatcher ignores a cron terminal before the primary's", () => {
|
|
163
|
+
const matcher = createMusePrimaryTerminalMatcher();
|
|
164
|
+
const lines = interleavedStdout.split("\n");
|
|
165
|
+
assert.equal(matcher.isTerminalLine(lines[0]), false, "primary linked");
|
|
166
|
+
assert.equal(matcher.isTerminalLine(lines[1]), false, "cron linked");
|
|
167
|
+
assert.equal(matcher.isTerminalLine(lines[2]), false, "cron terminal must not arm grace");
|
|
168
|
+
assert.equal(matcher.sawPrimaryTerminal(), false);
|
|
169
|
+
assert.equal(matcher.isTerminalLine(lines[3]), true, "primary terminal arms grace");
|
|
170
|
+
assert.equal(matcher.sawPrimaryTerminal(), true);
|
|
171
|
+
});
|
|
160
172
|
test("a cron run's model_completed cannot confirm the primary run's model", () => {
|
|
161
173
|
const log = modelCompletedLine("cron", { usage: { input_tokens: 1 } });
|
|
162
174
|
const r = parseMuseRun({ stdout: interleavedStdout, exitCode: 0, sessionLog: log, expectedModel: MODEL });
|
|
@@ -54,6 +54,7 @@ export async function spawnCli(runId, cmd, args, cwd, opts) {
|
|
|
54
54
|
const stderrChunks = [];
|
|
55
55
|
let timedOut = false;
|
|
56
56
|
let settled = false;
|
|
57
|
+
let lingeredAfterTerminal = false;
|
|
57
58
|
let killEscalation;
|
|
58
59
|
// NOT-307: idle watchdog state. lastActivityMs starts at spawn so a child that
|
|
59
60
|
// never emits anything is killed after exactly idleTimeoutMs. stdout bytes always
|
|
@@ -67,6 +68,14 @@ export async function spawnCli(runId, cmd, args, cwd, opts) {
|
|
|
67
68
|
const idleEnabled = typeof idleMs === "number" && Number.isFinite(idleMs) && idleMs > 0;
|
|
68
69
|
let idleTimer;
|
|
69
70
|
let progressTimer;
|
|
71
|
+
let terminalGraceTimer;
|
|
72
|
+
let lingerSettleTimer;
|
|
73
|
+
let terminalGraceArmed = false;
|
|
74
|
+
const terminalGraceMs = opts.terminalGrace?.graceMs;
|
|
75
|
+
const terminalGraceEnabled = typeof terminalGraceMs === "number" &&
|
|
76
|
+
Number.isFinite(terminalGraceMs) &&
|
|
77
|
+
terminalGraceMs > 0 &&
|
|
78
|
+
typeof opts.terminalGrace?.isTerminalLine === "function";
|
|
70
79
|
const clearIdle = () => {
|
|
71
80
|
if (idleTimer)
|
|
72
81
|
clearTimeout(idleTimer);
|
|
@@ -75,8 +84,16 @@ export async function spawnCli(runId, cmd, args, cwd, opts) {
|
|
|
75
84
|
idleTimer = undefined;
|
|
76
85
|
progressTimer = undefined;
|
|
77
86
|
};
|
|
87
|
+
const clearTerminalGrace = () => {
|
|
88
|
+
if (terminalGraceTimer)
|
|
89
|
+
clearTimeout(terminalGraceTimer);
|
|
90
|
+
terminalGraceTimer = undefined;
|
|
91
|
+
if (lingerSettleTimer)
|
|
92
|
+
clearTimeout(lingerSettleTimer);
|
|
93
|
+
lingerSettleTimer = undefined;
|
|
94
|
+
};
|
|
78
95
|
const rescheduleIdle = () => {
|
|
79
|
-
if (!idleEnabled || settled || aborted)
|
|
96
|
+
if (!idleEnabled || settled || aborted || lingeredAfterTerminal)
|
|
80
97
|
return;
|
|
81
98
|
if (idleTimer)
|
|
82
99
|
clearTimeout(idleTimer);
|
|
@@ -99,13 +116,69 @@ export async function spawnCli(runId, cmd, args, cwd, opts) {
|
|
|
99
116
|
// later) so salvage and infra-retry accounting treat an idle kill as a timeout.
|
|
100
117
|
// Never fires after an abort: a lost lease is not idleness.
|
|
101
118
|
function fireIdle() {
|
|
102
|
-
if (settled || aborted)
|
|
119
|
+
if (settled || aborted || lingeredAfterTerminal)
|
|
103
120
|
return;
|
|
104
121
|
idleTimedOut = true;
|
|
105
122
|
timedOut = true;
|
|
106
123
|
killRunProcess(runId);
|
|
107
124
|
setTimeout(() => finish(124), 500);
|
|
108
125
|
}
|
|
126
|
+
// Same polite SIGTERM → SIGKILL backstop the abort path uses: the CLI should
|
|
127
|
+
// reap its own children on SIGTERM. Does not settle the promise — `close` does.
|
|
128
|
+
const requestKill = () => {
|
|
129
|
+
try {
|
|
130
|
+
child.kill("SIGTERM");
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
// Already exited between the decision and this kill — nothing to signal.
|
|
134
|
+
}
|
|
135
|
+
if (killEscalation)
|
|
136
|
+
clearTimeout(killEscalation);
|
|
137
|
+
killEscalation = setTimeout(() => {
|
|
138
|
+
try {
|
|
139
|
+
child.kill("SIGKILL");
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
// Same race as above; the 'close' handler still settles the promise.
|
|
143
|
+
}
|
|
144
|
+
}, abortKillGraceMs());
|
|
145
|
+
killEscalation.unref?.();
|
|
146
|
+
};
|
|
147
|
+
function fireTerminalGrace() {
|
|
148
|
+
if (settled || aborted)
|
|
149
|
+
return;
|
|
150
|
+
lingeredAfterTerminal = true;
|
|
151
|
+
clearIdle();
|
|
152
|
+
requestKill();
|
|
153
|
+
// If a grandchild holds the stdout pipe open after SIGKILL, Node never
|
|
154
|
+
// emits 'close' — settle as lingered so the spawn slot and lease cannot
|
|
155
|
+
// leak until the wall clock. Close still wins when it arrives first.
|
|
156
|
+
if (lingerSettleTimer)
|
|
157
|
+
clearTimeout(lingerSettleTimer);
|
|
158
|
+
lingerSettleTimer = setTimeout(() => {
|
|
159
|
+
if (settled || aborted)
|
|
160
|
+
return;
|
|
161
|
+
finish(1);
|
|
162
|
+
}, abortKillGraceMs() + 500);
|
|
163
|
+
lingerSettleTimer.unref?.();
|
|
164
|
+
}
|
|
165
|
+
const maybeArmTerminalGrace = (line) => {
|
|
166
|
+
if (!terminalGraceEnabled || settled || aborted || terminalGraceArmed)
|
|
167
|
+
return;
|
|
168
|
+
let match = false;
|
|
169
|
+
try {
|
|
170
|
+
match = opts.terminalGrace.isTerminalLine(line) === true;
|
|
171
|
+
}
|
|
172
|
+
catch (err) {
|
|
173
|
+
console.error(`[spawn-cli] terminalGrace.isTerminalLine for ${runId}`, err);
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
if (!match)
|
|
177
|
+
return;
|
|
178
|
+
terminalGraceArmed = true;
|
|
179
|
+
terminalGraceTimer = setTimeout(fireTerminalGrace, terminalGraceMs);
|
|
180
|
+
terminalGraceTimer.unref?.();
|
|
181
|
+
};
|
|
109
182
|
const logStream = fs.createWriteStream(opts.logPath, { flags: "w" });
|
|
110
183
|
// A WriteStream's 'error' event has no default handler — left unguarded, any
|
|
111
184
|
// stream error (a write after end, ENOSPC, a permissions problem) is an uncaught
|
|
@@ -155,21 +228,8 @@ export async function spawnCli(runId, cmd, args, cwd, opts) {
|
|
|
155
228
|
return;
|
|
156
229
|
aborted = true;
|
|
157
230
|
clearIdle();
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
}
|
|
161
|
-
catch {
|
|
162
|
-
// Already exited between the abort firing and this kill — nothing to signal.
|
|
163
|
-
}
|
|
164
|
-
killEscalation = setTimeout(() => {
|
|
165
|
-
try {
|
|
166
|
-
child.kill("SIGKILL");
|
|
167
|
-
}
|
|
168
|
-
catch {
|
|
169
|
-
// Same race as above; the 'close' handler still settles the promise.
|
|
170
|
-
}
|
|
171
|
-
}, abortKillGraceMs());
|
|
172
|
-
killEscalation.unref?.();
|
|
231
|
+
clearTerminalGrace();
|
|
232
|
+
requestKill();
|
|
173
233
|
};
|
|
174
234
|
// Not cleared on settle alone — `cleanupAbort` also drops the listener, so a
|
|
175
235
|
// long-lived signal (one AbortController per work item, reused across the
|
|
@@ -211,6 +271,7 @@ export async function spawnCli(runId, cmd, args, cwd, opts) {
|
|
|
211
271
|
}
|
|
212
272
|
clearTimeout(timer);
|
|
213
273
|
clearIdle();
|
|
274
|
+
clearTerminalGrace();
|
|
214
275
|
cleanupAbort();
|
|
215
276
|
unregisterChild(runId);
|
|
216
277
|
// Write any stderr BEFORE ending the stream — writing after end() throws
|
|
@@ -238,6 +299,7 @@ export async function spawnCli(runId, cmd, args, cwd, opts) {
|
|
|
238
299
|
transcript: stdoutChunks.join(""),
|
|
239
300
|
timedOut,
|
|
240
301
|
idleTimedOut,
|
|
302
|
+
lingeredAfterTerminal,
|
|
241
303
|
firstOutputMs,
|
|
242
304
|
lastActivityAt: new Date(lastActivityMs).toISOString(),
|
|
243
305
|
idleForMs: idleTimedOut ? Math.max(0, Date.now() - lastActivityMs) : null,
|
|
@@ -269,6 +331,14 @@ export async function spawnCli(runId, cmd, args, cwd, opts) {
|
|
|
269
331
|
}
|
|
270
332
|
};
|
|
271
333
|
const timer = setTimeout(() => {
|
|
334
|
+
if (settled)
|
|
335
|
+
return;
|
|
336
|
+
// A terminal-grace kill already requested: settle as lingered (not timedOut)
|
|
337
|
+
// if 'close' still has not arrived — same backstop role as lingerSettleTimer.
|
|
338
|
+
if (lingeredAfterTerminal) {
|
|
339
|
+
finish(1);
|
|
340
|
+
return;
|
|
341
|
+
}
|
|
272
342
|
timedOut = true;
|
|
273
343
|
killRunProcess(runId);
|
|
274
344
|
setTimeout(() => finish(124), 500);
|
|
@@ -283,7 +353,7 @@ export async function spawnCli(runId, cmd, args, cwd, opts) {
|
|
|
283
353
|
// that the kill lands near the bound, sparse enough to never matter for I/O.
|
|
284
354
|
const pollMs = Math.min(Math.max(Math.floor(idleMs / 10), 50), 30_000);
|
|
285
355
|
progressTimer = setInterval(() => {
|
|
286
|
-
if (settled || aborted)
|
|
356
|
+
if (settled || aborted || lingeredAfterTerminal)
|
|
287
357
|
return;
|
|
288
358
|
let at = null;
|
|
289
359
|
try {
|
|
@@ -306,12 +376,14 @@ export async function spawnCli(runId, cmd, args, cwd, opts) {
|
|
|
306
376
|
// NOT-307: any stdout bytes are progress. (stderr deliberately does not count:
|
|
307
377
|
// the contract defines progress as stdout bytes or the external source.)
|
|
308
378
|
noteActivity();
|
|
309
|
-
if (opts.onStdoutLine) {
|
|
379
|
+
if (opts.onStdoutLine || terminalGraceEnabled) {
|
|
310
380
|
lineTail += chunk;
|
|
311
381
|
let idx;
|
|
312
382
|
while ((idx = lineTail.indexOf("\n")) >= 0) {
|
|
313
|
-
|
|
383
|
+
const line = lineTail.slice(0, idx);
|
|
314
384
|
lineTail = lineTail.slice(idx + 1);
|
|
385
|
+
emitLine(line);
|
|
386
|
+
maybeArmTerminalGrace(line);
|
|
315
387
|
}
|
|
316
388
|
}
|
|
317
389
|
});
|
|
@@ -324,6 +396,7 @@ export async function spawnCli(runId, cmd, args, cwd, opts) {
|
|
|
324
396
|
settled = true;
|
|
325
397
|
clearTimeout(timer);
|
|
326
398
|
clearIdle();
|
|
399
|
+
clearTerminalGrace();
|
|
327
400
|
cleanupAbort();
|
|
328
401
|
unregisterChild(runId);
|
|
329
402
|
logStream.end();
|
|
@@ -339,4 +339,42 @@ test("spawnCli without idleTimeoutMs times out on the wall clock with idleTimedO
|
|
|
339
339
|
});
|
|
340
340
|
assert.equal(result.timedOut, true);
|
|
341
341
|
assert.equal(result.idleTimedOut, false);
|
|
342
|
+
assert.equal(result.lingeredAfterTerminal, false);
|
|
343
|
+
});
|
|
344
|
+
const terminalFailedLine = JSON.stringify({
|
|
345
|
+
payload_type: "run.terminal.failed",
|
|
346
|
+
payload: { terminal: "failed", reason: "transport error [net-timeout]: timed out waiting for response data (meta stream)" },
|
|
347
|
+
});
|
|
348
|
+
const isTerminalLine = (line) => line.includes("run.terminal.failed") || line.includes("run.terminal.completed");
|
|
349
|
+
// NOT-342: a child that has already printed its terminal envelope but does not exit
|
|
350
|
+
// is SIGTERM'd after the grace — well before the wall clock — and is not a timeout.
|
|
351
|
+
test("spawnCli kills a lingering child after terminalGrace without timing out", { timeout: 10_000 }, async () => {
|
|
352
|
+
const logPath = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "dealer-spawn-cli-")), "out.ndjson");
|
|
353
|
+
const { cmd, args } = nodeChild(`console.log(${JSON.stringify(terminalFailedLine)}); setInterval(() => {}, 1000);`);
|
|
354
|
+
const startedAt = Date.now();
|
|
355
|
+
const result = await spawnCli("test-run-terminal-linger", cmd, args, process.cwd(), {
|
|
356
|
+
logPath,
|
|
357
|
+
timeoutMs: 30_000,
|
|
358
|
+
terminalGrace: { isTerminalLine, graceMs: 200 },
|
|
359
|
+
});
|
|
360
|
+
const elapsed = Date.now() - startedAt;
|
|
361
|
+
assert.ok(elapsed < 5_000, `settled on the grace, not the wall clock, took ${elapsed}ms`);
|
|
362
|
+
assert.equal(result.lingeredAfterTerminal, true);
|
|
363
|
+
assert.equal(result.timedOut, false);
|
|
364
|
+
assert.equal(result.idleTimedOut, false);
|
|
365
|
+
assert.match(result.transcript, /transport error/);
|
|
366
|
+
});
|
|
367
|
+
// NOT-342: exiting on its own inside the grace is the child's own close — no linger kill.
|
|
368
|
+
test("spawnCli does not linger-kill a child that exits inside the terminal grace", async () => {
|
|
369
|
+
const logPath = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "dealer-spawn-cli-")), "out.ndjson");
|
|
370
|
+
const { cmd, args } = nodeChild(`console.log(${JSON.stringify(terminalFailedLine)}); process.exit(0);`);
|
|
371
|
+
const result = await spawnCli("test-run-terminal-exits", cmd, args, process.cwd(), {
|
|
372
|
+
logPath,
|
|
373
|
+
timeoutMs: 10_000,
|
|
374
|
+
terminalGrace: { isTerminalLine, graceMs: 2_000 },
|
|
375
|
+
});
|
|
376
|
+
assert.equal(result.lingeredAfterTerminal, false);
|
|
377
|
+
assert.equal(result.timedOut, false);
|
|
378
|
+
assert.equal(result.idleTimedOut, false);
|
|
379
|
+
assert.equal(result.exitCode, 0);
|
|
342
380
|
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agent-dealer/server",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.10",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"files": [
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
"typecheck": "tsc --noEmit"
|
|
23
23
|
},
|
|
24
24
|
"dependencies": {
|
|
25
|
-
"@agent-dealer/shared": "1.2.
|
|
25
|
+
"@agent-dealer/shared": "1.2.10",
|
|
26
26
|
"@fastify/cors": "^11.0.1",
|
|
27
27
|
"@fastify/static": "^8.2.0",
|
|
28
28
|
"@modelcontextprotocol/sdk": "^1.29.0",
|