@yemi33/minions 0.1.2124 → 0.1.2126
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/js/settings.js +12 -0
- package/dashboard.js +62 -4
- package/docs/README.md +3 -1
- package/docs/design-state-storage.md +1 -1
- package/docs/watches.md +1 -0
- package/engine/dispatch.js +4 -0
- package/engine/shared.js +66 -0
- package/engine/spawn-phase-watchdog.js +271 -0
- package/engine/steering.js +191 -20
- package/engine.js +6 -2
- package/package.json +1 -1
package/dashboard/js/settings.js
CHANGED
|
@@ -276,6 +276,15 @@ async function openSettings() {
|
|
|
276
276
|
settingsField('Meeting Round Timeout', 'set-meetingRoundTimeout', e.meetingRoundTimeout || 900000, 'ms', 'Auto-advance meeting round after this') +
|
|
277
277
|
settingsField('Steering Deferred Max', 'set-steeringDeferredMaxMs', e.steeringDeferredMaxMs || 900000, 'ms', 'Max wait for a runtime to emit a resumable checkpoint before a deferred steering message is flagged stranded. After this, the engine warns to live-output, marks _steeringStranded on the dispatch, and (when the steering store is present) sets store status=stranded. Default 15min; range 60s–4h.') +
|
|
278
278
|
settingsField('Steering Max Kill Retries', 'set-steeringMaxKillRetries', e.steeringMaxKillRetries ?? 3, '', 'Cap on graceful+escalation kill attempts after a steering kill is issued. Ladder waits 30s → 60s → 120s between attempts (last interval reused). Attempt 1 is graceful; attempts 2..cap are platform hard kills (taskkill /F /T on Windows; descendant-tree SIGKILL + pkill on Unix). Past cap, the engine gives up with a [steering-stuck] log + inbox notice. Default 3; range 1–5.') +
|
|
279
|
+
'</div>' +
|
|
280
|
+
'<h4 style="margin:14px 0 4px;font-size:var(--text-md)">Spawn-phase watchdog</h4>' +
|
|
281
|
+
'<div class="settings-pane-sub">Kills agent processes that ran startup-only events (MCP init, hook fires) but never emitted real task progress, while CPU usage stayed below the threshold. Defense against MCP-init wedges that otherwise burn a worker slot until the 5h hard timeout. Fires only on fresh spawns (not engine-restart re-attaches).</div>' +
|
|
282
|
+
'<div class="settings-stack" style="margin-bottom:8px">' +
|
|
283
|
+
settingsToggle('Spawn-phase Watchdog', 'set-spawnPhaseWatchdogEnabled', e.spawnPhaseWatchdogEnabled !== false, 'Enable the spawn-phase MCP-init wedge detector. Default ON. Turn OFF only if you suspect false positives against a runtime whose legitimate startup exceeds the grace window.') +
|
|
284
|
+
'</div>' +
|
|
285
|
+
'<div class="settings-grid-2">' +
|
|
286
|
+
settingsField('Spawn-phase Grace', 'set-spawnPhaseGraceMs', e.spawnPhaseGraceMs || 120000, 'ms', 'Time after spawn before the watchdog can fire. Default 2min. A healthy startup ships a real event in seconds; this is generous headroom for slow MCP servers and corporate DNS.') +
|
|
287
|
+
settingsField('Spawn-phase CPU floor', 'set-spawnPhaseMaxCpuSeconds', e.spawnPhaseMaxCpuSeconds ?? 5, 's', 'Process is "idle" if cumulative CPU seconds ≤ this. Default 5s. A genuinely busy startup burns CPU on tokenizer bootstrap, MCP IPC, or model warmup; a wedged child sits in epoll near 0s.') +
|
|
279
288
|
'</div>';
|
|
280
289
|
|
|
281
290
|
const paneWorktree =
|
|
@@ -843,6 +852,9 @@ async function saveSettings() {
|
|
|
843
852
|
meetingRoundTimeout: document.getElementById('set-meetingRoundTimeout').value,
|
|
844
853
|
steeringDeferredMaxMs: document.getElementById('set-steeringDeferredMaxMs').value,
|
|
845
854
|
steeringMaxKillRetries: document.getElementById('set-steeringMaxKillRetries').value,
|
|
855
|
+
spawnPhaseWatchdogEnabled: document.getElementById('set-spawnPhaseWatchdogEnabled').checked,
|
|
856
|
+
spawnPhaseGraceMs: document.getElementById('set-spawnPhaseGraceMs').value,
|
|
857
|
+
spawnPhaseMaxCpuSeconds: document.getElementById('set-spawnPhaseMaxCpuSeconds').value,
|
|
846
858
|
operatorLogin: (document.getElementById('set-operatorLogin')?.value ?? '').trim(),
|
|
847
859
|
autoApprovePlans: document.getElementById('set-autoApprovePlans').checked,
|
|
848
860
|
evalLoop: document.getElementById('set-evalLoop').checked,
|
package/dashboard.js
CHANGED
|
@@ -9228,6 +9228,13 @@ What would you like to discuss or change? When you're happy, say "approve" and I
|
|
|
9228
9228
|
// W-mq066js7000fff1f-c (Gap B/C): steering safety-net knobs.
|
|
9229
9229
|
steeringDeferredMaxMs: [60000, 14400000],
|
|
9230
9230
|
steeringMaxKillRetries: [1, 5],
|
|
9231
|
+
// W-mq0e2dae000a003d — spawn-phase watchdog knobs. Grace must be
|
|
9232
|
+
// at least one tick so the watchdog never fires before the
|
|
9233
|
+
// runtime had a chance to emit. Upper bound 30min ≫ any legit
|
|
9234
|
+
// startup. CPU floor 0–60s (60s would let half-busy processes
|
|
9235
|
+
// through; the default 5s targets pure-idle epoll waits).
|
|
9236
|
+
spawnPhaseGraceMs: [10000, 1800000],
|
|
9237
|
+
spawnPhaseMaxCpuSeconds: [0, 60],
|
|
9231
9238
|
versionCheckInterval: [60000],
|
|
9232
9239
|
prPollStatusEvery: [1], prPollCommentsEvery: [1],
|
|
9233
9240
|
agentBusyReassignMs: [0],
|
|
@@ -11477,7 +11484,7 @@ What would you like to discuss or change? When you're happy, say "approve" and I
|
|
|
11477
11484
|
invalidateStatusCache();
|
|
11478
11485
|
return jsonReply(res, 200, { ok: true });
|
|
11479
11486
|
}},
|
|
11480
|
-
{ method: 'POST', path: '/api/agents/steer', desc: 'Inject steering message into a running agent', params: 'agent, message', handler: async (req, res) => {
|
|
11487
|
+
{ method: 'POST', path: '/api/agents/steer', desc: 'Inject steering message into a running agent', params: 'agent, message, supersede? (all|unacked|<steerId>), scope? (agent|current-dispatch)', handler: async (req, res) => {
|
|
11481
11488
|
const body = await readBody(req);
|
|
11482
11489
|
const { agent, message } = body;
|
|
11483
11490
|
if (!agent || !message) return jsonReply(res, 400, { error: 'agent and message required' });
|
|
@@ -11491,7 +11498,54 @@ What would you like to discuss or change? When you're happy, say "approve" and I
|
|
|
11491
11498
|
return jsonReply(res, 409, { error: 'Agent session is finishing; retry when the next session starts' });
|
|
11492
11499
|
}
|
|
11493
11500
|
|
|
11494
|
-
|
|
11501
|
+
// W-mq066js7000fff1f-e (Gap E) — supersede prior messages before
|
|
11502
|
+
// writing the new one. Accept 'all', 'unacked', or a specific
|
|
11503
|
+
// steerId. Bad shapes are silently ignored (best-effort).
|
|
11504
|
+
const supersede = body.supersede ? String(body.supersede).trim() : '';
|
|
11505
|
+
let superseded = [];
|
|
11506
|
+
if (supersede) {
|
|
11507
|
+
const provisionalSteerId = `steer-pending-${Date.now()}`;
|
|
11508
|
+
superseded = steering.supersedeMessages(agentId, supersede, { reason: `superseded by new steer (${supersede})`, newSteerId: provisionalSteerId });
|
|
11509
|
+
}
|
|
11510
|
+
|
|
11511
|
+
// W-mq066js7000fff1f-e — dedupe: if no supersede arg AND an
|
|
11512
|
+
// identical body is already pending within the 5-min window, echo
|
|
11513
|
+
// back the existing steerId instead of writing a duplicate file.
|
|
11514
|
+
if (!supersede) {
|
|
11515
|
+
const dup = steering.findRecentDuplicate(agentId, text);
|
|
11516
|
+
if (dup) {
|
|
11517
|
+
const delivery = _steeringDeliveryState(agentId);
|
|
11518
|
+
return jsonReply(res, 200, {
|
|
11519
|
+
ok: true,
|
|
11520
|
+
deduplicated: true,
|
|
11521
|
+
steerId: dup.steerId,
|
|
11522
|
+
status: dup.status || steering.STATUS.QUEUED,
|
|
11523
|
+
file: dup.file,
|
|
11524
|
+
// Gap D observability URL — points at the SQL delivery-state row
|
|
11525
|
+
// for /api/steering/:id (back-compat with master's contract).
|
|
11526
|
+
deliveryUrl: dup.steerId ? `/api/steering/${dup.steerId}` : null,
|
|
11527
|
+
message: 'Identical steering message already pending — returning existing entry',
|
|
11528
|
+
...delivery,
|
|
11529
|
+
inboxCount: steering.listUnreadSteeringMessages(agentId).length,
|
|
11530
|
+
});
|
|
11531
|
+
}
|
|
11532
|
+
}
|
|
11533
|
+
|
|
11534
|
+
// W-mq066js7000fff1f-f (Gap F) — per-dispatch scoping. With
|
|
11535
|
+
// scope='current-dispatch' we stamp the active dispatch id so the
|
|
11536
|
+
// engine filters this entry out of any future unrelated dispatch's
|
|
11537
|
+
// resume prompt. With scope='agent' (default) the message is
|
|
11538
|
+
// agent-wide and picks up on the next dispatch regardless of id.
|
|
11539
|
+
const scope = body.scope ? String(body.scope).trim().toLowerCase() : 'agent';
|
|
11540
|
+
let targetDispatchId = null;
|
|
11541
|
+
let scopeApplied = scope;
|
|
11542
|
+
if (scope === 'current-dispatch') {
|
|
11543
|
+
const active = (getDispatchQueue().active || []).find(d => d.agent === agentId);
|
|
11544
|
+
if (active?.id) targetDispatchId = String(active.id);
|
|
11545
|
+
else scopeApplied = 'agent'; // no active dispatch → fall back
|
|
11546
|
+
}
|
|
11547
|
+
|
|
11548
|
+
const entry = steering.writeSteeringMessage(agentId, text, { targetDispatchId });
|
|
11495
11549
|
const delivery = _steeringDeliveryState(agentId);
|
|
11496
11550
|
|
|
11497
11551
|
// Also append to live-output.log so it shows in the chat view
|
|
@@ -11506,12 +11560,16 @@ What would you like to discuss or change? When you're happy, say "approve" and I
|
|
|
11506
11560
|
const steerId = entry?.steerId || null;
|
|
11507
11561
|
return jsonReply(res, 200, {
|
|
11508
11562
|
ok: true,
|
|
11563
|
+
deduplicated: false,
|
|
11564
|
+
steerId,
|
|
11565
|
+
status: entry?.status || (steerId ? steering.STATUS.QUEUED : null),
|
|
11566
|
+
targetDispatchId: entry?.targetDispatchId || null,
|
|
11567
|
+
scope: scopeApplied,
|
|
11568
|
+
superseded: superseded.map(s => ({ steerId: s.steerId, previousStatus: s.previousStatus })),
|
|
11509
11569
|
message: delivery.pendingDelivery ? 'Steering message pending delivery' : 'Steering message queued',
|
|
11510
11570
|
...delivery,
|
|
11511
11571
|
file: entry?.file || null,
|
|
11512
11572
|
inboxCount: steering.listUnreadSteeringMessages(agentId).length,
|
|
11513
|
-
steerId,
|
|
11514
|
-
status: steerId ? 'queued' : null,
|
|
11515
11573
|
deliveryUrl: steerId ? `/api/steering/${steerId}` : null,
|
|
11516
11574
|
});
|
|
11517
11575
|
}},
|
package/docs/README.md
CHANGED
|
@@ -19,7 +19,9 @@ Architecture, design proposals, and lifecycle references for people working on t
|
|
|
19
19
|
- [constellation-bridge.md](constellation-bridge.md) — Read-only cross-repo bridge: `engine.constellationBridge.enabled` flag, marker-file contract, and the `minions bridge` subcommand for local debugging.
|
|
20
20
|
- [cooldown-merge-semantics.md](cooldown-merge-semantics.md) — Scoping deliverable defining merge semantics for `saveCooldowns` (longer-of TTL merge, key-level upserts, gitignored on-disk format).
|
|
21
21
|
- [copilot-cli-schema.md](copilot-cli-schema.md) — Behavior and schema reference for the GitHub Copilot CLI adapter (capability flags, stdin vs `-p`, model discovery, effort levels).
|
|
22
|
-
- [
|
|
22
|
+
- [dead-code-audit-retractions.md](dead-code-audit-retractions.md) — Retracted dead-code-audit findings (false positives) that future audits MUST read before re-citing.
|
|
23
|
+
- [design-state-storage.md](design-state-storage.md) — Design proposal evaluating five database options for replacing Minions' file-based JSON state; recommends `node:sqlite` as the medium-term target (accepted; implementation tracked in CHANGELOG.md Phases 0–9).
|
|
24
|
+
- [harness-mode.md](harness-mode.md) — Tri-Agent Harness Mode (`harness_mode: "tri_agent"` on scheduled tasks): Planner → Generator → Evaluator loop that iterates a shared on-disk artifact until a rubric passes or the iteration cap fires.
|
|
23
25
|
- [kb-sweep.md](kb-sweep.md) — Knowledge-base consolidation sweep (hash dedup → LLM batch dedup/reclassify → per-entry compress) and the detached runner that keeps it alive across `minions restart`.
|
|
24
26
|
- [keep-processes.md](keep-processes.md) — `meta.keep_processes` sidecar contract: when to use it vs managed-spawn, sidecar schema, caps, and the [`engine/keep-process-sweep.js`](../engine/keep-process-sweep.js) lifecycle.
|
|
25
27
|
- [managed-spawn.md](managed-spawn.md) — Engine-owned long-running services (managed-spawn primitive): sidecar schema, healthcheck examples, lifecycle, dashboard API, and the WI 1 (build) → WI 2 (test) chained-validation pattern.
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
> Author: Rebecca (Architect) | Date: 2026-04-07 | Status: **Accepted — implementation in progress**
|
|
4
4
|
|
|
5
|
-
> **Implementation status (as of 2026-06):** The `node:sqlite` recommendation in §3 has been adopted ahead of schedule. Phases 0–
|
|
5
|
+
> **Implementation status (as of 2026-06):** The `node:sqlite` recommendation in §3 has been adopted ahead of schedule. Phases 0–9 have shipped (events, dispatches, work_items, pull_requests, logs, metrics, watches, schedule_runs + pipeline_runs + managed_processes + worktree_pool, qa_runs + qa_sessions, pr_links, cooldowns + pending_rebases + cc_sessions + doc_sessions, and steering_deliveries — see `CHANGELOG.md` and `engine/db/migrations/`). The SQLite schema lives under `engine/db/migrations/` and the singleton opens `engine/state.db` in WAL mode. Phase 8 added the first opt-out toggle for the JSON sidecars (`engine.qaDualWriteJson`, default true). Phase 9.4 went further and deleted the silent SQL-unavailable JSON fallbacks in the engine — SQL is now the only reader/writer for everything migrated; the JSON mirror layer is dual-written as a passive mirror and slated for deletion in Phase 9.5. The "Phase 2: estimated Node 26 LTS" timeline in §3 is now historical context; treat sections 1–3 as design rationale rather than a forward plan.
|
|
6
6
|
|
|
7
7
|
## Executive Summary
|
|
8
8
|
|
package/docs/watches.md
CHANGED
|
@@ -173,6 +173,7 @@ I/O happens **outside the lock**: notifications via `writeToInbox`, follow-up ac
|
|
|
173
173
|
| `trigger-pipeline` | Start a new pipeline run (skipped if the pipeline already has an active run) |
|
|
174
174
|
| `archive-plan` | Set PRD `status="archived"` + `archivedAt` |
|
|
175
175
|
| `resume-plan` | Set PRD `status=PLAN_STATUS.ACTIVE` and clear `planStale` |
|
|
176
|
+
| `cc-triage` | Invoke Command Center headlessly via the loopback `POST /api/command-center/triage` endpoint with the trigger context (and optional completion-report / live-output artifacts). Wraps the prompt in `<UNTRUSTED-INPUT>`, uses a default 10-min timeout (capped at 1 h), and is isolated from the user CC session |
|
|
176
177
|
|
|
177
178
|
Constants live in `WATCH_ACTION_TYPE` (`engine/shared.js:2608`); handlers in `engine/watch-actions.js`.
|
|
178
179
|
|
package/engine/dispatch.js
CHANGED
|
@@ -467,6 +467,10 @@ const FORCE_DEMOTE_FAILURE_CLASSES = new Set([
|
|
|
467
467
|
FAILURE_CLASS.INVALID_KEEP_PROCESSES_SCHEMA,
|
|
468
468
|
FAILURE_CLASS.INVALID_MANAGED_SPAWN,
|
|
469
469
|
FAILURE_CLASS.MANAGED_SPAWN_HEALTHCHECK_FAILED,
|
|
470
|
+
// W-mq0e2dae000a003d: spawn-phase wedges should never claim success,
|
|
471
|
+
// but include here as defense-in-depth so a stray completion-report
|
|
472
|
+
// can't sneak through.
|
|
473
|
+
FAILURE_CLASS.SPAWN_PHASE_STALL,
|
|
470
474
|
]);
|
|
471
475
|
|
|
472
476
|
function readLiveWorkItem(meta) {
|
package/engine/shared.js
CHANGED
|
@@ -2255,6 +2255,18 @@ const ENGINE_DEFAULTS = {
|
|
|
2255
2255
|
allowedDashboardOrigins: [],
|
|
2256
2256
|
meetingRoundTimeout: 900000, // 15min per meeting round — soft signal; logs a "still waiting" warning each tick
|
|
2257
2257
|
meetingRoundHardTimeout: 3600000, // 60min hard backstop — non-terminal participants are marked failed and the round advances. Prevents permanent stalls if an agent's dispatch never spawns or its completion gets dropped.
|
|
2258
|
+
// W-mq0e2dae000a003d — spawn-phase progress watchdog. Detects the
|
|
2259
|
+
// MCP-init wedge where a runtime process starts, emits only startup-only
|
|
2260
|
+
// events (Copilot session.mcp_*/skills_loaded/tools_updated/info or Claude
|
|
2261
|
+
// system init/hook events), then sits idle indefinitely with no real task
|
|
2262
|
+
// events. The watchdog fires only when BOTH (a) elapsed since spawn
|
|
2263
|
+
// exceeds the grace and (b) per-process CPU seconds are below the
|
|
2264
|
+
// threshold — preventing kills of legitimately busy startups. Re-attached
|
|
2265
|
+
// processes (engine restart) are exempt; we only watchdog fresh spawns
|
|
2266
|
+
// whose live-output we can fully account for.
|
|
2267
|
+
spawnPhaseWatchdogEnabled: true,
|
|
2268
|
+
spawnPhaseGraceMs: 120000, // 2min — startup grace before the watchdog can fire
|
|
2269
|
+
spawnPhaseMaxCpuSeconds: 5, // process is "idle" if its CPU total is <= this many seconds
|
|
2258
2270
|
// W-mq066js7000fff1f-c (steering Gap B): max wall-clock a steering message may
|
|
2259
2271
|
// sit deferred (runtime hasn't emitted a resumable checkpoint yet — Copilot
|
|
2260
2272
|
// pre-first-checkpoint, etc.). Past this window the message is flagged
|
|
@@ -3506,6 +3518,7 @@ const FAILURE_CLASS = {
|
|
|
3506
3518
|
WORKSPACE_MANIFEST_REPO: 'workspace-manifest-repo-forbidden', // W-mq07avbk000m5543: dispatch routed an agent to a project/repo not present in its workspace_manifest.allowed_repos. Structural — never retryable until the manifest is widened or a different agent is chosen.
|
|
3507
3519
|
WORKSPACE_MANIFEST_TOOL: 'workspace-manifest-tool-forbidden', // W-mq07avbk000m5543: out-of-scope tool call (manifest enforcement at the runtime gate). Non-retryable as-is.
|
|
3508
3520
|
WORKSPACE_MANIFEST_URL: 'workspace-manifest-url-forbidden', // W-mq07avbk000m5543: out-of-scope external URL fetch. Non-retryable as-is.
|
|
3521
|
+
SPAWN_PHASE_STALL: 'spawn-phase-stall', // W-mq0e2dae000a003d: process spawned and ran startup-only events (MCP init / hooks) but never emitted real task progress; CPU usage stayed below threshold past the grace window. Engine kills the wedged child and treats this as retryable (fresh-session) so a re-spawn can clear a transient MCP wedge.
|
|
3509
3522
|
UNKNOWN: 'unknown', // Unclassified failure
|
|
3510
3523
|
};
|
|
3511
3524
|
const ESCALATION_POLICY = {
|
|
@@ -5552,6 +5565,58 @@ function killImmediate(proc) {
|
|
|
5552
5565
|
}
|
|
5553
5566
|
}
|
|
5554
5567
|
|
|
5568
|
+
// W-mq0e2dae000a003d — cross-platform CPU-seconds sampler used by the
|
|
5569
|
+
// spawn-phase watchdog to decide whether a process is genuinely wedged
|
|
5570
|
+
// vs busy. Returns the cumulative user+system CPU time in seconds, or
|
|
5571
|
+
// null when we can't determine it (process dead, sampler unavailable,
|
|
5572
|
+
// shell errored). Fail-open: a null result MUST NOT cause the watchdog
|
|
5573
|
+
// to fire — silence-from-the-OS is not the same as silence-from-the-process.
|
|
5574
|
+
function getProcessCpuSeconds(pid) {
|
|
5575
|
+
const n = Number(pid);
|
|
5576
|
+
if (!Number.isInteger(n) || n <= 0) return null;
|
|
5577
|
+
try {
|
|
5578
|
+
if (process.platform === 'win32') {
|
|
5579
|
+
// PowerShell. CPU = total processor time in seconds (sum of user + kernel).
|
|
5580
|
+
// 2.5s timeout keeps the tick budget bounded if PS is sluggish.
|
|
5581
|
+
const out = _execSync(
|
|
5582
|
+
`powershell -NoProfile -NonInteractive -Command "(Get-Process -Id ${n} -ErrorAction Stop).CPU"`,
|
|
5583
|
+
{ stdio: ['ignore', 'pipe', 'pipe'], timeout: 2500, windowsHide: true, encoding: 'utf8' }
|
|
5584
|
+
);
|
|
5585
|
+
const v = parseFloat(String(out).trim());
|
|
5586
|
+
return Number.isFinite(v) ? v : null;
|
|
5587
|
+
}
|
|
5588
|
+
if (process.platform === 'linux') {
|
|
5589
|
+
// /proc/<pid>/stat fields 14 (utime) + 15 (stime) in clock ticks.
|
|
5590
|
+
// _SC_CLK_TCK is 100 on virtually every Linux distro; sysconf isn't
|
|
5591
|
+
// exposed from Node so we use the conventional value. If a host
|
|
5592
|
+
// ever ships USER_HZ != 100 this would under/over-report by a
|
|
5593
|
+
// constant factor; safe within an order of magnitude for our gate.
|
|
5594
|
+
const raw = fs.readFileSync(`/proc/${n}/stat`, 'utf8');
|
|
5595
|
+
// The comm field can contain spaces and parens — slice past the last ')'
|
|
5596
|
+
const close = raw.lastIndexOf(')');
|
|
5597
|
+
if (close < 0) return null;
|
|
5598
|
+
const fields = raw.slice(close + 2).split(/\s+/);
|
|
5599
|
+
// After comm: state(0) ppid(1) pgrp(2) ... utime is index 11, stime 12
|
|
5600
|
+
const utime = Number(fields[11]);
|
|
5601
|
+
const stime = Number(fields[12]);
|
|
5602
|
+
if (!Number.isFinite(utime) || !Number.isFinite(stime)) return null;
|
|
5603
|
+
return (utime + stime) / 100;
|
|
5604
|
+
}
|
|
5605
|
+
if (process.platform === 'darwin') {
|
|
5606
|
+
// `ps -p N -o cputime=` → "MM:SS.ss" or "HH:MM:SS"
|
|
5607
|
+
const out = _execSync(`ps -p ${n} -o cputime=`, { stdio: ['ignore', 'pipe', 'pipe'], timeout: 2500, windowsHide: true, encoding: 'utf8' });
|
|
5608
|
+
const t = String(out).trim();
|
|
5609
|
+
if (!t) return null;
|
|
5610
|
+
const parts = t.split(':').map(s => parseFloat(s));
|
|
5611
|
+
if (!parts.every(p => Number.isFinite(p))) return null;
|
|
5612
|
+
let secs = 0;
|
|
5613
|
+
while (parts.length) secs = secs * 60 + parts.shift();
|
|
5614
|
+
return secs;
|
|
5615
|
+
}
|
|
5616
|
+
} catch { /* fail-open */ }
|
|
5617
|
+
return null;
|
|
5618
|
+
}
|
|
5619
|
+
|
|
5555
5620
|
// Single-PID kill (no /T tree walk) — used by the orphan-MCP sweep where we
|
|
5556
5621
|
// already enumerated descendants ourselves and the parent is dead, so /T would
|
|
5557
5622
|
// be a no-op anyway.
|
|
@@ -6441,6 +6506,7 @@ module.exports = {
|
|
|
6441
6506
|
sleepMs,
|
|
6442
6507
|
killGracefully,
|
|
6443
6508
|
killImmediate,
|
|
6509
|
+
getProcessCpuSeconds,
|
|
6444
6510
|
killByPidImmediate,
|
|
6445
6511
|
killByPidsImmediate,
|
|
6446
6512
|
isProcessCommandLineMatchingAgent,
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
// engine/spawn-phase-watchdog.js — W-mq0e2dae000a003d
|
|
2
|
+
//
|
|
3
|
+
// Per-tick check that kills agent processes wedged in the spawn/MCP-init
|
|
4
|
+
// phase. The classic failure mode is a Copilot or Claude child that emits
|
|
5
|
+
// startup-only events (MCP server load, skill registration, hook firing)
|
|
6
|
+
// and then sits idle indefinitely — no `assistant.*`, no `tool.*`, no
|
|
7
|
+
// `result`, no completion-report. The agent never makes progress so the
|
|
8
|
+
// hard timeout (5h) eventually kills it, wasting a worker slot for hours.
|
|
9
|
+
//
|
|
10
|
+
// Detection contract:
|
|
11
|
+
//
|
|
12
|
+
// 1. Process must be a fresh spawn (not re-attached after engine
|
|
13
|
+
// restart — `procInfo.reattached` is false/undefined). Re-attached
|
|
14
|
+
// processes already lost their early live-output and can look
|
|
15
|
+
// pre-wedged after the engine reads them mid-flight.
|
|
16
|
+
//
|
|
17
|
+
// 2. Time since spawn must exceed `spawnPhaseGraceMs` (default 2min).
|
|
18
|
+
// Short grace because a healthy startup ships a real event within
|
|
19
|
+
// seconds; 2min gives slow MCP servers and corp-network DNS plenty
|
|
20
|
+
// of headroom.
|
|
21
|
+
//
|
|
22
|
+
// 3. live-output.log tail must contain ZERO non-startup events. We tail
|
|
23
|
+
// ~16KB and parse JSON lines; any non-startup type means the runtime
|
|
24
|
+
// handed control to the task and the watchdog stands down.
|
|
25
|
+
//
|
|
26
|
+
// 4. Per-process CPU seconds must be <= `spawnPhaseMaxCpuSeconds`
|
|
27
|
+
// (default 5). A genuinely busy startup will burn CPU on tokenizer
|
|
28
|
+
// bootstrap, MCP IPC, or model warmup; a wedged child sits in epoll.
|
|
29
|
+
// The CPU gate is fail-open — if we can't sample, we DO NOT kill.
|
|
30
|
+
//
|
|
31
|
+
// On fire: kill the tracked process tree, complete the dispatch with
|
|
32
|
+
// SPAWN_PHASE_STALL, mark agent-retryable so the next tick re-spawns the
|
|
33
|
+
// agent with a fresh runtime invocation (resolves transient MCP wedges).
|
|
34
|
+
|
|
35
|
+
const fs = require('fs');
|
|
36
|
+
const path = require('path');
|
|
37
|
+
const shared = require('./shared');
|
|
38
|
+
const queries = require('./queries');
|
|
39
|
+
const { ENGINE_DEFAULTS, FAILURE_CLASS } = shared;
|
|
40
|
+
const AGENTS_DIR = queries.AGENTS_DIR;
|
|
41
|
+
const log = (level, msg) => shared.log ? shared.log(level, msg) : console.log(`[${level}] ${msg}`);
|
|
42
|
+
|
|
43
|
+
// Lazy require to break circular dep with dispatch.js (which lazy-requires engine.js).
|
|
44
|
+
function dispatch() { return require('./dispatch'); }
|
|
45
|
+
|
|
46
|
+
// ── Per-runtime startup-only event filters ──────────────────────────────────
|
|
47
|
+
//
|
|
48
|
+
// Anything NOT in these sets is treated as "real task activity" and the
|
|
49
|
+
// watchdog stands down. These match the runtimes/*.js KNOWN_EVENT_TYPES
|
|
50
|
+
// startup section — keep in sync.
|
|
51
|
+
|
|
52
|
+
const COPILOT_STARTUP_TYPES = new Set([
|
|
53
|
+
'session.mcp_server_status_changed',
|
|
54
|
+
'session.mcp_servers_loaded',
|
|
55
|
+
'session.skills_loaded',
|
|
56
|
+
'session.tools_updated',
|
|
57
|
+
'session.info',
|
|
58
|
+
]);
|
|
59
|
+
|
|
60
|
+
// Claude stream-json prints {"type":"system","subtype":"init|hook_started|hook_response"}
|
|
61
|
+
// during startup before any assistant/user/tool_use/result events fire.
|
|
62
|
+
const CLAUDE_STARTUP_SUBTYPES = new Set(['init', 'hook_started', 'hook_response']);
|
|
63
|
+
|
|
64
|
+
// Header line that engine.js writes at spawn: "[<iso>] pid: <n>"
|
|
65
|
+
const HEADER_LINE_RE = /^\[[^\]]+\]\s+(?:pid:|spawn-failed)/;
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Inspect a tail of `live-output.log` and decide whether the runtime has
|
|
69
|
+
* emitted any real task progress events yet.
|
|
70
|
+
*
|
|
71
|
+
* Returns true when we found at least one non-startup event. Returns
|
|
72
|
+
* false when the tail is empty, header-only, or contains ONLY recognized
|
|
73
|
+
* startup events.
|
|
74
|
+
*
|
|
75
|
+
* Conservative on parse errors: a line that fails JSON.parse is treated
|
|
76
|
+
* as a real progress signal (could be a `[engine-system]` notice or
|
|
77
|
+
* codex plain-text output), so the watchdog stands down. Only structured
|
|
78
|
+
* JSON whose `type` we positively recognize as startup is filtered out.
|
|
79
|
+
*/
|
|
80
|
+
function hasRealTaskActivity(tailText, runtimeName) {
|
|
81
|
+
if (!tailText) return false;
|
|
82
|
+
const lines = String(tailText).split(/\r?\n/);
|
|
83
|
+
for (const rawLine of lines) {
|
|
84
|
+
const line = rawLine.trim();
|
|
85
|
+
if (!line) continue;
|
|
86
|
+
if (HEADER_LINE_RE.test(line)) continue;
|
|
87
|
+
// Bracketed engine sentinels like `[process-exit] code=0` shouldn't
|
|
88
|
+
// reach this code path (timeout.js would have completed first), but
|
|
89
|
+
// if they do, treat them as real activity so we never race timeout.
|
|
90
|
+
if (/^\[(?:process-exit|engine-system|steering-)/.test(line)) return true;
|
|
91
|
+
// Try to parse JSON. Non-JSON lines (codex plain text, partial flushes)
|
|
92
|
+
// count as real activity — only positively-identified startup events
|
|
93
|
+
// are filtered out.
|
|
94
|
+
let evt;
|
|
95
|
+
try { evt = JSON.parse(line); }
|
|
96
|
+
catch { return true; }
|
|
97
|
+
if (!evt || typeof evt !== 'object') continue;
|
|
98
|
+
const type = typeof evt.type === 'string' ? evt.type : null;
|
|
99
|
+
if (!type) {
|
|
100
|
+
// Object with no `type` — unusual; treat as activity to stay safe.
|
|
101
|
+
return true;
|
|
102
|
+
}
|
|
103
|
+
if (runtimeName === 'copilot') {
|
|
104
|
+
if (COPILOT_STARTUP_TYPES.has(type)) continue;
|
|
105
|
+
// Anything else from Copilot is real (assistant.*, tool.*, result,
|
|
106
|
+
// session.task_complete, user.message, function).
|
|
107
|
+
return true;
|
|
108
|
+
}
|
|
109
|
+
if (runtimeName === 'claude' || !runtimeName) {
|
|
110
|
+
// Claude stream-json: only {type:'system', subtype:'init|hook_*'} is startup.
|
|
111
|
+
if (type === 'system') {
|
|
112
|
+
const sub = typeof evt.subtype === 'string' ? evt.subtype : '';
|
|
113
|
+
if (CLAUDE_STARTUP_SUBTYPES.has(sub)) continue;
|
|
114
|
+
// Other `system` subtypes (e.g. errors, banners) — treat as activity.
|
|
115
|
+
return true;
|
|
116
|
+
}
|
|
117
|
+
return true;
|
|
118
|
+
}
|
|
119
|
+
// Unknown runtime — be conservative.
|
|
120
|
+
return true;
|
|
121
|
+
}
|
|
122
|
+
return false;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Tick-loop hook. Iterates active processes, applies the four detection
|
|
127
|
+
* gates, kills + completes any that wedge.
|
|
128
|
+
*
|
|
129
|
+
* @param {Map<string, object>} activeProcesses Engine's tracked-process map
|
|
130
|
+
* @param {object} config Live engine config
|
|
131
|
+
* @param {object} [opts] Test seams
|
|
132
|
+
* @param {() => number} [opts.now] Clock injector
|
|
133
|
+
* @param {(pid:number) => number|null} [opts.cpuSampler] CPU-sec sampler
|
|
134
|
+
* @param {(p:string) => string|null} [opts.tailReader] live-output tail reader
|
|
135
|
+
* @param {(proc:object) => void} [opts.killer] killImmediate injector
|
|
136
|
+
* @param {(dispatchId, result, reason, summary, opts) => void} [opts.completer]
|
|
137
|
+
*/
|
|
138
|
+
function checkSpawnPhaseStalls(activeProcesses, config, opts = {}) {
|
|
139
|
+
if (!activeProcesses || typeof activeProcesses.entries !== 'function') return;
|
|
140
|
+
const enabled = config?.engine?.spawnPhaseWatchdogEnabled
|
|
141
|
+
?? ENGINE_DEFAULTS.spawnPhaseWatchdogEnabled;
|
|
142
|
+
if (enabled === false) return;
|
|
143
|
+
|
|
144
|
+
const graceMs = Math.max(1000, Number(config?.engine?.spawnPhaseGraceMs)
|
|
145
|
+
|| ENGINE_DEFAULTS.spawnPhaseGraceMs);
|
|
146
|
+
const maxCpuSec = Math.max(0, Number(config?.engine?.spawnPhaseMaxCpuSeconds
|
|
147
|
+
?? ENGINE_DEFAULTS.spawnPhaseMaxCpuSeconds));
|
|
148
|
+
|
|
149
|
+
const now = (opts.now || Date.now)();
|
|
150
|
+
const cpuSampler = opts.cpuSampler || ((pid) => shared.getProcessCpuSeconds(pid));
|
|
151
|
+
const tailReader = opts.tailReader || _defaultTailReader;
|
|
152
|
+
const killer = opts.killer || ((proc) => shared.killImmediate(proc));
|
|
153
|
+
const completer = opts.completer || ((id, result, reason, summary, o) => dispatch().completeDispatch(id, result, reason, summary, o));
|
|
154
|
+
|
|
155
|
+
for (const [id, procInfo] of activeProcesses.entries()) {
|
|
156
|
+
if (!procInfo) continue;
|
|
157
|
+
// Skip re-attached processes — we have no reliable spawn timestamp
|
|
158
|
+
// for their startup phase, and their live-output may be mid-stream.
|
|
159
|
+
if (procInfo.reattached) continue;
|
|
160
|
+
// Skip processes that have already been promoted past spawn phase
|
|
161
|
+
// by a prior watchdog tick (single-shot per spawn).
|
|
162
|
+
if (procInfo._spawnPhaseCleared) continue;
|
|
163
|
+
// Skip processes that the steering pipeline has already killed —
|
|
164
|
+
// a re-spawn will track itself.
|
|
165
|
+
if (procInfo._steeringAt) continue;
|
|
166
|
+
const proc = procInfo.proc;
|
|
167
|
+
if (!proc) continue;
|
|
168
|
+
// If the process already exited, let timeout.js / onAgentClose handle it.
|
|
169
|
+
if (Object.prototype.hasOwnProperty.call(proc, 'exitCode') && proc.exitCode !== null) continue;
|
|
170
|
+
|
|
171
|
+
const startedAtMs = procInfo.startedAt ? Date.parse(procInfo.startedAt) : NaN;
|
|
172
|
+
if (!Number.isFinite(startedAtMs)) continue;
|
|
173
|
+
const elapsed = now - startedAtMs;
|
|
174
|
+
if (elapsed < graceMs) continue;
|
|
175
|
+
|
|
176
|
+
const agentId = procInfo.agentId;
|
|
177
|
+
if (!agentId) continue;
|
|
178
|
+
|
|
179
|
+
const liveLogPath = path.join(AGENTS_DIR, agentId, 'live-output.log');
|
|
180
|
+
const tail = tailReader(liveLogPath);
|
|
181
|
+
|
|
182
|
+
if (hasRealTaskActivity(tail, procInfo.runtimeName)) {
|
|
183
|
+
// Promote out of spawn phase — never re-check this process again.
|
|
184
|
+
procInfo._spawnPhaseCleared = true;
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// Still in spawn phase past the grace. Now gate on CPU.
|
|
189
|
+
const pid = proc.pid;
|
|
190
|
+
if (!pid) continue;
|
|
191
|
+
const cpuSec = cpuSampler(pid);
|
|
192
|
+
if (cpuSec == null) {
|
|
193
|
+
// Fail-open: can't sample → don't kill. Try again next tick.
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
if (cpuSec > maxCpuSec) {
|
|
197
|
+
// Process is doing work; just slow to emit. Stand down.
|
|
198
|
+
continue;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// All four gates passed — this is a wedge. Kill + complete.
|
|
202
|
+
const elapsedSec = Math.round(elapsed / 1000);
|
|
203
|
+
const reason = `Spawn-phase stall: no task events after ${elapsedSec}s (CPU ${cpuSec.toFixed(2)}s ≤ ${maxCpuSec}s)`;
|
|
204
|
+
log('warn', `spawn-phase-watchdog: killing ${agentId} (${id}) — ${reason}`);
|
|
205
|
+
|
|
206
|
+
// Write a structured inbox note BEFORE killing so the dispatch
|
|
207
|
+
// completion's failure-report doesn't overwrite our diagnostic.
|
|
208
|
+
try {
|
|
209
|
+
const noteContent = [
|
|
210
|
+
`# Spawn-phase stall: ${agentId}`,
|
|
211
|
+
'',
|
|
212
|
+
`**Dispatch:** \`${id}\``,
|
|
213
|
+
`**Agent:** ${agentId}`,
|
|
214
|
+
`**Runtime:** ${procInfo.runtimeName || '(unknown)'}`,
|
|
215
|
+
`**PID:** ${pid}`,
|
|
216
|
+
`**Elapsed since spawn:** ${elapsedSec}s`,
|
|
217
|
+
`**CPU seconds:** ${cpuSec.toFixed(2)} (threshold ${maxCpuSec})`,
|
|
218
|
+
`**Grace window:** ${Math.round(graceMs / 1000)}s`,
|
|
219
|
+
'',
|
|
220
|
+
'## Tail of live-output.log',
|
|
221
|
+
'```',
|
|
222
|
+
(tail || '(empty)').slice(-2000),
|
|
223
|
+
'```',
|
|
224
|
+
'',
|
|
225
|
+
'The runtime emitted only startup-only events (MCP init, hook fires) ' +
|
|
226
|
+
'and burned no CPU past the grace window. Engine treated this as a wedged ' +
|
|
227
|
+
'startup and killed the child; the dispatch is marked retryable so the next ' +
|
|
228
|
+
'tick will respawn with a fresh runtime invocation.',
|
|
229
|
+
].join('\n');
|
|
230
|
+
shared.writeToInbox('engine', `spawn-phase-stall-${id}`, noteContent, null, {
|
|
231
|
+
dispatchId: id, agent: agentId, failureClass: FAILURE_CLASS.SPAWN_PHASE_STALL,
|
|
232
|
+
});
|
|
233
|
+
} catch (e) { log('warn', `spawn-phase-watchdog: inbox note: ${e.message}`); }
|
|
234
|
+
|
|
235
|
+
try { killer(proc); }
|
|
236
|
+
catch (e) { log('warn', `spawn-phase-watchdog: kill: ${e.message}`); }
|
|
237
|
+
|
|
238
|
+
try {
|
|
239
|
+
completer(id, shared.DISPATCH_RESULT.ERROR, reason, '', {
|
|
240
|
+
failureClass: FAILURE_CLASS.SPAWN_PHASE_STALL,
|
|
241
|
+
agentRetryable: true,
|
|
242
|
+
processWorkItemFailure: true,
|
|
243
|
+
});
|
|
244
|
+
} catch (e) { log('warn', `spawn-phase-watchdog: completeDispatch: ${e.message}`); }
|
|
245
|
+
|
|
246
|
+
// Clear the engine's tracking so the next discovery tick can rebuild.
|
|
247
|
+
try { activeProcesses.delete(id); } catch { /* defensive */ }
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
function _defaultTailReader(filePath) {
|
|
252
|
+
try {
|
|
253
|
+
const stat = fs.statSync(filePath);
|
|
254
|
+
if (!stat.size) return '';
|
|
255
|
+
const maxBytes = 16384;
|
|
256
|
+
const tailSize = Math.min(stat.size, maxBytes);
|
|
257
|
+
const fd = fs.openSync(filePath, 'r');
|
|
258
|
+
try {
|
|
259
|
+
const buf = Buffer.alloc(tailSize);
|
|
260
|
+
fs.readSync(fd, buf, 0, tailSize, Math.max(0, stat.size - tailSize));
|
|
261
|
+
return buf.toString('utf8');
|
|
262
|
+
} finally { fs.closeSync(fd); }
|
|
263
|
+
} catch { return null; }
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
module.exports = {
|
|
267
|
+
checkSpawnPhaseStalls,
|
|
268
|
+
hasRealTaskActivity, // exported for testing
|
|
269
|
+
COPILOT_STARTUP_TYPES, // exported for testing
|
|
270
|
+
CLAUDE_STARTUP_SUBTYPES, // exported for testing
|
|
271
|
+
};
|
package/engine/steering.js
CHANGED
|
@@ -19,6 +19,43 @@ function _generateSteerId() {
|
|
|
19
19
|
return `steer-${crypto.randomBytes(8).toString('hex').slice(0, 10)}`;
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
+
// W-mq066js7000fff1f-e (Gap E): identical-text dedupe window. POST
|
|
23
|
+
// /api/agents/steer collapses identical bodies within this window to
|
|
24
|
+
// the existing steerId instead of writing a duplicate inbox file.
|
|
25
|
+
const DEDUPE_WINDOW_MS = 5 * 60 * 1000;
|
|
26
|
+
|
|
27
|
+
// W-mq066js7000fff1f-e/f: status enum for the inbox frontmatter +
|
|
28
|
+
// supersede operations. Mirrors the steering_deliveries observability
|
|
29
|
+
// table (Gap D) so callers can speak the same vocabulary regardless
|
|
30
|
+
// of which storage backend is live.
|
|
31
|
+
const STATUS = Object.freeze({
|
|
32
|
+
QUEUED: 'queued',
|
|
33
|
+
LIVE_KILL: 'live_kill',
|
|
34
|
+
DEFERRED: 'deferred',
|
|
35
|
+
RE_SPAWNING: 're_spawning',
|
|
36
|
+
DELIVERED: 'delivered',
|
|
37
|
+
ACKNOWLEDGED: 'acknowledged',
|
|
38
|
+
STRANDED: 'stranded',
|
|
39
|
+
DROPPED: 'dropped',
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
const UNACKED_STATUSES = new Set([
|
|
43
|
+
STATUS.QUEUED,
|
|
44
|
+
STATUS.LIVE_KILL,
|
|
45
|
+
STATUS.DEFERRED,
|
|
46
|
+
STATUS.RE_SPAWNING,
|
|
47
|
+
STATUS.STRANDED,
|
|
48
|
+
]);
|
|
49
|
+
|
|
50
|
+
// Statuses that should still appear as "live" in the dedupe lookup.
|
|
51
|
+
const DEDUPE_CANDIDATE_STATUSES = new Set([
|
|
52
|
+
STATUS.QUEUED,
|
|
53
|
+
STATUS.LIVE_KILL,
|
|
54
|
+
STATUS.DEFERRED,
|
|
55
|
+
STATUS.RE_SPAWNING,
|
|
56
|
+
STATUS.DELIVERED,
|
|
57
|
+
]);
|
|
58
|
+
|
|
22
59
|
function agentInboxDir(agentId) {
|
|
23
60
|
return path.join(AGENTS_DIR, agentId, 'inbox');
|
|
24
61
|
}
|
|
@@ -73,15 +110,22 @@ function _readEntry(filePath, legacy = false) {
|
|
|
73
110
|
? fmCreatedAtMs
|
|
74
111
|
: _createdAtFromPath(filePath, stat);
|
|
75
112
|
const steerId = _frontmatterValue(raw, 'steerId') || null;
|
|
113
|
+
const status = _frontmatterValue(raw, 'status') || STATUS.QUEUED;
|
|
114
|
+
const targetDispatchId = _frontmatterValue(raw, 'targetDispatchId') || null;
|
|
115
|
+
const lastError = _frontmatterValue(raw, 'lastError') || null;
|
|
116
|
+
const source = _frontmatterValue(raw, 'source') || 'human';
|
|
76
117
|
return {
|
|
77
118
|
path: filePath,
|
|
78
119
|
file: path.basename(filePath),
|
|
79
120
|
createdAtMs,
|
|
80
121
|
createdAt: new Date(createdAtMs).toISOString(),
|
|
81
|
-
steerId,
|
|
82
122
|
raw,
|
|
83
123
|
message: _messageFromRaw(raw),
|
|
84
124
|
steerId,
|
|
125
|
+
status,
|
|
126
|
+
targetDispatchId,
|
|
127
|
+
lastError,
|
|
128
|
+
source,
|
|
85
129
|
legacy,
|
|
86
130
|
};
|
|
87
131
|
}
|
|
@@ -94,10 +138,6 @@ function _uniqueSteeringPath(inboxDir, createdAtMs) {
|
|
|
94
138
|
return filePath;
|
|
95
139
|
}
|
|
96
140
|
|
|
97
|
-
function _generateSteerId() {
|
|
98
|
-
return crypto.randomBytes(6).toString('hex');
|
|
99
|
-
}
|
|
100
|
-
|
|
101
141
|
// Contract block describing the ACK-file protocol. Injected into the prompt
|
|
102
142
|
// alongside any pending steering messages so the agent knows how to confirm
|
|
103
143
|
// it has read+addressed a labeled message. Mirrored verbatim into
|
|
@@ -111,9 +151,24 @@ function ackContractBlock() {
|
|
|
111
151
|
].join('\n');
|
|
112
152
|
}
|
|
113
153
|
|
|
154
|
+
function _renderFrontmatter(data) {
|
|
155
|
+
const createdAtMs = Number(data.createdAtMs) || Date.now();
|
|
156
|
+
const lines = [
|
|
157
|
+
'---',
|
|
158
|
+
`createdAt: ${new Date(createdAtMs).toISOString()}`,
|
|
159
|
+
`createdAtMs: ${createdAtMs}`,
|
|
160
|
+
`source: ${data.source || 'human'}`,
|
|
161
|
+
`steerId: ${data.steerId}`,
|
|
162
|
+
`status: ${data.status || STATUS.QUEUED}`,
|
|
163
|
+
];
|
|
164
|
+
if (data.targetDispatchId) lines.push(`targetDispatchId: ${data.targetDispatchId}`);
|
|
165
|
+
if (data.lastError) lines.push(`lastError: ${String(data.lastError).replace(/[\r\n]+/g, ' ').trim()}`);
|
|
166
|
+
lines.push('---', '', String(data.message || '').trim(), '');
|
|
167
|
+
return lines.join('\n');
|
|
168
|
+
}
|
|
169
|
+
|
|
114
170
|
function writeSteeringMessage(agentId, message, opts = {}) {
|
|
115
171
|
const createdAtMs = Number(opts.createdAtMs) || Date.now();
|
|
116
|
-
const createdAt = new Date(createdAtMs).toISOString();
|
|
117
172
|
const inboxDir = agentInboxDir(agentId);
|
|
118
173
|
fs.mkdirSync(inboxDir, { recursive: true });
|
|
119
174
|
const filePath = _uniqueSteeringPath(inboxDir, createdAtMs);
|
|
@@ -134,17 +189,15 @@ function writeSteeringMessage(agentId, message, opts = {}) {
|
|
|
134
189
|
steerId,
|
|
135
190
|
}));
|
|
136
191
|
}
|
|
137
|
-
const body =
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
'',
|
|
147
|
-
].join('\n');
|
|
192
|
+
const body = _renderFrontmatter({
|
|
193
|
+
createdAtMs,
|
|
194
|
+
source,
|
|
195
|
+
steerId,
|
|
196
|
+
status: opts.status || STATUS.QUEUED,
|
|
197
|
+
targetDispatchId: opts.targetDispatchId || null,
|
|
198
|
+
lastError: opts.lastError || null,
|
|
199
|
+
message: bodyText,
|
|
200
|
+
});
|
|
148
201
|
shared.safeWrite(filePath, body);
|
|
149
202
|
|
|
150
203
|
// W-mq066js7000fff1f-a (Gap D): insert a 'queued' row into the
|
|
@@ -169,7 +222,22 @@ function writeSteeringMessage(agentId, message, opts = {}) {
|
|
|
169
222
|
return _readEntry(filePath);
|
|
170
223
|
}
|
|
171
224
|
|
|
172
|
-
function
|
|
225
|
+
function _updateEntryStatus(entry, newStatus, opts = {}) {
|
|
226
|
+
if (!entry?.path) return null;
|
|
227
|
+
const body = _renderFrontmatter({
|
|
228
|
+
createdAtMs: entry.createdAtMs,
|
|
229
|
+
source: entry.source || 'human',
|
|
230
|
+
steerId: entry.steerId,
|
|
231
|
+
status: newStatus,
|
|
232
|
+
targetDispatchId: entry.targetDispatchId,
|
|
233
|
+
lastError: opts.lastError !== undefined ? opts.lastError : entry.lastError,
|
|
234
|
+
message: entry.message,
|
|
235
|
+
});
|
|
236
|
+
shared.safeWrite(entry.path, body);
|
|
237
|
+
return _readEntry(entry.path);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function listAllSteeringEntries(agentId, opts = {}) {
|
|
173
241
|
const includeLegacy = opts.includeLegacy !== false;
|
|
174
242
|
const entries = [];
|
|
175
243
|
const inboxDir = agentInboxDir(agentId);
|
|
@@ -189,8 +257,40 @@ function listUnreadSteeringMessages(agentId, opts = {}) {
|
|
|
189
257
|
return entries;
|
|
190
258
|
}
|
|
191
259
|
|
|
192
|
-
function
|
|
193
|
-
|
|
260
|
+
function listUnreadSteeringMessages(agentId, opts = {}) {
|
|
261
|
+
// Back-compat shape: "unread" = anything still pending agent attention
|
|
262
|
+
// (queued/live_kill/deferred/re_spawning/delivered/stranded). Explicit
|
|
263
|
+
// 'dropped' or 'acknowledged' rows are filtered so supersede/ack don't
|
|
264
|
+
// resurrect carry-over messages on the next dispatch.
|
|
265
|
+
return listAllSteeringEntries(agentId, opts).filter(entry => {
|
|
266
|
+
const status = entry.status || STATUS.QUEUED;
|
|
267
|
+
return status !== STATUS.DROPPED && status !== STATUS.ACKNOWLEDGED;
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// W-mq066js7000fff1f-f (Gap F): per-dispatch scoping. Callers that know
|
|
272
|
+
// which dispatch they're about to spawn can pass {currentDispatchId} so
|
|
273
|
+
// messages tagged for a different (older) dispatch are filtered out.
|
|
274
|
+
// Pre-spawn messages (targetDispatchId null) always pass through.
|
|
275
|
+
// Without currentDispatchId, no filtering happens — back-compat for
|
|
276
|
+
// callers that haven't been updated to pass the dispatch id yet.
|
|
277
|
+
function buildPendingSteeringPrompt(agentId, opts = {}) {
|
|
278
|
+
const includePrior = opts.includePrior === true;
|
|
279
|
+
const currentDispatchId = opts.currentDispatchId || null;
|
|
280
|
+
const allEntries = listUnreadSteeringMessages(agentId).filter(entry => entry.message.trim());
|
|
281
|
+
const entries = includePrior
|
|
282
|
+
? allEntries
|
|
283
|
+
: allEntries.filter(entry => {
|
|
284
|
+
// Pre-spawn / agent-scoped messages (null targetDispatchId) always
|
|
285
|
+
// pick up on the next dispatch — the agent never had a chance to
|
|
286
|
+
// hear them yet.
|
|
287
|
+
if (!entry.targetDispatchId) return true;
|
|
288
|
+
// No dispatch context → don't try to filter (back-compat).
|
|
289
|
+
if (!currentDispatchId) return true;
|
|
290
|
+
// Per-dispatch messages only belong to their tagged dispatch.
|
|
291
|
+
return entry.targetDispatchId === currentDispatchId;
|
|
292
|
+
});
|
|
293
|
+
|
|
194
294
|
if (entries.length === 0) return { entries, prompt: '' };
|
|
195
295
|
|
|
196
296
|
const sections = [
|
|
@@ -206,6 +306,69 @@ function buildPendingSteeringPrompt(agentId) {
|
|
|
206
306
|
return { entries, prompt: sections.join('\n') };
|
|
207
307
|
}
|
|
208
308
|
|
|
309
|
+
// W-mq066js7000fff1f-e (Gap E): dedupe lookup for POST /api/agents/steer.
|
|
310
|
+
// Returns the existing entry whose normalized body matches `message` and
|
|
311
|
+
// was created within `opts.windowMs` (default 5 min); null otherwise.
|
|
312
|
+
// Only entries in "live" delivery states qualify — dropped/acknowledged
|
|
313
|
+
// rows are ignored so a previously-superseded duplicate body is allowed
|
|
314
|
+
// to be re-sent.
|
|
315
|
+
function findRecentDuplicate(agentId, message, opts = {}) {
|
|
316
|
+
const trimmed = String(message || '').trim();
|
|
317
|
+
if (!trimmed) return null;
|
|
318
|
+
const windowMs = Number(opts.windowMs) > 0 ? Number(opts.windowMs) : DEDUPE_WINDOW_MS;
|
|
319
|
+
const now = Number(opts.now) > 0 ? Number(opts.now) : Date.now();
|
|
320
|
+
const entries = listAllSteeringEntries(agentId);
|
|
321
|
+
// Newest first — UI usually steers in a tight loop, the most recent
|
|
322
|
+
// identical message is the one we want to deduplicate against.
|
|
323
|
+
entries.sort((a, b) => b.createdAtMs - a.createdAtMs);
|
|
324
|
+
for (const entry of entries) {
|
|
325
|
+
const status = entry.status || STATUS.QUEUED;
|
|
326
|
+
if (!DEDUPE_CANDIDATE_STATUSES.has(status)) continue;
|
|
327
|
+
if (now - entry.createdAtMs > windowMs) continue;
|
|
328
|
+
if (entry.message.trim() !== trimmed) continue;
|
|
329
|
+
return entry;
|
|
330
|
+
}
|
|
331
|
+
return null;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
// W-mq066js7000fff1f-e (Gap E): supersede prior steering messages.
|
|
335
|
+
// mode:
|
|
336
|
+
// 'all' — drop every entry whose status is neither dropped nor
|
|
337
|
+
// acknowledged (queued/live_kill/deferred/re_spawning/
|
|
338
|
+
// delivered/stranded).
|
|
339
|
+
// 'unacked' — drop queued/live_kill/deferred/re_spawning/stranded but
|
|
340
|
+
// leave 'delivered' alone (agent already saw it).
|
|
341
|
+
// <steerId> — drop the single entry with that steerId.
|
|
342
|
+
// Returns the list of dropped {steerId, file, path, previousStatus}.
|
|
343
|
+
function supersedeMessages(agentId, mode, opts = {}) {
|
|
344
|
+
if (!mode) return [];
|
|
345
|
+
const newSteerId = opts.newSteerId || null;
|
|
346
|
+
const reason = opts.reason || (newSteerId ? `superseded by ${newSteerId}` : 'superseded');
|
|
347
|
+
const entries = listAllSteeringEntries(agentId);
|
|
348
|
+
const dropped = [];
|
|
349
|
+
for (const entry of entries) {
|
|
350
|
+
const status = entry.status || STATUS.QUEUED;
|
|
351
|
+
let target = false;
|
|
352
|
+
if (mode === 'all') {
|
|
353
|
+
if (status !== STATUS.ACKNOWLEDGED && status !== STATUS.DROPPED) target = true;
|
|
354
|
+
} else if (mode === 'unacked') {
|
|
355
|
+
if (UNACKED_STATUSES.has(status)) target = true;
|
|
356
|
+
} else {
|
|
357
|
+
// Specific steerId
|
|
358
|
+
if (entry.steerId && entry.steerId === mode) target = true;
|
|
359
|
+
}
|
|
360
|
+
if (!target) continue;
|
|
361
|
+
_updateEntryStatus(entry, STATUS.DROPPED, { lastError: reason });
|
|
362
|
+
dropped.push({
|
|
363
|
+
steerId: entry.steerId,
|
|
364
|
+
file: entry.file,
|
|
365
|
+
path: entry.path,
|
|
366
|
+
previousStatus: status,
|
|
367
|
+
});
|
|
368
|
+
}
|
|
369
|
+
return dropped;
|
|
370
|
+
}
|
|
371
|
+
|
|
209
372
|
function _eventTimestampMs(obj, observedAtMs) {
|
|
210
373
|
const value = obj?.timestamp || obj?.createdAt || obj?.created_at || obj?.time || obj?.data?.timestamp;
|
|
211
374
|
const parsed = value ? Date.parse(value) : NaN;
|
|
@@ -342,9 +505,17 @@ module.exports = {
|
|
|
342
505
|
ackContractBlock,
|
|
343
506
|
writeSteeringMessage,
|
|
344
507
|
listUnreadSteeringMessages,
|
|
508
|
+
listAllSteeringEntries,
|
|
345
509
|
buildPendingSteeringPrompt,
|
|
510
|
+
findRecentDuplicate,
|
|
511
|
+
supersedeMessages,
|
|
346
512
|
sessionIdFromEvent,
|
|
347
513
|
sessionIdFromOutputLine,
|
|
348
514
|
ackProcessedSteeringMessages,
|
|
349
515
|
ackSteeringFromAckDir,
|
|
516
|
+
STATUS,
|
|
517
|
+
DEDUPE_WINDOW_MS,
|
|
518
|
+
// Exposed for unit tests only.
|
|
519
|
+
_updateEntryStatus,
|
|
520
|
+
_generateSteerId,
|
|
350
521
|
};
|
package/engine.js
CHANGED
|
@@ -1326,7 +1326,7 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
1326
1326
|
// work-item prompts after setup because reused worktrees can live at arbitrary paths.
|
|
1327
1327
|
const systemPrompt = buildSystemPrompt(agentId, config, project);
|
|
1328
1328
|
const agentContext = buildAgentContext(agentId, config, project);
|
|
1329
|
-
const pendingSteering = steering.buildPendingSteeringPrompt(agentId);
|
|
1329
|
+
const pendingSteering = steering.buildPendingSteeringPrompt(agentId, { currentDispatchId: id });
|
|
1330
1330
|
const completionReportPath = shared.dispatchCompletionReportPath(id);
|
|
1331
1331
|
if (completionReportPath) {
|
|
1332
1332
|
try {
|
|
@@ -2793,7 +2793,7 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
2793
2793
|
// Write new prompt with all unACKed steering messages. This keeps delivery
|
|
2794
2794
|
// durable if the killed process had older pending messages that never
|
|
2795
2795
|
// produced processing evidence before the resume.
|
|
2796
|
-
const pendingForResume = steering.buildPendingSteeringPrompt(agentId);
|
|
2796
|
+
const pendingForResume = steering.buildPendingSteeringPrompt(agentId, { currentDispatchId: id });
|
|
2797
2797
|
const steerPromptBody = pendingForResume.prompt || steerMsg;
|
|
2798
2798
|
const steerPrompt = `Message from your human teammate:\n\n${steerPromptBody}\n\nRespond to this, then continue working on your current task.`;
|
|
2799
2799
|
const steerPromptPath = path.join(dispatchTmpDir, `prompt-steer-${safeId}.md`);
|
|
@@ -7254,6 +7254,10 @@ async function tickInner() {
|
|
|
7254
7254
|
// 1. Check for timed-out agents, steering messages, and idle threshold
|
|
7255
7255
|
safe('checkTimeouts', () => checkTimeouts(config));
|
|
7256
7256
|
safe('checkSteering', () => checkSteering(config));
|
|
7257
|
+
safe('checkSpawnPhaseStalls', () => {
|
|
7258
|
+
const { checkSpawnPhaseStalls } = require('./engine/spawn-phase-watchdog');
|
|
7259
|
+
checkSpawnPhaseStalls(activeProcesses, config);
|
|
7260
|
+
});
|
|
7257
7261
|
safe('checkIdleThreshold', () => checkIdleThreshold(config));
|
|
7258
7262
|
|
|
7259
7263
|
// 1b. Check for meeting round timeouts
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@yemi33/minions",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2126",
|
|
4
4
|
"description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
|
|
5
5
|
"bin": {
|
|
6
6
|
"minions": "bin/minions.js"
|