@adhdev/daemon-core 0.9.82-rc.552 → 0.9.82-rc.553
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-adapters/provider-cli-adapter.d.ts +80 -1
- package/dist/cli-adapters/provider-cli-shared.d.ts +55 -0
- package/dist/cli-adapters/terminal-screen.d.ts +5 -0
- package/dist/commands/stream-commands.d.ts +31 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +453 -9
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +449 -6
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-ledger.d.ts +1 -1
- package/dist/providers/cli-provider-instance.d.ts +82 -0
- package/dist/providers/provider-instance.d.ts +9 -0
- package/dist/repo-mesh-types.d.ts +23 -0
- package/package.json +3 -3
- package/src/cli-adapters/provider-cli-adapter.ts +135 -0
- package/src/cli-adapters/provider-cli-shared.ts +172 -0
- package/src/cli-adapters/terminal-screen.ts +5 -0
- package/src/commands/handler.ts +4 -0
- package/src/commands/router.ts +15 -0
- package/src/commands/stream-commands.ts +125 -0
- package/src/index.ts +1 -0
- package/src/mesh/coordinator-prompt.ts +2 -0
- package/src/mesh/mesh-events-utils.ts +15 -0
- package/src/mesh/mesh-ledger.ts +6 -0
- package/src/providers/cli-provider-instance.ts +196 -0
- package/src/providers/provider-instance-manager.ts +11 -0
- package/src/providers/provider-instance.ts +10 -0
- package/src/repo-mesh-types.ts +35 -0
|
@@ -134,6 +134,131 @@ export function handlePtyResize(_h: CommandHelpers, args: any): CommandResult {
|
|
|
134
134
|
return { success: false, error: 'PTY resize temporarily disabled', code: 'PTY_RESIZE_DISABLED' };
|
|
135
135
|
}
|
|
136
136
|
|
|
137
|
+
interface TerminalSnapshotInstance extends ProviderInstance {
|
|
138
|
+
getTerminalScreenSnapshot?(maxBytes?: number): {
|
|
139
|
+
text: string;
|
|
140
|
+
cursor: { col: number; row: number };
|
|
141
|
+
cols: number;
|
|
142
|
+
rows: number;
|
|
143
|
+
truncated: boolean;
|
|
144
|
+
originalBytes: number;
|
|
145
|
+
returnedBytes: number;
|
|
146
|
+
hash: string;
|
|
147
|
+
} | null;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* MESH-READ-TERMINAL (feature 2: RAW terminal read). Reads the CURRENT rendered
|
|
152
|
+
* PTY viewport of a specific mesh worker session for the mesh_read_terminal tool.
|
|
153
|
+
*
|
|
154
|
+
* This is the daemon-side `read_terminal` verb. It runs BOTH on the coordinator
|
|
155
|
+
* (for a locally-hosted worker) and, after router forwarding
|
|
156
|
+
* (MESH_FORWARDABLE_SESSION_COMMANDS + _meshDirectDispatch), on the OWNING remote
|
|
157
|
+
* worker daemon — where the live viewport actually exists. The instance's
|
|
158
|
+
* getTerminalScreenSnapshot() is gated on isMeshWorkerSession() (returns null for
|
|
159
|
+
* a non-mesh session), which the MCP layer complements with a mesh/session/node
|
|
160
|
+
* ownership cross-check. SECURITY: the raw viewport can contain tokens / args /
|
|
161
|
+
* env / user data — never logged here (only its byte size / truncation flag are).
|
|
162
|
+
*/
|
|
163
|
+
export function handleReadTerminal(h: CommandHelpers, args: any): CommandResult {
|
|
164
|
+
const targetSessionId = typeof args?.targetSessionId === 'string' ? args.targetSessionId.trim() : '';
|
|
165
|
+
const sessionId = targetSessionId || h.currentSession?.sessionId || '';
|
|
166
|
+
if (!sessionId) return { success: false, error: 'targetSessionId required' };
|
|
167
|
+
|
|
168
|
+
const session = h.ctx.sessionRegistry?.get(sessionId);
|
|
169
|
+
const instanceKey = session?.adapterKey || session?.instanceKey || sessionId;
|
|
170
|
+
const instance = h.ctx.instanceManager?.getInstance(instanceKey) as TerminalSnapshotInstance | undefined;
|
|
171
|
+
if (!instance) return { success: false, error: `Session not found: ${sessionId.split('_')[0]}` };
|
|
172
|
+
if (instance.category !== 'cli' || typeof instance.getTerminalScreenSnapshot !== 'function') {
|
|
173
|
+
return { success: false, error: 'read_terminal is only supported for CLI (PTY) sessions' };
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
const requestedMaxBytes = typeof args?.maxBytes === 'number' && Number.isFinite(args.maxBytes)
|
|
177
|
+
? args.maxBytes
|
|
178
|
+
: undefined;
|
|
179
|
+
const snapshot = instance.getTerminalScreenSnapshot(requestedMaxBytes);
|
|
180
|
+
if (!snapshot) {
|
|
181
|
+
// getTerminalScreenSnapshot returns null when the session is NOT a mesh
|
|
182
|
+
// worker — the raw-viewport read is scoped to coordinator-delegated workers.
|
|
183
|
+
return { success: false, error: 'read_terminal is only available for coordinator-spawned mesh worker sessions' };
|
|
184
|
+
}
|
|
185
|
+
// Log size/truncation ONLY — never the screen text (may carry secrets).
|
|
186
|
+
LOG.info('Command', `[readTerminal] session=${sessionId.split('_')[0]} bytes=${snapshot.returnedBytes}/${snapshot.originalBytes} truncated=${snapshot.truncated} cols=${snapshot.cols} rows=${snapshot.rows}`);
|
|
187
|
+
return { success: true, ...snapshot };
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
interface KeyInjectableInstance extends ProviderInstance {
|
|
191
|
+
injectKeys?(
|
|
192
|
+
items: Array<{ text: string } | { key: string }>,
|
|
193
|
+
opts?: { allowModalOverride?: boolean },
|
|
194
|
+
): Promise<
|
|
195
|
+
| { ok: true; keys: string[]; hasDestructive: boolean; submits: boolean; bytes: number }
|
|
196
|
+
| { ok: false; refused: string; keys: string[]; hasDestructive: boolean }
|
|
197
|
+
>;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* MESH-SEND-KEYS (feature 3: key injection). Inject a structured key sequence into
|
|
202
|
+
* a specific mesh worker session's PTY for the mesh_send_keys tool.
|
|
203
|
+
*
|
|
204
|
+
* Daemon-side `send_keys` verb. Like read_terminal it runs on the coordinator for
|
|
205
|
+
* a local worker and, after router forwarding (MESH_FORWARDABLE_SESSION_COMMANDS +
|
|
206
|
+
* _meshDirectDispatch), on the OWNING remote worker daemon. The instance's
|
|
207
|
+
* injectKeys() is gated on isMeshWorkerSession(); the MCP layer complements it with
|
|
208
|
+
* mesh/session/node ownership + the destructive-key double gate (confirm_destructive
|
|
209
|
+
* + mesh policy) + audit ledger.
|
|
210
|
+
*
|
|
211
|
+
* Defense-in-depth here: even though the MCP layer gates destructive keys, the
|
|
212
|
+
* daemon re-enforces confirm_destructive so a direct/forwarded send_keys that
|
|
213
|
+
* contains CTRL_C/ESC without confirm is refused at the boundary too.
|
|
214
|
+
* SECURITY: never logs the literal text body (only key enums / byte counts).
|
|
215
|
+
*/
|
|
216
|
+
export async function handleSendKeys(h: CommandHelpers, args: any): Promise<CommandResult> {
|
|
217
|
+
const targetSessionId = typeof args?.targetSessionId === 'string' ? args.targetSessionId.trim() : '';
|
|
218
|
+
const sessionId = targetSessionId || h.currentSession?.sessionId || '';
|
|
219
|
+
if (!sessionId) return { success: false, error: 'targetSessionId required' };
|
|
220
|
+
|
|
221
|
+
const items = Array.isArray(args?.sequence) ? args.sequence : null;
|
|
222
|
+
if (!items || items.length === 0) {
|
|
223
|
+
return { success: false, error: 'sequence (non-empty array of {text}|{key}) required' };
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
const session = h.ctx.sessionRegistry?.get(sessionId);
|
|
227
|
+
const instanceKey = session?.adapterKey || session?.instanceKey || sessionId;
|
|
228
|
+
const instance = h.ctx.instanceManager?.getInstance(instanceKey) as KeyInjectableInstance | undefined;
|
|
229
|
+
if (!instance) return { success: false, error: `Session not found: ${sessionId.split('_')[0]}` };
|
|
230
|
+
if (instance.category !== 'cli' || typeof instance.injectKeys !== 'function') {
|
|
231
|
+
return { success: false, error: 'send_keys is only supported for CLI (PTY) sessions' };
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// Defense-in-depth destructive gate: refuse CTRL_C/ESC without confirm_destructive
|
|
235
|
+
// even at the daemon boundary (the MCP layer is the primary gate + policy check).
|
|
236
|
+
const DESTRUCTIVE = new Set(['CTRL_C', 'ESC']);
|
|
237
|
+
const hasDestructiveRequested = items.some((it: any) => it && typeof it.key === 'string' && DESTRUCTIVE.has(it.key));
|
|
238
|
+
if (hasDestructiveRequested && args?.confirm_destructive !== true) {
|
|
239
|
+
return {
|
|
240
|
+
success: false,
|
|
241
|
+
error: 'destructive key (CTRL_C/ESC) requires confirm_destructive=true',
|
|
242
|
+
refused: 'destructive_unconfirmed',
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
try {
|
|
247
|
+
const result = await instance.injectKeys(items, {
|
|
248
|
+
allowModalOverride: args?.allow_modal_override === true,
|
|
249
|
+
});
|
|
250
|
+
if (!result.ok) {
|
|
251
|
+
LOG.info('Command', `[sendKeys] session=${sessionId.split('_')[0]} refused=${result.refused} keys=${result.keys.join(',')} destructive=${result.hasDestructive}`);
|
|
252
|
+
return { success: false, error: `send_keys refused: ${result.refused}`, refused: result.refused, keys: result.keys, hasDestructive: result.hasDestructive };
|
|
253
|
+
}
|
|
254
|
+
LOG.info('Command', `[sendKeys] session=${sessionId.split('_')[0]} injected keys=${result.keys.join(',') || '(text-only)'} bytes=${result.bytes} destructive=${result.hasDestructive} submits=${result.submits}`);
|
|
255
|
+
return { success: true, keys: result.keys, hasDestructive: result.hasDestructive, submits: result.submits, bytes: result.bytes };
|
|
256
|
+
} catch (e: any) {
|
|
257
|
+
// Encode/validation errors (unknown key, over-limit) surface as a clean failure.
|
|
258
|
+
return { success: false, error: `send_keys: ${e?.message || String(e)}` };
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
137
262
|
// ─── Provider Settings ────────────────────────
|
|
138
263
|
|
|
139
264
|
export function handleGetProviderSettings(h: CommandHelpers, args: any): CommandResult {
|
package/src/index.ts
CHANGED
|
@@ -743,6 +743,8 @@ const TOOLS_SECTION = `## Available Tools
|
|
|
743
743
|
| \`mesh_launch_session\` | Start a new agent session on a node |
|
|
744
744
|
| \`mesh_read_chat\` | Read recent chat messages from a delegated agent session |
|
|
745
745
|
| \`mesh_read_debug\` | Collect a daemon-side chat/parser debug bundle for a session |
|
|
746
|
+
| \`mesh_read_terminal\` | Read a worker session's CURRENT raw terminal screen (the live rendered PTY viewport — prompt/modal/spinner/unparsed output), not the parsed chat. Byte-bounded (32KiB default, 64KiB max; bottom of screen kept). Use to see exactly what a worker is showing when mesh_read_chat is not enough (e.g. after a stall alert). Screen text may contain secrets — treat as sensitive |
|
|
747
|
+
| \`mesh_send_keys\` | Inject a STRUCTURED key sequence into a worker's live PTY (text + named keys ENTER/ESC/CTRL_C/UP/DOWN/LEFT/RIGHT/TAB/BACKSPACE). For interactions mesh_send_task can't express — answer a non-approval prompt, navigate a picker, submit a typed line, or interrupt (CTRL_C). Use mesh_approve for approval modals (send_keys is refused on one). Destructive keys (CTRL_C/ESC) need confirm_destructive=true AND mesh policy allowSendKeysDestructive. Refused on a pending submit/echo race. Audited (key enums only) |
|
|
746
748
|
| \`mesh_task_history\` | Read the task ledger — dispatches, completions, failures. Use to understand what has been done before deciding next steps |
|
|
747
749
|
| \`mesh_ledger_query\` | Read-only ledger query along the kind/time/node axes (complement to task-axis mesh_task_history): filter by kind, since, node, tail — answer "what happened on node X / what failed since T" without scanning transcripts |
|
|
748
750
|
| \`mesh_reconcile_ledger\` | Reconcile daemon-local ledgers over P2P — import missing entries from remote nodes into the coordinator local ledger |
|
|
@@ -356,6 +356,21 @@ export function buildMeshSystemMessage(args: {
|
|
|
356
356
|
return `[System] ${args.nodeLabel} has stopped${metadata}. Use mesh_read_chat once if you need to inspect its last output.`;
|
|
357
357
|
}
|
|
358
358
|
if (args.event === 'monitor:no_progress') {
|
|
359
|
+
// MESH-STALL-WATCH (feature 1: STALL detection): the status-agnostic stall
|
|
360
|
+
// watchdog fires this event regardless of the reported status — a worker's
|
|
361
|
+
// raw PTY output was byte-for-byte unchanged past the stall bound. Surface
|
|
362
|
+
// the generalized "output unchanged" wording (with the observed status,
|
|
363
|
+
// stalled duration and taskId as context) and make explicit this is
|
|
364
|
+
// INFORMATIONAL — a quiet/idle worker can trip it; it is NOT a failure or
|
|
365
|
+
// auto-restart. The generating-only StatusMonitor copy keeps its original
|
|
366
|
+
// phrasing.
|
|
367
|
+
if (args.metadataEvent.meshWorkerStall === true) {
|
|
368
|
+
const observedStatus = readNonEmptyString(args.metadataEvent.observedStatus);
|
|
369
|
+
const stalledMs = typeof args.metadataEvent.stalledMs === 'number' ? args.metadataEvent.stalledMs : undefined;
|
|
370
|
+
const stalledSuffix = stalledMs !== undefined ? ` for ${Math.round(stalledMs / 1000)}s` : '';
|
|
371
|
+
const statusSuffix = observedStatus ? ` (observed status: ${observedStatus})` : '';
|
|
372
|
+
return `[System] ${args.nodeLabel}: PTY output unchanged${stalledSuffix}${statusSuffix}${metadata}. This is an informational stall — the worker's screen has been static regardless of its reported status; it may be genuinely idle, waiting, or wedged, so this is NOT a failure or auto-restart. Judge whether to inspect it: wait for pendingCoordinatorEvents/a completion event, or make one bounded mesh_read_chat check if you need to see its current screen, then wait again.`;
|
|
373
|
+
}
|
|
359
374
|
return `[System] ${args.nodeLabel} is still reported as generating after a long interval${metadata}. Wait for pendingCoordinatorEvents or a completion/status event; if the user explicitly asks for status, make one bounded status check and then wait again.`;
|
|
360
375
|
}
|
|
361
376
|
if (args.event === 'worktree_bootstrap_complete') {
|
package/src/mesh/mesh-ledger.ts
CHANGED
|
@@ -82,6 +82,12 @@ export type MeshLedgerKind =
|
|
|
82
82
|
// magi_synthesis payload: { source:'magi', consensusGroupId, missionId?, panel?, question?, synthesis }
|
|
83
83
|
| 'magi_dispatched'
|
|
84
84
|
| 'magi_synthesis'
|
|
85
|
+
// MESH-SEND-KEYS (feature 3): audit trail for coordinator PTY key injections
|
|
86
|
+
// via mesh_send_keys. Records the key ENUMS, destructive flag and result —
|
|
87
|
+
// NEVER the literal text body (may carry tokens / user data).
|
|
88
|
+
// payload: { keys: string[], hasDestructive: boolean, result: 'injected'|'refused'|'error',
|
|
89
|
+
// refused?: string, submits?: boolean, confirmDestructive?: boolean }
|
|
90
|
+
| 'key_injection'
|
|
85
91
|
;
|
|
86
92
|
|
|
87
93
|
export interface MeshLedgerEntry {
|
|
@@ -16,6 +16,7 @@ import { normalizeInteractivePrompt, normalizeInteractivePromptResponse, type In
|
|
|
16
16
|
import { ProviderCliAdapter } from '../cli-adapters/provider-cli-adapter.js';
|
|
17
17
|
import { shortHash } from '../system/hash.js';
|
|
18
18
|
import type { CliProviderModule } from '../cli-adapters/provider-cli-adapter.js';
|
|
19
|
+
import type { MeshSendKeyItem, MeshSendKeyName } from '../cli-adapters/provider-cli-shared.js';
|
|
19
20
|
import { createCliAdapter } from './spec/route.js';
|
|
20
21
|
import type { PtyRuntimeMetadata, PtyTransportFactory } from '../cli-adapters/pty-transport.js';
|
|
21
22
|
import { StatusMonitor } from './status-monitor.js';
|
|
@@ -252,6 +253,16 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
252
253
|
*/
|
|
253
254
|
private static readonly APPROVAL_RESUME_GRACE_MS = 18_000;
|
|
254
255
|
|
|
256
|
+
// MESH-STALL-WATCH (feature 1: STALL detection): how long a coordinator-spawned
|
|
257
|
+
// mesh worker's raw PTY output (lastOutputAt) may stay unchanged before the
|
|
258
|
+
// status-agnostic stall watchdog fires ONE informational monitor:no_progress
|
|
259
|
+
// event. Unlike the StatusMonitor no-progress watchdog (which only runs while a
|
|
260
|
+
// turn is generating), this observes pure screen stasis regardless of the
|
|
261
|
+
// reported status — a worker parked idle, wedged mid-turn, or spawned with no
|
|
262
|
+
// output at all. 180s matches DEFAULT_MONITOR_CONFIG.noProgressThresholdSec so
|
|
263
|
+
// the two watchdogs agree on the same "long interval" bound.
|
|
264
|
+
private static readonly MESH_WORKER_STALL_THRESHOLD_MS = 180_000;
|
|
265
|
+
|
|
255
266
|
private adapter: ProviderCliAdapter;
|
|
256
267
|
private context: InstanceContext | null = null;
|
|
257
268
|
private events: ProviderEvent[] = [];
|
|
@@ -265,6 +276,16 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
265
276
|
// first sets it; the other becomes a no-op.
|
|
266
277
|
private agentReadyEmitted = false;
|
|
267
278
|
private generatingStartedAt: number = 0;
|
|
279
|
+
// MESH-STALL-WATCH (feature 1): the lastOutputAt value the stall episode is
|
|
280
|
+
// currently armed against. A stall episode is "the raw PTY output has not
|
|
281
|
+
// advanced past this anchor". When the adapter has never emitted output
|
|
282
|
+
// (lastOutputAt === 0) the anchor is the spawn time (this.startedAt) so a
|
|
283
|
+
// worker that produced NOTHING is still caught. On any new output the anchor
|
|
284
|
+
// re-arms to the fresh lastOutputAt and meshStallEmittedForAnchor resets, so a
|
|
285
|
+
// single continuous stall fires AT MOST ONCE and a later stall re-arms cleanly.
|
|
286
|
+
// -1 = not yet initialised for this session.
|
|
287
|
+
private meshStallAnchorAt = -1;
|
|
288
|
+
private meshStallEmittedForAnchor = false;
|
|
268
289
|
// FALSE-IDLE continuity epoch: monotonically bumped on EVERY entry into a busy
|
|
269
290
|
// phase (→generating or →waiting_approval). The completedDebouncePending snapshots
|
|
270
291
|
// this value at arm time (busyEpochAtArm); the flush guard requires it UNCHANGED
|
|
@@ -1966,6 +1987,167 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
1966
1987
|
|| this.settings.meshNodeId || this.settings.launchedByCoordinator);
|
|
1967
1988
|
}
|
|
1968
1989
|
|
|
1990
|
+
/**
|
|
1991
|
+
* MESH-READ-TERMINAL (feature 2: RAW terminal read). Public read of the
|
|
1992
|
+
* CURRENT rendered PTY viewport for the mesh_read_terminal tool, delegating to
|
|
1993
|
+
* the adapter's narrow getTerminalScreenSnapshot() (viewport + cursor + size
|
|
1994
|
+
* only; no debug buffers / parser state / history; byte-bounded, bottom-tail
|
|
1995
|
+
* preserved).
|
|
1996
|
+
*
|
|
1997
|
+
* Gated on isMeshWorkerSession(): this raw viewport can expose tokens /
|
|
1998
|
+
* command args / env / user data, so only a coordinator-spawned worker session
|
|
1999
|
+
* is readable. The MCP layer ALSO cross-checks mesh/session/node ownership
|
|
2000
|
+
* (isMeshOwnedDelegateSession) — isMeshWorkerSession alone is a broad
|
|
2001
|
+
* "delegated" gate, so the two together block cross-mesh access. Returns null
|
|
2002
|
+
* for a non-mesh session so the daemon command surfaces a clean refusal.
|
|
2003
|
+
*/
|
|
2004
|
+
getTerminalScreenSnapshot(maxBytes?: number): {
|
|
2005
|
+
text: string;
|
|
2006
|
+
cursor: { col: number; row: number };
|
|
2007
|
+
cols: number;
|
|
2008
|
+
rows: number;
|
|
2009
|
+
truncated: boolean;
|
|
2010
|
+
originalBytes: number;
|
|
2011
|
+
returnedBytes: number;
|
|
2012
|
+
hash: string;
|
|
2013
|
+
} | null {
|
|
2014
|
+
if (!this.isMeshWorkerSession()) return null;
|
|
2015
|
+
return this.adapter.getTerminalScreenSnapshot(maxBytes);
|
|
2016
|
+
}
|
|
2017
|
+
|
|
2018
|
+
/**
|
|
2019
|
+
* MESH-SEND-KEYS (feature 3: key injection). Public entry for the
|
|
2020
|
+
* mesh_send_keys tool, delegating to the adapter's injectKeys() (structured
|
|
2021
|
+
* key encoding + atomic write + submit-race recheck + modal fail-closed).
|
|
2022
|
+
*
|
|
2023
|
+
* Gated on isMeshWorkerSession(): PTY input into a worker is a
|
|
2024
|
+
* coordinator-only capability. The MCP layer ALSO cross-checks mesh/session/
|
|
2025
|
+
* node ownership (isMeshOwnedDelegateSession) and owns the destructive-key
|
|
2026
|
+
* double gate (confirm_destructive + policy) and the audit ledger. Returns a
|
|
2027
|
+
* refusal object for a non-mesh session so the daemon command surfaces a clean
|
|
2028
|
+
* error (never silently writes to a non-worker PTY).
|
|
2029
|
+
*/
|
|
2030
|
+
async injectKeys(
|
|
2031
|
+
items: MeshSendKeyItem[],
|
|
2032
|
+
opts: { allowModalOverride?: boolean } = {},
|
|
2033
|
+
): Promise<
|
|
2034
|
+
| { ok: true; keys: MeshSendKeyName[]; hasDestructive: boolean; submits: boolean; bytes: number }
|
|
2035
|
+
| { ok: false; refused: 'submit_race' | 'actionable_modal' | 'not_mesh_worker'; keys: MeshSendKeyName[]; hasDestructive: boolean }
|
|
2036
|
+
> {
|
|
2037
|
+
if (!this.isMeshWorkerSession()) {
|
|
2038
|
+
return { ok: false, refused: 'not_mesh_worker', keys: [], hasDestructive: false };
|
|
2039
|
+
}
|
|
2040
|
+
return this.adapter.injectKeys(items, opts);
|
|
2041
|
+
}
|
|
2042
|
+
|
|
2043
|
+
/**
|
|
2044
|
+
* MESH-STALL-WATCH (feature 1: STALL detection). Status-agnostic stall
|
|
2045
|
+
* watchdog for coordinator-spawned mesh worker sessions. Driven by the
|
|
2046
|
+
* ProviderInstanceManager's existing 5s onTick loop (NO new timer) — see
|
|
2047
|
+
* ProviderInstanceManager.startTicking. Reuses the adapter's raw-PTY-output
|
|
2048
|
+
* clock (lastOutputAt, bumped on every output chunk) as the sole signal: if a
|
|
2049
|
+
* live worker's screen has been byte-for-byte unchanged for
|
|
2050
|
+
* MESH_WORKER_STALL_THRESHOLD_MS (180s), fire ONE informational
|
|
2051
|
+
* monitor:no_progress event down the existing task_stalled ledger +
|
|
2052
|
+
* pendingCoordinatorEvent path.
|
|
2053
|
+
*
|
|
2054
|
+
* Deliberately status-agnostic: it does NOT read getStatus()'s reported status
|
|
2055
|
+
* (which would couple it to the generating-only StatusMonitor and the
|
|
2056
|
+
* idle-timeout FSM). A normally-idle worker CAN trip this after 3 quiet
|
|
2057
|
+
* minutes; that is accepted and surfaced as an informational stall (NOT a
|
|
2058
|
+
* failure/auto-restart) so the coordinator judges. getStatus/getState are NOT
|
|
2059
|
+
* called here, so status heartbeats never move the stall anchor.
|
|
2060
|
+
*
|
|
2061
|
+
* Anchoring: the episode arms against the current lastOutputAt; a worker that
|
|
2062
|
+
* has emitted nothing yet (lastOutputAt === 0) anchors on this.startedAt (spawn
|
|
2063
|
+
* time) so a silent spawn is still caught. Any new output re-arms the anchor
|
|
2064
|
+
* and clears the emitted flag, so one continuous stall emits at most once and a
|
|
2065
|
+
* later stall re-arms cleanly.
|
|
2066
|
+
*/
|
|
2067
|
+
checkMeshWorkerStall(now: number = Date.now()): void {
|
|
2068
|
+
if (!this.isMeshWorkerSession()) {
|
|
2069
|
+
// Not (or no longer) a mesh worker — drop any armed episode so a session
|
|
2070
|
+
// whose mesh markers were detached does not carry a stale anchor.
|
|
2071
|
+
this.meshStallAnchorAt = -1;
|
|
2072
|
+
this.meshStallEmittedForAnchor = false;
|
|
2073
|
+
return;
|
|
2074
|
+
}
|
|
2075
|
+
if (!this.adapter.isAlive()) {
|
|
2076
|
+
// Dead PTY: nothing to watch. agent:stopped covers the exit; re-arm on
|
|
2077
|
+
// the next live session so a restart starts a fresh episode.
|
|
2078
|
+
this.meshStallAnchorAt = -1;
|
|
2079
|
+
this.meshStallEmittedForAnchor = false;
|
|
2080
|
+
return;
|
|
2081
|
+
}
|
|
2082
|
+
|
|
2083
|
+
let lastOutputAt: number;
|
|
2084
|
+
try {
|
|
2085
|
+
// allowParse:false — a cheap status read that must NOT trigger parsing
|
|
2086
|
+
// and must NOT mutate lastOutputAt (getState/getStatus never bump it).
|
|
2087
|
+
const status = this.adapter.getStatus({ allowParse: false }) as { lastOutputAt?: unknown };
|
|
2088
|
+
lastOutputAt = typeof status?.lastOutputAt === 'number' && Number.isFinite(status.lastOutputAt)
|
|
2089
|
+
? status.lastOutputAt
|
|
2090
|
+
: 0;
|
|
2091
|
+
} catch {
|
|
2092
|
+
return; // defensive: a failed status read just skips this tick
|
|
2093
|
+
}
|
|
2094
|
+
|
|
2095
|
+
// The anchor is the last raw output; before any output, the spawn time so a
|
|
2096
|
+
// silent worker is still caught.
|
|
2097
|
+
const anchor = lastOutputAt > 0 ? lastOutputAt : this.startedAt;
|
|
2098
|
+
|
|
2099
|
+
if (this.meshStallAnchorAt === -1) {
|
|
2100
|
+
// First observation this session — arm against the current anchor.
|
|
2101
|
+
this.meshStallAnchorAt = anchor;
|
|
2102
|
+
this.meshStallEmittedForAnchor = false;
|
|
2103
|
+
return;
|
|
2104
|
+
}
|
|
2105
|
+
|
|
2106
|
+
if (anchor > this.meshStallAnchorAt) {
|
|
2107
|
+
// New output advanced the clock — re-arm the episode against it.
|
|
2108
|
+
this.meshStallAnchorAt = anchor;
|
|
2109
|
+
this.meshStallEmittedForAnchor = false;
|
|
2110
|
+
return;
|
|
2111
|
+
}
|
|
2112
|
+
|
|
2113
|
+
if (this.meshStallEmittedForAnchor) return; // already fired for this stall
|
|
2114
|
+
|
|
2115
|
+
const stalledMs = now - this.meshStallAnchorAt;
|
|
2116
|
+
if (stalledMs < CliProviderInstance.MESH_WORKER_STALL_THRESHOLD_MS) return;
|
|
2117
|
+
|
|
2118
|
+
this.meshStallEmittedForAnchor = true;
|
|
2119
|
+
|
|
2120
|
+
// observedStatus is surfaced as context only — deliberately NOT stamped as
|
|
2121
|
+
// the reconciliation-triggering `status` field (which would let
|
|
2122
|
+
// buildNoProgressCompletionReconciliation mistake an idle stall for a
|
|
2123
|
+
// completion). See mesh-events-stale.buildNoProgressCompletionReconciliation.
|
|
2124
|
+
let observedStatus = 'unknown';
|
|
2125
|
+
try {
|
|
2126
|
+
const s = (this.adapter.getStatus({ allowParse: false }) as { status?: unknown })?.status;
|
|
2127
|
+
if (typeof s === 'string' && s) observedStatus = s;
|
|
2128
|
+
} catch { /* best-effort context only */ }
|
|
2129
|
+
|
|
2130
|
+
if (this.isMeshWorkerSession()) {
|
|
2131
|
+
traceMeshEventStage('fired', this.meshTraceCtx('monitor:no_progress'), 'mesh_worker_stall_watchdog');
|
|
2132
|
+
}
|
|
2133
|
+
|
|
2134
|
+
const stalledSec = Math.round(stalledMs / 1000);
|
|
2135
|
+
this.pushEvent({
|
|
2136
|
+
event: 'monitor:no_progress',
|
|
2137
|
+
agentKey: `${this.type}:cli`,
|
|
2138
|
+
elapsedSec: stalledSec,
|
|
2139
|
+
timestamp: now,
|
|
2140
|
+
// MESH-STALL-WATCH marker: buildMeshSystemMessage generalizes the
|
|
2141
|
+
// coordinator message (generating-specific → "PTY output unchanged")
|
|
2142
|
+
// when this is set, since this watchdog fires status-agnostically.
|
|
2143
|
+
meshWorkerStall: true,
|
|
2144
|
+
lastOutputAt: this.meshStallAnchorAt,
|
|
2145
|
+
stalledMs,
|
|
2146
|
+
observedStatus,
|
|
2147
|
+
taskId: this.completingTurnTaskId(),
|
|
2148
|
+
});
|
|
2149
|
+
}
|
|
2150
|
+
|
|
1969
2151
|
/**
|
|
1970
2152
|
* AUTOAPPROVE-FLAP-RECUR (Fix A+B): how long a busy blip / modal scroll-out may
|
|
1971
2153
|
* persist before the in-progress settle gate is torn down. For a delegated
|
|
@@ -3527,6 +3709,20 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
3527
3709
|
this.completedDebouncePending = null;
|
|
3528
3710
|
continue;
|
|
3529
3711
|
}
|
|
3712
|
+
// MESH-STALL-WATCH dedupe: for a coordinator-spawned mesh worker the
|
|
3713
|
+
// status-agnostic stall watchdog (checkMeshWorkerStall) owns the
|
|
3714
|
+
// monitor:no_progress alert. The StatusMonitor here fires its OWN
|
|
3715
|
+
// generating-only no-progress on the SAME 180s bound, which would
|
|
3716
|
+
// double-emit into the task_stalled ledger + coordinator inbox for the
|
|
3717
|
+
// one stall. Suppress the StatusMonitor's copy for mesh workers only —
|
|
3718
|
+
// the completion-reconciliation branch above (which turns a no-progress
|
|
3719
|
+
// WITH final-assistant into a real completion) still runs, so genuine
|
|
3720
|
+
// idle-reconciled completions are unaffected. Non-mesh sessions keep the
|
|
3721
|
+
// original StatusMonitor behavior untouched.
|
|
3722
|
+
if (me.type === 'monitor:no_progress' && this.isMeshWorkerSession()) {
|
|
3723
|
+
traceMeshEventDrop('mesh_worker_stall_watchdog_owns_no_progress', this.meshTraceCtx('monitor:no_progress'));
|
|
3724
|
+
continue;
|
|
3725
|
+
}
|
|
3530
3726
|
this.pushEvent({ event: me.type, agentKey: me.agentKey, message: me.message, elapsedSec: me.elapsedSec, timestamp: me.timestamp });
|
|
3531
3727
|
}
|
|
3532
3728
|
}
|
|
@@ -211,7 +211,18 @@ export class ProviderInstanceManager {
|
|
|
211
211
|
this.tickInterval = intervalMs || this.tickInterval;
|
|
212
212
|
|
|
213
213
|
this.tickTimer = setInterval(async () => {
|
|
214
|
+
const now = Date.now();
|
|
214
215
|
for (const [id, instance] of this.instances) {
|
|
216
|
+
// MESH-STALL-WATCH (feature 1: STALL detection): a status-agnostic
|
|
217
|
+
// watchdog for coordinator-spawned mesh worker sessions, driven by
|
|
218
|
+
// THIS existing 5s tick (no separate timer). Cheap (reads the
|
|
219
|
+
// adapter's raw-output clock only) and a no-op for non-mesh /
|
|
220
|
+
// non-CLI instances, so it runs before the per-instance onTick.
|
|
221
|
+
try {
|
|
222
|
+
instance.checkMeshWorkerStall?.(now);
|
|
223
|
+
} catch (e) {
|
|
224
|
+
LOG.warn('InstanceMgr', `[InstanceManager] Mesh stall check failed for ${id}: ${(e as Error).message}`);
|
|
225
|
+
}
|
|
215
226
|
try {
|
|
216
227
|
await instance.onTick();
|
|
217
228
|
} catch (e) {
|
|
@@ -195,6 +195,16 @@ export interface ProviderInstance {
|
|
|
195
195
|
/** Tick — periodic status refresh (IDE: readChat, Extension: stream collection) */
|
|
196
196
|
onTick(): Promise<void>;
|
|
197
197
|
|
|
198
|
+
/**
|
|
199
|
+
* MESH-STALL-WATCH (feature 1: STALL detection). Status-agnostic stall
|
|
200
|
+
* watchdog for coordinator-spawned mesh worker sessions, invoked from the
|
|
201
|
+
* ProviderInstanceManager's existing tick loop (no separate timer). Fires ONE
|
|
202
|
+
* informational monitor:no_progress event when a live worker's raw PTY output
|
|
203
|
+
* has been unchanged past the stall threshold. Optional — only CLI instances
|
|
204
|
+
* (which own a PTY / lastOutputAt clock) implement it; a no-op elsewhere.
|
|
205
|
+
*/
|
|
206
|
+
checkMeshWorkerStall?(now?: number): void;
|
|
207
|
+
|
|
198
208
|
/** Return current status */
|
|
199
209
|
getState(): ProviderState;
|
|
200
210
|
|
package/src/repo-mesh-types.ts
CHANGED
|
@@ -350,6 +350,17 @@ export interface RepoMeshPolicy {
|
|
|
350
350
|
* A node policy may override this per-node (RepoMeshNodePolicy.delegatedWorkerAutoApprove).
|
|
351
351
|
*/
|
|
352
352
|
delegatedWorkerAutoApprove?: boolean;
|
|
353
|
+
/**
|
|
354
|
+
* MESH-SEND-KEYS (feature 3): opt-in to allow the coordinator to inject
|
|
355
|
+
* DESTRUCTIVE keys (CTRL_C / ESC) into a worker PTY via mesh_send_keys. These
|
|
356
|
+
* can kill or derail the worker process, and delegatedWorkerAutoApprove is a
|
|
357
|
+
* TOOL-CONSENT policy, not a PTY-input authorization — so a destructive key
|
|
358
|
+
* injection additionally requires this explicit mesh-owner opt-in AND a
|
|
359
|
+
* per-call confirm_destructive=true. Defaults to false (destructive keys
|
|
360
|
+
* refused). Non-destructive keys (text/ENTER/arrows/TAB/BACKSPACE) are
|
|
361
|
+
* unaffected. A node policy may override per-node.
|
|
362
|
+
*/
|
|
363
|
+
allowSendKeysDestructive?: boolean;
|
|
353
364
|
/**
|
|
354
365
|
* What to do with delegated session-host records for a node when it is removed.
|
|
355
366
|
* Defaults to 'preserve' so completed work can be reviewed later and live
|
|
@@ -473,6 +484,11 @@ export interface RepoMeshNodePolicy {
|
|
|
473
484
|
* precedence over the mesh-level policy for worker sessions launched onto this node.
|
|
474
485
|
*/
|
|
475
486
|
delegatedWorkerAutoApprove?: boolean;
|
|
487
|
+
/**
|
|
488
|
+
* MESH-SEND-KEYS (feature 3): per-node override for
|
|
489
|
+
* RepoMeshPolicy.allowSendKeysDestructive.
|
|
490
|
+
*/
|
|
491
|
+
allowSendKeysDestructive?: boolean;
|
|
476
492
|
/**
|
|
477
493
|
* Optional associated/external repos that must be checked alongside this node.
|
|
478
494
|
* These are explicit policy/config entries only; Repo Mesh does not auto-discover
|
|
@@ -734,6 +750,25 @@ export function resolveDelegatedWorkerAutoApprove(
|
|
|
734
750
|
return true;
|
|
735
751
|
}
|
|
736
752
|
|
|
753
|
+
/**
|
|
754
|
+
* MESH-SEND-KEYS (feature 3): resolve whether DESTRUCTIVE key injection
|
|
755
|
+
* (CTRL_C/ESC via mesh_send_keys) is permitted for a node. Node policy overrides
|
|
756
|
+
* mesh policy; DEFAULTS TO FALSE (fail-closed) — a destructive key still requires
|
|
757
|
+
* a per-call confirm_destructive=true on top of this opt-in.
|
|
758
|
+
*/
|
|
759
|
+
export function resolveAllowSendKeysDestructive(
|
|
760
|
+
meshPolicy?: Pick<RepoMeshPolicy, 'allowSendKeysDestructive'> | null,
|
|
761
|
+
nodePolicy?: Pick<RepoMeshNodePolicy, 'allowSendKeysDestructive'> | null,
|
|
762
|
+
): boolean {
|
|
763
|
+
if (typeof nodePolicy?.allowSendKeysDestructive === 'boolean') {
|
|
764
|
+
return nodePolicy.allowSendKeysDestructive;
|
|
765
|
+
}
|
|
766
|
+
if (typeof meshPolicy?.allowSendKeysDestructive === 'boolean') {
|
|
767
|
+
return meshPolicy.allowSendKeysDestructive;
|
|
768
|
+
}
|
|
769
|
+
return false;
|
|
770
|
+
}
|
|
771
|
+
|
|
737
772
|
/**
|
|
738
773
|
* Resolve the enforced per-(node, provider) maxParallel cap from a node's resolved
|
|
739
774
|
* capability slots, or undefined when no matching slot declares a finite cap. Used
|