@adhdev/daemon-core 0.9.82-rc.381 → 0.9.82-rc.383
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/router.d.ts +11 -1
- package/dist/index.js +234 -117
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +234 -117
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-event-forwarding.d.ts +1 -0
- package/dist/providers/cli-provider-instance.d.ts +18 -0
- package/package.json +2 -2
- package/src/commands/high-family/mesh-status.ts +36 -0
- package/src/commands/router.ts +50 -17
- package/src/mesh/mesh-event-forwarding.ts +46 -1
- package/src/mesh/mesh-reconcile-loop.ts +74 -30
- package/src/providers/cli-provider-instance.ts +27 -0
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
|
|
2
|
+
export declare function recoverMeshIdByCoordinatorAndNode(coordinatorDaemonId: string, nodeId: string): string;
|
|
2
3
|
export declare function resolveForwardEventMeshId(components: DaemonComponents, payload: Record<string, unknown>): string;
|
|
3
4
|
export declare function __resetMeshWorkspaceCacheForTests(): void;
|
|
4
5
|
export declare function buildRelayMetadataEvent(payload: Record<string, unknown>): Record<string, unknown>;
|
|
@@ -150,6 +150,24 @@ export declare class CliProviderInstance implements ProviderInstance {
|
|
|
150
150
|
* terminal state. Leaving meshNodeFor pinned would route this session's
|
|
151
151
|
* subsequent unrelated turns (e.g. ad-hoc dashboard chats) to the
|
|
152
152
|
* coordinator as if they were task completions.
|
|
153
|
+
*
|
|
154
|
+
* MESHID-DROP-ON-DETACH (Fix C): a coordinator-LAUNCHED worker session
|
|
155
|
+
* (launchedByCoordinator) holds its mesh membership (meshNodeFor / meshNodeId /
|
|
156
|
+
* meshCoordinatorDaemonId) at the SESSION level — set once at launch
|
|
157
|
+
* (mesh_launch_session / queue auto-launch), independent of any single task.
|
|
158
|
+
* The original detach wiped meshNodeFor + meshNodeId together with the
|
|
159
|
+
* task-level meshActiveTaskId, so the FIRST task completion stripped the
|
|
160
|
+
* membership and EVERY subsequent completion forwarded with meshId absent —
|
|
161
|
+
* resolveWorkerDelegateRouting fell to mesh_unresolved and the coordinator
|
|
162
|
+
* rejected the forward "meshId required". For a launched member we therefore
|
|
163
|
+
* clear ONLY the task-level marker (meshActiveTaskId) and preserve the
|
|
164
|
+
* session-level membership so its next task's completion still resolves.
|
|
165
|
+
* A task-less ad-hoc turn on a preserved-membership session is NOT misrouted:
|
|
166
|
+
* its completion carries no taskId and the session holds no active assignment,
|
|
167
|
+
* so the forwarder's WARMUPGAP guard skips the dispatch-row flip (it only
|
|
168
|
+
* injects a benign task-less notification). A NON-launched session (a plain CLI
|
|
169
|
+
* session adopted by mesh_send_task --direct, launchedByCoordinator falsy)
|
|
170
|
+
* keeps the original full clear so an ad-hoc session is never left pinned.
|
|
153
171
|
*/
|
|
154
172
|
detachMeshAssignment(): void;
|
|
155
173
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adhdev/daemon-core",
|
|
3
|
-
"version": "0.9.82-rc.
|
|
3
|
+
"version": "0.9.82-rc.383",
|
|
4
4
|
"description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
"author": "vilmire",
|
|
47
47
|
"license": "AGPL-3.0-or-later",
|
|
48
48
|
"dependencies": {
|
|
49
|
-
"@adhdev/mesh-shared": "0.9.82-rc.
|
|
49
|
+
"@adhdev/mesh-shared": "0.9.82-rc.383",
|
|
50
50
|
"@adhdev/session-host-core": "*",
|
|
51
51
|
"@agentclientprotocol/sdk": "^0.16.1",
|
|
52
52
|
"ajv": "^8.20.0",
|
|
@@ -107,6 +107,19 @@ export const meshStatusHandlers: Record<string, HighFamilyHandler> = {
|
|
|
107
107
|
const queue = getQueue(meshId);
|
|
108
108
|
const queueSummary = getMeshQueueStats(meshId);
|
|
109
109
|
|
|
110
|
+
// Scheduling-runtime projection — the load-balancer's live view (tie-break
|
|
111
|
+
// strategy, global parallel caps + consumption, per-node load/priority/provider
|
|
112
|
+
// caps). Built from the SAME helper + args the MCP `mesh_status` tool uses
|
|
113
|
+
// (mesh-tools-status.ts: buildMeshSchedulingRuntime(mesh, getQueue(mesh.id)))
|
|
114
|
+
// so the dashboard surface and the coordinator MCP surface render an identical
|
|
115
|
+
// runtime. Without this the dashboard Status/Runtime tab showed the
|
|
116
|
+
// "not reported by this daemon (older build)" fallback (RepoMeshStatus.scheduling
|
|
117
|
+
// was never populated by this daemon-command producer). Computed once so each
|
|
118
|
+
// node entry can attach its slice and statusResult can carry the mesh rollup.
|
|
119
|
+
const { buildMeshSchedulingRuntime } = await import('../../mesh/mesh-scheduling-runtime.js');
|
|
120
|
+
const schedulingRuntime = buildMeshSchedulingRuntime(mesh, queue);
|
|
121
|
+
const schedulingByNode = new Map(schedulingRuntime.nodes.map(n => [n.nodeId, n]));
|
|
122
|
+
|
|
110
123
|
const { readLedgerEntries, getLedgerSummary } = await import('../../mesh/mesh-ledger.js');
|
|
111
124
|
const ledgerEntries = readLedgerEntries(meshId, { tail: 20 });
|
|
112
125
|
const asyncRefineLedgerEntries = readLedgerEntries(meshId, { tail: 100 });
|
|
@@ -260,6 +273,16 @@ export const meshStatusHandlers: Record<string, HighFamilyHandler> = {
|
|
|
260
273
|
activeSessionDetails: [],
|
|
261
274
|
launchReady: false,
|
|
262
275
|
};
|
|
276
|
+
// Per-node scheduling slice (load / priority / provider caps / claim-block
|
|
277
|
+
// reasons) read by the dashboard's MeshNodeSchedulingBadges. Full shape —
|
|
278
|
+
// unlike the MCP compact path this is a dashboard surface, so it keeps the
|
|
279
|
+
// whole RepoMeshNodeSchedulingStatus. Redundant nodeId dropped (the entry
|
|
280
|
+
// already carries it).
|
|
281
|
+
const nodeScheduling = schedulingByNode.get(nodeId);
|
|
282
|
+
if (nodeScheduling) {
|
|
283
|
+
const { nodeId: _omitNodeId, ...nodeSchedulingRest } = nodeScheduling;
|
|
284
|
+
status.scheduling = nodeSchedulingRest;
|
|
285
|
+
}
|
|
263
286
|
if (isSelfNode) {
|
|
264
287
|
status.connection = {
|
|
265
288
|
perspective: 'selected_coordinator',
|
|
@@ -511,6 +534,19 @@ export const meshStatusHandlers: Record<string, HighFamilyHandler> = {
|
|
|
511
534
|
branchConvergenceSummary: summarizeInlineMeshBranchConvergence(nodeStatuses),
|
|
512
535
|
...(previewFreshness ? { previewFreshness, deployFreshness: previewFreshness } : {}),
|
|
513
536
|
nodes: nodeStatuses,
|
|
537
|
+
// Mesh-level scheduling rollup (strategy + global cap consumption). Mirrors
|
|
538
|
+
// the MCP `mesh_status` tool's `scheduling` block field-for-field so both
|
|
539
|
+
// surfaces read the same runtime; per-node detail lives on each
|
|
540
|
+
// nodes[].scheduling above.
|
|
541
|
+
scheduling: {
|
|
542
|
+
strategy: schedulingRuntime.strategy,
|
|
543
|
+
maxParallelTasks: schedulingRuntime.maxParallelTasks,
|
|
544
|
+
maxReadonlyParallelTasks: schedulingRuntime.maxReadonlyParallelTasks,
|
|
545
|
+
activeWriteAssigned: schedulingRuntime.activeWriteAssigned,
|
|
546
|
+
activeReadonlyAssigned: schedulingRuntime.activeReadonlyAssigned,
|
|
547
|
+
globalWriteCapReached: schedulingRuntime.globalWriteCapReached,
|
|
548
|
+
globalReadonlyCapReached: schedulingRuntime.globalReadonlyCapReached,
|
|
549
|
+
},
|
|
514
550
|
queue: { tasks: queue, summary: queueSummary },
|
|
515
551
|
ledger: { entries: ledgerEntries, summary: ledgerSummary },
|
|
516
552
|
...(missions.length > 0 ? { missions } : {}),
|
package/src/commands/router.ts
CHANGED
|
@@ -491,25 +491,51 @@ export class DaemonCommandRouter {
|
|
|
491
491
|
* the wider plural-shape scan so a controlbar/modal command targeting a non-primary remote
|
|
492
492
|
* session still resolves its owner — the singular readCachedInlineMeshActiveSessions semantics
|
|
493
493
|
* other consumers depend on stay untouched.
|
|
494
|
+
*
|
|
495
|
+
* CANCEL-STOP-RELAY: the session-id cache scan above only matches when the coordinator's
|
|
496
|
+
* cached status snapshot already lists the worker's session id in a recognized active-sessions
|
|
497
|
+
* shape. A worktree-clone worker session whose id form/timing differs from the cached snapshot
|
|
498
|
+
* (or is simply not yet reflected) misses the scan, so a stop that carries the authoritative
|
|
499
|
+
* owning nodeId (mesh_queue_cancel knows assignedNodeId) used to silently fail to forward.
|
|
500
|
+
* `ownerNodeIdHint` adds a deterministic fallback: when the session-id scan misses, resolve the
|
|
501
|
+
* owner daemonId by matching the node by id (meshNodeIdMatches — same form-tolerant compare the
|
|
502
|
+
* rest of the router uses, no new raw compare). The same self-loopback guard applies to both
|
|
503
|
+
* paths, so a coordinator-hosted node is still never force-forwarded to a remote form of itself.
|
|
494
504
|
*/
|
|
495
|
-
public resolveRemoteMeshSessionOwnerDaemonId(sessionId: string): string | undefined {
|
|
505
|
+
public resolveRemoteMeshSessionOwnerDaemonId(sessionId: string, ownerNodeIdHint?: string): string | undefined {
|
|
496
506
|
const trimmed = typeof sessionId === 'string' ? sessionId.trim() : '';
|
|
497
|
-
|
|
507
|
+
const nodeHint = typeof ownerNodeIdHint === 'string' ? ownerNodeIdHint.trim() : '';
|
|
508
|
+
if (!trimmed && !nodeHint) return undefined;
|
|
498
509
|
const selfDaemonId = this.deps.statusInstanceId;
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
const
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
510
|
+
const candidates = this.collectMeshSessionOwnerCandidateNodes();
|
|
511
|
+
if (trimmed) {
|
|
512
|
+
for (const node of candidates) {
|
|
513
|
+
if (!collectMeshNodeHostedSessionIds(node).has(trimmed)) continue;
|
|
514
|
+
const nodeDaemonId = readMeshNodeDaemonId(readObjectRecord(node));
|
|
515
|
+
// A matching node with no readable daemonId can't be attributed — keep scanning
|
|
516
|
+
// the remaining candidates (e.g. the same session on an aggregate node that does
|
|
517
|
+
// carry the daemonId) rather than bailing on the whole resolution.
|
|
518
|
+
if (!nodeDaemonId) continue;
|
|
519
|
+
// Only forward to a genuinely remote daemon. When the owning node is this
|
|
520
|
+
// coordinator itself (locally hosted worker), fall through to local handling.
|
|
521
|
+
// id-form robust: the node daemonId and selfDaemonId may be stored in different
|
|
522
|
+
// forms of the same machine — a strict `===` would miss the self-match and forward
|
|
523
|
+
// a local session to a remote form of THIS daemon (loopback).
|
|
524
|
+
if (selfDaemonId && daemonIdsEquivalent(nodeDaemonId, selfDaemonId)) return undefined;
|
|
525
|
+
return nodeDaemonId;
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
// Deterministic fallback: the session-id scan missed (cache lag / id-form mismatch on a
|
|
529
|
+
// worktree-clone worker), but the caller knows the authoritative owning nodeId. Resolve the
|
|
530
|
+
// owner daemonId straight off that node — never the fuzzy session cache.
|
|
531
|
+
if (nodeHint) {
|
|
532
|
+
for (const node of candidates) {
|
|
533
|
+
if (!meshNodeIdMatches(node, nodeHint)) continue;
|
|
534
|
+
const nodeDaemonId = readMeshNodeDaemonId(readObjectRecord(node));
|
|
535
|
+
if (!nodeDaemonId) continue;
|
|
536
|
+
if (selfDaemonId && daemonIdsEquivalent(nodeDaemonId, selfDaemonId)) return undefined;
|
|
537
|
+
return nodeDaemonId;
|
|
538
|
+
}
|
|
513
539
|
}
|
|
514
540
|
return undefined;
|
|
515
541
|
}
|
|
@@ -3092,7 +3118,14 @@ export class DaemonCommandRouter {
|
|
|
3092
3118
|
const localInstance = this.deps.instanceManager?.getInstance(targetSessionId);
|
|
3093
3119
|
const localRegistry = this.deps.sessionRegistry?.get?.(targetSessionId);
|
|
3094
3120
|
if (!localInstance && !localRegistry) {
|
|
3095
|
-
|
|
3121
|
+
// CANCEL-STOP-RELAY: pass the authoritative owning nodeId (when the caller
|
|
3122
|
+
// shipped one in meshContext, e.g. mesh_queue_cancel's assignedNodeId) as the
|
|
3123
|
+
// deterministic owner-resolution fallback. The session-id cache scan stays the
|
|
3124
|
+
// primary path; the hint only kicks in when that scan misses (worktree-clone
|
|
3125
|
+
// worker session not yet in / form-mismatched against the cached snapshot).
|
|
3126
|
+
const meshContext = readObjectRecord(args?.meshContext);
|
|
3127
|
+
const ownerNodeIdHint = readStringValue(meshContext.nodeId);
|
|
3128
|
+
const ownerDaemonId = this.resolveRemoteMeshSessionOwnerDaemonId(targetSessionId, ownerNodeIdHint);
|
|
3096
3129
|
if (ownerDaemonId) {
|
|
3097
3130
|
LOG.info('Mesh', `[Mesh] Forwarding session-scoped '${cmd}' for remote worker session ${targetSessionId.split('_')[0]} → daemon ${ownerDaemonId.slice(0, 12)}`);
|
|
3098
3131
|
const forwarded = await this.deps.dispatchMeshCommand(ownerDaemonId, cmd, {
|
|
@@ -9,6 +9,7 @@ import { markSessionDeliveriesTerminal, updateSessionDeliveryStatus, recordCompl
|
|
|
9
9
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
10
10
|
import { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents } from './mesh-events-pending.js';
|
|
11
11
|
import { resolveWorkerDelegateRouting, recordUnroutableDelegateEvent, isUnroutableDelegateRejection } from './mesh-routing.js';
|
|
12
|
+
import { resolveMeshHostStatus } from './mesh-host-ownership.js';
|
|
12
13
|
import { enqueueUnresolvedDelegateForward, peekUnresolvedDelegateForwards, ackUnresolvedDelegateForward } from './mesh-unresolved-forward-outbox.js';
|
|
13
14
|
import { traceMeshEventStage, traceMeshEventDrop } from './mesh-event-trace.js';
|
|
14
15
|
import { getLastDisplayMessage } from '../status/snapshot.js';
|
|
@@ -92,6 +93,38 @@ function recoverMeshIdByNodeId(nodeId: string): string {
|
|
|
92
93
|
return '';
|
|
93
94
|
}
|
|
94
95
|
|
|
96
|
+
// MESHID-DROP coordinator-anchor recovery (Fix B): the last-resort meshId recovery for an
|
|
97
|
+
// unresolved-delegate forward whose payload carries the worker's coordinator anchor
|
|
98
|
+
// (meshCoordinatorDaemonId) but no resolvable meshId — neither workspace nor nodeId scan
|
|
99
|
+
// matched (a freshly-cloned worktree node not yet registered under the forwarded id form, or
|
|
100
|
+
// an empty payload nodeId). The receiving daemon IS the coordinator/host, so scope to the
|
|
101
|
+
// meshes IT hosts whose host daemon matches the anchor (daemonIdsEquivalent — never a raw
|
|
102
|
+
// compare, so daemon_mach_/mach_/standalone_ forms all match the same machine). Among those:
|
|
103
|
+
// • a nodeId → the mesh whose nodes contain it (meshNodeIdMatches 3-form) wins;
|
|
104
|
+
// • no nodeId → fall back to the anchor's SINGLE hosted mesh only. Ambiguity (the anchor
|
|
105
|
+
// hosts >1 mesh and no node disambiguates) returns '' rather than guess — a wrong meshId
|
|
106
|
+
// would inject the completion into an unrelated mesh's ledger, worse than the retry-cap drop.
|
|
107
|
+
export function recoverMeshIdByCoordinatorAndNode(coordinatorDaemonId: string, nodeId: string): string {
|
|
108
|
+
if (!coordinatorDaemonId) return '';
|
|
109
|
+
const hosted = listMeshes().filter(mesh => {
|
|
110
|
+
const host = resolveMeshHostStatus(mesh);
|
|
111
|
+
return host.role === 'host'
|
|
112
|
+
&& (!host.hostDaemonId || daemonIdsEquivalent(host.hostDaemonId, coordinatorDaemonId));
|
|
113
|
+
});
|
|
114
|
+
if (hosted.length === 0) return '';
|
|
115
|
+
if (nodeId) {
|
|
116
|
+
const byNode = hosted.find(mesh =>
|
|
117
|
+
Array.isArray(mesh.nodes) && mesh.nodes.some((n: any) => meshNodeIdMatches(n, nodeId)));
|
|
118
|
+
if (byNode) return readNonEmptyString(byNode.id);
|
|
119
|
+
// nodeId present but matched no hosted mesh — do NOT fall through to the single-mesh
|
|
120
|
+
// guess; the node belongs to a mesh we don't host (or under a different id), and
|
|
121
|
+
// guessing would misroute. Stay unresolved.
|
|
122
|
+
return '';
|
|
123
|
+
}
|
|
124
|
+
// No nodeId to disambiguate: only safe when the anchor hosts exactly one mesh.
|
|
125
|
+
return hosted.length === 1 ? readNonEmptyString(hosted[0].id) : '';
|
|
126
|
+
}
|
|
127
|
+
|
|
95
128
|
// RECONCILE-MESHID-DROP: WORKER-side meshId resolution for an unresolved-delegate
|
|
96
129
|
// forward payload. forwardUnresolvedDelegateEvent omits meshId by design (the worker
|
|
97
130
|
// "can't resolve it") and relies on the COORDINATOR recovering it from workspace/nodeId.
|
|
@@ -1234,7 +1267,14 @@ export function handleMeshForwardEvent(components: DaemonComponents, payload: Re
|
|
|
1234
1267
|
// The nodeId is a stable coordinator-side fact and resolves timing-independently.
|
|
1235
1268
|
const meshId = readNonEmptyString(payload.meshId)
|
|
1236
1269
|
|| (workspace ? readNonEmptyString(getCachedMeshByWorkspace(workspace)?.id) : '')
|
|
1237
|
-
|| recoverMeshIdByNodeId(nodeId)
|
|
1270
|
+
|| recoverMeshIdByNodeId(nodeId)
|
|
1271
|
+
// Fix B last resort: workspace + nodeId scan both missed, but the worker stamped
|
|
1272
|
+
// its coordinator anchor onto the forward (forwardUnresolvedDelegateEvent). Recover
|
|
1273
|
+
// via the hosted mesh that anchor owns (+ node disambiguation), guarding ambiguity.
|
|
1274
|
+
|| recoverMeshIdByCoordinatorAndNode(
|
|
1275
|
+
readNonEmptyString(payload.meshCoordinatorDaemonId) || readNonEmptyString(payload.coordinatorDaemonId),
|
|
1276
|
+
nodeId,
|
|
1277
|
+
);
|
|
1238
1278
|
if (!meshId) {
|
|
1239
1279
|
// EVTTRACE: forwarded event rejected at receive — no meshId could be resolved
|
|
1240
1280
|
// (no payload.meshId, no workspace→mesh, no nodeId→mesh). Observation only.
|
|
@@ -1364,6 +1404,11 @@ function forwardUnresolvedDelegateEvent(
|
|
|
1364
1404
|
event: eventName,
|
|
1365
1405
|
nodeId: readNonEmptyString(routing.nodeId) || readNonEmptyString(event.meshNodeId) || undefined,
|
|
1366
1406
|
workspace: readNonEmptyString(routing.workspace) || readNonEmptyString(event.workspace) || undefined,
|
|
1407
|
+
// Fix B: carry the resolved coordinator anchor so the coordinator's receive-side
|
|
1408
|
+
// recovery (recoverMeshIdByCoordinatorAndNode) can match this forward to one of the
|
|
1409
|
+
// meshes it hosts when workspace + nodeId both miss. routing.coordinatorDaemonId is
|
|
1410
|
+
// the same anchor this forward is addressed to (coordinatorDaemonId below).
|
|
1411
|
+
meshCoordinatorDaemonId: coordinatorDaemonId,
|
|
1367
1412
|
};
|
|
1368
1413
|
// RECONCILE-MESHID-DROP: stamp meshId when the WORKER can resolve it (member node /
|
|
1369
1414
|
// live-session meshNodeFor). Historically omitted "because the worker can't resolve
|
|
@@ -49,7 +49,7 @@ import { drainPendingMeshCoordinatorEvents, getPendingMeshCoordinatorEvents, bui
|
|
|
49
49
|
import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
50
50
|
import { appendLedgerEntry } from './mesh-ledger.js';
|
|
51
51
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
52
|
-
import { handleMeshForwardEvent, shouldForceInjectMeshEvent,
|
|
52
|
+
import { handleMeshForwardEvent, shouldForceInjectMeshEvent, triggerMeshQueue, resolveForwardEventMeshId } from './mesh-events-coordinator.js';
|
|
53
53
|
import {
|
|
54
54
|
peekUnresolvedDelegateForwards,
|
|
55
55
|
ackUnresolvedDelegateForward,
|
|
@@ -589,35 +589,53 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
589
589
|
for (const [meshId, meshCoordinators] of byMesh) {
|
|
590
590
|
// Drain the local queue scoped to this coordinator daemon and inject.
|
|
591
591
|
// - If an idle coordinator exists, FULL-drain and deliver every event to it
|
|
592
|
-
// (
|
|
593
|
-
//
|
|
594
|
-
//
|
|
595
|
-
//
|
|
596
|
-
//
|
|
597
|
-
//
|
|
598
|
-
//
|
|
592
|
+
// (the idle input box accepts the prompt as a real next turn). The drain
|
|
593
|
+
// marks consumed rows drained=1 atomically, so the pull path can't re-deliver.
|
|
594
|
+
// - If only GENERATING coordinators exist (no idle target), we HOLD: leave the
|
|
595
|
+
// events queued (drained=0) for the coordinator's next idle/turn-end tick.
|
|
596
|
+
//
|
|
597
|
+
// NOTIF-SURFACE-LOCAL (false-idle hold): we used to force-inject terminal events
|
|
598
|
+
// (completion/approval/stop/refine·bootstrap) straight into a *generating*
|
|
599
|
+
// coordinator's PTY (forceSendMessage → atomic content+\r write), on the theory it
|
|
600
|
+
// bypassed the busy send-guard and broke the await-result deadlock. But a raw PTY
|
|
601
|
+
// write into a claude-cli that is mid-generation is NOT consumed as a new turn — the
|
|
602
|
+
// bytes land in the terminal input buffer and the LLM never reads them on its next
|
|
603
|
+
// turn. The `surfaced/force-inject` trace fired, the row was marked drained=1, and the
|
|
604
|
+
// genuine completion was lost forever (the exact same-daemon local-worktree miss: the
|
|
605
|
+
// coordinator's OWN session is generating at the moment its worker completes). The
|
|
606
|
+
// deadlock the force path guarded against does not actually require force: a
|
|
607
|
+
// coordinator that dispatched a task via mesh_send_task returns to idle when that
|
|
608
|
+
// tool call resolves (dispatch is fire-and-forget; the worker runs for minutes while
|
|
609
|
+
// the coordinator is idle/between turns), so the completion lands on the very next
|
|
610
|
+
// idle tick (≤ one reconcile interval). Holding the event undrained for that idle
|
|
611
|
+
// tick is therefore the single, reliable delivery — and it is the SAME skip-and-hold
|
|
612
|
+
// the modal-park branch below already uses. This also makes double-injection
|
|
613
|
+
// structurally impossible: there is exactly one delivery path (the idle full-drain),
|
|
614
|
+
// so we never need a surface-time fingerprint to dedup a force-write against a re-drain.
|
|
599
615
|
const idleCoordinators = meshCoordinators.filter(c => c.idle);
|
|
600
|
-
// A coordinator parked on a harness modal (waiting_choice / waiting_approval)
|
|
601
|
-
//
|
|
602
|
-
//
|
|
603
|
-
//
|
|
604
|
-
//
|
|
605
|
-
//
|
|
606
|
-
// targets are the non-idle, non-modal-parked coordinators.
|
|
616
|
+
// A coordinator parked on a harness modal (waiting_choice / waiting_approval) is
|
|
617
|
+
// non-idle; it is held under the modal-park branch (a force-inject into a modal would
|
|
618
|
+
// write raw keystrokes the modal key handler eats, silently selecting a choice the
|
|
619
|
+
// user never made). A plainly-generating coordinator (non-idle, non-modal-parked) is
|
|
620
|
+
// ALSO held now — for the false-idle reason above — but separately, so the C1 ledger
|
|
621
|
+
// audit and the operator-facing skip log can name the right hold reason.
|
|
607
622
|
const generatingCoordinators = meshCoordinators.filter(c => !c.idle && !c.modalParked);
|
|
608
623
|
const modalParkedCoordinators = meshCoordinators.filter(c => !c.idle && c.modalParked);
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
//
|
|
614
|
-
//
|
|
615
|
-
//
|
|
616
|
-
//
|
|
617
|
-
//
|
|
618
|
-
//
|
|
619
|
-
//
|
|
620
|
-
//
|
|
624
|
+
// Only an IDLE coordinator is a deliverable target. A generating coordinator's PTY
|
|
625
|
+
// does not consume an injected prompt as a turn, so it is held (not a target).
|
|
626
|
+
const targetCoordinators = idleCoordinators;
|
|
627
|
+
|
|
628
|
+
// ── no-idle-target short-circuit (MUST precede the drain) ─────────────────
|
|
629
|
+
// When there is no IDLE coordinator for this mesh — only generating and/or
|
|
630
|
+
// modal-parked ones — there is nowhere a queued event can land as a real turn.
|
|
631
|
+
// We skip-and-hold: by NOT draining we leave the events at drained=0 in the queue,
|
|
632
|
+
// so a later tick (once a coordinator returns to idle) delivers them. This
|
|
633
|
+
// short-circuit MUST run BEFORE drainPendingMeshCoordinatorEvents — the drain marks
|
|
634
|
+
// rows drained=1 atomically, which would lose the events for a coordinator that is
|
|
635
|
+
// only transiently busy (the false-idle local-worktree miss). Both the generating
|
|
636
|
+
// hold and the modal-park hold record a C1 ledger audit copy so a held completion's
|
|
637
|
+
// worker summary is recoverable even if the coordinator never returns or the pending
|
|
638
|
+
// file is later trimmed.
|
|
621
639
|
if (targetCoordinators.length === 0) {
|
|
622
640
|
if (modalParkedCoordinators.length > 0) {
|
|
623
641
|
// ── orphan escape (MUST precede the blanket modal-park hold) ──────────
|
|
@@ -705,6 +723,30 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
705
723
|
modalParkedCoordinators.length,
|
|
706
724
|
);
|
|
707
725
|
}
|
|
726
|
+
} else if (generatingCoordinators.length > 0) {
|
|
727
|
+
// ── generating hold (NOTIF-SURFACE-LOCAL false-idle fix) ─────────────
|
|
728
|
+
// The only coordinator(s) for this mesh are plainly generating (no idle, no
|
|
729
|
+
// modal). A raw force-write into a generating claude-cli PTY is not consumed
|
|
730
|
+
// as a turn, so we do NOT inject and do NOT drain — the events stay queued
|
|
731
|
+
// (drained=0) and the next tick that finds the coordinator idle full-drains
|
|
732
|
+
// them as real turns (the coordinator returns to idle when its current
|
|
733
|
+
// tool-call/turn resolves; a dispatched worker runs for minutes while the
|
|
734
|
+
// coordinator is idle, so this lands within one reconcile interval). C1: mirror
|
|
735
|
+
// any held terminal events into the ledger so a completion's worker summary is
|
|
736
|
+
// recoverable even before that idle tick. Idempotent per process; O(1)-gated.
|
|
737
|
+
let hasPending = true;
|
|
738
|
+
if (store) {
|
|
739
|
+
try { hasPending = store.pendingEventCount(meshId) > 0; } catch { /* peek below */ }
|
|
740
|
+
}
|
|
741
|
+
if (hasPending) {
|
|
742
|
+
LOG.info('MeshReconcile', `Reconcile skip → generating: holding pending event(s) for mesh ${meshId} (${generatingCoordinators.length} coordinator(s) busy; events left queued for the next idle tick)`);
|
|
743
|
+
recordHeldTerminalEventsToLedger(
|
|
744
|
+
meshId,
|
|
745
|
+
drainDaemonIds.length > 0 ? drainDaemonIds : (localDaemonId ? [localDaemonId] : []),
|
|
746
|
+
'generating_no_idle_coordinator',
|
|
747
|
+
generatingCoordinators.length,
|
|
748
|
+
);
|
|
749
|
+
}
|
|
708
750
|
}
|
|
709
751
|
continue;
|
|
710
752
|
}
|
|
@@ -716,12 +758,15 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
716
758
|
} catch { /* fall through to drain */ }
|
|
717
759
|
}
|
|
718
760
|
|
|
761
|
+
// An idle coordinator is present (targetCoordinators.length > 0): FULL-drain every
|
|
762
|
+
// queued event and deliver it to the idle input box as a real turn. The no-idle case
|
|
763
|
+
// (generating/modal-only) was already held above and never reaches here, so there is
|
|
764
|
+
// no force-drain-into-generating path left — the single delivery is the idle drain.
|
|
719
765
|
let pendingEvents: PendingMeshCoordinatorEvent[] = [];
|
|
720
766
|
try {
|
|
721
767
|
pendingEvents = drainPendingMeshCoordinatorEvents(
|
|
722
768
|
meshId,
|
|
723
769
|
drainDaemonIds.length > 0 ? drainDaemonIds : localDaemonId,
|
|
724
|
-
forceOnly ? { onlyEvents: MESH_FORCE_INJECT_EVENTS } : undefined,
|
|
725
770
|
);
|
|
726
771
|
} catch (e: any) {
|
|
727
772
|
LOG.warn('MeshReconcile', `Drain failed for mesh ${meshId}: ${e?.message || e}`);
|
|
@@ -729,8 +774,7 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
729
774
|
}
|
|
730
775
|
if (pendingEvents.length === 0) continue;
|
|
731
776
|
|
|
732
|
-
|
|
733
|
-
LOG.info('MeshReconcile', `Reconcile ${mode}: ${pendingEvents.length} pending event(s) → ${targetCoordinators.length} coordinator(s) for mesh ${meshId}`);
|
|
777
|
+
LOG.info('MeshReconcile', `Reconcile inject → idle: ${pendingEvents.length} pending event(s) → ${targetCoordinators.length} coordinator(s) for mesh ${meshId}`);
|
|
734
778
|
for (const pending of pendingEvents) {
|
|
735
779
|
// Strict session routing (multi-coordinator): when the event names an
|
|
736
780
|
// originating coordinator session, deliver ONLY to the live coordinator whose
|
|
@@ -972,9 +972,36 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
972
972
|
* terminal state. Leaving meshNodeFor pinned would route this session's
|
|
973
973
|
* subsequent unrelated turns (e.g. ad-hoc dashboard chats) to the
|
|
974
974
|
* coordinator as if they were task completions.
|
|
975
|
+
*
|
|
976
|
+
* MESHID-DROP-ON-DETACH (Fix C): a coordinator-LAUNCHED worker session
|
|
977
|
+
* (launchedByCoordinator) holds its mesh membership (meshNodeFor / meshNodeId /
|
|
978
|
+
* meshCoordinatorDaemonId) at the SESSION level — set once at launch
|
|
979
|
+
* (mesh_launch_session / queue auto-launch), independent of any single task.
|
|
980
|
+
* The original detach wiped meshNodeFor + meshNodeId together with the
|
|
981
|
+
* task-level meshActiveTaskId, so the FIRST task completion stripped the
|
|
982
|
+
* membership and EVERY subsequent completion forwarded with meshId absent —
|
|
983
|
+
* resolveWorkerDelegateRouting fell to mesh_unresolved and the coordinator
|
|
984
|
+
* rejected the forward "meshId required". For a launched member we therefore
|
|
985
|
+
* clear ONLY the task-level marker (meshActiveTaskId) and preserve the
|
|
986
|
+
* session-level membership so its next task's completion still resolves.
|
|
987
|
+
* A task-less ad-hoc turn on a preserved-membership session is NOT misrouted:
|
|
988
|
+
* its completion carries no taskId and the session holds no active assignment,
|
|
989
|
+
* so the forwarder's WARMUPGAP guard skips the dispatch-row flip (it only
|
|
990
|
+
* injects a benign task-less notification). A NON-launched session (a plain CLI
|
|
991
|
+
* session adopted by mesh_send_task --direct, launchedByCoordinator falsy)
|
|
992
|
+
* keeps the original full clear so an ad-hoc session is never left pinned.
|
|
975
993
|
*/
|
|
976
994
|
detachMeshAssignment(): void {
|
|
977
995
|
if (!this.settings.meshNodeFor && !this.settings.meshActiveTaskId && !this.settings.meshNodeId) return;
|
|
996
|
+
// Session-level member: keep membership, drop only the task-level marker.
|
|
997
|
+
if (this.settings.launchedByCoordinator === true) {
|
|
998
|
+
if (!this.settings.meshActiveTaskId) return;
|
|
999
|
+
const { meshActiveTaskId, ...rest } = this.settings;
|
|
1000
|
+
void meshActiveTaskId;
|
|
1001
|
+
this.settings = rest;
|
|
1002
|
+
this.adapter.updateRuntimeSettings?.(this.settings);
|
|
1003
|
+
return;
|
|
1004
|
+
}
|
|
978
1005
|
const { meshNodeFor, meshNodeId, meshActiveTaskId, ...rest } = this.settings;
|
|
979
1006
|
void meshNodeFor; void meshActiveTaskId;
|
|
980
1007
|
// WTCLAIM (A): clear the active binding but PRESERVE the last bound node id
|