@adhdev/daemon-core 0.9.82-rc.420 → 0.9.82-rc.422
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -1
- package/dist/index.js +149 -41
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +148 -41
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-event-forwarding.d.ts +22 -0
- package/dist/providers/cli-provider-instance.d.ts +11 -0
- package/dist/providers/provider-instance-manager.d.ts +18 -3
- package/package.json +2 -2
- package/src/commands/cli-manager.ts +10 -2
- package/src/index.ts +1 -1
- package/src/mesh/mesh-event-forwarding.ts +95 -1
- package/src/mesh/mesh-queue-assignment.ts +55 -0
- package/src/mesh/mesh-reconcile-loop.ts +11 -3
- package/src/providers/cli-provider-instance.ts +33 -0
- package/src/providers/provider-instance-manager.ts +54 -13
|
@@ -16,4 +16,26 @@ export declare function handleMeshForwardEvent(components: DaemonComponents, pay
|
|
|
16
16
|
success: boolean;
|
|
17
17
|
error: string;
|
|
18
18
|
};
|
|
19
|
+
/**
|
|
20
|
+
* NOTIF-HELD-DRAIN (Fix 2): event-driven coordinator drain. The reconcile loop delivers a
|
|
21
|
+
* worker's queued completion to an IDLE local coordinator only on its periodic poll. When a
|
|
22
|
+
* coordinator is sitting idle awaiting exactly that completion, waiting up to a full poll
|
|
23
|
+
* interval is the avoidable delivery latency the RCA flags — and combined with the (now-fixed)
|
|
24
|
+
* modal-park false-positive it stretched into the multi-minute notification stall. So the
|
|
25
|
+
* MOMENT a worker delegate event is persisted for a mesh, attempt the same idle-coordinator
|
|
26
|
+
* drain immediately, mirroring the event-driven worker-claim path (agent:ready /
|
|
27
|
+
* agent:generating_completed → triggerMeshQueue).
|
|
28
|
+
*
|
|
29
|
+
* Safety:
|
|
30
|
+
* - drainPendingMeshCoordinatorEvents marks rows drained=1 atomically, so this races the
|
|
31
|
+
* reconcile poll and the coordinator's own idle auto-flush harmlessly — exactly one consumes
|
|
32
|
+
* each row.
|
|
33
|
+
* - Only IDLE, non-modal-parked coordinators are delivery targets (never a generating /
|
|
34
|
+
* consent-modal PTY).
|
|
35
|
+
* - Strict session routing is honoured: an event naming an originating coordinator session is
|
|
36
|
+
* delivered only to that live idle session; anything not currently deliverable here
|
|
37
|
+
* (wrong/absent session, or a message-less lifecycle event) is RE-QUEUED — never dropped —
|
|
38
|
+
* so the reconcile loop's strict hold/expire path remains the single authority for it.
|
|
39
|
+
*/
|
|
40
|
+
export declare function flushPendingForMeshIdleCoordinators(components: DaemonComponents, meshId: string): void;
|
|
19
41
|
export declare function setupMeshEventForwarding(components: DaemonComponents): void;
|
|
@@ -205,6 +205,17 @@ export declare class CliProviderInstance implements ProviderInstance {
|
|
|
205
205
|
* absent from some of them.
|
|
206
206
|
*/
|
|
207
207
|
resolveModalParkStatus(): 'waiting_choice' | 'waiting_approval' | null;
|
|
208
|
+
/**
|
|
209
|
+
* NOTIF-HELD-DRAIN: true when this `waiting_approval` is a routine, transient tool-consent
|
|
210
|
+
* of an autonomously-progressing mesh session rather than a genuine human-await modal —
|
|
211
|
+
* i.e. it is a mesh coordinator/worker session, a turn is actively in flight
|
|
212
|
+
* (hasAdapterPendingResponse), and no human is attending it by hand. Such a consent is
|
|
213
|
+
* driven to resolution by the harness/operator as part of the in-flight turn, so holding
|
|
214
|
+
* the mesh's completion events behind it (modal_parked) is the false-positive that stalls
|
|
215
|
+
* delivery. Narrow by design: manual attendance or a non-progressing session falls through
|
|
216
|
+
* to the genuine-modal classification.
|
|
217
|
+
*/
|
|
218
|
+
private isTransientToolConsent;
|
|
208
219
|
/** True when this session is parked on a modal awaiting a human answer. */
|
|
209
220
|
isModalParked(): boolean;
|
|
210
221
|
onEvent(event: string, data?: any): void;
|
|
@@ -84,15 +84,30 @@ export declare class ProviderInstanceManager {
|
|
|
84
84
|
updateInstanceSettings(providerType: string, settings: Record<string, any>): number;
|
|
85
85
|
/** Stamp a mesh assignment on a single instance (used by mesh_send_task
|
|
86
86
|
* --direct so the worker's completion event has a coordinator routing
|
|
87
|
-
* marker in state.settings). Returns true
|
|
88
|
-
*
|
|
87
|
+
* marker in state.settings). Returns `{ stamped: true }` when the stamp was
|
|
88
|
+
* applied, or `{ stamped: false, reason }` when it was refused — the instance
|
|
89
|
+
* was missing / has no attach method, or the DOUBLE-DISPATCH idempotence guard
|
|
90
|
+
* fired (the same task is already running on another live session here). */
|
|
89
91
|
attachMeshAssignmentToInstance(instanceId: string, assignment: {
|
|
90
92
|
meshId: string;
|
|
91
93
|
nodeId?: string;
|
|
92
94
|
taskId?: string;
|
|
93
95
|
coordinatorDaemonId?: string;
|
|
94
96
|
coordinatorSessionId?: string;
|
|
95
|
-
}):
|
|
97
|
+
}): {
|
|
98
|
+
stamped: boolean;
|
|
99
|
+
reason?: string;
|
|
100
|
+
};
|
|
101
|
+
/**
|
|
102
|
+
* DOUBLE-DISPATCH support: the id of another LIVE, actively-working instance that already
|
|
103
|
+
* holds (meshId, taskId), or null. "Live working" = stamped with this exact mesh+task AND
|
|
104
|
+
* currently mid-turn / booting toward it (generating / waiting on approval-or-choice /
|
|
105
|
+
* starting) — NOT idle, stopped, or errored. A stale/dead/idle holder is deliberately
|
|
106
|
+
* ignored so a legitimate re-dispatch (e.g. after a dispatch failure) is never blocked.
|
|
107
|
+
* The instance being stamped (excludeInstanceId) is skipped so re-stamping the same
|
|
108
|
+
* session stays idempotent. O(n) over instances — the count is small.
|
|
109
|
+
*/
|
|
110
|
+
private findLiveWorkingTaskHolder;
|
|
96
111
|
/** Clear a mesh assignment after the dispatched task reaches a terminal
|
|
97
112
|
* state (generating_completed / stopped / failed). */
|
|
98
113
|
detachMeshAssignmentFromInstance(instanceId: string): boolean;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adhdev/daemon-core",
|
|
3
|
-
"version": "0.9.82-rc.
|
|
3
|
+
"version": "0.9.82-rc.422",
|
|
4
4
|
"description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
"author": "vilmire",
|
|
47
47
|
"license": "AGPL-3.0-or-later",
|
|
48
48
|
"dependencies": {
|
|
49
|
-
"@adhdev/mesh-shared": "0.9.82-rc.
|
|
49
|
+
"@adhdev/mesh-shared": "0.9.82-rc.422",
|
|
50
50
|
"@adhdev/session-host-core": "*",
|
|
51
51
|
"@agentclientprotocol/sdk": "^0.16.1",
|
|
52
52
|
"ajv": "^8.20.0",
|
|
@@ -1535,14 +1535,22 @@ export class DaemonCliManager {
|
|
|
1535
1535
|
const meshContext = (args as any)?.meshContext;
|
|
1536
1536
|
if (meshContext && typeof meshContext === 'object' && typeof meshContext.meshId === 'string' && meshContext.meshId) {
|
|
1537
1537
|
const targetInstanceId = key;
|
|
1538
|
+
let stampResult: { stamped: boolean; reason?: string } | undefined;
|
|
1538
1539
|
try {
|
|
1539
|
-
this.deps.getInstanceManager()?.attachMeshAssignmentToInstance(targetInstanceId, {
|
|
1540
|
+
stampResult = this.deps.getInstanceManager()?.attachMeshAssignmentToInstance(targetInstanceId, {
|
|
1540
1541
|
meshId: meshContext.meshId,
|
|
1541
1542
|
...(typeof meshContext.nodeId === 'string' && meshContext.nodeId ? { nodeId: meshContext.nodeId } : {}),
|
|
1542
1543
|
...(typeof meshContext.taskId === 'string' && meshContext.taskId ? { taskId: meshContext.taskId } : {}),
|
|
1543
1544
|
...(typeof meshContext.coordinatorDaemonId === 'string' && meshContext.coordinatorDaemonId ? { coordinatorDaemonId: meshContext.coordinatorDaemonId } : {}),
|
|
1544
1545
|
});
|
|
1545
|
-
} catch { /* best-effort */ }
|
|
1546
|
+
} catch { /* best-effort — stamping is a routing aid, not a hard requirement */ }
|
|
1547
|
+
// DOUBLE-DISPATCH stamp guard: the instance manager refused this stamp because
|
|
1548
|
+
// the SAME task is already running on another live session on this daemon.
|
|
1549
|
+
// Sending the prompt anyway would double-execute the task — fail closed so the
|
|
1550
|
+
// coordinator does not duplicate the work onto a second session.
|
|
1551
|
+
if (stampResult && stampResult.stamped === false && stampResult.reason === 'task_already_stamped_on_live_instance') {
|
|
1552
|
+
throw new Error(`Refusing duplicate mesh dispatch: task ${meshContext.taskId} is already being worked by a live session on this daemon`);
|
|
1553
|
+
}
|
|
1546
1554
|
}
|
|
1547
1555
|
const input = normalizeInputEnvelope(args?.input ? { input: args.input } : args);
|
|
1548
1556
|
const provider = this.providerLoader.resolve(agentType) || this.providerLoader.getMeta(agentType);
|
package/src/index.ts
CHANGED
|
@@ -293,7 +293,7 @@ export type { PendingMeshCoordinatorEvent } from './mesh/mesh-events.js';
|
|
|
293
293
|
// (finalSummary / workerResult.summary / lastMessagePreview). Same data the mobile
|
|
294
294
|
// inbox is fed; reused by mesh_read_chat's cache fallback when the live P2P read path
|
|
295
295
|
// is unavailable (saturated/unreachable peer).
|
|
296
|
-
export { resolveMeshSurfacedSessionPreview, readMeshCompletionSummary } from './mesh/mesh-events-utils.js';
|
|
296
|
+
export { resolveMeshSurfacedSessionPreview, readMeshCompletionSummary, isWeakCompletionEvidence } from './mesh/mesh-events-utils.js';
|
|
297
297
|
|
|
298
298
|
// ── Mesh Delivery Policy ──
|
|
299
299
|
export { resolveDeliveryDecision, createSessionDelivery, updateSessionDeliveryStatus, getActiveSessionDeliveries, markSessionDeliveriesTerminal, recordCompletionConflict, getRecentCompletionConflicts } from './mesh/mesh-delivery-policy.js';
|
|
@@ -7,7 +7,8 @@ import type { SessionRecoveryContext } from './mesh-ledger.js';
|
|
|
7
7
|
import { updateSessionTaskStatus, enqueueTask, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, getActiveDirectDispatches, hasPendingDependents, getQueue } from './mesh-work-queue.js';
|
|
8
8
|
import { markSessionDeliveriesTerminal, updateSessionDeliveryStatus, recordCompletionConflict } from './mesh-delivery-policy.js';
|
|
9
9
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
10
|
-
import { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents } from './mesh-events-pending.js';
|
|
10
|
+
import { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents, type PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
11
|
+
import type { ProviderInstance } from '../providers/provider-instance.js';
|
|
11
12
|
import { resolveWorkerDelegateRouting, recordUnroutableDelegateEvent, isUnroutableDelegateRejection } from './mesh-routing.js';
|
|
12
13
|
import { resolveMeshHostStatus } from './mesh-host-ownership.js';
|
|
13
14
|
import { enqueueUnresolvedDelegateForward, peekUnresolvedDelegateForwards, ackUnresolvedDelegateForward } from './mesh-unresolved-forward-outbox.js';
|
|
@@ -1670,6 +1671,93 @@ function ackUnresolvedDelegateForwardByFingerprint(
|
|
|
1670
1671
|
if (match) ackUnresolvedDelegateForward(match.id);
|
|
1671
1672
|
}
|
|
1672
1673
|
|
|
1674
|
+
/**
|
|
1675
|
+
* NOTIF-HELD-DRAIN (Fix 2): event-driven coordinator drain. The reconcile loop delivers a
|
|
1676
|
+
* worker's queued completion to an IDLE local coordinator only on its periodic poll. When a
|
|
1677
|
+
* coordinator is sitting idle awaiting exactly that completion, waiting up to a full poll
|
|
1678
|
+
* interval is the avoidable delivery latency the RCA flags — and combined with the (now-fixed)
|
|
1679
|
+
* modal-park false-positive it stretched into the multi-minute notification stall. So the
|
|
1680
|
+
* MOMENT a worker delegate event is persisted for a mesh, attempt the same idle-coordinator
|
|
1681
|
+
* drain immediately, mirroring the event-driven worker-claim path (agent:ready /
|
|
1682
|
+
* agent:generating_completed → triggerMeshQueue).
|
|
1683
|
+
*
|
|
1684
|
+
* Safety:
|
|
1685
|
+
* - drainPendingMeshCoordinatorEvents marks rows drained=1 atomically, so this races the
|
|
1686
|
+
* reconcile poll and the coordinator's own idle auto-flush harmlessly — exactly one consumes
|
|
1687
|
+
* each row.
|
|
1688
|
+
* - Only IDLE, non-modal-parked coordinators are delivery targets (never a generating /
|
|
1689
|
+
* consent-modal PTY).
|
|
1690
|
+
* - Strict session routing is honoured: an event naming an originating coordinator session is
|
|
1691
|
+
* delivered only to that live idle session; anything not currently deliverable here
|
|
1692
|
+
* (wrong/absent session, or a message-less lifecycle event) is RE-QUEUED — never dropped —
|
|
1693
|
+
* so the reconcile loop's strict hold/expire path remains the single authority for it.
|
|
1694
|
+
*/
|
|
1695
|
+
export function flushPendingForMeshIdleCoordinators(components: DaemonComponents, meshId: string): void {
|
|
1696
|
+
// O(1) gate: skip the (relatively expensive) per-instance getState scan when the queue is
|
|
1697
|
+
// empty for this mesh.
|
|
1698
|
+
try {
|
|
1699
|
+
const store = MeshRuntimeStore.getInstance();
|
|
1700
|
+
if (store.pendingEventCount(meshId) === 0) return;
|
|
1701
|
+
} catch { /* store unavailable — fall through and let the drain decide */ }
|
|
1702
|
+
|
|
1703
|
+
const idleCoordinators: { instance: ProviderInstance; sessionId: string }[] = [];
|
|
1704
|
+
try {
|
|
1705
|
+
for (const inst of components.instanceManager.getByCategory('cli')) {
|
|
1706
|
+
const state = inst.getState();
|
|
1707
|
+
const settings = state.settings && typeof state.settings === 'object'
|
|
1708
|
+
? state.settings as Record<string, unknown>
|
|
1709
|
+
: {};
|
|
1710
|
+
if (readNonEmptyString(settings.meshCoordinatorFor) !== meshId) continue;
|
|
1711
|
+
const status = readNonEmptyString(state.status).toLowerCase();
|
|
1712
|
+
const modalParked = typeof (inst as any).isModalParked === 'function'
|
|
1713
|
+
? (inst as any).isModalParked() === true
|
|
1714
|
+
: (status === 'waiting_choice' || status === 'waiting_approval');
|
|
1715
|
+
if (status === 'idle' && !modalParked) {
|
|
1716
|
+
idleCoordinators.push({ instance: inst, sessionId: readNonEmptyString(state.instanceId) });
|
|
1717
|
+
}
|
|
1718
|
+
}
|
|
1719
|
+
} catch { return; }
|
|
1720
|
+
if (idleCoordinators.length === 0) return; // no idle target now → leave for the reconcile poll
|
|
1721
|
+
|
|
1722
|
+
const drainDaemonIds = resolveCoordinatorDrainDaemonIds(components);
|
|
1723
|
+
let pendingEvents: PendingMeshCoordinatorEvent[];
|
|
1724
|
+
try {
|
|
1725
|
+
pendingEvents = drainPendingMeshCoordinatorEvents(meshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
|
|
1726
|
+
} catch (e: any) {
|
|
1727
|
+
LOG.warn('MeshEvents', `Event-driven coordinator drain failed for mesh ${meshId}: ${e?.message || e}`);
|
|
1728
|
+
return;
|
|
1729
|
+
}
|
|
1730
|
+
if (pendingEvents.length === 0) return;
|
|
1731
|
+
|
|
1732
|
+
let delivered = 0;
|
|
1733
|
+
for (const pending of pendingEvents) {
|
|
1734
|
+
const wantSession = readNonEmptyString(pending.targetCoordinatorSessionId);
|
|
1735
|
+
const targets = wantSession
|
|
1736
|
+
? idleCoordinators.filter(c => c.sessionId === wantSession)
|
|
1737
|
+
: idleCoordinators;
|
|
1738
|
+
// Not deliverable into an idle target here (wrong/absent session), or a message-less
|
|
1739
|
+
// lifecycle event (agent:ready / generating_started carry no coordinatorMessage and
|
|
1740
|
+
// must not be injected): re-queue so the reconcile loop owns it (lazy-synth / strict
|
|
1741
|
+
// hold/expire). Re-queue preserves queuedAt so the strict TTL measures true age.
|
|
1742
|
+
if (targets.length === 0 || !pending.coordinatorMessage) {
|
|
1743
|
+
try { queuePendingMeshCoordinatorEvent(pending); } catch { /* best-effort re-queue */ }
|
|
1744
|
+
continue;
|
|
1745
|
+
}
|
|
1746
|
+
const message = pending.coordinatorMessage;
|
|
1747
|
+
const force = shouldForceInjectMeshEvent(pending.event);
|
|
1748
|
+
for (const c of targets) {
|
|
1749
|
+
c.instance.onEvent('send_message', {
|
|
1750
|
+
input: { text: message, textFallback: message },
|
|
1751
|
+
...(force ? { force: true } : {}),
|
|
1752
|
+
});
|
|
1753
|
+
delivered++;
|
|
1754
|
+
}
|
|
1755
|
+
}
|
|
1756
|
+
if (delivered > 0) {
|
|
1757
|
+
LOG.info('MeshEvents', `Event-driven drain delivered ${delivered} pending event(s) to ${idleCoordinators.length} idle coordinator(s) for mesh ${meshId}`);
|
|
1758
|
+
}
|
|
1759
|
+
}
|
|
1760
|
+
|
|
1673
1761
|
export function setupMeshEventForwarding(components: DaemonComponents) {
|
|
1674
1762
|
components.instanceManager.onEvent((event) => {
|
|
1675
1763
|
// --- Coordinator idle auto-flush (fast path) ---
|
|
@@ -1785,5 +1873,11 @@ export function setupMeshEventForwarding(components: DaemonComponents) {
|
|
|
1785
1873
|
event: event.event,
|
|
1786
1874
|
metadataEvent: event,
|
|
1787
1875
|
});
|
|
1876
|
+
|
|
1877
|
+
// NOTIF-HELD-DRAIN (Fix 2): the worker's event is now persisted in the pending queue.
|
|
1878
|
+
// If a local coordinator for this mesh is sitting idle awaiting it, deliver immediately
|
|
1879
|
+
// instead of waiting up to a full reconcile interval (event-driven, mirrors the
|
|
1880
|
+
// worker-claim path). No-op when no idle coordinator is present (held for reconcile).
|
|
1881
|
+
flushPendingForMeshIdleCoordinators(components, routing.meshId);
|
|
1788
1882
|
});
|
|
1789
1883
|
}
|
|
@@ -1098,6 +1098,47 @@ function liveSessionCountForNode(components: DaemonComponents, meshId: string, n
|
|
|
1098
1098
|
}).length;
|
|
1099
1099
|
}
|
|
1100
1100
|
|
|
1101
|
+
/**
|
|
1102
|
+
* DOUBLE-DISPATCH auto-launch gate: does this node already have a LIVE mesh session that
|
|
1103
|
+
* is NOT holding an assigned queue task — i.e. one that is idle, booting toward its first
|
|
1104
|
+
* claim, or in a momentary non-idle flip? Such a session WILL claim a still-pending task
|
|
1105
|
+
* on its own via the idle→claim / agent:ready drain, so spawning a NEW session here only
|
|
1106
|
+
* races it and yields a duplicate worker that double-stamps the same taskId (the
|
|
1107
|
+
* enqueue → drain-miss → auto-launch RCA: the drain skipped a momentarily-non-idle idle
|
|
1108
|
+
* session as a candidate, and the write-only nodeHasActiveAssignment gate — which only
|
|
1109
|
+
* inspects status='assigned' rows — could not see the about-to-claim session either).
|
|
1110
|
+
*
|
|
1111
|
+
* A session that already HOLDS an assigned queue task is genuine concurrent work, not a
|
|
1112
|
+
* free claimer, and is excluded — so a read-only auto-launch onto a busy-but-no-idle node
|
|
1113
|
+
* is still allowed. A node with NO live mesh session at all (dead, or never launched) does
|
|
1114
|
+
* not match, preserving the legitimate first-session spawn.
|
|
1115
|
+
*/
|
|
1116
|
+
function nodeHasLiveSessionPendingClaim(components: DaemonComponents, meshId: string, nodeId: string): boolean {
|
|
1117
|
+
// Session ids currently holding an assigned queue task on this node — those are busy,
|
|
1118
|
+
// not pending claimers, so they must NOT suppress a (read-only) launch.
|
|
1119
|
+
const busySessionIds = new Set(
|
|
1120
|
+
getQueue(meshId, { status: ['assigned'] as any })
|
|
1121
|
+
.filter(task => daemonIdsEquivalent(task.assignedNodeId, nodeId))
|
|
1122
|
+
.map(task => readNonEmptyString(task.assignedSessionId))
|
|
1123
|
+
.filter(Boolean),
|
|
1124
|
+
);
|
|
1125
|
+
return components.instanceManager.getByCategory('cli').some((inst: any) => {
|
|
1126
|
+
const state = inst.getState();
|
|
1127
|
+
const settings = state.settings as Record<string, unknown> || {};
|
|
1128
|
+
if (readNonEmptyString(settings.meshNodeFor) !== meshId) return false;
|
|
1129
|
+
const instNodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
|
|
1130
|
+
// Canonical-form match (see nodeHasActiveMeshWork / liveSessionCountForNode): a
|
|
1131
|
+
// daemon-id form skew must not make a present session look absent and reopen the
|
|
1132
|
+
// duplicate-launch hole.
|
|
1133
|
+
if (!daemonIdsEquivalent(instNodeId, nodeId)) return false;
|
|
1134
|
+
const status = readNonEmptyString(state.status).toLowerCase();
|
|
1135
|
+
if (isTerminalSessionStatus(status)) return false; // dead → no claimer here, allow launch
|
|
1136
|
+
const sessionId = readNonEmptyString(state.instanceId);
|
|
1137
|
+
if (sessionId && busySessionIds.has(sessionId)) return false; // busy with its own assigned task
|
|
1138
|
+
return true; // live + unassigned → will claim the pending task itself
|
|
1139
|
+
});
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1101
1142
|
function recordAutoLaunchEvent(meshId: string, args: {
|
|
1102
1143
|
phase: 'skipped' | 'started' | 'failed' | 'completed';
|
|
1103
1144
|
taskId: string;
|
|
@@ -1417,6 +1458,20 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
1417
1458
|
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS); sweepExpiredCooldowns();
|
|
1418
1459
|
continue;
|
|
1419
1460
|
}
|
|
1461
|
+
// DOUBLE-DISPATCH auto-launch gate (see nodeHasLiveSessionPendingClaim): when this
|
|
1462
|
+
// node already has a live session on its way to claim (idle / booting / momentary
|
|
1463
|
+
// non-idle flip), do NOT spawn a second one — that session pulls the pending task
|
|
1464
|
+
// via the normal idle→claim / agent:ready drain. Launching here races it and yields
|
|
1465
|
+
// a duplicate worker that double-stamps the same taskId. Applies to read-only tasks
|
|
1466
|
+
// too: an idle session can claim either kind, while a genuinely BUSY session (holding
|
|
1467
|
+
// its own assigned task) is excluded by the helper, so a read-only launch onto a
|
|
1468
|
+
// busy-but-no-idle node is still allowed. Skip with a transient (non-actionable)
|
|
1469
|
+
// reason so the coordinator is not paged; the 4s reconcile retries, and once the
|
|
1470
|
+
// existing session goes terminal this gate clears and a legitimate launch proceeds.
|
|
1471
|
+
if (nodeHasLiveSessionPendingClaim(components, meshId, nodeId)) {
|
|
1472
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_has_live_session_pending_claim', nodeId });
|
|
1473
|
+
continue;
|
|
1474
|
+
}
|
|
1420
1475
|
// Write tasks keep the one-active-per-node invariant (worktree isolation);
|
|
1421
1476
|
// read-only diagnoses may auto-launch onto a node that already has an active
|
|
1422
1477
|
// assignment. Classified by the shared isTaskReadonly predicate.
|
|
@@ -307,9 +307,17 @@ function findLiveCoordinators(components: DaemonComponents): LiveCoordinator[] {
|
|
|
307
307
|
const status = readNonEmptyString(state.status).toLowerCase();
|
|
308
308
|
// getState() overlays the modal-park statuses: an active AskUserQuestion
|
|
309
309
|
// prompt surfaces as waiting_choice, a tool-consent prompt as waiting_approval.
|
|
310
|
-
//
|
|
311
|
-
//
|
|
312
|
-
|
|
310
|
+
// NOTIF-HELD-DRAIN (Fix 1): consult the instance's own isModalParked() rather than the
|
|
311
|
+
// raw status literal so the corrected classification flows here — a busy mesh
|
|
312
|
+
// coordinator's routine, in-flight tool-consent (auto-approve off) is NOT a human-await
|
|
313
|
+
// modal and must NOT wedge the mesh's pending completion events under `modal_parked`.
|
|
314
|
+
// resolveModalParkStatus() (which isModalParked wraps) already encodes that distinction
|
|
315
|
+
// and the waiting_choice/stalled-auto-approve genuine-modal cases. Fall back to the
|
|
316
|
+
// status literal for any instance that does not expose the method. Lowercase compare —
|
|
317
|
+
// the SessionStatus enum is forked across modules and waiting_choice is absent from some.
|
|
318
|
+
const modalParked = typeof (inst as any).isModalParked === 'function'
|
|
319
|
+
? (inst as any).isModalParked() === true
|
|
320
|
+
: (status === 'waiting_choice' || status === 'waiting_approval');
|
|
313
321
|
const sessionId = readNonEmptyString(state.instanceId);
|
|
314
322
|
// Modal-park transition observability: a coordinator entering modal-park is what
|
|
315
323
|
// begins holding completion events under `modal_parked`; one leaving it is what
|
|
@@ -1134,11 +1134,44 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
1134
1134
|
// as modal-parked so its events are held/surfaced rather than masked behind generating.
|
|
1135
1135
|
if (adapterStatus.status === 'waiting_approval'
|
|
1136
1136
|
&& (!this.autoApproveEffectivelyActive(adapterStatus.status) || this.autoApproveMaskStalled())) {
|
|
1137
|
+
// NOTIF-HELD-DRAIN (Fix 1): an autonomous mesh session (coordinator or worker)
|
|
1138
|
+
// that is actively progressing a turn — a tool call is in flight
|
|
1139
|
+
// (hasAdapterPendingResponse) and NO human is attending it by hand — surfaces a
|
|
1140
|
+
// routine tool-consent `waiting_approval` on EVERY tool call when auto-approve is
|
|
1141
|
+
// off. That transient consent is part of the turn the harness/operator drives to
|
|
1142
|
+
// completion, NOT a session genuinely wedged awaiting a human's modal answer.
|
|
1143
|
+
// Classifying it modal-parked makes findLiveCoordinators hold the mesh's pending
|
|
1144
|
+
// completion events under `modal_parked` across a busy coordinator's whole work
|
|
1145
|
+
// batch, which is the multi-minute notification stall. Treat such a transient
|
|
1146
|
+
// consent as NOT modal-parked so it is held as ordinary "generating" (released on
|
|
1147
|
+
// the next idle) instead. A manually-attended session, a stalled auto-approve, or a
|
|
1148
|
+
// non-progressing session (no turn in flight) still parks — those are the genuine
|
|
1149
|
+
// human-await cases the guard must keep holding.
|
|
1150
|
+
if (this.isTransientToolConsent()) {
|
|
1151
|
+
return null;
|
|
1152
|
+
}
|
|
1137
1153
|
return 'waiting_approval';
|
|
1138
1154
|
}
|
|
1139
1155
|
return null;
|
|
1140
1156
|
}
|
|
1141
1157
|
|
|
1158
|
+
/**
|
|
1159
|
+
* NOTIF-HELD-DRAIN: true when this `waiting_approval` is a routine, transient tool-consent
|
|
1160
|
+
* of an autonomously-progressing mesh session rather than a genuine human-await modal —
|
|
1161
|
+
* i.e. it is a mesh coordinator/worker session, a turn is actively in flight
|
|
1162
|
+
* (hasAdapterPendingResponse), and no human is attending it by hand. Such a consent is
|
|
1163
|
+
* driven to resolution by the harness/operator as part of the in-flight turn, so holding
|
|
1164
|
+
* the mesh's completion events behind it (modal_parked) is the false-positive that stalls
|
|
1165
|
+
* delivery. Narrow by design: manual attendance or a non-progressing session falls through
|
|
1166
|
+
* to the genuine-modal classification.
|
|
1167
|
+
*/
|
|
1168
|
+
private isTransientToolConsent(now = Date.now()): boolean {
|
|
1169
|
+
const isAutonomousMeshSession = this.isMeshWorkerSession() || !!this.settings.meshCoordinatorFor;
|
|
1170
|
+
return isAutonomousMeshSession
|
|
1171
|
+
&& this.hasAdapterPendingResponse()
|
|
1172
|
+
&& !this.manualAttendance.isAttended(now);
|
|
1173
|
+
}
|
|
1174
|
+
|
|
1142
1175
|
/** True when this session is parked on a modal awaiting a human answer. */
|
|
1143
1176
|
isModalParked(): boolean {
|
|
1144
1177
|
return this.resolveModalParkStatus() !== null;
|
|
@@ -308,23 +308,64 @@ export class ProviderInstanceManager {
|
|
|
308
308
|
|
|
309
309
|
/** Stamp a mesh assignment on a single instance (used by mesh_send_task
|
|
310
310
|
* --direct so the worker's completion event has a coordinator routing
|
|
311
|
-
* marker in state.settings). Returns true
|
|
312
|
-
*
|
|
313
|
-
|
|
311
|
+
* marker in state.settings). Returns `{ stamped: true }` when the stamp was
|
|
312
|
+
* applied, or `{ stamped: false, reason }` when it was refused — the instance
|
|
313
|
+
* was missing / has no attach method, or the DOUBLE-DISPATCH idempotence guard
|
|
314
|
+
* fired (the same task is already running on another live session here). */
|
|
315
|
+
attachMeshAssignmentToInstance(instanceId: string, assignment: { meshId: string; nodeId?: string; taskId?: string; coordinatorDaemonId?: string; coordinatorSessionId?: string }): { stamped: boolean; reason?: string } {
|
|
314
316
|
const inst = this.instances.get(instanceId);
|
|
315
317
|
if (!inst || typeof inst.attachMeshAssignment !== 'function') {
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
318
|
+
LOG.warn('MeshDispatch', `attachMeshAssignment skipped: instance ${instanceId} ${inst ? 'has no attach method' : 'not found'}`);
|
|
319
|
+
return { stamped: false, reason: inst ? 'instance_has_no_attach_method' : 'instance_not_found' };
|
|
320
|
+
}
|
|
321
|
+
// DOUBLE-DISPATCH stamp idempotence guard (defense in depth): refuse to stamp this
|
|
322
|
+
// (meshId, taskId) onto a SECOND instance when a DIFFERENT, still-live and actively
|
|
323
|
+
// working instance already holds the exact same task. Two sessions carrying one taskId
|
|
324
|
+
// double-execute the work (the auto-launch race RCA: a delayed claim by the original
|
|
325
|
+
// session plus the new session's post-boot claim sequentially stamp the same task —
|
|
326
|
+
// the atomic claim only blocks SIMULTANEOUS claims). A stale/dead/idle prior holder is
|
|
327
|
+
// NOT a conflict — a legitimate re-dispatch after a dispatch failure must still stamp.
|
|
328
|
+
if (assignment.taskId) {
|
|
329
|
+
const conflict = this.findLiveWorkingTaskHolder(assignment.meshId, assignment.taskId, instanceId);
|
|
330
|
+
if (conflict) {
|
|
331
|
+
LOG.warn('MeshDispatch', `attachMeshAssignment refused: task ${assignment.taskId} (mesh ${assignment.meshId}) is already being worked by live session ${conflict} — skipping duplicate stamp on ${instanceId}`);
|
|
332
|
+
return { stamped: false, reason: 'task_already_stamped_on_live_instance' };
|
|
333
|
+
}
|
|
321
334
|
}
|
|
322
335
|
inst.attachMeshAssignment(assignment);
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
336
|
+
LOG.info('MeshDispatch', `stamped mesh assignment on ${instanceId}: mesh=${assignment.meshId} node=${assignment.nodeId || ''} task=${assignment.taskId || ''} coordinator=${assignment.coordinatorDaemonId || ''}`);
|
|
337
|
+
return { stamped: true };
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* DOUBLE-DISPATCH support: the id of another LIVE, actively-working instance that already
|
|
342
|
+
* holds (meshId, taskId), or null. "Live working" = stamped with this exact mesh+task AND
|
|
343
|
+
* currently mid-turn / booting toward it (generating / waiting on approval-or-choice /
|
|
344
|
+
* starting) — NOT idle, stopped, or errored. A stale/dead/idle holder is deliberately
|
|
345
|
+
* ignored so a legitimate re-dispatch (e.g. after a dispatch failure) is never blocked.
|
|
346
|
+
* The instance being stamped (excludeInstanceId) is skipped so re-stamping the same
|
|
347
|
+
* session stays idempotent. O(n) over instances — the count is small.
|
|
348
|
+
*/
|
|
349
|
+
private findLiveWorkingTaskHolder(meshId: string, taskId: string, excludeInstanceId: string): string | null {
|
|
350
|
+
// Mid-turn / booting statuses (top-level or activeChat). Anything else — idle, stopped,
|
|
351
|
+
// error — is not a live worker actively holding the task.
|
|
352
|
+
const working = new Set(['generating', 'waiting_approval', 'waiting_choice', 'starting', 'streaming', 'working', 'no_progress', 'long_generating']);
|
|
353
|
+
for (const [id, inst] of this.instances) {
|
|
354
|
+
if (id === excludeInstanceId) continue;
|
|
355
|
+
let state: ProviderState;
|
|
356
|
+
try {
|
|
357
|
+
state = inst.getState();
|
|
358
|
+
} catch {
|
|
359
|
+
continue;
|
|
360
|
+
}
|
|
361
|
+
const settings = (state.settings as Record<string, unknown>) || {};
|
|
362
|
+
if (settings.meshNodeFor !== meshId) continue;
|
|
363
|
+
if (settings.meshActiveTaskId !== taskId) continue;
|
|
364
|
+
const status = (typeof state.status === 'string' ? state.status : '').toLowerCase();
|
|
365
|
+
const chatStatus = (typeof state.activeChat?.status === 'string' ? state.activeChat.status : '').toLowerCase();
|
|
366
|
+
if (working.has(status) || working.has(chatStatus)) return id;
|
|
367
|
+
}
|
|
368
|
+
return null;
|
|
328
369
|
}
|
|
329
370
|
|
|
330
371
|
/** Clear a mesh assignment after the dispatched task reaches a terminal
|