@adhdev/daemon-core 0.9.82-rc.195 → 0.9.82-rc.197
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-adapter-types.d.ts +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +594 -78
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +593 -83
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/contracts.d.ts +1 -1
- package/dist/mesh/mesh-active-work.d.ts +1 -1
- package/dist/mesh/mesh-delivery-policy.d.ts +126 -0
- package/dist/mesh/{beads-db.d.ts → mesh-runtime-store.d.ts} +68 -2
- package/dist/mesh/mesh-work-queue.d.ts +3 -3
- package/dist/providers/provider-instance.d.ts +1 -1
- package/dist/providers/spec/driver.d.ts +4 -1
- package/dist/providers/spec/schema.gen.d.ts +46 -0
- package/dist/providers/spec/types.d.ts +39 -0
- package/dist/shared-types-extra.d.ts +1 -1
- package/dist/status/normalize.d.ts +1 -1
- package/dist/status/normalize.js +1 -0
- package/dist/status/normalize.js.map +1 -1
- package/dist/status/normalize.mjs +1 -0
- package/dist/status/normalize.mjs.map +1 -1
- package/package.json +1 -1
- package/src/cli-adapter-types.ts +1 -0
- package/src/cli-adapters/cli-state-engine.ts +44 -2
- package/src/index.ts +4 -0
- package/src/mesh/contracts.ts +1 -1
- package/src/mesh/mesh-active-work.ts +8 -8
- package/src/mesh/mesh-delivery-policy.ts +298 -0
- package/src/mesh/mesh-events.ts +64 -15
- package/src/mesh/{beads-db.ts → mesh-runtime-store.ts} +249 -7
- package/src/mesh/mesh-work-queue.ts +33 -33
- package/src/providers/cli-provider-instance.ts +31 -8
- package/src/providers/provider-instance.ts +1 -1
- package/src/providers/spec/driver.ts +34 -3
- package/src/providers/spec/evaluator.ts +32 -3
- package/src/providers/spec/schema.gen.ts +22 -2
- package/src/providers/spec/schema.json +1 -0
- package/src/providers/spec/types.ts +39 -0
- package/src/providers/types/interactive-prompt.ts +21 -7
- package/src/shared-types-extra.ts +1 -1
- package/src/status/normalize.ts +2 -0
package/src/index.ts
CHANGED
|
@@ -232,6 +232,10 @@ export { buildMeshHostRequiredFailure, createDefaultMeshHostMetadata, isMeshHost
|
|
|
232
232
|
export { triggerMeshQueue, drainPendingMeshCoordinatorEvents, getPendingMeshCoordinatorEvents, clearPendingMeshCoordinatorEvents, queuePendingMeshCoordinatorEvent, reconcileDirectDispatchCompletionFromTranscript } from './mesh/mesh-events.js';
|
|
233
233
|
export type { PendingMeshCoordinatorEvent } from './mesh/mesh-events.js';
|
|
234
234
|
|
|
235
|
+
// ── Mesh Delivery Policy ──
|
|
236
|
+
export { resolveDeliveryDecision, createSessionDelivery, updateSessionDeliveryStatus, getActiveSessionDeliveries, recordCompletionConflict, getRecentCompletionConflicts } from './mesh/mesh-delivery-policy.js';
|
|
237
|
+
export type { MeshSessionDeliveryStatus, MeshSessionDeliveryKind, MeshDeliveryDecision, MeshDeliveryPolicyResult, SessionDeliveryRecord } from './mesh/mesh-delivery-policy.js';
|
|
238
|
+
|
|
235
239
|
// ── Mesh P2P Relay Failure Classification ──
|
|
236
240
|
export {
|
|
237
241
|
P2pRelayFailureError,
|
package/src/mesh/contracts.ts
CHANGED
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* - Session identifier keys diverged across stores: targetSessionId /
|
|
17
17
|
* assignedSessionId / instanceId / runtimeSessionId / providerSessionId.
|
|
18
18
|
* resolveEventSessionId() tried four fallbacks per call.
|
|
19
|
-
* - No protocol version on the JSONL ledger or
|
|
19
|
+
* - No protocol version on the JSONL ledger or MeshRuntimeStore. Schema
|
|
20
20
|
* evolutions had no guard rail.
|
|
21
21
|
* - mesh_reconcile_ledger existed as a routine recovery tool, not as
|
|
22
22
|
* an incident-response escape hatch. That itself signals the routing
|
|
@@ -70,7 +70,7 @@ export interface BuildMeshActiveWorkOptions {
|
|
|
70
70
|
queue?: MeshWorkQueueEntry[];
|
|
71
71
|
ledgerEntries?: MeshLedgerEntry[];
|
|
72
72
|
/**
|
|
73
|
-
* Active direct dispatches from
|
|
73
|
+
* Active direct dispatches from MeshRuntimeStore. When provided, these are used instead of
|
|
74
74
|
* scanning ledger entries for direct dispatches — eliminates the O(n_ledger) scan.
|
|
75
75
|
* Falls back to ledger scanning when not provided.
|
|
76
76
|
*/
|
|
@@ -224,9 +224,9 @@ export function buildMeshActiveWork(opts: BuildMeshActiveWorkOptions): { activeW
|
|
|
224
224
|
});
|
|
225
225
|
}
|
|
226
226
|
|
|
227
|
-
// When
|
|
228
|
-
// ALSO scan ledger for remote dispatches (P2P) whose taskIds are not in
|
|
229
|
-
// are never written to the local
|
|
227
|
+
// When MeshRuntimeStore direct dispatches are provided, use them for LOCAL dispatches (O(1) indexed).
|
|
228
|
+
// ALSO scan ledger for remote dispatches (P2P) whose taskIds are not in MeshRuntimeStore — these
|
|
229
|
+
// are never written to the local MeshRuntimeStore since they're dispatched from a remote daemon.
|
|
230
230
|
if (opts.directDispatches !== undefined) {
|
|
231
231
|
const dbTaskIds = new Set(opts.directDispatches.map(d => d.taskId));
|
|
232
232
|
for (const dispatch of opts.directDispatches) {
|
|
@@ -274,13 +274,13 @@ export function buildMeshActiveWork(opts: BuildMeshActiveWorkOptions): { activeW
|
|
|
274
274
|
}
|
|
275
275
|
records.push(record);
|
|
276
276
|
}
|
|
277
|
-
// Also scan ledger for remote dispatches (via p2p_direct) whose taskIds are NOT in
|
|
278
|
-
// Remote daemons write their own local
|
|
277
|
+
// Also scan ledger for remote dispatches (via p2p_direct) whose taskIds are NOT in MeshRuntimeStore.
|
|
278
|
+
// Remote daemons write their own local MeshRuntimeStore; this coordinator's MeshRuntimeStore only has local dispatches.
|
|
279
279
|
const ledgerEntries = (opts.ledgerEntries || []).slice().sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime());
|
|
280
280
|
const terminals = ledgerEntries.filter(entry => TERMINAL_LEDGER_KINDS.has(entry.kind) || entry.kind === 'task_approval_needed');
|
|
281
281
|
for (const dispatch of ledgerEntries.filter(isDirectDispatch)) {
|
|
282
282
|
const taskId = directDispatchTaskId(dispatch);
|
|
283
|
-
if (dbTaskIds.has(taskId)) continue; // already covered by
|
|
283
|
+
if (dbTaskIds.has(taskId)) continue; // already covered by MeshRuntimeStore path above
|
|
284
284
|
const terminal = terminals
|
|
285
285
|
.filter(entry => new Date(entry.timestamp).getTime() >= new Date(dispatch.timestamp).getTime())
|
|
286
286
|
.find(entry => terminalMatchesDispatch(entry, dispatch, taskId));
|
|
@@ -329,7 +329,7 @@ export function buildMeshActiveWork(opts: BuildMeshActiveWorkOptions): { activeW
|
|
|
329
329
|
records.push(record);
|
|
330
330
|
}
|
|
331
331
|
} else {
|
|
332
|
-
// Full ledger scan: no
|
|
332
|
+
// Full ledger scan: no MeshRuntimeStore direct dispatches available (standalone mode or empty).
|
|
333
333
|
const ledgerEntries = (opts.ledgerEntries || []).slice().sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime());
|
|
334
334
|
const terminals = ledgerEntries.filter(entry => TERMINAL_LEDGER_KINDS.has(entry.kind) || entry.kind === 'task_approval_needed');
|
|
335
335
|
for (const dispatch of ledgerEntries.filter(isDirectDispatch)) {
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
import { randomUUID } from 'crypto';
|
|
2
|
+
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Possible delivery statuses for a session delivery record.
|
|
6
|
+
*/
|
|
7
|
+
export type MeshSessionDeliveryStatus =
|
|
8
|
+
| 'queued'
|
|
9
|
+
| 'delivering'
|
|
10
|
+
| 'delivered'
|
|
11
|
+
| 'acked'
|
|
12
|
+
| 'completed'
|
|
13
|
+
| 'failed'
|
|
14
|
+
| 'expired'
|
|
15
|
+
| 'cancelled';
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Kind of delivery — controls priority and policy handling.
|
|
19
|
+
*/
|
|
20
|
+
export type MeshSessionDeliveryKind =
|
|
21
|
+
| 'task'
|
|
22
|
+
| 'followup'
|
|
23
|
+
| 'approval'
|
|
24
|
+
| 'recovery'
|
|
25
|
+
| 'system_notice';
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* A session delivery decision — what to do when a task arrives for a session.
|
|
29
|
+
*/
|
|
30
|
+
export type MeshDeliveryDecision =
|
|
31
|
+
| 'immediate' // Session is idle: deliver now, create an 'acked' delivery record
|
|
32
|
+
| 'queued' // Session is busy: hold delivery until session becomes idle
|
|
33
|
+
| 'rejected'; // Session is terminal or unknown: cannot deliver
|
|
34
|
+
|
|
35
|
+
export interface MeshDeliveryPolicyResult {
|
|
36
|
+
decision: MeshDeliveryDecision;
|
|
37
|
+
reason: string;
|
|
38
|
+
/** When decision='queued', estimated deliver-after ISO timestamp if known. */
|
|
39
|
+
deliverAfter?: string;
|
|
40
|
+
/** Human-readable explanation for coordinator/operator. */
|
|
41
|
+
message: string;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Session statuses where immediate delivery is allowed.
|
|
46
|
+
* The session is ready to accept new work.
|
|
47
|
+
*/
|
|
48
|
+
const IMMEDIATE_DELIVERY_STATUSES = new Set([
|
|
49
|
+
'idle',
|
|
50
|
+
'waiting_input',
|
|
51
|
+
'ready',
|
|
52
|
+
]);
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Session statuses that indicate the session is busy but still alive.
|
|
56
|
+
* Delivery is queued rather than attempted immediately.
|
|
57
|
+
*/
|
|
58
|
+
const BUSY_DELIVERY_STATUSES = new Set([
|
|
59
|
+
'generating',
|
|
60
|
+
'running',
|
|
61
|
+
'streaming',
|
|
62
|
+
'busy',
|
|
63
|
+
'starting',
|
|
64
|
+
'initializing',
|
|
65
|
+
'waiting_approval',
|
|
66
|
+
]);
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Session statuses that indicate the session is permanently unavailable.
|
|
70
|
+
* Delivery should be rejected.
|
|
71
|
+
*/
|
|
72
|
+
const TERMINAL_DELIVERY_STATUSES = new Set([
|
|
73
|
+
'stopped',
|
|
74
|
+
'failed',
|
|
75
|
+
'terminated',
|
|
76
|
+
'exited',
|
|
77
|
+
'closed',
|
|
78
|
+
'deleted',
|
|
79
|
+
'error',
|
|
80
|
+
]);
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Determine whether to deliver immediately, queue, or reject based on session status.
|
|
84
|
+
*
|
|
85
|
+
* This is a pure function — it does not write to any store.
|
|
86
|
+
*/
|
|
87
|
+
export function resolveDeliveryDecision(
|
|
88
|
+
sessionStatus: string | undefined,
|
|
89
|
+
opts?: {
|
|
90
|
+
kind?: MeshSessionDeliveryKind;
|
|
91
|
+
/** When true, busy session immediate injection is allowed (provider-specific capability). */
|
|
92
|
+
allowBusyInjection?: boolean;
|
|
93
|
+
},
|
|
94
|
+
): MeshDeliveryPolicyResult {
|
|
95
|
+
const status = (sessionStatus || '').trim().toLowerCase();
|
|
96
|
+
|
|
97
|
+
if (!status) {
|
|
98
|
+
return {
|
|
99
|
+
decision: 'rejected',
|
|
100
|
+
reason: 'unknown_session_status',
|
|
101
|
+
message: 'Session status is unknown. Delivery rejected (fail-closed). Use mesh_launch_session to start a fresh session.',
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
if (IMMEDIATE_DELIVERY_STATUSES.has(status)) {
|
|
106
|
+
return {
|
|
107
|
+
decision: 'immediate',
|
|
108
|
+
reason: `session_${status}`,
|
|
109
|
+
message: `Session is ${status} — delivery allowed immediately.`,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
if (BUSY_DELIVERY_STATUSES.has(status)) {
|
|
114
|
+
if (opts?.allowBusyInjection) {
|
|
115
|
+
return {
|
|
116
|
+
decision: 'immediate',
|
|
117
|
+
reason: `session_${status}_busy_injection_allowed`,
|
|
118
|
+
message: `Session is ${status} but provider supports busy injection. Delivered immediately.`,
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
// approval-kind may be delivered to waiting_approval sessions
|
|
122
|
+
if (status === 'waiting_approval' && opts?.kind === 'approval') {
|
|
123
|
+
return {
|
|
124
|
+
decision: 'immediate',
|
|
125
|
+
reason: 'session_waiting_approval_approval_message',
|
|
126
|
+
message: 'Session is waiting for approval — approval message delivered immediately.',
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
return {
|
|
130
|
+
decision: 'queued',
|
|
131
|
+
reason: `session_${status}_busy`,
|
|
132
|
+
message: `Session is ${status}. Task queued for delivery when session becomes idle. Do not inject directly into a busy session.`,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
if (TERMINAL_DELIVERY_STATUSES.has(status)) {
|
|
137
|
+
return {
|
|
138
|
+
decision: 'rejected',
|
|
139
|
+
reason: `session_${status}_terminal`,
|
|
140
|
+
message: `Session is ${status} (terminal). Delivery rejected. Launch a new session before dispatching tasks.`,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// Unknown/unrecognized status: fail-closed
|
|
145
|
+
return {
|
|
146
|
+
decision: 'rejected',
|
|
147
|
+
reason: 'unrecognized_session_status',
|
|
148
|
+
message: `Session status '${sessionStatus}' is not recognized. Delivery rejected (fail-closed). Inspect session state before retrying.`,
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
export interface SessionDeliveryRecord {
|
|
153
|
+
id: string;
|
|
154
|
+
meshId: string;
|
|
155
|
+
nodeId?: string;
|
|
156
|
+
sessionId?: string;
|
|
157
|
+
providerType?: string;
|
|
158
|
+
taskId?: string;
|
|
159
|
+
kind: MeshSessionDeliveryKind;
|
|
160
|
+
priority: number;
|
|
161
|
+
message: string;
|
|
162
|
+
status: MeshSessionDeliveryStatus;
|
|
163
|
+
deliverAfter?: string;
|
|
164
|
+
expiresAt?: string;
|
|
165
|
+
attemptCount: number;
|
|
166
|
+
sourceCoordinatorSessionId?: string;
|
|
167
|
+
sourceCoordinatorDaemonId?: string;
|
|
168
|
+
lastError?: string;
|
|
169
|
+
createdAt: string;
|
|
170
|
+
updatedAt: string;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Create a delivery record in the store.
|
|
175
|
+
*/
|
|
176
|
+
export function createSessionDelivery(opts: {
|
|
177
|
+
meshId: string;
|
|
178
|
+
nodeId?: string;
|
|
179
|
+
sessionId?: string;
|
|
180
|
+
providerType?: string;
|
|
181
|
+
taskId?: string;
|
|
182
|
+
kind: MeshSessionDeliveryKind;
|
|
183
|
+
message: string;
|
|
184
|
+
status: MeshSessionDeliveryStatus;
|
|
185
|
+
priority?: number;
|
|
186
|
+
deliverAfter?: string;
|
|
187
|
+
expiresAt?: string;
|
|
188
|
+
sourceCoordinatorSessionId?: string;
|
|
189
|
+
sourceCoordinatorDaemonId?: string;
|
|
190
|
+
}): SessionDeliveryRecord {
|
|
191
|
+
const now = new Date().toISOString();
|
|
192
|
+
const id = randomUUID();
|
|
193
|
+
const record: SessionDeliveryRecord = {
|
|
194
|
+
id,
|
|
195
|
+
meshId: opts.meshId,
|
|
196
|
+
nodeId: opts.nodeId,
|
|
197
|
+
sessionId: opts.sessionId,
|
|
198
|
+
providerType: opts.providerType,
|
|
199
|
+
taskId: opts.taskId,
|
|
200
|
+
kind: opts.kind,
|
|
201
|
+
priority: opts.priority ?? 0,
|
|
202
|
+
message: opts.message,
|
|
203
|
+
status: opts.status,
|
|
204
|
+
deliverAfter: opts.deliverAfter,
|
|
205
|
+
expiresAt: opts.expiresAt,
|
|
206
|
+
attemptCount: 0,
|
|
207
|
+
sourceCoordinatorSessionId: opts.sourceCoordinatorSessionId,
|
|
208
|
+
sourceCoordinatorDaemonId: opts.sourceCoordinatorDaemonId,
|
|
209
|
+
createdAt: now,
|
|
210
|
+
updatedAt: now,
|
|
211
|
+
};
|
|
212
|
+
MeshRuntimeStore.getInstance().insertSessionDelivery({
|
|
213
|
+
id,
|
|
214
|
+
meshId: opts.meshId,
|
|
215
|
+
nodeId: opts.nodeId,
|
|
216
|
+
sessionId: opts.sessionId,
|
|
217
|
+
providerType: opts.providerType,
|
|
218
|
+
taskId: opts.taskId,
|
|
219
|
+
kind: opts.kind,
|
|
220
|
+
priority: opts.priority ?? 0,
|
|
221
|
+
message: opts.message,
|
|
222
|
+
status: opts.status,
|
|
223
|
+
deliverAfter: opts.deliverAfter,
|
|
224
|
+
expiresAt: opts.expiresAt,
|
|
225
|
+
sourceCoordinatorSessionId: opts.sourceCoordinatorSessionId,
|
|
226
|
+
sourceCoordinatorDaemonId: opts.sourceCoordinatorDaemonId,
|
|
227
|
+
createdAt: now,
|
|
228
|
+
updatedAt: now,
|
|
229
|
+
});
|
|
230
|
+
return record;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Update the status of a delivery record.
|
|
235
|
+
*/
|
|
236
|
+
export function updateSessionDeliveryStatus(
|
|
237
|
+
id: string,
|
|
238
|
+
status: MeshSessionDeliveryStatus,
|
|
239
|
+
opts?: { lastError?: string; incrementAttempt?: boolean },
|
|
240
|
+
): void {
|
|
241
|
+
try {
|
|
242
|
+
MeshRuntimeStore.getInstance().updateSessionDeliveryStatus(id, status, opts);
|
|
243
|
+
} catch { /* best-effort */ }
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* Get active (non-terminal) deliveries for a mesh, optionally filtered by session.
|
|
248
|
+
*/
|
|
249
|
+
export function getActiveSessionDeliveries(meshId: string, sessionId?: string) {
|
|
250
|
+
try {
|
|
251
|
+
return MeshRuntimeStore.getInstance().getActiveSessionDeliveries(meshId, sessionId);
|
|
252
|
+
} catch {
|
|
253
|
+
return [];
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* Record a completion conflict diagnostic when a duplicate event points to
|
|
259
|
+
* different task/session than the already-seen event with the same fingerprint.
|
|
260
|
+
*/
|
|
261
|
+
export function recordCompletionConflict(opts: {
|
|
262
|
+
meshId: string;
|
|
263
|
+
fingerprint: string;
|
|
264
|
+
conflictingTaskId?: string;
|
|
265
|
+
conflictingSessionId?: string;
|
|
266
|
+
originalTaskId?: string;
|
|
267
|
+
originalSessionId?: string;
|
|
268
|
+
event: string;
|
|
269
|
+
}): void {
|
|
270
|
+
try {
|
|
271
|
+
MeshRuntimeStore.getInstance().recordCompletionConflict({
|
|
272
|
+
id: randomUUID(),
|
|
273
|
+
meshId: opts.meshId,
|
|
274
|
+
fingerprint: opts.fingerprint,
|
|
275
|
+
conflictingTaskId: opts.conflictingTaskId,
|
|
276
|
+
conflictingSessionId: opts.conflictingSessionId,
|
|
277
|
+
originalTaskId: opts.originalTaskId,
|
|
278
|
+
originalSessionId: opts.originalSessionId,
|
|
279
|
+
event: opts.event,
|
|
280
|
+
createdAt: new Date().toISOString(),
|
|
281
|
+
});
|
|
282
|
+
} catch { /* best-effort diagnostics */ }
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/**
|
|
286
|
+
* Get recent completion conflicts for diagnostic inspection.
|
|
287
|
+
*/
|
|
288
|
+
export function getRecentCompletionConflicts(meshId: string, limitMs?: number) {
|
|
289
|
+
try {
|
|
290
|
+
return MeshRuntimeStore.getInstance().getRecentCompletionConflicts(meshId, limitMs);
|
|
291
|
+
} catch {
|
|
292
|
+
return [];
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
export function __clearSessionDeliveriesForTests(meshId: string): void {
|
|
297
|
+
MeshRuntimeStore.getInstance().deleteSessionDeliveries(meshId);
|
|
298
|
+
}
|
package/src/mesh/mesh-events.ts
CHANGED
|
@@ -8,8 +8,9 @@ import { LOG } from '../logging/logger.js';
|
|
|
8
8
|
import { appendLedgerEntry, buildTaskCompletionEvidence, getLedgerDir, getSessionRecoveryContext, isIntentionalCleanupStopEntry, readLedgerEntries } from './mesh-ledger.js';
|
|
9
9
|
import type { MeshLedgerKind, SessionRecoveryContext } from './mesh-ledger.js';
|
|
10
10
|
import { buildMeshNodeCapabilityTags, claimNextTask, updateSessionTaskStatus, enqueueTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, getActiveDirectDispatches } from './mesh-work-queue.js';
|
|
11
|
-
import {
|
|
11
|
+
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
12
12
|
import { fastForwardMeshNode } from './mesh-fast-forward.js';
|
|
13
|
+
import { createSessionDelivery, updateSessionDeliveryStatus, recordCompletionConflict } from './mesh-delivery-policy.js';
|
|
13
14
|
|
|
14
15
|
// ---------------------------------------------------------------------------
|
|
15
16
|
// Remote Node Idle Session Tracking
|
|
@@ -49,7 +50,7 @@ export function __resetIdleAutoFastForwardForTests(): void {
|
|
|
49
50
|
|
|
50
51
|
function sweepExpiredRemoteIdleSessions(): void {
|
|
51
52
|
try {
|
|
52
|
-
|
|
53
|
+
MeshRuntimeStore.getInstance().pruneExpiredRemoteIdleSessions();
|
|
53
54
|
} catch { /* best-effort */ }
|
|
54
55
|
}
|
|
55
56
|
|
|
@@ -438,7 +439,7 @@ const RECENT_COMPLETION_FINGERPRINT_TTL_MS = 10 * 60 * 1000;
|
|
|
438
439
|
|
|
439
440
|
function hasFingerprintSeen(fingerprint: string): boolean {
|
|
440
441
|
try {
|
|
441
|
-
return
|
|
442
|
+
return MeshRuntimeStore.getInstance().hasCompletionFingerprint(fingerprint);
|
|
442
443
|
} catch {
|
|
443
444
|
return false;
|
|
444
445
|
}
|
|
@@ -446,7 +447,7 @@ function hasFingerprintSeen(fingerprint: string): boolean {
|
|
|
446
447
|
|
|
447
448
|
function recordFingerprintSeen(fingerprint: string): void {
|
|
448
449
|
try {
|
|
449
|
-
const db =
|
|
450
|
+
const db = MeshRuntimeStore.getInstance();
|
|
450
451
|
db.recordCompletionFingerprint(fingerprint, RECENT_COMPLETION_FINGERPRINT_TTL_MS);
|
|
451
452
|
db.sweepExpiredFingerprints();
|
|
452
453
|
} catch { /* best-effort; duplicate events are preferable to a crash */ }
|
|
@@ -499,10 +500,27 @@ function isDuplicateMeshCompletionEvent(args: {
|
|
|
499
500
|
timestamp?: number | null;
|
|
500
501
|
finalSummary?: string;
|
|
501
502
|
coordinatorDaemonId?: string;
|
|
503
|
+
taskId?: string;
|
|
504
|
+
nodeId?: string;
|
|
502
505
|
}): boolean {
|
|
503
506
|
const fingerprint = buildMeshCompletionFingerprint(args);
|
|
504
507
|
if (!fingerprint) return false;
|
|
505
|
-
if (hasFingerprintSeen(fingerprint))
|
|
508
|
+
if (hasFingerprintSeen(fingerprint)) {
|
|
509
|
+
// Suppressed duplicate — but if we have a taskId and it differs from what the
|
|
510
|
+
// fingerprint was stamped for, record a conflict diagnostic so it doesn't disappear silently.
|
|
511
|
+
// (We can't recover the original taskId from the fingerprint alone, so we record
|
|
512
|
+
// the conflicting taskId/session as a diagnostic for coordinator inspection.)
|
|
513
|
+
if (args.taskId) {
|
|
514
|
+
recordCompletionConflict({
|
|
515
|
+
meshId: args.meshId,
|
|
516
|
+
fingerprint,
|
|
517
|
+
conflictingTaskId: args.taskId,
|
|
518
|
+
conflictingSessionId: args.sessionId,
|
|
519
|
+
event: args.event,
|
|
520
|
+
});
|
|
521
|
+
}
|
|
522
|
+
return true;
|
|
523
|
+
}
|
|
506
524
|
recordFingerprintSeen(fingerprint);
|
|
507
525
|
return false;
|
|
508
526
|
}
|
|
@@ -550,7 +568,7 @@ function findRecentTerminalLedgerEvidence(args: {
|
|
|
550
568
|
if (!args.sessionId && !args.nodeId) return null;
|
|
551
569
|
// Tail-limit: 200 entries gives a wide enough window to catch terminal events for active
|
|
552
570
|
// sessions while avoiding a full O(n) scan. If a terminal is older than 200 entries,
|
|
553
|
-
// the
|
|
571
|
+
// the MeshRuntimeStore fingerprint dedup will still block duplicate processing downstream.
|
|
554
572
|
const entries = readLedgerEntries(args.meshId, { tail: 200 });
|
|
555
573
|
for (let i = entries.length - 1; i >= 0; i--) {
|
|
556
574
|
const entry = entries[i];
|
|
@@ -837,13 +855,27 @@ export function tryAssignQueueTask(
|
|
|
837
855
|
if (node?.daemonId && components.dispatchMeshCommand) {
|
|
838
856
|
const isLocalNode = components.cliManager.adapters.has(sessionId);
|
|
839
857
|
if (!isLocalNode) {
|
|
858
|
+
// Create delivery record before attempting P2P send
|
|
859
|
+
const delivery = createSessionDelivery({
|
|
860
|
+
meshId,
|
|
861
|
+
nodeId,
|
|
862
|
+
sessionId,
|
|
863
|
+
providerType,
|
|
864
|
+
taskId: task.id,
|
|
865
|
+
kind: 'task',
|
|
866
|
+
message: task.message,
|
|
867
|
+
status: 'delivering',
|
|
868
|
+
});
|
|
840
869
|
components.dispatchMeshCommand(node.daemonId, 'agent_command', {
|
|
841
870
|
targetSessionId: sessionId,
|
|
842
871
|
cliType: providerType,
|
|
843
872
|
action: 'send_chat',
|
|
844
873
|
message: task.message,
|
|
874
|
+
}).then(() => {
|
|
875
|
+
updateSessionDeliveryStatus(delivery.id, 'delivered');
|
|
845
876
|
}).catch((e: any) => {
|
|
846
877
|
LOG.error('MeshQueue', `Failed to dispatch task via P2P to remote node ${nodeId}: ${e?.message}`);
|
|
878
|
+
updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
|
|
847
879
|
// Revert to pending so the task can be retried rather than permanently failing
|
|
848
880
|
updateTaskStatus(meshId, task.id, 'pending');
|
|
849
881
|
try {
|
|
@@ -851,7 +883,7 @@ export function tryAssignQueueTask(
|
|
|
851
883
|
kind: 'dispatch_failed' as any,
|
|
852
884
|
nodeId,
|
|
853
885
|
sessionId,
|
|
854
|
-
payload: { taskId: task.id, error: e?.message, retryable: true },
|
|
886
|
+
payload: { taskId: task.id, deliveryId: delivery.id, error: e?.message, retryable: true },
|
|
855
887
|
});
|
|
856
888
|
} catch { /* ledger write is best-effort */ }
|
|
857
889
|
});
|
|
@@ -859,14 +891,27 @@ export function tryAssignQueueTask(
|
|
|
859
891
|
}
|
|
860
892
|
}
|
|
861
893
|
|
|
862
|
-
// Local routing
|
|
894
|
+
// Local routing — create delivery record before send_chat
|
|
895
|
+
const delivery = createSessionDelivery({
|
|
896
|
+
meshId,
|
|
897
|
+
nodeId,
|
|
898
|
+
sessionId,
|
|
899
|
+
providerType,
|
|
900
|
+
taskId: task.id,
|
|
901
|
+
kind: 'task',
|
|
902
|
+
message: task.message,
|
|
903
|
+
status: 'delivering',
|
|
904
|
+
});
|
|
863
905
|
components.cliManager.handleCliCommand('agent_command', {
|
|
864
906
|
targetSessionId: sessionId,
|
|
865
907
|
cliType: providerType,
|
|
866
908
|
action: 'send_chat',
|
|
867
909
|
message: task.message,
|
|
910
|
+
}).then(() => {
|
|
911
|
+
updateSessionDeliveryStatus(delivery.id, 'delivered');
|
|
868
912
|
}).catch((e: any) => {
|
|
869
913
|
LOG.error('MeshQueue', `Failed to dispatch task locally to node ${nodeId}: ${e?.message}`);
|
|
914
|
+
updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
|
|
870
915
|
updateTaskStatus(meshId, task.id, 'failed');
|
|
871
916
|
});
|
|
872
917
|
|
|
@@ -1280,7 +1325,7 @@ export async function triggerMeshQueue(components: DaemonComponents, meshId: str
|
|
|
1280
1325
|
// Also check known idle remote sessions
|
|
1281
1326
|
let remoteSessions: Array<{ nodeId: string; sessionId: string; providerType: string }> = [];
|
|
1282
1327
|
try {
|
|
1283
|
-
remoteSessions =
|
|
1328
|
+
remoteSessions = MeshRuntimeStore.getInstance().getRemoteIdleSessions();
|
|
1284
1329
|
} catch { /* best-effort */ }
|
|
1285
1330
|
|
|
1286
1331
|
for (const idle of remoteSessions) {
|
|
@@ -1291,7 +1336,7 @@ export async function triggerMeshQueue(components: DaemonComponents, meshId: str
|
|
|
1291
1336
|
const assigned = tryAssignQueueTask(components, meshId, idle.nodeId, idle.sessionId, idle.providerType);
|
|
1292
1337
|
if (assigned) {
|
|
1293
1338
|
try {
|
|
1294
|
-
|
|
1339
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(idle.nodeId, idle.sessionId);
|
|
1295
1340
|
} catch { /* best-effort */ }
|
|
1296
1341
|
}
|
|
1297
1342
|
}
|
|
@@ -1534,7 +1579,7 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
1534
1579
|
if (intentionalCleanupStop) {
|
|
1535
1580
|
if (eventSessionId && eventNodeId) {
|
|
1536
1581
|
try {
|
|
1537
|
-
|
|
1582
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(eventNodeId, eventSessionId);
|
|
1538
1583
|
} catch { /* best-effort */ }
|
|
1539
1584
|
}
|
|
1540
1585
|
LOG.info('MeshEvents', `Suppressed ${args.event} for intentionally cleanup-stopped session ${eventSessionId || '(unknown session)'}`);
|
|
@@ -1625,6 +1670,8 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
1625
1670
|
// Scope dedup to the coordinator daemon so two coordinators for the same mesh
|
|
1626
1671
|
// don't suppress each other's completion events via shared fingerprint table.
|
|
1627
1672
|
coordinatorDaemonId: workerCoordinatorDaemonId || undefined,
|
|
1673
|
+
taskId: readNonEmptyString(args.metadataEvent.taskId) || undefined,
|
|
1674
|
+
nodeId: eventNodeId || undefined,
|
|
1628
1675
|
});
|
|
1629
1676
|
if (duplicateCompletion) {
|
|
1630
1677
|
LOG.info('MeshEvents', `Suppressed duplicate completion for mesh ${args.meshId} session ${eventSessionId}`);
|
|
@@ -1641,6 +1688,8 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
1641
1688
|
timestamp: eventTimestamp,
|
|
1642
1689
|
finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
|
|
1643
1690
|
coordinatorDaemonId: workerCoordinatorDaemonId || undefined,
|
|
1691
|
+
taskId: readNonEmptyString(args.metadataEvent.taskId) || undefined,
|
|
1692
|
+
nodeId: eventNodeId || undefined,
|
|
1644
1693
|
});
|
|
1645
1694
|
if (duplicateStopped) {
|
|
1646
1695
|
LOG.info('MeshEvents', `Suppressed duplicate stopped event for mesh ${args.meshId} session ${eventSessionId}`);
|
|
@@ -1718,14 +1767,14 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
1718
1767
|
if (sessionId && nodeId && providerType) {
|
|
1719
1768
|
sweepExpiredRemoteIdleSessions();
|
|
1720
1769
|
try {
|
|
1721
|
-
|
|
1770
|
+
MeshRuntimeStore.getInstance().setRemoteIdleSession(nodeId, sessionId, providerType, Date.now() + REMOTE_IDLE_SESSION_TTL_MS);
|
|
1722
1771
|
} catch { /* best-effort */ }
|
|
1723
1772
|
setImmediate(() => {
|
|
1724
1773
|
maybeAutoFastForwardIdleNode(components, { meshId: args.meshId, nodeId, sessionId, providerType })
|
|
1725
1774
|
.finally(() => {
|
|
1726
1775
|
try {
|
|
1727
1776
|
const assigned = tryAssignQueueTask(components, args.meshId, nodeId, sessionId, providerType);
|
|
1728
|
-
if (assigned)
|
|
1777
|
+
if (assigned) MeshRuntimeStore.getInstance().deleteRemoteIdleSession(nodeId, sessionId);
|
|
1729
1778
|
} catch (e: any) {
|
|
1730
1779
|
LOG.warn('MeshQueue', `Failed to assign idle queue task after maintenance for ${nodeId}: ${e?.message || e}`);
|
|
1731
1780
|
}
|
|
@@ -1737,7 +1786,7 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
1737
1786
|
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
1738
1787
|
if (sessionId && nodeId) {
|
|
1739
1788
|
try {
|
|
1740
|
-
|
|
1789
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(nodeId, sessionId);
|
|
1741
1790
|
} catch { /* best-effort */ }
|
|
1742
1791
|
}
|
|
1743
1792
|
if (sessionId) {
|
|
@@ -1748,7 +1797,7 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
1748
1797
|
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
1749
1798
|
if (sessionId && nodeId) {
|
|
1750
1799
|
try {
|
|
1751
|
-
|
|
1800
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(nodeId, sessionId);
|
|
1752
1801
|
} catch { /* best-effort */ }
|
|
1753
1802
|
}
|
|
1754
1803
|
if (sessionId) {
|