@adhdev/daemon-core 0.9.82-rc.274 → 0.9.82-rc.276
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/boot/daemon-lifecycle.d.ts +1 -0
- package/dist/index.js +148 -25
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +148 -25
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-events-coordinator.d.ts +1 -0
- package/dist/mesh/mesh-events-pending.d.ts +15 -3
- package/dist/mesh/mesh-runtime-store.d.ts +14 -2
- package/package.json +2 -2
- package/src/boot/daemon-lifecycle.ts +9 -0
- package/src/mesh/mesh-events-coordinator.ts +21 -3
- package/src/mesh/mesh-events-pending.ts +128 -17
- package/src/mesh/mesh-reconcile-loop.ts +56 -14
- package/src/mesh/mesh-runtime-store.ts +50 -12
|
@@ -28,6 +28,7 @@ export interface MeshQueueTriggerResult {
|
|
|
28
28
|
}
|
|
29
29
|
export declare function triggerMeshQueue(components: DaemonComponents, meshId: string): Promise<MeshQueueTriggerResult>;
|
|
30
30
|
export declare function isMeshCoordinatorEvent(eventName: unknown): eventName is string;
|
|
31
|
+
export declare const MESH_FORCE_INJECT_EVENTS: ReadonlySet<string>;
|
|
31
32
|
export declare function shouldForceInjectMeshEvent(eventName: unknown): boolean;
|
|
32
33
|
export declare function handleMeshForwardEvent(components: DaemonComponents, payload: Record<string, unknown>): {
|
|
33
34
|
success: boolean;
|
|
@@ -20,10 +20,22 @@ export declare function readRefineJobId(event: {
|
|
|
20
20
|
export declare function buildPendingEventFingerprint(event: PendingMeshCoordinatorEvent): string;
|
|
21
21
|
export declare function hasPendingCoordinatorEventDuplicate(event: PendingMeshCoordinatorEvent): boolean;
|
|
22
22
|
export declare function queuePendingMeshCoordinatorEvent(event: PendingMeshCoordinatorEvent): boolean;
|
|
23
|
-
/**
|
|
24
|
-
|
|
23
|
+
/**
|
|
24
|
+
* Drain and return pending coordinator events for meshId, removing the drained
|
|
25
|
+
* ones from both the SQLite inbox and the JSONL legacy file.
|
|
26
|
+
*
|
|
27
|
+
* When `opts.onlyEvents` is supplied, ONLY events whose name is in that set are
|
|
28
|
+
* drained; every other event stays queued (undrained in SQLite, rewritten back to
|
|
29
|
+
* the JSONL file). The reconcile loop uses this to force-drain terminal/force-inject
|
|
30
|
+
* events into a *generating* coordinator while leaving non-force progress events for
|
|
31
|
+
* the coordinator's next idle transition. The atomic SQLite drained=1 marking and the
|
|
32
|
+
* atomic JSONL rename keep force-drain and a concurrent full drain from double-consuming.
|
|
33
|
+
*/
|
|
34
|
+
export declare function drainPendingMeshCoordinatorEvents(meshId?: string, coordinatorDaemonId?: string | ReadonlyArray<string>, opts?: {
|
|
35
|
+
onlyEvents?: ReadonlySet<string>;
|
|
36
|
+
}): PendingMeshCoordinatorEvent[];
|
|
25
37
|
/** Peek at pending coordinator events without draining (non-destructive). */
|
|
26
|
-
export declare function getPendingMeshCoordinatorEvents(meshId?: string, coordinatorDaemonId?: string): readonly PendingMeshCoordinatorEvent[];
|
|
38
|
+
export declare function getPendingMeshCoordinatorEvents(meshId?: string, coordinatorDaemonId?: string | ReadonlyArray<string>): readonly PendingMeshCoordinatorEvent[];
|
|
27
39
|
/**
|
|
28
40
|
* Test helper: purge all pending-event state for a mesh — SQLite rows
|
|
29
41
|
* (including drained fingerprint history) and JSONL files.
|
|
@@ -288,13 +288,25 @@ export declare class MeshRuntimeStore {
|
|
|
288
288
|
fingerprint?: string | null;
|
|
289
289
|
queuedAt: number;
|
|
290
290
|
}): boolean;
|
|
291
|
-
|
|
291
|
+
/**
|
|
292
|
+
* Drain undrained pending events for a mesh, atomically marking them drained.
|
|
293
|
+
* When `opts.onlyEvents` is supplied, ONLY rows whose `event` is in that set are
|
|
294
|
+
* drained — the rest stay queued (drained=0) for a later drain. This is how the
|
|
295
|
+
* reconcile loop force-drains terminal/force-inject events into a *generating*
|
|
296
|
+
* coordinator while leaving non-force progress events for the coordinator's next
|
|
297
|
+
* idle transition. Filtering happens inside the same transaction as the
|
|
298
|
+
* drained=1 marking, so force-drain + a concurrent full drain can never both
|
|
299
|
+
* consume the same row.
|
|
300
|
+
*/
|
|
301
|
+
drainPendingEvents(meshId: string, coordinatorDaemonId?: string | null | ReadonlyArray<string>, opts?: {
|
|
302
|
+
onlyEvents?: ReadonlySet<string>;
|
|
303
|
+
}): Array<{
|
|
292
304
|
id: string;
|
|
293
305
|
event: string;
|
|
294
306
|
payload: unknown;
|
|
295
307
|
}>;
|
|
296
308
|
/** Non-destructive peek — returns undrained events without marking them drained. */
|
|
297
|
-
peekPendingEvents(meshId: string, coordinatorDaemonId?: string | null): Array<{
|
|
309
|
+
peekPendingEvents(meshId: string, coordinatorDaemonId?: string | null | ReadonlyArray<string>): Array<{
|
|
298
310
|
id: string;
|
|
299
311
|
event: string;
|
|
300
312
|
payload: unknown;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adhdev/daemon-core",
|
|
3
|
-
"version": "0.9.82-rc.
|
|
3
|
+
"version": "0.9.82-rc.276",
|
|
4
4
|
"description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
"author": "vilmire",
|
|
47
47
|
"license": "AGPL-3.0-or-later",
|
|
48
48
|
"dependencies": {
|
|
49
|
-
"@adhdev/mesh-shared": "0.9.82-rc.
|
|
49
|
+
"@adhdev/mesh-shared": "0.9.82-rc.276",
|
|
50
50
|
"@adhdev/session-host-core": "*",
|
|
51
51
|
"@agentclientprotocol/sdk": "^0.16.1",
|
|
52
52
|
"ajv": "^8.20.0",
|
|
@@ -123,6 +123,14 @@ export interface DaemonComponents {
|
|
|
123
123
|
// single-model (queue + polling) delivery: drains the pending-events queue
|
|
124
124
|
// on a fixed interval and injects into live CLI coordinators when idle.
|
|
125
125
|
meshReconcileLoop?: { stop(): void };
|
|
126
|
+
// Canonical status/daemon identity (e.g. `standalone_<machineId>` /
|
|
127
|
+
// `daemon_<machineId>`). This is the SAME id the MCP layer stamps as a
|
|
128
|
+
// worker's meshCoordinatorDaemonId (ctx.localDaemonId, sourced from
|
|
129
|
+
// getStatus().status.instanceId), so the reconcile loop MUST drain with it —
|
|
130
|
+
// draining with bare loadConfig().machineId never matches a unicast event
|
|
131
|
+
// stamped with the prefixed status id. Absent → reconcile falls back to
|
|
132
|
+
// machineId only.
|
|
133
|
+
statusInstanceId?: string;
|
|
126
134
|
}
|
|
127
135
|
|
|
128
136
|
export interface DaemonDevSupportOptions {
|
|
@@ -362,6 +370,7 @@ export async function initDaemonComponents(config: DaemonInitConfig): Promise<Da
|
|
|
362
370
|
refreshProviderAvailability,
|
|
363
371
|
dispatchMeshCommand: config.dispatchMeshCommand,
|
|
364
372
|
onMeshCoordinatorEventForwarded: config.onMeshCoordinatorEventForwarded,
|
|
373
|
+
statusInstanceId: config.statusInstanceId,
|
|
365
374
|
};
|
|
366
375
|
|
|
367
376
|
// 11. Setup Mesh Event Forwarding (queue persistence) + periodic reconcile loop.
|
|
@@ -29,6 +29,19 @@ import {
|
|
|
29
29
|
readWorkerResultMetadata,
|
|
30
30
|
} from './mesh-events-utils.js';
|
|
31
31
|
|
|
32
|
+
// The set of coordinator-daemon ids this daemon answers to when draining the
|
|
33
|
+
// pending-events queue (canonical status id + bare machineId). Mirrors
|
|
34
|
+
// resolveCoordinatorDaemonIds in mesh-reconcile-loop — a unicast event may be
|
|
35
|
+
// stamped with either id depending on which dispatch path created the worker.
|
|
36
|
+
function resolveCoordinatorDrainDaemonIds(components: DaemonComponents): string[] {
|
|
37
|
+
const ids = new Set<string>();
|
|
38
|
+
const statusInstanceId = readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId);
|
|
39
|
+
if (statusInstanceId) ids.add(statusInstanceId);
|
|
40
|
+
const machineId = readNonEmptyString(loadConfig().machineId);
|
|
41
|
+
if (machineId) ids.add(machineId);
|
|
42
|
+
return [...ids];
|
|
43
|
+
}
|
|
44
|
+
|
|
32
45
|
// ---------------------------------------------------------------------------
|
|
33
46
|
// Remote Node Idle Session Tracking
|
|
34
47
|
// ---------------------------------------------------------------------------
|
|
@@ -975,7 +988,7 @@ export function isMeshCoordinatorEvent(eventName: unknown): eventName is string
|
|
|
975
988
|
// only flushed on the coordinator's OWN idle transition. That transition can't
|
|
976
989
|
// happen until it receives this very event → deadlock. We force-inject these so
|
|
977
990
|
// they bypass the busy send-guard and land in the PTY while generating.
|
|
978
|
-
const MESH_FORCE_INJECT_EVENTS = new Set([
|
|
991
|
+
export const MESH_FORCE_INJECT_EVENTS: ReadonlySet<string> = new Set([
|
|
979
992
|
'agent:generating_completed',
|
|
980
993
|
'agent:stopped',
|
|
981
994
|
'agent:waiting_approval',
|
|
@@ -1506,8 +1519,13 @@ export function setupMeshEventForwarding(components: DaemonComponents) {
|
|
|
1506
1519
|
const status = readNonEmptyString(flushState.status).toLowerCase();
|
|
1507
1520
|
if (status === 'idle') {
|
|
1508
1521
|
try {
|
|
1509
|
-
|
|
1510
|
-
|
|
1522
|
+
// Drain with the daemon's full coordinator-id set (status id + machineId).
|
|
1523
|
+
// The MCP layer stamps the prefixed status id (`standalone_<machineId>` /
|
|
1524
|
+
// `daemon_<machineId>`) as the worker's meshCoordinatorDaemonId; draining
|
|
1525
|
+
// with bare machineId alone would miss those unicast events. Mirrors
|
|
1526
|
+
// resolveCoordinatorDaemonIds in mesh-reconcile-loop.
|
|
1527
|
+
const drainDaemonIds = resolveCoordinatorDrainDaemonIds(components);
|
|
1528
|
+
const pendingEvents = drainPendingMeshCoordinatorEvents(coordinatorMeshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
|
|
1511
1529
|
if (pendingEvents.length > 0) {
|
|
1512
1530
|
LOG.info('MeshEvents', `Auto-flushing ${pendingEvents.length} pending coordinator event(s) for mesh ${coordinatorMeshId} on coordinator idle`);
|
|
1513
1531
|
for (const pending of pendingEvents) {
|
|
@@ -37,6 +37,27 @@ export interface PendingMeshCoordinatorEvent {
|
|
|
37
37
|
|
|
38
38
|
const REFINE_TERMINAL_EVENTS = new Set(['refine:completed', 'refine:failed']);
|
|
39
39
|
|
|
40
|
+
/** Normalise a coordinator-daemon-id argument (single id, list, or undefined) into a
|
|
41
|
+
* de-duplicated list of non-empty strings. The first entry is treated as primary for
|
|
42
|
+
* per-daemon JSONL file naming; all entries are accepted by drain/peek targeting. */
|
|
43
|
+
function normalizeCoordinatorDaemonIds(
|
|
44
|
+
coordinatorDaemonId?: string | null | ReadonlyArray<string>,
|
|
45
|
+
): string[] {
|
|
46
|
+
const raw = Array.isArray(coordinatorDaemonId)
|
|
47
|
+
? coordinatorDaemonId
|
|
48
|
+
: coordinatorDaemonId != null ? [coordinatorDaemonId] : [];
|
|
49
|
+
const seen = new Set<string>();
|
|
50
|
+
const out: string[] = [];
|
|
51
|
+
for (const id of raw) {
|
|
52
|
+
if (typeof id !== 'string') continue;
|
|
53
|
+
const trimmed = id.trim();
|
|
54
|
+
if (!trimmed || seen.has(trimmed)) continue;
|
|
55
|
+
seen.add(trimmed);
|
|
56
|
+
out.push(trimmed);
|
|
57
|
+
}
|
|
58
|
+
return out;
|
|
59
|
+
}
|
|
60
|
+
|
|
40
61
|
export function readRefineJobId(event: { metadataEvent?: Record<string, unknown> } | Record<string, unknown>): string {
|
|
41
62
|
const metadata = readRecord((event as any).metadataEvent) || event as Record<string, unknown>;
|
|
42
63
|
const result = readRecord(metadata.result);
|
|
@@ -108,11 +129,13 @@ function getPendingEventsPath(meshId: string, coordinatorDaemonId?: string): str
|
|
|
108
129
|
return join(getLedgerDir(), `${safe}.pending-events.jsonl`);
|
|
109
130
|
}
|
|
110
131
|
|
|
111
|
-
function readPendingMeshCoordinatorEventsFromDisk(meshId?: string, coordinatorDaemonId?: string): PendingMeshCoordinatorEvent[] {
|
|
132
|
+
function readPendingMeshCoordinatorEventsFromDisk(meshId?: string, coordinatorDaemonId?: string | ReadonlyArray<string>): PendingMeshCoordinatorEvent[] {
|
|
112
133
|
if (!meshId) return [];
|
|
134
|
+
const daemonIds = normalizeCoordinatorDaemonIds(coordinatorDaemonId);
|
|
135
|
+
const primaryDaemonId = daemonIds[0];
|
|
113
136
|
// Read coordinator-scoped file first; fall back to legacy shared file.
|
|
114
|
-
const paths =
|
|
115
|
-
? [getPendingEventsPath(meshId,
|
|
137
|
+
const paths = primaryDaemonId
|
|
138
|
+
? [getPendingEventsPath(meshId, primaryDaemonId), getPendingEventsPath(meshId)]
|
|
116
139
|
: [getPendingEventsPath(meshId)];
|
|
117
140
|
const events: PendingMeshCoordinatorEvent[] = [];
|
|
118
141
|
for (const path of paths) {
|
|
@@ -123,8 +146,8 @@ function readPendingMeshCoordinatorEventsFromDisk(meshId?: string, coordinatorDa
|
|
|
123
146
|
try { return [JSON.parse(line) as PendingMeshCoordinatorEvent]; } catch { return []; }
|
|
124
147
|
});
|
|
125
148
|
// If reading the shared file, filter to events that target this coordinator or are unscoped.
|
|
126
|
-
const filtered = (
|
|
127
|
-
? parsed.filter(e => !e.targetCoordinatorDaemonId || e.targetCoordinatorDaemonId
|
|
149
|
+
const filtered = (primaryDaemonId && path === getPendingEventsPath(meshId))
|
|
150
|
+
? parsed.filter(e => !e.targetCoordinatorDaemonId || daemonIds.includes(e.targetCoordinatorDaemonId))
|
|
128
151
|
: parsed;
|
|
129
152
|
events.push(...filtered);
|
|
130
153
|
} catch { /* skip unreadable files */ }
|
|
@@ -279,10 +302,84 @@ function atomicDrainFile(path: string): string | null {
|
|
|
279
302
|
}
|
|
280
303
|
}
|
|
281
304
|
|
|
282
|
-
|
|
283
|
-
|
|
305
|
+
// Selectively drain a JSONL pending-events file: atomically claim it (rename), then
|
|
306
|
+
// consume only the lines whose parsed event matches `predicate` and rewrite the
|
|
307
|
+
// remaining (kept) lines back to the original path. Unparseable lines are kept
|
|
308
|
+
// untouched. Returns the consumed events. The rename makes claiming exclusive —
|
|
309
|
+
// only one concurrent caller wins, so there is no double-consume of the same lines.
|
|
310
|
+
function selectiveDrainFile(
|
|
311
|
+
path: string,
|
|
312
|
+
predicate: (event: PendingMeshCoordinatorEvent) => boolean,
|
|
313
|
+
): PendingMeshCoordinatorEvent[] {
|
|
314
|
+
const tmpPath = `${path}.draining`;
|
|
315
|
+
try {
|
|
316
|
+
renameSync(path, tmpPath);
|
|
317
|
+
} catch {
|
|
318
|
+
return []; // another drain claimed it, or the file doesn't exist
|
|
319
|
+
}
|
|
320
|
+
let content: string;
|
|
321
|
+
try {
|
|
322
|
+
content = readFileSync(tmpPath, 'utf-8');
|
|
323
|
+
} catch {
|
|
324
|
+
try { unlinkSync(tmpPath); } catch { /* best-effort */ }
|
|
325
|
+
return [];
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
const consumed: PendingMeshCoordinatorEvent[] = [];
|
|
329
|
+
const keptLines: string[] = [];
|
|
330
|
+
for (const line of content.split('\n')) {
|
|
331
|
+
if (!line) continue;
|
|
332
|
+
let parsed: PendingMeshCoordinatorEvent | undefined;
|
|
333
|
+
try { parsed = JSON.parse(line) as PendingMeshCoordinatorEvent; } catch { parsed = undefined; }
|
|
334
|
+
if (parsed && predicate(parsed)) {
|
|
335
|
+
consumed.push(parsed);
|
|
336
|
+
} else {
|
|
337
|
+
keptLines.push(line); // non-matching or unparseable → leave queued
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
try {
|
|
342
|
+
if (keptLines.length > 0) {
|
|
343
|
+
writeFileSync(path, keptLines.join('\n') + '\n', 'utf-8');
|
|
344
|
+
}
|
|
345
|
+
unlinkSync(tmpPath);
|
|
346
|
+
} catch {
|
|
347
|
+
// If the rewrite/cleanup fails, restore the claimed file so no events are
|
|
348
|
+
// lost — the next drain retries the whole file.
|
|
349
|
+
try { if (existsSync(tmpPath) && !existsSync(path)) renameSync(tmpPath, path); } catch { /* best-effort */ }
|
|
350
|
+
return [];
|
|
351
|
+
}
|
|
352
|
+
return consumed;
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
/**
|
|
356
|
+
* Drain and return pending coordinator events for meshId, removing the drained
|
|
357
|
+
* ones from both the SQLite inbox and the JSONL legacy file.
|
|
358
|
+
*
|
|
359
|
+
* When `opts.onlyEvents` is supplied, ONLY events whose name is in that set are
|
|
360
|
+
* drained; every other event stays queued (undrained in SQLite, rewritten back to
|
|
361
|
+
* the JSONL file). The reconcile loop uses this to force-drain terminal/force-inject
|
|
362
|
+
* events into a *generating* coordinator while leaving non-force progress events for
|
|
363
|
+
* the coordinator's next idle transition. The atomic SQLite drained=1 marking and the
|
|
364
|
+
* atomic JSONL rename keep force-drain and a concurrent full drain from double-consuming.
|
|
365
|
+
*/
|
|
366
|
+
export function drainPendingMeshCoordinatorEvents(
|
|
367
|
+
meshId?: string,
|
|
368
|
+
coordinatorDaemonId?: string | ReadonlyArray<string>,
|
|
369
|
+
opts?: { onlyEvents?: ReadonlySet<string> },
|
|
370
|
+
): PendingMeshCoordinatorEvent[] {
|
|
284
371
|
if (!meshId) return [];
|
|
285
372
|
|
|
373
|
+
// A daemon may answer to more than one coordinator-id form (its canonical
|
|
374
|
+
// status id like `standalone_<machineId>` AND the bare machineId). Normalise
|
|
375
|
+
// to a list so both the SQLite IN-filter and the JSONL targeting predicate
|
|
376
|
+
// accept any of them.
|
|
377
|
+
const daemonIds = normalizeCoordinatorDaemonIds(coordinatorDaemonId);
|
|
378
|
+
const primaryDaemonId = daemonIds[0];
|
|
379
|
+
|
|
380
|
+
const onlyEvents = opts?.onlyEvents;
|
|
381
|
+
const matchesFilter = (eventName: string): boolean => !onlyEvents || onlyEvents.has(eventName);
|
|
382
|
+
|
|
286
383
|
// Dual-write means SQLite and JSONL hold the same events. Both stores must be
|
|
287
384
|
// emptied in one drain call — draining only one leaves the other to re-deliver
|
|
288
385
|
// the same events on the next call. Merge with fingerprint dedup.
|
|
@@ -301,7 +398,7 @@ export function drainPendingMeshCoordinatorEvents(meshId?: string, coordinatorDa
|
|
|
301
398
|
try {
|
|
302
399
|
const store = MeshRuntimeStore.getInstance();
|
|
303
400
|
if (store.pendingEventCount(meshId) > 0) {
|
|
304
|
-
for (const row of store.drainPendingEvents(meshId,
|
|
401
|
+
for (const row of store.drainPendingEvents(meshId, daemonIds.length > 0 ? daemonIds : undefined, onlyEvents ? { onlyEvents } : undefined)) {
|
|
305
402
|
const event = row.payload as PendingMeshCoordinatorEvent;
|
|
306
403
|
if (event) pushUnique(event);
|
|
307
404
|
}
|
|
@@ -310,20 +407,33 @@ export function drainPendingMeshCoordinatorEvents(meshId?: string, coordinatorDa
|
|
|
310
407
|
// SQLite drain failed — JSONL below still drains
|
|
311
408
|
}
|
|
312
409
|
|
|
313
|
-
// JSONL (legacy / migration path) — always drained alongside SQLite
|
|
314
|
-
|
|
315
|
-
|
|
410
|
+
// JSONL (legacy / migration path) — always drained alongside SQLite.
|
|
411
|
+
// The scoped per-daemon file is keyed by a single id; use the primary. The
|
|
412
|
+
// shared (unscoped) file's targeting predicate accepts ANY of this daemon's ids.
|
|
413
|
+
const paths = primaryDaemonId
|
|
414
|
+
? [getPendingEventsPath(meshId, primaryDaemonId), getPendingEventsPath(meshId)]
|
|
316
415
|
: [getPendingEventsPath(meshId)];
|
|
317
416
|
for (const path of paths) {
|
|
417
|
+
const isSharedFile = !!primaryDaemonId && path === getPendingEventsPath(meshId);
|
|
418
|
+
// Targeting predicate for the shared (unscoped) file: only this coordinator's
|
|
419
|
+
// events (or legacy untargeted ones) are eligible.
|
|
420
|
+
const targets = (e: PendingMeshCoordinatorEvent): boolean =>
|
|
421
|
+
!isSharedFile || !e.targetCoordinatorDaemonId || daemonIds.includes(e.targetCoordinatorDaemonId);
|
|
422
|
+
|
|
423
|
+
if (onlyEvents) {
|
|
424
|
+
// Selective JSONL drain: consume only matching events, rewrite the rest back.
|
|
425
|
+
for (const event of selectiveDrainFile(path, e => targets(e) && matchesFilter(e.event))) {
|
|
426
|
+
pushUnique(event);
|
|
427
|
+
}
|
|
428
|
+
continue;
|
|
429
|
+
}
|
|
318
430
|
const content = atomicDrainFile(path);
|
|
319
431
|
if (!content) continue;
|
|
320
432
|
const parsed = content.split('\n').filter(Boolean).flatMap(line => {
|
|
321
433
|
try { return [JSON.parse(line) as PendingMeshCoordinatorEvent]; } catch { return []; }
|
|
322
434
|
});
|
|
323
435
|
// If reading the shared file, filter to events that target this coordinator or are unscoped.
|
|
324
|
-
const filtered =
|
|
325
|
-
? parsed.filter(e => !e.targetCoordinatorDaemonId || e.targetCoordinatorDaemonId === coordinatorDaemonId)
|
|
326
|
-
: parsed;
|
|
436
|
+
const filtered = isSharedFile ? parsed.filter(targets) : parsed;
|
|
327
437
|
for (const event of filtered) pushUnique(event);
|
|
328
438
|
}
|
|
329
439
|
if (merged.length === 0) return [];
|
|
@@ -335,8 +445,9 @@ export function drainPendingMeshCoordinatorEvents(meshId?: string, coordinatorDa
|
|
|
335
445
|
}
|
|
336
446
|
|
|
337
447
|
/** Peek at pending coordinator events without draining (non-destructive). */
|
|
338
|
-
export function getPendingMeshCoordinatorEvents(meshId?: string, coordinatorDaemonId?: string): readonly PendingMeshCoordinatorEvent[] {
|
|
448
|
+
export function getPendingMeshCoordinatorEvents(meshId?: string, coordinatorDaemonId?: string | ReadonlyArray<string>): readonly PendingMeshCoordinatorEvent[] {
|
|
339
449
|
if (!meshId) return [];
|
|
450
|
+
const daemonIds = normalizeCoordinatorDaemonIds(coordinatorDaemonId);
|
|
340
451
|
|
|
341
452
|
// Merge SQLite (primary) + JSONL (legacy) with fingerprint dedup.
|
|
342
453
|
const merged: PendingMeshCoordinatorEvent[] = [];
|
|
@@ -354,7 +465,7 @@ export function getPendingMeshCoordinatorEvents(meshId?: string, coordinatorDaem
|
|
|
354
465
|
try {
|
|
355
466
|
const store = MeshRuntimeStore.getInstance();
|
|
356
467
|
if (store.pendingEventCount(meshId) > 0) {
|
|
357
|
-
for (const row of store.peekPendingEvents(meshId,
|
|
468
|
+
for (const row of store.peekPendingEvents(meshId, daemonIds.length > 0 ? daemonIds : undefined)) {
|
|
358
469
|
const event = row.payload as PendingMeshCoordinatorEvent;
|
|
359
470
|
if (event) pushUnique(event);
|
|
360
471
|
}
|
|
@@ -362,7 +473,7 @@ export function getPendingMeshCoordinatorEvents(meshId?: string, coordinatorDaem
|
|
|
362
473
|
} catch { /* SQLite unavailable — JSONL fallback below */ }
|
|
363
474
|
|
|
364
475
|
// JSONL (legacy)
|
|
365
|
-
for (const event of readPendingMeshCoordinatorEventsFromDisk(meshId,
|
|
476
|
+
for (const event of readPendingMeshCoordinatorEventsFromDisk(meshId, daemonIds)) {
|
|
366
477
|
pushUnique(event);
|
|
367
478
|
}
|
|
368
479
|
|
|
@@ -14,10 +14,16 @@
|
|
|
14
14
|
//
|
|
15
15
|
// This loop is that drainer. On a fixed interval it:
|
|
16
16
|
// 1. Finds live CLI coordinator sessions on THIS daemon (meshCoordinatorFor
|
|
17
|
-
// stamp). For each, drains the local queue scoped to this daemon and
|
|
18
|
-
// injects pending events into the coordinator
|
|
19
|
-
//
|
|
20
|
-
//
|
|
17
|
+
// stamp). For each mesh, drains the local queue scoped to this daemon and
|
|
18
|
+
// injects pending events into the coordinator. When a coordinator is idle it
|
|
19
|
+
// receives every queued event. When ONLY generating coordinators exist (the
|
|
20
|
+
// common case while the coordinator is blocked awaiting a worker result), the
|
|
21
|
+
// loop force-drains ONLY the force-inject events (completion / approval / stop /
|
|
22
|
+
// refine·bootstrap terminal) and force-writes them into the generating PTY —
|
|
23
|
+
// the same busy-bypass send-guard escape the live-CLI inject used to use.
|
|
24
|
+
// Non-force progress events stay queued for the next idle tick (injecting them
|
|
25
|
+
// mid-generation would be noise). This is what makes a coordinator parked in
|
|
26
|
+
// `generating` while awaiting a worker's completion actually receive it.
|
|
21
27
|
// 2. In cloud mode (dispatchMeshCommand present), pulls each remote worker
|
|
22
28
|
// node daemon's queue over P2P (get_pending_mesh_events) and re-injects via
|
|
23
29
|
// handleMeshForwardEvent — the same pull the MCP drainCoordinatorPendingEvents
|
|
@@ -41,7 +47,7 @@ import { LOG } from '../logging/logger.js';
|
|
|
41
47
|
import { drainPendingMeshCoordinatorEvents } from './mesh-events-pending.js';
|
|
42
48
|
import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
43
49
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
44
|
-
import { handleMeshForwardEvent, shouldForceInjectMeshEvent } from './mesh-events-coordinator.js';
|
|
50
|
+
import { handleMeshForwardEvent, shouldForceInjectMeshEvent, MESH_FORCE_INJECT_EVENTS } from './mesh-events-coordinator.js';
|
|
45
51
|
import { readNonEmptyString } from './mesh-events-utils.js';
|
|
46
52
|
|
|
47
53
|
// Default reconcile cadence. approval/completion notifications to a live CLI
|
|
@@ -63,6 +69,24 @@ interface LiveCoordinator {
|
|
|
63
69
|
idle: boolean;
|
|
64
70
|
}
|
|
65
71
|
|
|
72
|
+
// The set of coordinator-daemon ids THIS daemon answers to when draining the
|
|
73
|
+
// pending-events queue. A unicast completion event is stamped with the worker's
|
|
74
|
+
// meshCoordinatorDaemonId, which can be either:
|
|
75
|
+
// - the daemon's canonical status id (`standalone_<machineId>` / `daemon_<machineId>`),
|
|
76
|
+
// stamped by the MCP layer via ctx.localDaemonId (= getStatus().status.instanceId), or
|
|
77
|
+
// - the bare machineId, stamped by the local queue-assignment path (loadConfig().machineId).
|
|
78
|
+
// Draining with only one of these silently misses events stamped with the other —
|
|
79
|
+
// the exact reason a generating coordinator never self-received local completions.
|
|
80
|
+
// We accept BOTH so the drain matches regardless of which path stamped the event.
|
|
81
|
+
function resolveCoordinatorDaemonIds(components: DaemonComponents): string[] {
|
|
82
|
+
const ids = new Set<string>();
|
|
83
|
+
const statusInstanceId = readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId);
|
|
84
|
+
if (statusInstanceId) ids.add(statusInstanceId);
|
|
85
|
+
const machineId = readNonEmptyString(loadConfig().machineId);
|
|
86
|
+
if (machineId) ids.add(machineId);
|
|
87
|
+
return [...ids];
|
|
88
|
+
}
|
|
89
|
+
|
|
66
90
|
// Find live CLI coordinator instances on THIS daemon, keyed by mesh.
|
|
67
91
|
function findLiveCoordinators(components: DaemonComponents): LiveCoordinator[] {
|
|
68
92
|
const out: LiveCoordinator[] = [];
|
|
@@ -79,7 +103,9 @@ function findLiveCoordinators(components: DaemonComponents): LiveCoordinator[] {
|
|
|
79
103
|
return out;
|
|
80
104
|
}
|
|
81
105
|
|
|
82
|
-
// Inject a drained pending event into a live
|
|
106
|
+
// Inject a drained pending event into a live coordinator session. Force-inject
|
|
107
|
+
// events carry force:true so they bypass the busy send-guard and land in the PTY
|
|
108
|
+
// even while the coordinator is generating (see shouldForceInjectMeshEvent).
|
|
83
109
|
function injectPendingIntoCoordinator(
|
|
84
110
|
coordinator: LiveCoordinator['instance'],
|
|
85
111
|
pending: PendingMeshCoordinatorEvent,
|
|
@@ -111,6 +137,10 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
111
137
|
}
|
|
112
138
|
|
|
113
139
|
const localDaemonId = readNonEmptyString(loadConfig().machineId) || undefined;
|
|
140
|
+
// The id-set used to scope the local queue drain (status id + machineId). See
|
|
141
|
+
// resolveCoordinatorDaemonIds — the status id is what the MCP layer stamps and
|
|
142
|
+
// is mandatory here for a generating CLI coordinator to self-receive completions.
|
|
143
|
+
const drainDaemonIds = resolveCoordinatorDaemonIds(components);
|
|
114
144
|
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
115
145
|
const store = (() => {
|
|
116
146
|
try { return MeshRuntimeStore.getInstance(); } catch { return undefined; }
|
|
@@ -130,12 +160,19 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
130
160
|
}
|
|
131
161
|
}
|
|
132
162
|
|
|
133
|
-
// (b) Drain the local queue scoped to this coordinator daemon and inject
|
|
134
|
-
//
|
|
135
|
-
//
|
|
136
|
-
//
|
|
163
|
+
// (b) Drain the local queue scoped to this coordinator daemon and inject.
|
|
164
|
+
// - If an idle coordinator exists, FULL-drain and deliver every event to it
|
|
165
|
+
// (it can receive non-force progress events without deadlocking).
|
|
166
|
+
// - If only GENERATING coordinators exist, force-drain ONLY the force-inject
|
|
167
|
+
// events (completion/approval/stop/refine·bootstrap terminal) and force-inject
|
|
168
|
+
// them so a coordinator parked in `generating` while awaiting that very event
|
|
169
|
+
// is not deadlocked. Non-force progress events stay queued for the next idle
|
|
170
|
+
// tick — injecting them would be noise mid-generation. Both drains mark the
|
|
171
|
+
// consumed rows drained=1 atomically, so the pull path can't re-deliver.
|
|
137
172
|
const idleCoordinators = meshCoordinators.filter(c => c.idle);
|
|
138
|
-
|
|
173
|
+
const generatingCoordinators = meshCoordinators.filter(c => !c.idle);
|
|
174
|
+
const targetCoordinators = idleCoordinators.length > 0 ? idleCoordinators : generatingCoordinators;
|
|
175
|
+
const forceOnly = idleCoordinators.length === 0;
|
|
139
176
|
|
|
140
177
|
// O(1) guard: skip the drain entirely when the queue is empty.
|
|
141
178
|
if (store) {
|
|
@@ -146,16 +183,21 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
146
183
|
|
|
147
184
|
let pendingEvents: PendingMeshCoordinatorEvent[] = [];
|
|
148
185
|
try {
|
|
149
|
-
pendingEvents = drainPendingMeshCoordinatorEvents(
|
|
186
|
+
pendingEvents = drainPendingMeshCoordinatorEvents(
|
|
187
|
+
meshId,
|
|
188
|
+
drainDaemonIds.length > 0 ? drainDaemonIds : localDaemonId,
|
|
189
|
+
forceOnly ? { onlyEvents: MESH_FORCE_INJECT_EVENTS } : undefined,
|
|
190
|
+
);
|
|
150
191
|
} catch (e: any) {
|
|
151
192
|
LOG.warn('MeshReconcile', `Drain failed for mesh ${meshId}: ${e?.message || e}`);
|
|
152
193
|
continue;
|
|
153
194
|
}
|
|
154
195
|
if (pendingEvents.length === 0) continue;
|
|
155
196
|
|
|
156
|
-
|
|
197
|
+
const mode = forceOnly ? 'force-drain → generating' : 'inject → idle';
|
|
198
|
+
LOG.info('MeshReconcile', `Reconcile ${mode}: ${pendingEvents.length} pending event(s) → ${targetCoordinators.length} coordinator(s) for mesh ${meshId}`);
|
|
157
199
|
for (const pending of pendingEvents) {
|
|
158
|
-
for (const c of
|
|
200
|
+
for (const c of targetCoordinators) {
|
|
159
201
|
injectPendingIntoCoordinator(c.instance, pending);
|
|
160
202
|
}
|
|
161
203
|
}
|
|
@@ -1241,16 +1241,50 @@ export class MeshRuntimeStore {
|
|
|
1241
1241
|
return result.changes > 0;
|
|
1242
1242
|
}
|
|
1243
1243
|
|
|
1244
|
-
|
|
1244
|
+
/**
|
|
1245
|
+
* Drain undrained pending events for a mesh, atomically marking them drained.
|
|
1246
|
+
* When `opts.onlyEvents` is supplied, ONLY rows whose `event` is in that set are
|
|
1247
|
+
* drained — the rest stay queued (drained=0) for a later drain. This is how the
|
|
1248
|
+
* reconcile loop force-drains terminal/force-inject events into a *generating*
|
|
1249
|
+
* coordinator while leaving non-force progress events for the coordinator's next
|
|
1250
|
+
* idle transition. Filtering happens inside the same transaction as the
|
|
1251
|
+
* drained=1 marking, so force-drain + a concurrent full drain can never both
|
|
1252
|
+
* consume the same row.
|
|
1253
|
+
*/
|
|
1254
|
+
drainPendingEvents(
|
|
1255
|
+
meshId: string,
|
|
1256
|
+
coordinatorDaemonId?: string | null | ReadonlyArray<string>,
|
|
1257
|
+
opts?: { onlyEvents?: ReadonlySet<string> },
|
|
1258
|
+
): Array<{ id: string; event: string; payload: unknown }> {
|
|
1245
1259
|
return this.transaction(() => {
|
|
1246
|
-
const
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
const
|
|
1250
|
-
|
|
1251
|
-
|
|
1260
|
+
const onlyEvents = opts?.onlyEvents;
|
|
1261
|
+
// An explicit-but-empty filter means "drain nothing" (no event name can match).
|
|
1262
|
+
if (onlyEvents && onlyEvents.size === 0) return [];
|
|
1263
|
+
const eventList = onlyEvents ? [...onlyEvents] : [];
|
|
1264
|
+
// A coordinator daemon can answer to more than one id form: its canonical
|
|
1265
|
+
// status id (e.g. `standalone_<machineId>` / `daemon_<machineId>`, which the
|
|
1266
|
+
// MCP layer stamps via ctx.localDaemonId) AND the bare machineId (stamped by
|
|
1267
|
+
// the local queue-assignment path). Accept ANY of them so a unicast event
|
|
1268
|
+
// stamped with either id is drained here. Unscoped (NULL) rows always match.
|
|
1269
|
+
const daemonIds = (Array.isArray(coordinatorDaemonId)
|
|
1270
|
+
? coordinatorDaemonId
|
|
1271
|
+
: coordinatorDaemonId ? [coordinatorDaemonId] : [])
|
|
1272
|
+
.filter((id): id is string => typeof id === 'string' && id.length > 0);
|
|
1273
|
+
// Filter by event name IN-SQL when onlyEvents is set so the LIMIT applies to
|
|
1274
|
+
// matching rows — a long run of non-force events ahead in the queue must not
|
|
1275
|
+
// crowd a force event out of the 100-row window.
|
|
1276
|
+
const clauses = ['mesh_id = ?', 'drained = 0'];
|
|
1277
|
+
const params: unknown[] = [meshId];
|
|
1278
|
+
if (daemonIds.length > 0) {
|
|
1279
|
+
clauses.push(`(coordinator_daemon_id IS NULL OR coordinator_daemon_id IN (${daemonIds.map(() => '?').join(',')}))`);
|
|
1280
|
+
params.push(...daemonIds);
|
|
1281
|
+
}
|
|
1282
|
+
if (eventList.length > 0) {
|
|
1283
|
+
clauses.push(`event IN (${eventList.map(() => '?').join(',')})`);
|
|
1284
|
+
params.push(...eventList);
|
|
1285
|
+
}
|
|
1252
1286
|
const rows = this.db.prepare(
|
|
1253
|
-
`SELECT id, event, payload FROM mesh_pending_events ${
|
|
1287
|
+
`SELECT id, event, payload FROM mesh_pending_events WHERE ${clauses.join(' AND ')} ORDER BY queued_at ASC LIMIT 100`
|
|
1254
1288
|
).all(...params) as Array<{ id: string; event: string; payload: string }>;
|
|
1255
1289
|
if (rows.length === 0) return [];
|
|
1256
1290
|
const ids = rows.map(r => r.id);
|
|
@@ -1267,11 +1301,15 @@ export class MeshRuntimeStore {
|
|
|
1267
1301
|
}
|
|
1268
1302
|
|
|
1269
1303
|
/** Non-destructive peek — returns undrained events without marking them drained. */
|
|
1270
|
-
peekPendingEvents(meshId: string, coordinatorDaemonId?: string | null): Array<{ id: string; event: string; payload: unknown }> {
|
|
1271
|
-
const
|
|
1272
|
-
?
|
|
1304
|
+
peekPendingEvents(meshId: string, coordinatorDaemonId?: string | null | ReadonlyArray<string>): Array<{ id: string; event: string; payload: unknown }> {
|
|
1305
|
+
const daemonIds = (Array.isArray(coordinatorDaemonId)
|
|
1306
|
+
? coordinatorDaemonId
|
|
1307
|
+
: coordinatorDaemonId ? [coordinatorDaemonId] : [])
|
|
1308
|
+
.filter((id): id is string => typeof id === 'string' && id.length > 0);
|
|
1309
|
+
const whereClause = daemonIds.length > 0
|
|
1310
|
+
? `WHERE mesh_id = ? AND drained = 0 AND (coordinator_daemon_id IS NULL OR coordinator_daemon_id IN (${daemonIds.map(() => '?').join(',')}))`
|
|
1273
1311
|
: `WHERE mesh_id = ? AND drained = 0`;
|
|
1274
|
-
const params: unknown[] =
|
|
1312
|
+
const params: unknown[] = daemonIds.length > 0 ? [meshId, ...daemonIds] : [meshId];
|
|
1275
1313
|
const rows = this.db.prepare(
|
|
1276
1314
|
`SELECT id, event, payload FROM mesh_pending_events ${whereClause} ORDER BY queued_at ASC LIMIT 100`
|
|
1277
1315
|
).all(...params) as Array<{ id: string; event: string; payload: string }>;
|