@adhdev/daemon-core 0.9.82-rc.468 → 0.9.82-rc.469
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +7 -5
- package/dist/index.js +683 -495
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +674 -493
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-completion-synthesis.d.ts +4 -0
- package/dist/mesh/mesh-delivery-policy.d.ts +0 -27
- package/dist/mesh/mesh-events-pending.d.ts +40 -0
- package/dist/mesh/mesh-events.d.ts +2 -2
- package/dist/mesh/mesh-ledger.d.ts +1 -1
- package/dist/mesh/mesh-reconcile-config.d.ts +6 -0
- package/dist/mesh/mesh-remote-event-pull.d.ts +15 -0
- package/dist/mesh/mesh-runtime-store.d.ts +0 -22
- package/dist/mesh/mesh-work-queue.d.ts +45 -0
- package/package.json +3 -3
- package/src/index.ts +11 -5
- package/src/mesh/coordinator-prompt.ts +15 -0
- package/src/mesh/mesh-completion-synthesis.ts +398 -0
- package/src/mesh/mesh-delivery-policy.ts +7 -38
- package/src/mesh/mesh-event-forwarding.ts +9 -10
- package/src/mesh/mesh-events-pending.ts +176 -0
- package/src/mesh/mesh-events.ts +6 -1
- package/src/mesh/mesh-ledger.ts +5 -0
- package/src/mesh/mesh-queue-assignment.ts +16 -2
- package/src/mesh/mesh-reconcile-config.ts +66 -0
- package/src/mesh/mesh-reconcile-loop.ts +21 -674
- package/src/mesh/mesh-remote-event-pull.ts +279 -0
- package/src/mesh/mesh-runtime-store.ts +46 -82
- package/src/mesh/mesh-work-queue.ts +90 -0
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// mesh-remote-event-pull — cloud P2P remote-node pull helpers for the reconcile loop
|
|
3
|
+
// ---------------------------------------------------------------------------
|
|
4
|
+
// Extracted from mesh-reconcile-loop.ts (A-3 god-module decomposition, pure move,
|
|
5
|
+
// no behavior change). These helpers implement the reconcile loop's cloud-only
|
|
6
|
+
// PHASE that pulls pending coordinator events + worker status from REMOTE worker
|
|
7
|
+
// node daemons over P2P (get_pending_mesh_events / read_chat / get_status_metadata)
|
|
8
|
+
// and the payload-unwrapping utilities that tolerate the varied transport envelope
|
|
9
|
+
// shapes a local commandHandler vs. a remote dispatchMeshCommand returns.
|
|
10
|
+
//
|
|
11
|
+
// mesh-completion-synthesis.ts (the PHASE-4 synth) consumes several of these
|
|
12
|
+
// (unwrapReadChatPayload, readChatPayloadStatus, reprobeWorkerStatus,
|
|
13
|
+
// realTerminalEmitPendingForTask, collectLiveNodesWithSessions); the reconcile
|
|
14
|
+
// loop itself consumes pullRemoteNodeQueues.
|
|
15
|
+
// ---------------------------------------------------------------------------
|
|
16
|
+
|
|
17
|
+
import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
|
|
18
|
+
import type { LocalMeshEntry } from '../repo-mesh-types.js';
|
|
19
|
+
import { getPendingMeshCoordinatorEvents, serializeV2EnvelopeToWire } from './mesh-events-pending.js';
|
|
20
|
+
import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
21
|
+
import { handleMeshForwardEvent } from './mesh-events-coordinator.js';
|
|
22
|
+
import { readNonEmptyString } from './mesh-events-utils.js';
|
|
23
|
+
import { daemonIdsEquivalent } from '@adhdev/mesh-shared';
|
|
24
|
+
import { daemonIdListIncludes } from './mesh-reconcile-identity.js';
|
|
25
|
+
|
|
26
|
+
// Cloud-only: poll each remote worker node daemon for pending coordinator events
|
|
27
|
+
// and re-inject them locally via handleMeshForwardEvent (which re-queues +
|
|
28
|
+
// surfaces to the live coordinator on the next tick / immediately if idle).
|
|
29
|
+
//
|
|
30
|
+
// Scoping: the remote handler (get_pending_mesh_events) drains its queue filtered
|
|
31
|
+
// by coordinatorDaemonId — returning events targeted at that id OR unscoped, and
|
|
32
|
+
// leaving events targeted at a *different* coordinator. A remote worker stamps the
|
|
33
|
+
// coordinator id in one of SEVERAL forms (the canonical status id `standalone_`/
|
|
34
|
+
// `daemon_<machineId>` stamped by the MCP layer, the bare machineId stamped by the
|
|
35
|
+
// local queue path, OR — most commonly for remote launches — the coordinator mesh
|
|
36
|
+
// node's config-form `daemonId`, which resolveCoordinatorDaemonId prefers and which
|
|
37
|
+
// is NOT canonicalised). `candidateDaemonIds` is the already-expanded self-identity
|
|
38
|
+
// set (resolveCoordinatorSelfIds: runtime drain ids ∪ this daemon's mesh-config node/
|
|
39
|
+
// host id forms), so we pull ONCE PER candidate id and a completion stamped with any
|
|
40
|
+
// of them is recovered. The remote drain is atomic (drained=1), so issuing multiple
|
|
41
|
+
// pulls cannot double-deliver — the first pull that matches consumes the event; the
|
|
42
|
+
// rest see nothing. When no ids resolve we fall back to a single unscoped pull.
|
|
43
|
+
export async function pullRemoteNodeQueues(
|
|
44
|
+
components: DaemonComponents,
|
|
45
|
+
mesh: LocalMeshEntry,
|
|
46
|
+
localDaemonId: string | undefined,
|
|
47
|
+
candidateDaemonIds: string[],
|
|
48
|
+
): Promise<void> {
|
|
49
|
+
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
50
|
+
if (!dispatchMeshCommand) return;
|
|
51
|
+
const meshId = mesh.id;
|
|
52
|
+
|
|
53
|
+
// One args object per candidate coordinator-id form, or a single unscoped pull
|
|
54
|
+
// when none resolve.
|
|
55
|
+
const pulls: Array<Record<string, unknown>> = candidateDaemonIds.length > 0
|
|
56
|
+
? candidateDaemonIds.map(id => ({ meshId, coordinatorDaemonId: id }))
|
|
57
|
+
: [{ meshId }];
|
|
58
|
+
|
|
59
|
+
// Parallelize across nodes: a single connected-but-slow node must not serially
|
|
60
|
+
// block the other nodes for the rest of the tick. Each node callback is fully
|
|
61
|
+
// self-contained (local/candidate skip, peer-connected pre-check, per-candidate
|
|
62
|
+
// pulls, extract→re-inject) and best-effort — allSettled swallows per-node errors.
|
|
63
|
+
await Promise.allSettled(mesh.nodes.map(async (node) => {
|
|
64
|
+
const nodeDaemonId = readNonEmptyString(node.daemonId);
|
|
65
|
+
// Skip nodes without a daemon, and nodes on THIS daemon (their events are
|
|
66
|
+
// already in the local queue drained in PHASE 2). "This daemon" is matched
|
|
67
|
+
// against the full self-identity set (candidateDaemonIds), not just the bare
|
|
68
|
+
// localDaemonId — a self node can be registered under the config-form daemonId
|
|
69
|
+
// (`daemon_<machineId>`) which would NOT equal bare localDaemonId, and pulling
|
|
70
|
+
// from ourselves over P2P is both wasteful and a self-dispatch hazard.
|
|
71
|
+
if (!nodeDaemonId) return;
|
|
72
|
+
if (daemonIdsEquivalent(nodeDaemonId, localDaemonId)) return;
|
|
73
|
+
if (daemonIdListIncludes(candidateDaemonIds, nodeDaemonId)) return;
|
|
74
|
+
|
|
75
|
+
// Peer-connected pre-check (EVENT-DELIVERY-DELAY fix(a)): a degraded peer whose
|
|
76
|
+
// DataChannel is not open would sink this pull into peer.connectQueue and stall
|
|
77
|
+
// until CONNECT_TIMEOUT_MS (90s), formerly freezing the whole serial loop and
|
|
78
|
+
// delaying completion-event recovery from healthy nodes. Skip such a node THIS
|
|
79
|
+
// tick and retry next tick — LOSSLESS: an unconnected peer has not drained
|
|
80
|
+
// anything (drained=0 preserved), so its events are recovered whole on the next
|
|
81
|
+
// successful tick. Skip = delay, never loss.
|
|
82
|
+
// • snapshot present and state !== 'connected' → skip (continue next tick).
|
|
83
|
+
// • snapshot null/undefined (getter unwired, e.g. standalone) → DO NOT skip;
|
|
84
|
+
// fall through to the legacy path so this stays regression-free.
|
|
85
|
+
const peerSnapshot = components.getMeshPeerConnectionStatus?.(nodeDaemonId);
|
|
86
|
+
if (peerSnapshot && String(peerSnapshot.state) !== 'connected') return;
|
|
87
|
+
|
|
88
|
+
for (const pendingEventArgs of pulls) {
|
|
89
|
+
let events: unknown;
|
|
90
|
+
try {
|
|
91
|
+
events = await dispatchMeshCommand(nodeDaemonId, 'get_pending_mesh_events', pendingEventArgs);
|
|
92
|
+
} catch {
|
|
93
|
+
// Remote pull is best-effort; the node may be offline. Retry next tick.
|
|
94
|
+
break; // node unreachable — don't bother with the other id form this tick.
|
|
95
|
+
}
|
|
96
|
+
const list = extractPendingEvents(events).filter(e => readNonEmptyString(e?.meshId) === meshId);
|
|
97
|
+
for (const event of list) {
|
|
98
|
+
const payload = buildForwardPayloadFromPending(event);
|
|
99
|
+
if (!payload.event || !payload.meshId) continue;
|
|
100
|
+
try {
|
|
101
|
+
handleMeshForwardEvent(components, payload);
|
|
102
|
+
} catch { /* best-effort re-inject */ }
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}));
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Pull the read_chat payload out of whatever envelope the transport returned.
|
|
109
|
+
// A local commandHandler.handle() returns the CommandResult directly; a remote
|
|
110
|
+
// dispatchMeshCommand returns it possibly wrapped in { payload } / { result }.
|
|
111
|
+
export function unwrapReadChatPayload(raw: unknown): Record<string, unknown> | null {
|
|
112
|
+
let cursor: unknown = raw;
|
|
113
|
+
for (let depth = 0; depth < 4 && cursor && typeof cursor === 'object'; depth++) {
|
|
114
|
+
const record = cursor as Record<string, unknown>;
|
|
115
|
+
if (Array.isArray(record.messages)) return record;
|
|
116
|
+
if (record.payload && typeof record.payload === 'object') { cursor = record.payload; continue; }
|
|
117
|
+
if (record.result && typeof record.result === 'object') { cursor = record.result; continue; }
|
|
118
|
+
if (record.data && typeof record.data === 'object') { cursor = record.data; continue; }
|
|
119
|
+
break;
|
|
120
|
+
}
|
|
121
|
+
return cursor && typeof cursor === 'object' ? cursor as Record<string, unknown> : null;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export function readChatPayloadStatus(payload: Record<string, unknown> | null): string {
|
|
125
|
+
return readNonEmptyString(payload?.status).toLowerCase();
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// R4e fix (3): peek the pending-events queue for a REAL (worker-emitted) terminal completion
|
|
129
|
+
// already queued for a task — used to yield the in-flight synth to the worker's own emit. Broad
|
|
130
|
+
// peek (no daemon-id scoping) matched precisely by taskId, so a worker stamp in any daemon-id form
|
|
131
|
+
// is still recognized. Best-effort: a peek failure returns false (proceed to synth — never block
|
|
132
|
+
// delivery). A prior SYNTH's still-queued pending event also names this taskId, but a synth always
|
|
133
|
+
// writes its terminal ledger atomically, so hasTerminalLedgerAfterDispatch downstream already
|
|
134
|
+
// no-ops that case — this guard is specifically for an as-yet-unledgered worker emit in flight.
|
|
135
|
+
export function realTerminalEmitPendingForTask(meshId: string, taskId: string): boolean {
|
|
136
|
+
let pending: readonly PendingMeshCoordinatorEvent[];
|
|
137
|
+
try {
|
|
138
|
+
pending = getPendingMeshCoordinatorEvents(meshId);
|
|
139
|
+
} catch {
|
|
140
|
+
return false;
|
|
141
|
+
}
|
|
142
|
+
return pending.some(e =>
|
|
143
|
+
readNonEmptyString(e.metadataEvent?.taskId) === taskId
|
|
144
|
+
&& (e.event === 'agent:generating_completed' || e.event === 'agent:stopped'));
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// R4e fix (2): one fresh read_chat status read for the worker session, via the same local/remote
|
|
148
|
+
// transport PHASE 4 uses. Returns the lowercased status, or null when the read is inconclusive
|
|
149
|
+
// (transport error, success:false, no payload) — callers treat null as "no new evidence, proceed".
|
|
150
|
+
export async function reprobeWorkerStatus(
|
|
151
|
+
components: DaemonComponents,
|
|
152
|
+
args: { isLocalNode: boolean; nodeDaemonId: string; readArgs: Record<string, unknown> },
|
|
153
|
+
): Promise<string | null> {
|
|
154
|
+
try {
|
|
155
|
+
if (args.isLocalNode) {
|
|
156
|
+
const r = await components.commandHandler.handle('read_chat', args.readArgs);
|
|
157
|
+
if (r && (r as { success?: boolean }).success === false) return null;
|
|
158
|
+
return readChatPayloadStatus(unwrapReadChatPayload(r));
|
|
159
|
+
}
|
|
160
|
+
if (components.dispatchMeshCommand) {
|
|
161
|
+
const r = await components.dispatchMeshCommand(args.nodeDaemonId, 'read_chat', args.readArgs);
|
|
162
|
+
const p = unwrapReadChatPayload(r);
|
|
163
|
+
if (p && (p as { success?: boolean }).success === false) return null;
|
|
164
|
+
return readChatPayloadStatus(p);
|
|
165
|
+
}
|
|
166
|
+
} catch {
|
|
167
|
+
return null;
|
|
168
|
+
}
|
|
169
|
+
return null;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// Probe each node for its live session list (get_status_metadata) and return mesh.nodes
|
|
173
|
+
// decorated with a `sessions` array — the shape buildMeshActiveWork / sessionStatusFromNodes
|
|
174
|
+
// consume to decide whether a dispatched session is still present. Best-effort: an unreachable
|
|
175
|
+
// node yields an empty session list rather than throwing.
|
|
176
|
+
export async function collectLiveNodesWithSessions(
|
|
177
|
+
components: DaemonComponents,
|
|
178
|
+
mesh: LocalMeshEntry,
|
|
179
|
+
selfIds: string[],
|
|
180
|
+
localDaemonId: string | undefined,
|
|
181
|
+
): Promise<any[]> {
|
|
182
|
+
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
183
|
+
return Promise.all(mesh.nodes.map(async (node) => {
|
|
184
|
+
const nodeDaemonId = readNonEmptyString(node.daemonId);
|
|
185
|
+
const isLocalNode = !nodeDaemonId
|
|
186
|
+
|| daemonIdListIncludes(selfIds, nodeDaemonId)
|
|
187
|
+
|| daemonIdsEquivalent(nodeDaemonId, localDaemonId);
|
|
188
|
+
// Peer-connected pre-check (EVENT-DELIVERY-DELAY fix(a)): mirror pullRemoteNodeQueues.
|
|
189
|
+
// Without this the 90s connect-deadline block re-enters via this Promise.all —
|
|
190
|
+
// a degraded remote's get_status_metadata sinks into peer.connectQueue and stalls
|
|
191
|
+
// the whole prune probe. Only call the remote when the peer is 'connected'; an
|
|
192
|
+
// unconnected peer is left undecorated (empty session list), same as unreachable.
|
|
193
|
+
// Getter unwired (null/undefined) → do NOT skip, fall through (regression-free).
|
|
194
|
+
if (!isLocalNode) {
|
|
195
|
+
const peerSnapshot = components.getMeshPeerConnectionStatus?.(nodeDaemonId);
|
|
196
|
+
if (peerSnapshot && String(peerSnapshot.state) !== 'connected') return node;
|
|
197
|
+
}
|
|
198
|
+
let statusResult: unknown;
|
|
199
|
+
try {
|
|
200
|
+
if (isLocalNode) {
|
|
201
|
+
statusResult = await components.commandHandler.handle('get_status_metadata', {});
|
|
202
|
+
} else if (dispatchMeshCommand) {
|
|
203
|
+
statusResult = await dispatchMeshCommand(nodeDaemonId, 'get_status_metadata', {});
|
|
204
|
+
} else {
|
|
205
|
+
return node; // remote node, no P2P transport — leave undecorated
|
|
206
|
+
}
|
|
207
|
+
} catch {
|
|
208
|
+
return node; // unreachable — leave undecorated (empty session list)
|
|
209
|
+
}
|
|
210
|
+
const sessions = extractStatusMetadataSessions(statusResult);
|
|
211
|
+
return sessions.length > 0 ? { ...node, sessions } : node;
|
|
212
|
+
}));
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// Pull the live session list out of a get_status_metadata result, tolerating the same
|
|
216
|
+
// envelope shapes unwrapReadChatPayload handles (direct CommandResult or { payload }/{ result }).
|
|
217
|
+
export function extractStatusMetadataSessions(raw: unknown): any[] {
|
|
218
|
+
let cursor: unknown = raw;
|
|
219
|
+
for (let depth = 0; depth < 4 && cursor && typeof cursor === 'object'; depth++) {
|
|
220
|
+
const record = cursor as Record<string, unknown>;
|
|
221
|
+
const status = record.status && typeof record.status === 'object' ? record.status as Record<string, unknown> : undefined;
|
|
222
|
+
if (status && Array.isArray(status.sessions)) return status.sessions;
|
|
223
|
+
if (Array.isArray(record.sessions)) return record.sessions;
|
|
224
|
+
if (record.payload && typeof record.payload === 'object') { cursor = record.payload; continue; }
|
|
225
|
+
if (record.result && typeof record.result === 'object') { cursor = record.result; continue; }
|
|
226
|
+
if (record.data && typeof record.data === 'object') { cursor = record.data; continue; }
|
|
227
|
+
break;
|
|
228
|
+
}
|
|
229
|
+
return [];
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
export function extractPendingEvents(raw: unknown): any[] {
|
|
233
|
+
if (Array.isArray(raw)) return raw;
|
|
234
|
+
if (raw && typeof raw === 'object') {
|
|
235
|
+
const events = (raw as Record<string, unknown>).events;
|
|
236
|
+
if (Array.isArray(events)) return events;
|
|
237
|
+
}
|
|
238
|
+
return [];
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
// Flatten a queued PendingMeshCoordinatorEvent into the flat payload shape
|
|
242
|
+
// handleMeshForwardEvent expects (mirrors the MCP buildMeshForwardPayloadFromPendingEvent).
|
|
243
|
+
export function buildForwardPayloadFromPending(event: any): Record<string, unknown> {
|
|
244
|
+
const metadata = event?.metadataEvent && typeof event.metadataEvent === 'object'
|
|
245
|
+
? event.metadataEvent as Record<string, unknown>
|
|
246
|
+
: {};
|
|
247
|
+
return {
|
|
248
|
+
event: readNonEmptyString(event?.event),
|
|
249
|
+
meshId: readNonEmptyString(event?.meshId),
|
|
250
|
+
nodeId: readNonEmptyString(event?.nodeId) || readNonEmptyString(metadata.meshNodeId),
|
|
251
|
+
workspace: readNonEmptyString(event?.workspace) || readNonEmptyString(metadata.workspace),
|
|
252
|
+
// Preserve the originating coordinator session id across the relay. It is normally
|
|
253
|
+
// carried inside metadataEvent.meshCoordinatorSessionId (spread below), but pass the
|
|
254
|
+
// top-level field through explicitly too so the handleMeshForwardEvent whitelist
|
|
255
|
+
// recovers it regardless of which carrier the producing daemon used.
|
|
256
|
+
...(readNonEmptyString(event?.targetCoordinatorSessionId)
|
|
257
|
+
? { targetCoordinatorSessionId: readNonEmptyString(event.targetCoordinatorSessionId) }
|
|
258
|
+
: {}),
|
|
259
|
+
...metadata,
|
|
260
|
+
// NOTIF-MISS (FIX 3): surface the dispatch task id at the TOP LEVEL so the relay's
|
|
261
|
+
// received-stage trace (and buildRelayMetadataEvent) recovers it regardless of which
|
|
262
|
+
// carrier the producing daemon used. The metadata spread above may carry the id only as
|
|
263
|
+
// `meshActiveTaskId` (a worker provider event), leaving top-level `taskId` unset and the
|
|
264
|
+
// received stage rendering `task=-`. Resolve both carriers into an explicit `taskId` so
|
|
265
|
+
// dedup stays task-scoped end-to-end. Only set when a non-empty id exists (no clobber to
|
|
266
|
+
// undefined when neither is present).
|
|
267
|
+
...((): Record<string, unknown> => {
|
|
268
|
+
const tid = readNonEmptyString(metadata.taskId) || readNonEmptyString(metadata.meshActiveTaskId);
|
|
269
|
+
return tid ? { taskId: tid } : {};
|
|
270
|
+
})(),
|
|
271
|
+
// T4 (B3b): carry the v2 envelope (protocolVersion/eventId/scope/dispatchedBy/
|
|
272
|
+
// intendedFor) across the P2P relay boundary at the TOP LEVEL. These live on the
|
|
273
|
+
// pending event itself, not inside metadataEvent, so without this the remote pull
|
|
274
|
+
// re-queue would re-stamp a fresh eventId — breaking cross-machine idempotency and
|
|
275
|
+
// downgrading the relayed completion to v1 broadcast routing. Spread LAST so the
|
|
276
|
+
// authoritative envelope always wins over any stale key the metadata spread carried.
|
|
277
|
+
...serializeV2EnvelopeToWire(event as PendingMeshCoordinatorEvent),
|
|
278
|
+
};
|
|
279
|
+
}
|
|
@@ -4,7 +4,7 @@ import { LOG } from '../logging/logger.js';
|
|
|
4
4
|
import { loadBetterSqlite3 } from '../system/load-better-sqlite3.js';
|
|
5
5
|
import { getConfigDir } from '../config/config.js';
|
|
6
6
|
import { getLedgerDir } from './mesh-ledger.js';
|
|
7
|
-
import { nodeSatisfiesRequiredTags, isTaskReadonly, taskDependenciesSatisfied } from './mesh-work-queue.js';
|
|
7
|
+
import { nodeSatisfiesRequiredTags, isTaskReadonly, taskDependenciesSatisfied, meshTaskNotBeforeReady, meshTaskPriorityRank } from './mesh-work-queue.js';
|
|
8
8
|
import { meshNodeIdMatches, daemonIdsEquivalent, expandDaemonIdForms, sessionIdsEquivalent } from '@adhdev/mesh-shared';
|
|
9
9
|
import type { MeshTaskStatus, MeshWorkQueueEntry } from './mesh-work-queue.js';
|
|
10
10
|
import type BetterSqlite3 from 'better-sqlite3';
|
|
@@ -270,20 +270,9 @@ export class MeshRuntimeStore {
|
|
|
270
270
|
CREATE INDEX IF NOT EXISTS idx_mesh_session_delivery_task
|
|
271
271
|
ON mesh_session_delivery(mesh_id, task_id);
|
|
272
272
|
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
fingerprint TEXT NOT NULL,
|
|
277
|
-
conflicting_task_id TEXT,
|
|
278
|
-
conflicting_session_id TEXT,
|
|
279
|
-
original_task_id TEXT,
|
|
280
|
-
original_session_id TEXT,
|
|
281
|
-
event TEXT NOT NULL,
|
|
282
|
-
created_at TEXT NOT NULL
|
|
283
|
-
);
|
|
284
|
-
|
|
285
|
-
CREATE INDEX IF NOT EXISTS idx_mesh_completion_conflicts_mesh
|
|
286
|
-
ON mesh_completion_conflicts(mesh_id, created_at);
|
|
273
|
+
-- MESH-COMPLEXITY-AUDIT Part 8-2: mesh_completion_conflicts removed
|
|
274
|
+
-- (write-only fingerprint-collision diagnostic, no production reader,
|
|
275
|
+
-- no no-loss role). Dropped in migrateMeshIsolationColumns step 6.
|
|
287
276
|
|
|
288
277
|
CREATE TABLE IF NOT EXISTS mesh_tool_call_log (
|
|
289
278
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -504,6 +493,16 @@ export class MeshRuntimeStore {
|
|
|
504
493
|
// the dormant table has it removed once. Idempotent — DROP TABLE IF EXISTS is
|
|
505
494
|
// a no-op on every subsequent boot.
|
|
506
495
|
this.db.exec(`DROP TABLE IF EXISTS mesh_direct_delivered_events`);
|
|
496
|
+
|
|
497
|
+
// 6. MESH-COMPLEXITY-AUDIT Part 8-2: drop the mesh_completion_conflicts
|
|
498
|
+
// diagnostic table. It recorded which task lost a completion-fingerprint
|
|
499
|
+
// dedup collision but had NO production reader (getRecentCompletionConflicts
|
|
500
|
+
// was test-only) and played NO part in the no-loss delivery contract — the
|
|
501
|
+
// dedup DECISION is the fingerprint match in mesh-event-forwarding.ts and is
|
|
502
|
+
// unchanged. Pure runtime-residue cleanup with no behavior change: a fresh
|
|
503
|
+
// store never creates it; an old install drops the dormant table once.
|
|
504
|
+
// Idempotent — DROP TABLE IF EXISTS is a no-op on every subsequent boot.
|
|
505
|
+
this.db.exec(`DROP TABLE IF EXISTS mesh_completion_conflicts`);
|
|
507
506
|
} catch (err: any) {
|
|
508
507
|
// Best-effort: a failed isolation migration must not brick the store. The
|
|
509
508
|
// CREATE-TABLE definitions above already carry the new schema for fresh DBs;
|
|
@@ -912,31 +911,35 @@ export class MeshRuntimeStore {
|
|
|
912
911
|
// targetMatches() JS gate above re-validates each fetched row.
|
|
913
912
|
const nodeIdForms = expandDaemonIdForms(nodeId);
|
|
914
913
|
const nodePinnedPlaceholders = nodeIdForms.map(() => '?').join(', ');
|
|
915
|
-
// Priority: session-targeted > node-targeted (no session) > unconstrained
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
914
|
+
// Priority: session-targeted > node-targeted (no session) > unconstrained.
|
|
915
|
+
// G6: WITHIN each targeting tier, a higher task-level priority is pulled first;
|
|
916
|
+
// created_at ASC (from the SQL ORDER BY) is the intra-priority tie-break. The
|
|
917
|
+
// tier ordering is preserved (a high-priority unconstrained task never jumps
|
|
918
|
+
// ahead of a session/node-pinned task) so targeting stays the outer key and
|
|
919
|
+
// priority is the inner key. Sort is stable, so equal-priority rows keep FIFO.
|
|
920
|
+
const parseTier = (query: string, ...params: unknown[]): MeshWorkQueueEntry[] => {
|
|
921
|
+
const tierRows = this.db.prepare(query).all(...params) as Array<{ payload: string }>;
|
|
922
|
+
return tierRows
|
|
923
|
+
.map(row => JSON.parse(row.payload) as MeshWorkQueueEntry)
|
|
924
|
+
.sort((a, b) => meshTaskPriorityRank(b.priority) - meshTaskPriorityRank(a.priority));
|
|
925
|
+
};
|
|
926
|
+
const candidates = [
|
|
927
|
+
...parseTier(`
|
|
919
928
|
SELECT payload FROM mesh_queue
|
|
920
929
|
WHERE mesh_id = ? AND status = 'pending' AND target_session_id = ?
|
|
921
930
|
ORDER BY created_at ASC
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
...(
|
|
925
|
-
this.db.prepare(`
|
|
931
|
+
`, meshId, sessionId),
|
|
932
|
+
...parseTier(`
|
|
926
933
|
SELECT payload FROM mesh_queue
|
|
927
934
|
WHERE mesh_id = ? AND status = 'pending' AND target_node_id IN (${nodePinnedPlaceholders}) AND target_session_id IS NULL
|
|
928
935
|
ORDER BY created_at ASC
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
...(
|
|
932
|
-
this.db.prepare(`
|
|
936
|
+
`, meshId, ...nodeIdForms),
|
|
937
|
+
...parseTier(`
|
|
933
938
|
SELECT payload FROM mesh_queue
|
|
934
939
|
WHERE mesh_id = ? AND status = 'pending' AND target_node_id IS NULL AND target_session_id IS NULL
|
|
935
940
|
ORDER BY created_at ASC
|
|
936
|
-
|
|
937
|
-
),
|
|
941
|
+
`, meshId),
|
|
938
942
|
];
|
|
939
|
-
const candidates = rows.map(row => JSON.parse(row.payload) as MeshWorkQueueEntry);
|
|
940
943
|
|
|
941
944
|
// M1: a task with unmet dependencies (or a system blockedReason) is not claimable.
|
|
942
945
|
// Resolve dependency statuses in one query over the union of referenced ids.
|
|
@@ -964,6 +967,13 @@ export class MeshRuntimeStore {
|
|
|
964
967
|
return !nodeBusy;
|
|
965
968
|
};
|
|
966
969
|
|
|
970
|
+
// G7: delayed execution. A task with a notBefore in the future is held pending
|
|
971
|
+
// (skipped as a claim candidate) until the wall clock passes it. Fail-open on an
|
|
972
|
+
// unparseable timestamp (meshTaskNotBeforeReady) so a bad value never strands work.
|
|
973
|
+
const claimNowMs = Date.now();
|
|
974
|
+
const notBeforeReady = (candidate: MeshWorkQueueEntry): boolean =>
|
|
975
|
+
meshTaskNotBeforeReady(candidate, claimNowMs);
|
|
976
|
+
|
|
967
977
|
// WTDISPATCH-FANOUT: a `convergence` task lands its work onto base (merge →
|
|
968
978
|
// push → cleanup against the real checkout). It must NEVER be claimed by a
|
|
969
979
|
// co-located worktree-clone session — N sibling worktree sessions on one daemon
|
|
@@ -1006,6 +1016,7 @@ export class MeshRuntimeStore {
|
|
|
1006
1016
|
const entry = candidates.find(candidate =>
|
|
1007
1017
|
nodeSatisfiesRequiredTags(candidate.requiredTags, capabilityTags)
|
|
1008
1018
|
&& dependenciesSatisfied(candidate)
|
|
1019
|
+
&& notBeforeReady(candidate)
|
|
1009
1020
|
&& convergenceAllows(candidate)
|
|
1010
1021
|
&& targetMatches(candidate)
|
|
1011
1022
|
&& nodeConflictAllows(candidate));
|
|
@@ -1478,58 +1489,11 @@ export class MeshRuntimeStore {
|
|
|
1478
1489
|
|
|
1479
1490
|
// ── Completion Conflict Diagnostics ──────────────────────────────────────
|
|
1480
1491
|
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
conflictingSessionId?: string;
|
|
1487
|
-
originalTaskId?: string;
|
|
1488
|
-
originalSessionId?: string;
|
|
1489
|
-
event: string;
|
|
1490
|
-
createdAt: string;
|
|
1491
|
-
}): void {
|
|
1492
|
-
this.db.prepare(`
|
|
1493
|
-
INSERT OR IGNORE INTO mesh_completion_conflicts
|
|
1494
|
-
(id, mesh_id, fingerprint, conflicting_task_id, conflicting_session_id,
|
|
1495
|
-
original_task_id, original_session_id, event, created_at)
|
|
1496
|
-
VALUES (@id, @meshId, @fingerprint, @conflictingTaskId, @conflictingSessionId,
|
|
1497
|
-
@originalTaskId, @originalSessionId, @event, @createdAt)
|
|
1498
|
-
`).run({
|
|
1499
|
-
id: entry.id,
|
|
1500
|
-
meshId: entry.meshId,
|
|
1501
|
-
fingerprint: entry.fingerprint,
|
|
1502
|
-
conflictingTaskId: entry.conflictingTaskId ?? null,
|
|
1503
|
-
conflictingSessionId: entry.conflictingSessionId ?? null,
|
|
1504
|
-
originalTaskId: entry.originalTaskId ?? null,
|
|
1505
|
-
originalSessionId: entry.originalSessionId ?? null,
|
|
1506
|
-
event: entry.event,
|
|
1507
|
-
createdAt: entry.createdAt,
|
|
1508
|
-
});
|
|
1509
|
-
this.maybeCheckpointWal();
|
|
1510
|
-
}
|
|
1511
|
-
|
|
1512
|
-
getRecentCompletionConflicts(meshId: string, limitMs: number = 60 * 60 * 1000): Array<{
|
|
1513
|
-
id: string; meshId: string; fingerprint: string; conflictingTaskId: string | null;
|
|
1514
|
-
conflictingSessionId: string | null; originalTaskId: string | null;
|
|
1515
|
-
originalSessionId: string | null; event: string; createdAt: string;
|
|
1516
|
-
}> {
|
|
1517
|
-
const cutoff = new Date(Date.now() - limitMs).toISOString();
|
|
1518
|
-
const rows = this.db.prepare(
|
|
1519
|
-
'SELECT * FROM mesh_completion_conflicts WHERE mesh_id = ? AND created_at >= ? ORDER BY created_at DESC LIMIT 50'
|
|
1520
|
-
).all(meshId, cutoff) as Array<Record<string, unknown>>;
|
|
1521
|
-
return rows.map(r => ({
|
|
1522
|
-
id: r.id as string,
|
|
1523
|
-
meshId: r.mesh_id as string,
|
|
1524
|
-
fingerprint: r.fingerprint as string,
|
|
1525
|
-
conflictingTaskId: r.conflicting_task_id as string | null,
|
|
1526
|
-
conflictingSessionId: r.conflicting_session_id as string | null,
|
|
1527
|
-
originalTaskId: r.original_task_id as string | null,
|
|
1528
|
-
originalSessionId: r.original_session_id as string | null,
|
|
1529
|
-
event: r.event as string,
|
|
1530
|
-
createdAt: r.created_at as string,
|
|
1531
|
-
}));
|
|
1532
|
-
}
|
|
1492
|
+
// MESH-COMPLEXITY-AUDIT Part 8-2: recordCompletionConflict /
|
|
1493
|
+
// getRecentCompletionConflicts (and their mesh_completion_conflicts table)
|
|
1494
|
+
// were removed. They were a write-only diagnostic of fingerprint-dedup
|
|
1495
|
+
// collisions with no production reader and no part in the no-loss delivery
|
|
1496
|
+
// contract; the table is dropped in migrateMeshIsolationColumns (step 6).
|
|
1533
1497
|
|
|
1534
1498
|
/**
|
|
1535
1499
|
* Record a mesh tool call and check whether this mesh+tool combination is
|
|
@@ -16,9 +16,66 @@ export type MeshActiveTaskStatus = Extract<MeshTaskStatus, 'pending' | 'assigned
|
|
|
16
16
|
export type MeshHistoricalTaskStatus = Extract<MeshTaskStatus, 'completed' | 'failed' | 'cancelled'>;
|
|
17
17
|
export type MeshTaskMode = 'code_change' | 'validation' | 'live_debug_readonly' | 'launch_app' | 'convergence';
|
|
18
18
|
|
|
19
|
+
/** G6: task-level scheduling priority. Ranks which task a node pulls first (created_at tie-break). */
|
|
20
|
+
export type MeshTaskPriority = 'low' | 'normal' | 'high';
|
|
21
|
+
|
|
19
22
|
export const ACTIVE_MESH_QUEUE_STATUSES: MeshActiveTaskStatus[] = ['pending', 'assigned'];
|
|
20
23
|
export const HISTORICAL_MESH_QUEUE_STATUSES: MeshHistoricalTaskStatus[] = ['completed', 'failed', 'cancelled'];
|
|
21
24
|
export const MESH_TASK_MODES: MeshTaskMode[] = ['code_change', 'validation', 'live_debug_readonly', 'launch_app', 'convergence'];
|
|
25
|
+
export const MESH_TASK_PRIORITIES: MeshTaskPriority[] = ['low', 'normal', 'high'];
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* G6: numeric rank of a task priority (higher = pulled first). Absent/unknown → 'normal' (1).
|
|
29
|
+
* Shared by the claim-candidate ordering and any surface that must sort by task priority.
|
|
30
|
+
*/
|
|
31
|
+
export function meshTaskPriorityRank(priority: unknown): number {
|
|
32
|
+
switch (priority) {
|
|
33
|
+
case 'high': return 2;
|
|
34
|
+
case 'low': return 0;
|
|
35
|
+
default: return 1; // 'normal' and any absent/unknown value
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** G6: coerce an arbitrary input to a valid MeshTaskPriority, or undefined when not one of the three. */
|
|
40
|
+
export function normalizeMeshTaskPriority(value: unknown): MeshTaskPriority | undefined {
|
|
41
|
+
return value === 'low' || value === 'normal' || value === 'high' ? value : undefined;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* G7: resolve a not_before input to a stored ISO string (or undefined when absent/invalid).
|
|
46
|
+
* Accepts an ISO/date string, an absolute epoch-ms number, or a small relative-ms offset from
|
|
47
|
+
* `nowMs`. Disambiguation for numbers: a value below {@link NOT_BEFORE_RELATIVE_THRESHOLD_MS}
|
|
48
|
+
* (~1 year in ms) is treated as a relative offset added to now; a larger value is an absolute
|
|
49
|
+
* epoch-ms timestamp. A past/negative result is normalized to now (immediately claimable).
|
|
50
|
+
*/
|
|
51
|
+
export const NOT_BEFORE_RELATIVE_THRESHOLD_MS = 365 * 24 * 60 * 60 * 1000;
|
|
52
|
+
export function resolveNotBefore(value: unknown, nowMs: number = Date.now()): string | undefined {
|
|
53
|
+
if (value === undefined || value === null) return undefined;
|
|
54
|
+
let absMs: number;
|
|
55
|
+
if (typeof value === 'number' && Number.isFinite(value)) {
|
|
56
|
+
absMs = value < NOT_BEFORE_RELATIVE_THRESHOLD_MS ? nowMs + value : value;
|
|
57
|
+
} else if (typeof value === 'string' && value.trim()) {
|
|
58
|
+
const parsed = Date.parse(value.trim());
|
|
59
|
+
if (Number.isNaN(parsed)) return undefined;
|
|
60
|
+
absMs = parsed;
|
|
61
|
+
} else {
|
|
62
|
+
return undefined;
|
|
63
|
+
}
|
|
64
|
+
if (absMs <= nowMs) return new Date(nowMs).toISOString();
|
|
65
|
+
return new Date(absMs).toISOString();
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** G7: is a task claimable now, or is it still held back by its notBefore gate? */
|
|
69
|
+
export function meshTaskNotBeforeReady(
|
|
70
|
+
task: { notBefore?: string } | null | undefined,
|
|
71
|
+
nowMs: number = Date.now(),
|
|
72
|
+
): boolean {
|
|
73
|
+
const nb = task?.notBefore;
|
|
74
|
+
if (!nb) return true;
|
|
75
|
+
const parsed = Date.parse(nb);
|
|
76
|
+
if (Number.isNaN(parsed)) return true; // unparseable → do not block (fail-open)
|
|
77
|
+
return parsed <= nowMs;
|
|
78
|
+
}
|
|
22
79
|
|
|
23
80
|
/**
|
|
24
81
|
* QUEUE-NODE-SERIALIZATION: single source of truth for "is this task read-only?".
|
|
@@ -488,6 +545,22 @@ export interface MeshWorkQueueEntry {
|
|
|
488
545
|
targetSessionId?: string;
|
|
489
546
|
/** If specified, a node must expose all tags before it can claim the task. */
|
|
490
547
|
requiredTags?: string[];
|
|
548
|
+
/**
|
|
549
|
+
* G6 (task-level scheduling priority): 'low' | 'normal' | 'high'. Orders the
|
|
550
|
+
* claim candidate list so a high-priority task is pulled ahead of an older
|
|
551
|
+
* normal/low task within the same claim tier (created_at is the tie-break).
|
|
552
|
+
* Absent → treated as 'normal'. This is the TASK-level priority, distinct from
|
|
553
|
+
* the NODE-level schedulingPriority (resolveNodeSchedulingPriority), which ranks
|
|
554
|
+
* which node a task goes to, not which task a node pulls first.
|
|
555
|
+
*/
|
|
556
|
+
priority?: MeshTaskPriority;
|
|
557
|
+
/**
|
|
558
|
+
* G7 (delayed execution): ISO timestamp before which the task is NOT claimable.
|
|
559
|
+
* The claim gate holds the task pending while now < notBefore; once the wall
|
|
560
|
+
* clock passes it the task becomes a normal claim candidate. A pure time gate —
|
|
561
|
+
* cron/webhook triggers are out of scope. Absent → immediately claimable.
|
|
562
|
+
*/
|
|
563
|
+
notBefore?: string;
|
|
491
564
|
/**
|
|
492
565
|
* M1: ids of tasks that must reach 'completed' before this task is claimable.
|
|
493
566
|
* Forward references (ids not yet enqueued) are allowed for batch flows and
|
|
@@ -777,6 +850,12 @@ export function enqueueTask(
|
|
|
777
850
|
requiredTags?: string[];
|
|
778
851
|
/** M1: tasks that must complete before this one is claimable. */
|
|
779
852
|
dependsOn?: string[];
|
|
853
|
+
/** G6: task-level scheduling priority ('low' | 'normal' | 'high'). Absent → 'normal'. */
|
|
854
|
+
priority?: MeshTaskPriority | string;
|
|
855
|
+
/** G7: hold the task pending until this time. ISO string, absolute epoch-ms, or relative-ms offset from now. */
|
|
856
|
+
notBefore?: string | number;
|
|
857
|
+
/** P3: max automatic requeue attempts before the task auto-fails. Absent → policy default (1). */
|
|
858
|
+
maxRetries?: number;
|
|
780
859
|
/** M1/M3: mission this task belongs to. */
|
|
781
860
|
missionId?: string;
|
|
782
861
|
/** MAGI: consensus group id shared by every replica of a mesh_magi_review fan-out. */
|
|
@@ -797,6 +876,11 @@ export function enqueueTask(
|
|
|
797
876
|
}
|
|
798
877
|
const id = typeof opts?.id === 'string' && opts.id.trim() ? opts.id.trim() : randomUUID();
|
|
799
878
|
const dependsOn = normalizeDependsOn(opts?.dependsOn);
|
|
879
|
+
const priority = normalizeMeshTaskPriority(opts?.priority);
|
|
880
|
+
const notBefore = resolveNotBefore(opts?.notBefore);
|
|
881
|
+
const maxRetries = typeof opts?.maxRetries === 'number' && Number.isFinite(opts.maxRetries) && opts.maxRetries >= 0
|
|
882
|
+
? Math.floor(opts.maxRetries)
|
|
883
|
+
: undefined;
|
|
800
884
|
return withQueueLock(meshId, () => {
|
|
801
885
|
if (MeshRuntimeStore.getInstance().findQueueEntryById(meshId, id)) {
|
|
802
886
|
throw new Error(`duplicate_task_id: task '${id}' already exists in mesh '${meshId}'`);
|
|
@@ -825,6 +909,12 @@ export function enqueueTask(
|
|
|
825
909
|
targetSessionId: opts?.targetSessionId,
|
|
826
910
|
requiredTags: resolvedRequiredTags,
|
|
827
911
|
...(dependsOn.length > 0 ? { dependsOn } : {}),
|
|
912
|
+
// G6: only persist a non-default priority so legacy/normal rows stay minimal.
|
|
913
|
+
...(priority && priority !== 'normal' ? { priority } : {}),
|
|
914
|
+
// G7: hold-until gate (stored ISO). Omitted when absent/immediate.
|
|
915
|
+
...(notBefore ? { notBefore } : {}),
|
|
916
|
+
// P3: explicit retry cap. Omitted → requeue path falls back to policy default.
|
|
917
|
+
...(maxRetries !== undefined ? { maxRetries } : {}),
|
|
828
918
|
...(typeof opts?.missionId === 'string' && opts.missionId.trim() ? { missionId: opts.missionId.trim() } : {}),
|
|
829
919
|
...(typeof opts?.consensusGroupId === 'string' && opts.consensusGroupId.trim() ? { consensusGroupId: opts.consensusGroupId.trim() } : {}),
|
|
830
920
|
...(typeof opts?.model === 'string' && opts.model.trim() ? { model: opts.model.trim() } : {}),
|