@adhdev/daemon-core 0.9.82-rc.388 → 0.9.82-rc.389
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +152 -39
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +152 -39
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-reconcile-loop.d.ts +1 -0
- package/dist/mesh/mesh-runtime-store.d.ts +13 -5
- package/package.json +2 -2
- package/src/mesh/mesh-event-forwarding.ts +62 -17
- package/src/mesh/mesh-queue-assignment.ts +2 -2
- package/src/mesh/mesh-reconcile-loop.ts +68 -1
- package/src/mesh/mesh-runtime-store.ts +103 -16
- package/src/mesh/mesh-work-queue.ts +36 -0
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
|
|
2
|
+
export declare function __resetReconcileInFlightSynthDebounceForTests(): void;
|
|
2
3
|
/**
|
|
3
4
|
* DRAIN-WITHOUT-INJECT guard. Classify, for a mesh on THIS daemon, whether a
|
|
4
5
|
* queue-drain caller (the MCP `get_pending_mesh_events` poll) may safely consume
|
|
@@ -16,8 +16,16 @@ export declare class MeshRuntimeStore {
|
|
|
16
16
|
close(): void;
|
|
17
17
|
transaction<T>(fn: () => T): T;
|
|
18
18
|
private migrate;
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
private tableColumns;
|
|
20
|
+
/**
|
|
21
|
+
* MESH-ISOLATION-LEAK migration. Two tables historically lacked a `mesh_id` column,
|
|
22
|
+
* letting one machine that belongs to multiple meshes (multiple repos) leak rows
|
|
23
|
+
* across meshes. Both migrations are idempotent and run on every boot — the column
|
|
24
|
+
* check short-circuits once the new schema is in place.
|
|
25
|
+
*/
|
|
26
|
+
private migrateMeshIsolationColumns;
|
|
27
|
+
hasCompletionFingerprint(meshId: string, fingerprint: string): boolean;
|
|
28
|
+
recordCompletionFingerprint(meshId: string, fingerprint: string, ttlMs: number): void;
|
|
21
29
|
sweepExpiredFingerprints(): void;
|
|
22
30
|
private maybeCheckpointWal;
|
|
23
31
|
private ensureLegacyQueueMigrated;
|
|
@@ -149,15 +157,15 @@ export declare class MeshRuntimeStore {
|
|
|
149
157
|
*/
|
|
150
158
|
deleteDirectDispatchesByTaskId(meshId: string, taskIds: string[]): number;
|
|
151
159
|
markStaleDirectDispatches(meshId: string, olderThanMs: number): void;
|
|
152
|
-
setRemoteIdleSession(nodeId: string, sessionId: string, providerType: string, expiresAt: number, metadata?: any): void;
|
|
153
|
-
getRemoteIdleSessions(): Array<{
|
|
160
|
+
setRemoteIdleSession(meshId: string, nodeId: string, sessionId: string, providerType: string, expiresAt: number, metadata?: any): void;
|
|
161
|
+
getRemoteIdleSessions(meshId: string): Array<{
|
|
154
162
|
nodeId: string;
|
|
155
163
|
sessionId: string;
|
|
156
164
|
providerType: string;
|
|
157
165
|
expiresAt: number;
|
|
158
166
|
metadata?: any;
|
|
159
167
|
}>;
|
|
160
|
-
deleteRemoteIdleSession(nodeId: string, sessionId: string): void;
|
|
168
|
+
deleteRemoteIdleSession(meshId: string, nodeId: string, sessionId: string): void;
|
|
161
169
|
pruneExpiredRemoteIdleSessions(): void;
|
|
162
170
|
insertSessionDelivery(entry: {
|
|
163
171
|
id: string;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adhdev/daemon-core",
|
|
3
|
-
"version": "0.9.82-rc.
|
|
3
|
+
"version": "0.9.82-rc.389",
|
|
4
4
|
"description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
"author": "vilmire",
|
|
47
47
|
"license": "AGPL-3.0-or-later",
|
|
48
48
|
"dependencies": {
|
|
49
|
-
"@adhdev/mesh-shared": "0.9.82-rc.
|
|
49
|
+
"@adhdev/mesh-shared": "0.9.82-rc.389",
|
|
50
50
|
"@adhdev/session-host-core": "*",
|
|
51
51
|
"@agentclientprotocol/sdk": "^0.16.1",
|
|
52
52
|
"ajv": "^8.20.0",
|
|
@@ -226,18 +226,18 @@ function shouldSuppressIntentionalCleanupStop(args: {
|
|
|
226
226
|
|
|
227
227
|
const RECENT_COMPLETION_FINGERPRINT_TTL_MS = 10 * 60 * 1000;
|
|
228
228
|
|
|
229
|
-
function hasFingerprintSeen(fingerprint: string): boolean {
|
|
229
|
+
function hasFingerprintSeen(meshId: string, fingerprint: string): boolean {
|
|
230
230
|
try {
|
|
231
|
-
return MeshRuntimeStore.getInstance().hasCompletionFingerprint(fingerprint);
|
|
231
|
+
return MeshRuntimeStore.getInstance().hasCompletionFingerprint(meshId, fingerprint);
|
|
232
232
|
} catch {
|
|
233
233
|
return false;
|
|
234
234
|
}
|
|
235
235
|
}
|
|
236
236
|
|
|
237
|
-
function recordFingerprintSeen(fingerprint: string): void {
|
|
237
|
+
function recordFingerprintSeen(meshId: string, fingerprint: string): void {
|
|
238
238
|
try {
|
|
239
239
|
const db = MeshRuntimeStore.getInstance();
|
|
240
|
-
db.recordCompletionFingerprint(fingerprint, RECENT_COMPLETION_FINGERPRINT_TTL_MS);
|
|
240
|
+
db.recordCompletionFingerprint(meshId, fingerprint, RECENT_COMPLETION_FINGERPRINT_TTL_MS);
|
|
241
241
|
db.sweepExpiredFingerprints();
|
|
242
242
|
} catch { /* best-effort; duplicate events are preferable to a crash */ }
|
|
243
243
|
}
|
|
@@ -291,7 +291,7 @@ function isDuplicateMeshCompletionEvent(args: {
|
|
|
291
291
|
}): boolean {
|
|
292
292
|
const fingerprint = buildMeshCompletionFingerprint(args);
|
|
293
293
|
if (!fingerprint) return false;
|
|
294
|
-
if (hasFingerprintSeen(fingerprint)) {
|
|
294
|
+
if (hasFingerprintSeen(args.meshId, fingerprint)) {
|
|
295
295
|
if (args.taskId) {
|
|
296
296
|
recordCompletionConflict({
|
|
297
297
|
meshId: args.meshId,
|
|
@@ -303,7 +303,7 @@ function isDuplicateMeshCompletionEvent(args: {
|
|
|
303
303
|
}
|
|
304
304
|
return true;
|
|
305
305
|
}
|
|
306
|
-
recordFingerprintSeen(fingerprint);
|
|
306
|
+
recordFingerprintSeen(args.meshId, fingerprint);
|
|
307
307
|
return false;
|
|
308
308
|
}
|
|
309
309
|
|
|
@@ -329,8 +329,8 @@ function isDuplicateMeshApprovalEvent(args: {
|
|
|
329
329
|
args.providerType || '',
|
|
330
330
|
approvalIdentity,
|
|
331
331
|
].join('::');
|
|
332
|
-
if (hasFingerprintSeen(fingerprint)) return true;
|
|
333
|
-
recordFingerprintSeen(fingerprint);
|
|
332
|
+
if (hasFingerprintSeen(args.meshId, fingerprint)) return true;
|
|
333
|
+
recordFingerprintSeen(args.meshId, fingerprint);
|
|
334
334
|
return false;
|
|
335
335
|
}
|
|
336
336
|
|
|
@@ -338,8 +338,8 @@ function isDuplicateRefineTerminalEvent(meshId: string, eventName: string, metad
|
|
|
338
338
|
const jobId = readRefineJobId({ metadataEvent });
|
|
339
339
|
const fingerprint = jobId && new Set(['refine:completed', 'refine:failed']).has(eventName) ? `${meshId}::${eventName}::${jobId}` : '';
|
|
340
340
|
if (!fingerprint) return false;
|
|
341
|
-
if (hasFingerprintSeen(fingerprint)) return true;
|
|
342
|
-
recordFingerprintSeen(fingerprint);
|
|
341
|
+
if (hasFingerprintSeen(meshId, fingerprint)) return true;
|
|
342
|
+
recordFingerprintSeen(meshId, fingerprint);
|
|
343
343
|
return false;
|
|
344
344
|
}
|
|
345
345
|
|
|
@@ -485,7 +485,7 @@ function evaluateMeshEventSuppression(
|
|
|
485
485
|
if (intentionalCleanupStop) {
|
|
486
486
|
if (eventSessionId && eventNodeId) {
|
|
487
487
|
try {
|
|
488
|
-
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(eventNodeId, eventSessionId);
|
|
488
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(args.meshId, eventNodeId, eventSessionId);
|
|
489
489
|
} catch { /* best-effort */ }
|
|
490
490
|
}
|
|
491
491
|
LOG.info('MeshEvents', `Suppressed ${args.event} for intentionally cleanup-stopped session ${eventSessionId || '(unknown session)'}`);
|
|
@@ -597,9 +597,19 @@ function evaluateMeshEventSuppression(
|
|
|
597
597
|
// permanently mask the real completion: let the real event through (it is re-recorded by
|
|
598
598
|
// the normal task_completed path, superseding the synthesized one). A second SYNTHESIZED
|
|
599
599
|
// re-arrival is NOT a real event and still dedups normally.
|
|
600
|
+
// RECONCILE-SYNTH-PREEMPTS-COMPLETION: the bar here is deliberately LOWER than the
|
|
601
|
+
// weak/truncated supersessions above — a synthesized terminal is a coordinator-side
|
|
602
|
+
// reconstruction, never a real provider event, so ANY genuine real provider completion
|
|
603
|
+
// for the session must win over it. Requiring the full isGenuineCompletionEvidence
|
|
604
|
+
// (finalSummary OR workerResult present) dropped the real event whenever the relay did
|
|
605
|
+
// not re-populate finalSummary at this layer — exactly the observed 71s task whose real
|
|
606
|
+
// generating_completed hit drop:duplicate_completion_terminal_ledger after a premature
|
|
607
|
+
// synth. We require only that the incoming event is a REAL provider completion that is
|
|
608
|
+
// not itself a false idle (missing-final-assistant); a real-but-false-idle event still
|
|
609
|
+
// does not supersede (it is not trustworthy terminal evidence either).
|
|
600
610
|
const supersedesSynthesizedTerminal = isSynthesizedReconciledTerminal(terminal.payload)
|
|
601
611
|
&& isRealProviderCompletionEvent(args.metadataEvent)
|
|
602
|
-
&&
|
|
612
|
+
&& !isFalseIdleCompletion(args.metadataEvent);
|
|
603
613
|
if (!newDispatchAfterTerminal && !supersedesWeakTerminal && !distinctTaskCompletion && !supersedesTruncatedTerminal && !supersedesSynthesizedTerminal) {
|
|
604
614
|
const terminalProviderSessionId = readNonEmptyString(terminal.payload.providerSessionId);
|
|
605
615
|
const terminalFinalSummary = readNonEmptyString(terminal.payload.finalSummary);
|
|
@@ -825,7 +835,42 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
825
835
|
const isFalseIdle = isFalseIdleCompletion(args.metadataEvent);
|
|
826
836
|
completedTaskForLedger = markSessionTerminal(sessionId, 'completed', eventTimestamp, { tentativeIfDirect: isFalseIdle });
|
|
827
837
|
if (nodeId && providerType) {
|
|
828
|
-
|
|
838
|
+
// OVEREAGER-REMOTE-IDLE (Defect A+B): re-register the now-idle remote session into
|
|
839
|
+
// the remote-idle store, symmetric with the agent:ready branch. Previously
|
|
840
|
+
// setRemoteIdleSession ran ONLY on agent:ready, while agent:generating_started
|
|
841
|
+
// DELETES the entry — so the FIRST turn a remote worker runs permanently evicts it
|
|
842
|
+
// from the store, and generating_completed never re-added it. A later
|
|
843
|
+
// mesh_enqueue_task's triggerMeshQueue then read getRemoteIdleSessions() == 0
|
|
844
|
+
// (remoteIdleSessionsChecked:0) for a session that is genuinely live-idle, needlessly
|
|
845
|
+
// auto-launching a second worker (Defect A). Worse, the two idle-session sources then
|
|
846
|
+
// disagreed: the enqueue drain saw 0 (store empty) and auto-launched + left the queue
|
|
847
|
+
// task pending, while THIS completing session's runIdleMaintenanceThenAssignQueue
|
|
848
|
+
// claimed the same still-pending row straight from SQL — so the task body injected into
|
|
849
|
+
// BOTH the reused idle session and the auto-launched one (Defect B). Re-registering here
|
|
850
|
+
// unifies the source: the next triggerMeshQueue drain sees the live idle session, reuses
|
|
851
|
+
// it (claimed:true, no auto-launch), and injects exactly once. Skip a false-idle
|
|
852
|
+
// (mid-turn / no-final-assistant) completion — that session is NOT genuinely idle.
|
|
853
|
+
if (!isFalseIdle) {
|
|
854
|
+
sweepExpiredRemoteIdleSessions();
|
|
855
|
+
try {
|
|
856
|
+
MeshRuntimeStore.getInstance().setRemoteIdleSession(args.meshId, nodeId, sessionId, providerType, Date.now() + REMOTE_IDLE_SESSION_TTL_MS);
|
|
857
|
+
} catch { /* best-effort */ }
|
|
858
|
+
setImmediate(() => {
|
|
859
|
+
maybeAutoFastForwardIdleNode(components, { meshId: args.meshId, nodeId, sessionId, providerType })
|
|
860
|
+
.finally(() => {
|
|
861
|
+
try {
|
|
862
|
+
// Claim for THIS session first; on success drop the just-registered
|
|
863
|
+
// idle entry so the enqueue drain doesn't re-pick an already-busy session.
|
|
864
|
+
const assigned = tryAssignQueueTask(components, args.meshId, nodeId, sessionId, providerType);
|
|
865
|
+
if (assigned) MeshRuntimeStore.getInstance().deleteRemoteIdleSession(args.meshId, nodeId, sessionId);
|
|
866
|
+
} catch (e: any) {
|
|
867
|
+
LOG.warn('MeshQueue', `Failed to assign idle queue task after completion for ${nodeId}: ${e?.message || e}`);
|
|
868
|
+
}
|
|
869
|
+
});
|
|
870
|
+
});
|
|
871
|
+
} else {
|
|
872
|
+
runIdleMaintenanceThenAssignQueue(components, { meshId: args.meshId, nodeId, sessionId, providerType });
|
|
873
|
+
}
|
|
829
874
|
}
|
|
830
875
|
// M1-3: wake dependents of the completed task. The maintenance path above
|
|
831
876
|
// only assigns to the completing session; dependents may be claimable by
|
|
@@ -884,14 +929,14 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
884
929
|
if (sessionId && nodeId && providerType) {
|
|
885
930
|
sweepExpiredRemoteIdleSessions();
|
|
886
931
|
try {
|
|
887
|
-
MeshRuntimeStore.getInstance().setRemoteIdleSession(nodeId, sessionId, providerType, Date.now() + REMOTE_IDLE_SESSION_TTL_MS);
|
|
932
|
+
MeshRuntimeStore.getInstance().setRemoteIdleSession(args.meshId, nodeId, sessionId, providerType, Date.now() + REMOTE_IDLE_SESSION_TTL_MS);
|
|
888
933
|
} catch { /* best-effort */ }
|
|
889
934
|
setImmediate(() => {
|
|
890
935
|
maybeAutoFastForwardIdleNode(components, { meshId: args.meshId, nodeId, sessionId, providerType })
|
|
891
936
|
.finally(() => {
|
|
892
937
|
try {
|
|
893
938
|
const assigned = tryAssignQueueTask(components, args.meshId, nodeId, sessionId, providerType);
|
|
894
|
-
if (assigned) MeshRuntimeStore.getInstance().deleteRemoteIdleSession(nodeId, sessionId);
|
|
939
|
+
if (assigned) MeshRuntimeStore.getInstance().deleteRemoteIdleSession(args.meshId, nodeId, sessionId);
|
|
895
940
|
} catch (e: any) {
|
|
896
941
|
LOG.warn('MeshQueue', `Failed to assign idle queue task after maintenance for ${nodeId}: ${e?.message || e}`);
|
|
897
942
|
}
|
|
@@ -903,7 +948,7 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
903
948
|
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
904
949
|
if (sessionId && nodeId) {
|
|
905
950
|
try {
|
|
906
|
-
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(nodeId, sessionId);
|
|
951
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(args.meshId, nodeId, sessionId);
|
|
907
952
|
} catch { /* best-effort */ }
|
|
908
953
|
}
|
|
909
954
|
if (sessionId) {
|
|
@@ -954,7 +999,7 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
954
999
|
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
955
1000
|
if (sessionId && nodeId) {
|
|
956
1001
|
try {
|
|
957
|
-
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(nodeId, sessionId);
|
|
1002
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(args.meshId, nodeId, sessionId);
|
|
958
1003
|
} catch { /* best-effort */ }
|
|
959
1004
|
}
|
|
960
1005
|
if (sessionId) {
|
|
@@ -1298,7 +1298,7 @@ export async function triggerMeshQueue(components: DaemonComponents, meshId: str
|
|
|
1298
1298
|
|
|
1299
1299
|
let remoteSessions: Array<{ nodeId: string; sessionId: string; providerType: string }> = [];
|
|
1300
1300
|
try {
|
|
1301
|
-
remoteSessions = MeshRuntimeStore.getInstance().getRemoteIdleSessions();
|
|
1301
|
+
remoteSessions = MeshRuntimeStore.getInstance().getRemoteIdleSessions(meshId);
|
|
1302
1302
|
} catch { /* best-effort */ }
|
|
1303
1303
|
|
|
1304
1304
|
const remoteCandidates: IdleCandidate[] = [];
|
|
@@ -1318,7 +1318,7 @@ export async function triggerMeshQueue(components: DaemonComponents, meshId: str
|
|
|
1318
1318
|
const assigned = tryAssignQueueTask(components, meshId, candidate.nodeId, candidate.sessionId, candidate.providerType);
|
|
1319
1319
|
if (assigned && candidate.origin === 'remote') {
|
|
1320
1320
|
try {
|
|
1321
|
-
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(candidate.nodeId, candidate.sessionId);
|
|
1321
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(meshId, candidate.nodeId, candidate.sessionId);
|
|
1322
1322
|
} catch { /* best-effort */ }
|
|
1323
1323
|
}
|
|
1324
1324
|
};
|
|
@@ -97,6 +97,35 @@ function resolveReconcileIntervalMs(): number {
|
|
|
97
97
|
return DEFAULT_RECONCILE_INTERVAL_MS;
|
|
98
98
|
}
|
|
99
99
|
|
|
100
|
+
// RECONCILE-SYNTH-PREEMPTS-COMPLETION in-flight debounce. PHASE 4 only synthesizes a missing
|
|
101
|
+
// completion when the worker session reads `idle`. But a worker that is GENUINELY generating
|
|
102
|
+
// (it emitted agent:generating_started — the dispatch row is 'acked' — and has not yet
|
|
103
|
+
// completed) can momentarily read `idle` mid-turn: a CLI PTY parser sees an inter-turn blip
|
|
104
|
+
// between tool calls, or the read_chat probe races a brief settle. A single such idle read
|
|
105
|
+
// must NOT fabricate a completion for an in-flight task — doing so writes a synthesized
|
|
106
|
+
// terminal that then masks the REAL completion when it lands seconds later
|
|
107
|
+
// (drop:duplicate_completion_terminal_ledger; the observed 71s task a250fb44 lost its [System]
|
|
108
|
+
// notification this way). We require the session to read `idle` on CONSECUTIVE reconcile ticks
|
|
109
|
+
// before synthesizing for an `acked` (started-generating) dispatch — a transient mid-turn idle
|
|
110
|
+
// flicker clears on the next ~4s tick, whereas a genuinely-settled (completed-but-lost) or dead
|
|
111
|
+
// session reads idle every tick. A dispatch that was never acked (worker never started) is NOT
|
|
112
|
+
// debounced here: there is no in-flight generation to protect, and the downstream grace gate +
|
|
113
|
+
// stale-summary guard remain the backstops for that lost-dispatch case.
|
|
114
|
+
//
|
|
115
|
+
// Keyed by `${meshId}::${taskId}`. The map is pruned each PHASE-4 pass to the set of currently
|
|
116
|
+
// active dispatches, so a completed/pruned task's counter is dropped (no unbounded growth).
|
|
117
|
+
const REQUIRED_CONSECUTIVE_IDLE_TICKS_FOR_INFLIGHT_SYNTH = 2;
|
|
118
|
+
const inFlightIdleObservationCounts = new Map<string, number>();
|
|
119
|
+
|
|
120
|
+
function inFlightSynthKey(meshId: string, taskId: string): string {
|
|
121
|
+
return `${meshId}::${taskId}`;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Test hook: clear the in-flight idle debounce state between cases.
|
|
125
|
+
export function __resetReconcileInFlightSynthDebounceForTests(): void {
|
|
126
|
+
inFlightIdleObservationCounts.clear();
|
|
127
|
+
}
|
|
128
|
+
|
|
100
129
|
interface LiveCoordinator {
|
|
101
130
|
meshId: string;
|
|
102
131
|
instance: ReturnType<DaemonComponents['instanceManager']['getInstance']>;
|
|
@@ -1195,6 +1224,20 @@ async function reconcileUnterminatedDirectDispatches(
|
|
|
1195
1224
|
const dispatches = getActiveDirectDispatches(mesh.id);
|
|
1196
1225
|
if (dispatches.length === 0) return; // cheap exit — nothing dispatched, nothing to reconcile
|
|
1197
1226
|
|
|
1227
|
+
// Prune the in-flight idle debounce map to the tasks still active in THIS mesh, so a
|
|
1228
|
+
// completed/pruned task's counter is dropped (the map never grows without bound).
|
|
1229
|
+
const activeTaskKeys = new Set(
|
|
1230
|
+
dispatches
|
|
1231
|
+
.map(d => readNonEmptyString(d.taskId))
|
|
1232
|
+
.filter(Boolean)
|
|
1233
|
+
.map(taskId => inFlightSynthKey(mesh.id, taskId)),
|
|
1234
|
+
);
|
|
1235
|
+
for (const key of inFlightIdleObservationCounts.keys()) {
|
|
1236
|
+
if (key.startsWith(`${mesh.id}::`) && !activeTaskKeys.has(key)) {
|
|
1237
|
+
inFlightIdleObservationCounts.delete(key);
|
|
1238
|
+
}
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1198
1241
|
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
1199
1242
|
const nodeById = new Map(mesh.nodes.map(n => [n.id, n] as const));
|
|
1200
1243
|
|
|
@@ -1243,7 +1286,31 @@ async function reconcileUnterminatedDirectDispatches(
|
|
|
1243
1286
|
// Only act on a session that has actually settled to idle. A generating /
|
|
1244
1287
|
// waiting_approval session is mid-turn — synthesizing a completion now would
|
|
1245
1288
|
// be wrong. (idle is the only status the MCP poll path reconciles too.)
|
|
1246
|
-
|
|
1289
|
+
const synthKey = inFlightSynthKey(mesh.id, taskId);
|
|
1290
|
+
if (readChatPayloadStatus(payload) !== 'idle') {
|
|
1291
|
+
// Not idle → the worker is mid-turn. Reset any partial idle streak so a single
|
|
1292
|
+
// idle blip during a long generation never accumulates toward the synth threshold.
|
|
1293
|
+
inFlightIdleObservationCounts.delete(synthKey);
|
|
1294
|
+
continue;
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
// RECONCILE-SYNTH-PREEMPTS-COMPLETION: a dispatch whose worker was OBSERVED to start
|
|
1298
|
+
// generating (the agent:generating_started ack flipped the row to 'acked') and has no
|
|
1299
|
+
// terminal yet is potentially still in-flight — its `idle` read here may be a transient
|
|
1300
|
+
// mid-turn flicker, not a settled completion. Require CONSECUTIVE idle observations
|
|
1301
|
+
// before synthesizing for such a task: a flicker clears next tick (counter reset above),
|
|
1302
|
+
// while a genuinely-settled (completed-but-lost) or dead session reads idle every tick
|
|
1303
|
+
// and crosses the threshold within ~one extra interval. A never-acked dispatch (worker
|
|
1304
|
+
// never started) is exempt — there is no in-flight generation to pre-empt, and its
|
|
1305
|
+
// lost-dispatch case is still covered by the downstream grace + stale-summary guards.
|
|
1306
|
+
if (dispatch.status === 'acked') {
|
|
1307
|
+
const idleStreak = (inFlightIdleObservationCounts.get(synthKey) ?? 0) + 1;
|
|
1308
|
+
inFlightIdleObservationCounts.set(synthKey, idleStreak);
|
|
1309
|
+
if (idleStreak < REQUIRED_CONSECUTIVE_IDLE_TICKS_FOR_INFLIGHT_SYNTH) {
|
|
1310
|
+
LOG.info('MeshReconcile', `In-flight synth hold: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read idle ${idleStreak}/${REQUIRED_CONSECUTIVE_IDLE_TICKS_FOR_INFLIGHT_SYNTH} consecutive tick(s) after generating_started — deferring completion synth until the idle settle is confirmed (guards against a mid-turn idle flicker pre-empting the real completion)`);
|
|
1311
|
+
continue;
|
|
1312
|
+
}
|
|
1313
|
+
}
|
|
1247
1314
|
|
|
1248
1315
|
const messages = Array.isArray(payload.messages) ? payload.messages as ChatMessage[] : [];
|
|
1249
1316
|
const evidence = extractFinalAssistantSummaryEvidence(messages);
|
|
@@ -147,10 +147,20 @@ export class MeshRuntimeStore {
|
|
|
147
147
|
CREATE INDEX IF NOT EXISTS idx_mesh_queue_assignment
|
|
148
148
|
ON mesh_queue(mesh_id, assigned_node_id, assigned_session_id, status);
|
|
149
149
|
|
|
150
|
+
-- mesh_id is DB-level isolation (defense-in-depth). The fingerprint STRING
|
|
151
|
+
-- also carries meshId as its first '::'-joined segment (see
|
|
152
|
+
-- buildMeshCompletionFingerprint) — that string-prefix defense is kept; this
|
|
153
|
+
-- column makes cross-mesh suppression impossible even if the string format
|
|
154
|
+
-- drifts or two meshes ever collide on a fingerprint body.
|
|
150
155
|
CREATE TABLE IF NOT EXISTS mesh_completion_fingerprints (
|
|
151
156
|
fingerprint TEXT PRIMARY KEY,
|
|
152
|
-
expires_at INTEGER NOT NULL
|
|
157
|
+
expires_at INTEGER NOT NULL,
|
|
158
|
+
mesh_id TEXT NOT NULL DEFAULT ''
|
|
153
159
|
);
|
|
160
|
+
-- NOTE: the (mesh_id, fingerprint) index is created in migrateMeshIsolationColumns,
|
|
161
|
+
-- NOT here. A pre-isolation DB still has the legacy table (CREATE IF NOT EXISTS is a
|
|
162
|
+
-- no-op), so referencing mesh_id in an index before the ALTER ADD COLUMN runs would
|
|
163
|
+
-- fail with "no such column". The migration adds the column then the index.
|
|
154
164
|
|
|
155
165
|
CREATE TABLE IF NOT EXISTS mesh_direct_dispatches (
|
|
156
166
|
task_id TEXT PRIMARY KEY,
|
|
@@ -170,13 +180,18 @@ export class MeshRuntimeStore {
|
|
|
170
180
|
CREATE INDEX IF NOT EXISTS idx_direct_dispatches_mesh_session
|
|
171
181
|
ON mesh_direct_dispatches(mesh_id, session_id, status);
|
|
172
182
|
|
|
183
|
+
-- MESH-ISOLATION-LEAK: mesh_id is part of the PK so a nodeId shared across two
|
|
184
|
+
-- meshes (same machine in multiple repos) keeps a separate idle-session row per
|
|
185
|
+
-- mesh, and getRemoteIdleSessions(meshId) can never surface another mesh's
|
|
186
|
+
-- session for a queue claim.
|
|
173
187
|
CREATE TABLE IF NOT EXISTS remote_idle_sessions (
|
|
188
|
+
mesh_id TEXT NOT NULL,
|
|
174
189
|
node_id TEXT NOT NULL,
|
|
175
190
|
session_id TEXT NOT NULL,
|
|
176
191
|
provider_type TEXT NOT NULL,
|
|
177
192
|
expires_at INTEGER NOT NULL,
|
|
178
193
|
metadata TEXT,
|
|
179
|
-
PRIMARY KEY (node_id, session_id)
|
|
194
|
+
PRIMARY KEY (mesh_id, node_id, session_id)
|
|
180
195
|
);
|
|
181
196
|
|
|
182
197
|
CREATE TABLE IF NOT EXISTS mesh_session_delivery (
|
|
@@ -300,13 +315,83 @@ export class MeshRuntimeStore {
|
|
|
300
315
|
cursor INTEGER NOT NULL DEFAULT 0
|
|
301
316
|
);
|
|
302
317
|
`);
|
|
318
|
+
this.migrateMeshIsolationColumns();
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
private tableColumns(table: string): Set<string> {
|
|
322
|
+
const rows = this.db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>;
|
|
323
|
+
return new Set(rows.map(r => r.name));
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
/**
|
|
327
|
+
* MESH-ISOLATION-LEAK migration. Two tables historically lacked a `mesh_id` column,
|
|
328
|
+
* letting one machine that belongs to multiple meshes (multiple repos) leak rows
|
|
329
|
+
* across meshes. Both migrations are idempotent and run on every boot — the column
|
|
330
|
+
* check short-circuits once the new schema is in place.
|
|
331
|
+
*/
|
|
332
|
+
private migrateMeshIsolationColumns(): void {
|
|
333
|
+
try {
|
|
334
|
+
// 1. mesh_completion_fingerprints: ADD COLUMN + backfill mesh_id from the
|
|
335
|
+
// fingerprint string's first '::'-joined segment (buildMeshCompletionFingerprint
|
|
336
|
+
// prefixes meshId). A row whose fingerprint has no '::' (legacy/foreign format)
|
|
337
|
+
// backfills to '' — still strictly tighter than the prior global query.
|
|
338
|
+
const fpCols = this.tableColumns('mesh_completion_fingerprints');
|
|
339
|
+
if (!fpCols.has('mesh_id')) {
|
|
340
|
+
this.db.exec(`ALTER TABLE mesh_completion_fingerprints ADD COLUMN mesh_id TEXT NOT NULL DEFAULT ''`);
|
|
341
|
+
this.db.exec(`
|
|
342
|
+
UPDATE mesh_completion_fingerprints
|
|
343
|
+
SET mesh_id = substr(fingerprint, 1, instr(fingerprint, '::') - 1)
|
|
344
|
+
WHERE instr(fingerprint, '::') > 0 AND mesh_id = ''
|
|
345
|
+
`);
|
|
346
|
+
}
|
|
347
|
+
// The mesh_id column is now guaranteed to exist (fresh DB had it from CREATE TABLE,
|
|
348
|
+
// legacy DB just got it via ALTER). Create the index unconditionally — IF NOT EXISTS
|
|
349
|
+
// makes it a no-op once present.
|
|
350
|
+
this.db.exec(`
|
|
351
|
+
CREATE INDEX IF NOT EXISTS idx_mesh_completion_fingerprints_mesh
|
|
352
|
+
ON mesh_completion_fingerprints(mesh_id, fingerprint)
|
|
353
|
+
`);
|
|
354
|
+
|
|
355
|
+
// 2. remote_idle_sessions: the mesh_id is part of the PRIMARY KEY, which SQLite
|
|
356
|
+
// cannot add via ALTER. The rows are ephemeral — sessions re-register on the
|
|
357
|
+
// next agent:ready / agent:generating_completed — so a safe DROP+recreate is
|
|
358
|
+
// acceptable (per fix spec) rather than a full table rebuild + un-backfillable
|
|
359
|
+
// mesh_id. Only rebuild when the legacy (no mesh_id) schema is detected.
|
|
360
|
+
const idleCols = this.tableColumns('remote_idle_sessions');
|
|
361
|
+
if (!idleCols.has('mesh_id')) {
|
|
362
|
+
this.db.exec(`
|
|
363
|
+
DROP TABLE IF EXISTS remote_idle_sessions;
|
|
364
|
+
CREATE TABLE remote_idle_sessions (
|
|
365
|
+
mesh_id TEXT NOT NULL,
|
|
366
|
+
node_id TEXT NOT NULL,
|
|
367
|
+
session_id TEXT NOT NULL,
|
|
368
|
+
provider_type TEXT NOT NULL,
|
|
369
|
+
expires_at INTEGER NOT NULL,
|
|
370
|
+
metadata TEXT,
|
|
371
|
+
PRIMARY KEY (mesh_id, node_id, session_id)
|
|
372
|
+
);
|
|
373
|
+
`);
|
|
374
|
+
}
|
|
375
|
+
} catch (err: any) {
|
|
376
|
+
// Best-effort: a failed isolation migration must not brick the store. The
|
|
377
|
+
// CREATE-TABLE definitions above already carry the new schema for fresh DBs;
|
|
378
|
+
// an existing DB that fails here keeps the old (leaky-but-functional) schema
|
|
379
|
+
// until the next boot retries. Surface one warn for diagnosability.
|
|
380
|
+
if (!loggedMigrationFailure) {
|
|
381
|
+
loggedMigrationFailure = true;
|
|
382
|
+
LOG.warn('MeshRuntimeStore', `mesh-isolation column migration failed: ${err?.message || err}`);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
303
385
|
}
|
|
304
386
|
|
|
305
|
-
hasCompletionFingerprint(fingerprint: string): boolean {
|
|
387
|
+
hasCompletionFingerprint(meshId: string, fingerprint: string): boolean {
|
|
306
388
|
const now = Date.now();
|
|
389
|
+
// Scope by mesh_id (defense-in-depth) AS WELL AS the fingerprint string, whose
|
|
390
|
+
// first '::' segment already encodes meshId. A fingerprint can only suppress a
|
|
391
|
+
// duplicate within its own mesh.
|
|
307
392
|
const row = this.db
|
|
308
|
-
.prepare('SELECT 1 FROM mesh_completion_fingerprints WHERE fingerprint = ? AND expires_at > ?')
|
|
309
|
-
.get(fingerprint, now) as { 1: number } | undefined;
|
|
393
|
+
.prepare('SELECT 1 FROM mesh_completion_fingerprints WHERE mesh_id = ? AND fingerprint = ? AND expires_at > ?')
|
|
394
|
+
.get(meshId, fingerprint, now) as { 1: number } | undefined;
|
|
310
395
|
// Sweep expired fingerprints every 100 reads so stale rows don't accumulate
|
|
311
396
|
// even during read-heavy (non-write) periods when recordFingerprintSeen is idle.
|
|
312
397
|
if (++this.fingerprintSweepCounter >= 100) {
|
|
@@ -316,10 +401,10 @@ export class MeshRuntimeStore {
|
|
|
316
401
|
return row !== undefined;
|
|
317
402
|
}
|
|
318
403
|
|
|
319
|
-
recordCompletionFingerprint(fingerprint: string, ttlMs: number): void {
|
|
404
|
+
recordCompletionFingerprint(meshId: string, fingerprint: string, ttlMs: number): void {
|
|
320
405
|
const expiresAt = Date.now() + ttlMs;
|
|
321
|
-
this.db.prepare('INSERT OR REPLACE INTO mesh_completion_fingerprints (fingerprint, expires_at) VALUES (?, ?)')
|
|
322
|
-
.run(fingerprint, expiresAt);
|
|
406
|
+
this.db.prepare('INSERT OR REPLACE INTO mesh_completion_fingerprints (fingerprint, expires_at, mesh_id) VALUES (?, ?, ?)')
|
|
407
|
+
.run(fingerprint, expiresAt, meshId);
|
|
323
408
|
this.maybeCheckpointWal();
|
|
324
409
|
}
|
|
325
410
|
|
|
@@ -969,15 +1054,17 @@ export class MeshRuntimeStore {
|
|
|
969
1054
|
|
|
970
1055
|
// ── Remote Idle Sessions ─────────────────────────────────────────────────
|
|
971
1056
|
|
|
972
|
-
setRemoteIdleSession(nodeId: string, sessionId: string, providerType: string, expiresAt: number, metadata?: any): void {
|
|
1057
|
+
setRemoteIdleSession(meshId: string, nodeId: string, sessionId: string, providerType: string, expiresAt: number, metadata?: any): void {
|
|
973
1058
|
this.db.prepare(`
|
|
974
|
-
INSERT OR REPLACE INTO remote_idle_sessions (node_id, session_id, provider_type, expires_at, metadata)
|
|
975
|
-
VALUES (?, ?, ?, ?, ?)
|
|
976
|
-
`).run(nodeId, sessionId, providerType, expiresAt, metadata ? JSON.stringify(metadata) : null);
|
|
1059
|
+
INSERT OR REPLACE INTO remote_idle_sessions (mesh_id, node_id, session_id, provider_type, expires_at, metadata)
|
|
1060
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
1061
|
+
`).run(meshId, nodeId, sessionId, providerType, expiresAt, metadata ? JSON.stringify(metadata) : null);
|
|
977
1062
|
}
|
|
978
1063
|
|
|
979
|
-
getRemoteIdleSessions(): Array<{ nodeId: string; sessionId: string; providerType: string; expiresAt: number; metadata?: any }> {
|
|
980
|
-
|
|
1064
|
+
getRemoteIdleSessions(meshId: string): Array<{ nodeId: string; sessionId: string; providerType: string; expiresAt: number; metadata?: any }> {
|
|
1065
|
+
// MESH-ISOLATION-LEAK: always mesh-scoped. A bare (cross-mesh) read here is what
|
|
1066
|
+
// let mesh B claim mesh A's idle session when both share a nodeId.
|
|
1067
|
+
const rows = this.db.prepare('SELECT node_id, session_id, provider_type, expires_at, metadata FROM remote_idle_sessions WHERE mesh_id = ?').all(meshId) as Array<any>;
|
|
981
1068
|
return rows.map(r => ({
|
|
982
1069
|
nodeId: r.node_id,
|
|
983
1070
|
sessionId: r.session_id,
|
|
@@ -987,8 +1074,8 @@ export class MeshRuntimeStore {
|
|
|
987
1074
|
}));
|
|
988
1075
|
}
|
|
989
1076
|
|
|
990
|
-
deleteRemoteIdleSession(nodeId: string, sessionId: string): void {
|
|
991
|
-
this.db.prepare('DELETE FROM remote_idle_sessions WHERE node_id = ? AND session_id = ?').run(nodeId, sessionId);
|
|
1077
|
+
deleteRemoteIdleSession(meshId: string, nodeId: string, sessionId: string): void {
|
|
1078
|
+
this.db.prepare('DELETE FROM remote_idle_sessions WHERE mesh_id = ? AND node_id = ? AND session_id = ?').run(meshId, nodeId, sessionId);
|
|
992
1079
|
}
|
|
993
1080
|
|
|
994
1081
|
pruneExpiredRemoteIdleSessions(): void {
|
|
@@ -273,6 +273,38 @@ function isInsideQuotedSpan(text: string, matchStart: number, matchEnd: number):
|
|
|
273
273
|
return false;
|
|
274
274
|
}
|
|
275
275
|
|
|
276
|
+
/**
|
|
277
|
+
* True if the keyword at [matchStart, matchEnd) is glued directly to a Unicode
|
|
278
|
+
* *letter* on either side with no separator — i.e. it is word-internal, part of a
|
|
279
|
+
* larger word rather than a standalone command token.
|
|
280
|
+
*
|
|
281
|
+
* The keyword regexes use `\b` boundaries, which already exclude an *ASCII-letter*
|
|
282
|
+
* suffix/prefix: `\bdeploy\b` does NOT match the "deploy" inside "deployed"
|
|
283
|
+
* (`y`→`e` is not a boundary). But `\b` fires a boundary between a Latin letter
|
|
284
|
+
* and a non-Latin letter, so `deploy된` ("deployed", Korean passive), `release한`,
|
|
285
|
+
* or `version-bump됨` DO match even though they are single conjugated/compound
|
|
286
|
+
* words in prose, not shell commands. That is the exact i18n asymmetry behind
|
|
287
|
+
* GUARDRAIL-I18N-FP: the English "deployed worker" passes (mid-prose, no command
|
|
288
|
+
* context) while the Korean "deploy된 워커" was flagged because the glued CJK
|
|
289
|
+
* suffix left the bare keyword sitting at line-start, which the command-context
|
|
290
|
+
* heuristic reads as a command invocation.
|
|
291
|
+
*
|
|
292
|
+
* We restore the symmetry `\b` provides for ASCII: a forbidden keyword fused to
|
|
293
|
+
* ANY Unicode letter (Hangul, Han, Kana, accented Latin, …) is a word-internal
|
|
294
|
+
* occurrence → descriptive prose, never a command. A genuine command keyword is
|
|
295
|
+
* always followed by a non-letter — whitespace, end-of-input, a path separator,
|
|
296
|
+
* a flag/redirect, or a shell metachar (`npm run deploy`, `wrangler deploy\n`,
|
|
297
|
+
* `version-bump.sh`, `npm publish && ...`).
|
|
298
|
+
*/
|
|
299
|
+
function isGluedToLetterSuffix(text: string, matchStart: number, matchEnd: number): boolean {
|
|
300
|
+
// \p{L} = any Unicode letter; \p{M} = combining mark (e.g. Jamo/diacritics
|
|
301
|
+
// that compose with the adjacent letter). Either side counts as "glued".
|
|
302
|
+
const letterOrMark = /[\p{L}\p{M}]/u;
|
|
303
|
+
const next = matchEnd < text.length ? text[matchEnd] : '';
|
|
304
|
+
const prev = matchStart > 0 ? text[matchStart - 1] : '';
|
|
305
|
+
return letterOrMark.test(next) || letterOrMark.test(prev);
|
|
306
|
+
}
|
|
307
|
+
|
|
276
308
|
/**
|
|
277
309
|
* Decides whether a forbidden keyword match at [matchStart, matchEnd) should
|
|
278
310
|
* count as a real violation. A match counts only when it is in command context
|
|
@@ -283,6 +315,10 @@ function isInsideQuotedSpan(text: string, matchStart: number, matchEnd: number):
|
|
|
283
315
|
function isRealMutationMatch(text: string, matchStart: number, matchEnd: number): boolean {
|
|
284
316
|
if (hasNegationBefore(text, matchStart)) return false;
|
|
285
317
|
if (hasTrailingNegation(text, matchEnd)) return false;
|
|
318
|
+
// A keyword fused to a non-ASCII letter ("deploy된", "release한") is a single
|
|
319
|
+
// conjugated/compound word in prose, not a command — `\b` only guards the
|
|
320
|
+
// ASCII-letter case, so restore the same symmetry for all Unicode letters.
|
|
321
|
+
if (isGluedToLetterSuffix(text, matchStart, matchEnd)) return false;
|
|
286
322
|
// A keyword that is part of a file path (build/Release, dist/release/) or
|
|
287
323
|
// sits inside quoted prose (a commit-message citation, "포인터 bump") is a
|
|
288
324
|
// description, not a command — suppress it even if it lands at line-start.
|