@adhdev/daemon-core 0.9.82-rc.487 → 0.9.82-rc.489
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/router-refine.d.ts +14 -0
- package/dist/index.js +148 -13
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +148 -13
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-event-forwarding.d.ts +9 -0
- package/dist/mesh/mesh-work-queue.d.ts +11 -0
- package/dist/providers/cli-provider-instance.d.ts +9 -3
- package/dist/providers/provider-instance-manager.d.ts +1 -0
- package/dist/providers/provider-instance.d.ts +1 -0
- package/package.json +3 -3
- package/src/commands/cli-manager.ts +4 -0
- package/src/commands/router-refine.ts +60 -2
- package/src/mesh/mesh-event-forwarding.ts +92 -0
- package/src/mesh/mesh-queue-assignment.ts +6 -0
- package/src/mesh/mesh-runtime-store.ts +5 -0
- package/src/mesh/mesh-work-queue.ts +18 -0
- package/src/providers/cli-provider-instance.ts +27 -11
- package/src/providers/provider-instance-manager.ts +1 -1
- package/src/providers/provider-instance.ts +1 -1
|
@@ -13,12 +13,21 @@ export declare function handleMeshForwardEvent(components: DaemonComponents, pay
|
|
|
13
13
|
forwarded: number;
|
|
14
14
|
suppressed: boolean;
|
|
15
15
|
autoApprovingWorkerApproval: boolean;
|
|
16
|
+
staleDispatchRejected?: undefined;
|
|
17
|
+
error?: undefined;
|
|
18
|
+
} | {
|
|
19
|
+
success: boolean;
|
|
20
|
+
forwarded: number;
|
|
21
|
+
suppressed: boolean;
|
|
22
|
+
staleDispatchRejected: boolean;
|
|
23
|
+
autoApprovingWorkerApproval?: undefined;
|
|
16
24
|
error?: undefined;
|
|
17
25
|
} | {
|
|
18
26
|
success: boolean;
|
|
19
27
|
forwarded: number;
|
|
20
28
|
suppressed?: undefined;
|
|
21
29
|
autoApprovingWorkerApproval?: undefined;
|
|
30
|
+
staleDispatchRejected?: undefined;
|
|
22
31
|
error?: undefined;
|
|
23
32
|
} | {
|
|
24
33
|
success: boolean;
|
|
@@ -179,6 +179,17 @@ export interface MeshWorkQueueEntry {
|
|
|
179
179
|
};
|
|
180
180
|
/** ISO timestamp when the task was dispatched (assigned) to a node/session. Used for precise matching on completion. */
|
|
181
181
|
dispatchTimestamp?: string;
|
|
182
|
+
/**
|
|
183
|
+
* REDRIVE-DUP: monotonic per-task dispatch nonce. Bumped on every (re)dispatch of
|
|
184
|
+
* this task (assignQueueTask) AND on every reclaim (reclaimStrandedAssignedTask), and
|
|
185
|
+
* carried to the worker in meshContext.dispatchNonce. The worker echoes it back on
|
|
186
|
+
* agent:generating_started (metadataEvent.dispatchNonce). When a delivered-not-consumed
|
|
187
|
+
* task is reclaimed and re-dispatched to a different node, the ORIGINAL inject to the
|
|
188
|
+
* first node still carries the now-stale nonce; the coordinator rejects that node's
|
|
189
|
+
* generating_started ack (and stops it) so the SAME taskId is never executed twice.
|
|
190
|
+
* Absent on legacy rows → the coordinator skips the stale-nonce guard (backward safe).
|
|
191
|
+
*/
|
|
192
|
+
dispatchNonce?: number;
|
|
182
193
|
/**
|
|
183
194
|
* (3) The ORIGINATING coordinator session that enqueued this task. Stamped onto the
|
|
184
195
|
* worker at dispatch (meshCoordinatorSessionId) so the task's completion routes back to
|
|
@@ -102,9 +102,14 @@ export declare class CliProviderInstance implements ProviderInstance {
|
|
|
102
102
|
* INVARIANT (do not regress): must be STRICTLY GREATER than
|
|
103
103
|
* AUTO_APPROVE_FLAP_CONTINUITY_MS + max_busy_phase + AUTO_APPROVE_SETTLE_MS so
|
|
104
104
|
* that during a flap the settle clock (which FLAP_CONTINUITY keeps alive across
|
|
105
|
-
* each
|
|
106
|
-
*
|
|
107
|
-
* busy ~4.3–4.5s
|
|
105
|
+
* each busy phase) gets to accrue its 600ms on the RETURNING approval frame
|
|
106
|
+
* before this stall bound can trip. Observed geometry — worker: approval ~1.5s,
|
|
107
|
+
* busy ~4.3–4.5s; coordinator self-session: approval ~1.5s, busy ~2.85s. Both
|
|
108
|
+
* now use the extended window (isAutonomousMeshSession covers worker +
|
|
109
|
+
* meshCoordinatorFor). Worst case: CONTINUITY(6000) + busy(~4.5s) + SETTLE(600)
|
|
110
|
+
* = ~11100ms, so the stall bound must exceed that. 10500ms satisfies the invariant
|
|
111
|
+
* for coordinator (6000 + 2850 + 600 = 9450 < 10500) and was previously 9000ms
|
|
112
|
+
* (which failed for a worker busy phase of 4.5s: 6000+4500+600=11100 > 9000).
|
|
108
113
|
* the old 4500ms tripped inside the very first busy phase (while modal=none, so
|
|
109
114
|
* the nudge was NOT deferred) and leaked to the coordinator.
|
|
110
115
|
*/
|
|
@@ -201,6 +206,7 @@ export declare class CliProviderInstance implements ProviderInstance {
|
|
|
201
206
|
meshId: string;
|
|
202
207
|
nodeId?: string;
|
|
203
208
|
taskId?: string;
|
|
209
|
+
dispatchNonce?: number;
|
|
204
210
|
coordinatorDaemonId?: string;
|
|
205
211
|
coordinatorSessionId?: string;
|
|
206
212
|
}): void;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adhdev/daemon-core",
|
|
3
|
-
"version": "0.9.82-rc.
|
|
3
|
+
"version": "0.9.82-rc.489",
|
|
4
4
|
"description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -47,8 +47,8 @@
|
|
|
47
47
|
"author": "vilmire",
|
|
48
48
|
"license": "AGPL-3.0-or-later",
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@adhdev/mesh-shared": "0.9.82-rc.
|
|
51
|
-
"@adhdev/session-host-core": "0.9.82-rc.
|
|
50
|
+
"@adhdev/mesh-shared": "0.9.82-rc.489",
|
|
51
|
+
"@adhdev/session-host-core": "0.9.82-rc.489",
|
|
52
52
|
"@agentclientprotocol/sdk": "^0.16.1",
|
|
53
53
|
"ajv": "^8.20.0",
|
|
54
54
|
"ajv-formats": "^3.0.1",
|
|
@@ -1725,6 +1725,10 @@ export class DaemonCliManager {
|
|
|
1725
1725
|
meshId: meshContext.meshId,
|
|
1726
1726
|
...(typeof meshContext.nodeId === 'string' && meshContext.nodeId ? { nodeId: meshContext.nodeId } : {}),
|
|
1727
1727
|
...(typeof meshContext.taskId === 'string' && meshContext.taskId ? { taskId: meshContext.taskId } : {}),
|
|
1728
|
+
// REDRIVE-DUP: carry the dispatch nonce onto the worker session so
|
|
1729
|
+
// its generating_started event echoes it back for the coordinator's
|
|
1730
|
+
// stale-nonce guard.
|
|
1731
|
+
...(typeof meshContext.dispatchNonce === 'number' ? { dispatchNonce: meshContext.dispatchNonce } : {}),
|
|
1728
1732
|
...(typeof meshContext.coordinatorDaemonId === 'string' && meshContext.coordinatorDaemonId ? { coordinatorDaemonId: meshContext.coordinatorDaemonId } : {}),
|
|
1729
1733
|
});
|
|
1730
1734
|
} catch { /* best-effort — stamping is a routing aid, not a hard requirement */ }
|
|
@@ -82,7 +82,65 @@ export function buildRefineJobHandle(self: DaemonCommandRouter, args: {
|
|
|
82
82
|
};
|
|
83
83
|
}
|
|
84
84
|
|
|
85
|
+
/**
|
|
86
|
+
* Slim the terminal-stage refine result down to the fields a coordinator needs to
|
|
87
|
+
* decide next-step, dropping the heavy per-command / per-entry detail.
|
|
88
|
+
*
|
|
89
|
+
* The full `CommandRouterResult` (with `validationSummary.commandsRun[]` carrying
|
|
90
|
+
* per-command stdout/stderr, `rejectedCommands`, `suggestions`, `suggestedConfig`,
|
|
91
|
+
* the full `patchEquivalence`, and `submoduleReachability.entries[]`/`.unreachable[]`)
|
|
92
|
+
* routinely exceeds 70KB and overflows the coordinator token limit when it rides on a
|
|
93
|
+
* `mesh_wait_events` payload. The full detail is still persisted verbatim to the ledger
|
|
94
|
+
* (`appendRefineJobLedger`) and `terminalRefineJobs`, so slimming only the EVENT loses
|
|
95
|
+
* nothing — the coordinator can pull the full record on demand via
|
|
96
|
+
* `evidence.ledgerCommand` / `taskHistoryKind`.
|
|
97
|
+
*/
|
|
98
|
+
export function slimRefineEventResult(result: Record<string, unknown>): Record<string, unknown> {
|
|
99
|
+
const slim: Record<string, unknown> = {};
|
|
100
|
+
// Top-level scalars the coordinator branches on.
|
|
101
|
+
for (const key of [
|
|
102
|
+
'success', 'code', 'error', 'convergenceStatus', 'blockedReason',
|
|
103
|
+
'branch', 'into', 'terminalKind', 'nextStep', 'finalBranchConvergenceState',
|
|
104
|
+
] as const) {
|
|
105
|
+
if (result[key] !== undefined) slim[key] = result[key];
|
|
106
|
+
}
|
|
107
|
+
// Mapped subset of the unreachable-submodule commits (path + autoPublishAllowed),
|
|
108
|
+
// not the full commit records.
|
|
109
|
+
if (Array.isArray(result.unreachableSubmoduleCommits)) {
|
|
110
|
+
slim.unreachableSubmoduleCommits = (result.unreachableSubmoduleCommits as Array<Record<string, unknown>>)
|
|
111
|
+
.map(e => ({ path: e?.path, autoPublishAllowed: e?.autoPublishAllowed }));
|
|
112
|
+
}
|
|
113
|
+
// Reduced validation summary — status + failure classification + config source
|
|
114
|
+
// + a count of commands run (drop the full commandsRun/rejectedCommands/
|
|
115
|
+
// suggestions/suggestedConfig detail).
|
|
116
|
+
if (result.validationSummary && typeof result.validationSummary === 'object') {
|
|
117
|
+
const vs = result.validationSummary as Record<string, unknown>;
|
|
118
|
+
slim.validationSummary = {
|
|
119
|
+
status: vs.status,
|
|
120
|
+
failureCode: vs.failureCode,
|
|
121
|
+
configSource: vs.configSource,
|
|
122
|
+
configSourceType: vs.configSourceType,
|
|
123
|
+
commandsRunCount: Array.isArray(vs.commandsRun) ? vs.commandsRun.length : undefined,
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
// Reduce patch-equivalence to just its verdict.
|
|
127
|
+
if (result.patchEquivalence && typeof result.patchEquivalence === 'object') {
|
|
128
|
+
const pe = result.patchEquivalence as Record<string, unknown>;
|
|
129
|
+
slim.patchEquivalence = { status: pe.status, equivalent: pe.equivalent };
|
|
130
|
+
}
|
|
131
|
+
// Reduce submodule reachability to counts; drop the full entries/unreachable arrays.
|
|
132
|
+
if (result.submoduleReachability && typeof result.submoduleReachability === 'object') {
|
|
133
|
+
const sr = result.submoduleReachability as Record<string, unknown>;
|
|
134
|
+
slim.submoduleReachability = {
|
|
135
|
+
checked: Array.isArray(sr.entries) ? sr.entries.length : undefined,
|
|
136
|
+
unreachable: Array.isArray(sr.unreachable) ? sr.unreachable.length : undefined,
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
return slim;
|
|
140
|
+
}
|
|
141
|
+
|
|
85
142
|
export function queueRefineJobEvent(self: DaemonCommandRouter, event: 'refine:accepted' | 'refine:completed' | 'refine:failed', handle: MeshRefineJobHandle, result?: Record<string, unknown>): void {
|
|
143
|
+
const slimResult = result ? slimRefineEventResult(result) : undefined;
|
|
86
144
|
const metadataEvent = {
|
|
87
145
|
source: 'refine_mesh_node_async_job',
|
|
88
146
|
jobId: handle.jobId,
|
|
@@ -95,7 +153,7 @@ export function queueRefineJobEvent(self: DaemonCommandRouter, event: 'refine:ac
|
|
|
95
153
|
startedAt: handle.startedAt,
|
|
96
154
|
completedAt: handle.completedAt,
|
|
97
155
|
retryOfJobId: handle.retryOfJobId,
|
|
98
|
-
...(
|
|
156
|
+
...(slimResult ? { result: slimResult } : {}),
|
|
99
157
|
};
|
|
100
158
|
const eventPayload = {
|
|
101
159
|
event,
|
|
@@ -122,7 +180,7 @@ export function queueRefineJobEvent(self: DaemonCommandRouter, event: 'refine:ac
|
|
|
122
180
|
startedAt: handle.startedAt,
|
|
123
181
|
completedAt: handle.completedAt,
|
|
124
182
|
retryOfJobId: handle.retryOfJobId,
|
|
125
|
-
...(
|
|
183
|
+
...(slimResult ? { result: slimResult } : {}),
|
|
126
184
|
},
|
|
127
185
|
);
|
|
128
186
|
if (forwarded?.success === true) return;
|
|
@@ -24,6 +24,7 @@ import {
|
|
|
24
24
|
buildNoProgressCompletionReconciliation,
|
|
25
25
|
} from './mesh-events-stale.js';
|
|
26
26
|
import { endTaskDispatchInFlight } from './mesh-task-inflight.js';
|
|
27
|
+
import { readMeshNodeDaemonId } from './mesh-node-identity.js';
|
|
27
28
|
import {
|
|
28
29
|
buildMeshSystemMessage,
|
|
29
30
|
readNonEmptyString,
|
|
@@ -716,6 +717,58 @@ function sourceWorkerAutoApproves(components: DaemonComponents, sessionId: strin
|
|
|
716
717
|
}
|
|
717
718
|
}
|
|
718
719
|
|
|
720
|
+
/**
|
|
721
|
+
* REDRIVE-DUP: stop a worker session that started a STALE (reclaimed) mesh dispatch,
|
|
722
|
+
* so it discards the reclaimed task before it double-executes it. Prefers the local
|
|
723
|
+
* transport when the session's adapter lives on this daemon; otherwise forwards a
|
|
724
|
+
* `stop_cli` to the worker node's daemon over P2P (best-effort — a failed stop only
|
|
725
|
+
* loses the belt-and-suspenders stop; the ack was already rejected, so the coordinator
|
|
726
|
+
* never treats the stale run as the authoritative execution).
|
|
727
|
+
*/
|
|
728
|
+
function stopStaleMeshWorker(
|
|
729
|
+
components: DaemonComponents,
|
|
730
|
+
args: { meshId: string; sessionId: string; nodeId?: string; providerType?: string; daemonId?: string },
|
|
731
|
+
): void {
|
|
732
|
+
const { meshId, sessionId, providerType } = args;
|
|
733
|
+
const stopArgs: Record<string, unknown> = {
|
|
734
|
+
targetSessionId: sessionId,
|
|
735
|
+
...(providerType ? { cliType: providerType } : {}),
|
|
736
|
+
mode: 'hard',
|
|
737
|
+
reason: 'stale_mesh_dispatch_reclaimed',
|
|
738
|
+
};
|
|
739
|
+
try {
|
|
740
|
+
const isLocal = components.cliManager?.adapters?.has?.(sessionId) === true;
|
|
741
|
+
if (isLocal) {
|
|
742
|
+
// cliType is required by stop_cli; resolve it from the local adapter when the
|
|
743
|
+
// event carried no providerType.
|
|
744
|
+
if (!stopArgs.cliType) {
|
|
745
|
+
const localType = components.cliManager?.adapters?.get?.(sessionId)?.cliType;
|
|
746
|
+
if (localType) stopArgs.cliType = localType;
|
|
747
|
+
}
|
|
748
|
+
Promise.resolve(components.cliManager?.handleCliCommand?.('stop_cli', stopArgs))
|
|
749
|
+
.catch((e: any) => LOG.warn('MeshQueue', `Local stop of stale worker ${sessionId} failed: ${e?.message || e}`));
|
|
750
|
+
return;
|
|
751
|
+
}
|
|
752
|
+
// Remote: resolve the worker node's daemon id (event metadata first, then the mesh node).
|
|
753
|
+
let daemonId = args.daemonId;
|
|
754
|
+
if (!daemonId && args.nodeId) {
|
|
755
|
+
try {
|
|
756
|
+
const mesh = getMeshWithCache(components, meshId);
|
|
757
|
+
const node = mesh?.nodes?.find((n: any) => meshNodeIdMatches(n, args.nodeId!));
|
|
758
|
+
daemonId = node ? readMeshNodeDaemonId(node) || undefined : undefined;
|
|
759
|
+
} catch { /* best-effort */ }
|
|
760
|
+
}
|
|
761
|
+
if (daemonId && components.dispatchMeshCommand) {
|
|
762
|
+
Promise.resolve(components.dispatchMeshCommand(daemonId, 'stop_cli', stopArgs))
|
|
763
|
+
.catch((e: any) => LOG.warn('MeshQueue', `Remote stop of stale worker ${sessionId} on daemon ${daemonId} failed: ${e?.message || e}`));
|
|
764
|
+
} else {
|
|
765
|
+
LOG.warn('MeshQueue', `Cannot stop stale worker ${sessionId}: no local adapter and no resolvable remote daemon id (node ${args.nodeId ?? '?'}). Ack already rejected — task will re-strand-and-fail if the worker completes.`);
|
|
766
|
+
}
|
|
767
|
+
} catch (e: any) {
|
|
768
|
+
LOG.warn('MeshQueue', `stopStaleMeshWorker error for ${sessionId}: ${e?.message || e}`);
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
|
|
719
772
|
function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
720
773
|
meshId: string;
|
|
721
774
|
sourceInstanceId?: string;
|
|
@@ -1115,6 +1168,45 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
1115
1168
|
// sibling must keep that row 'dispatched' so its own confirm can match it; acking
|
|
1116
1169
|
// by session would mark it 'acked' prematurely and hide a genuine non-delivery.
|
|
1117
1170
|
const startedTaskId = readNonEmptyString(args.metadataEvent.taskId) || undefined;
|
|
1171
|
+
// REDRIVE-DUP: reject a STALE dispatch. When a delivered-but-unconsumed task is
|
|
1172
|
+
// reclaimed (reclaimStrandedAssignedTask) and re-dispatched to another node, the
|
|
1173
|
+
// ORIGINAL inject to the first node is not cancelled — it can still fire and make
|
|
1174
|
+
// that worker start the SAME taskId, double-executing it. The reclaim bumped the
|
|
1175
|
+
// task row's dispatchNonce, so the stranded inject's generating_started echoes a
|
|
1176
|
+
// nonce STRICTLY LESS than the row's current value. Detect that here: skip the ack
|
|
1177
|
+
// (do NOT resurrect the row onto this stale session) and stop the worker so it
|
|
1178
|
+
// discards the reclaimed task. A matching/greater nonce, or an absent nonce
|
|
1179
|
+
// (legacy worker), falls through to the normal ack — backward safe.
|
|
1180
|
+
const startedNonce = typeof args.metadataEvent.dispatchNonce === 'number'
|
|
1181
|
+
? args.metadataEvent.dispatchNonce
|
|
1182
|
+
: undefined;
|
|
1183
|
+
if (startedTaskId && startedNonce !== undefined) {
|
|
1184
|
+
const currentRow = (() => {
|
|
1185
|
+
try { return MeshRuntimeStore.getInstance().findQueueEntryById(args.meshId, startedTaskId); }
|
|
1186
|
+
catch { return null; }
|
|
1187
|
+
})();
|
|
1188
|
+
const currentNonce = typeof currentRow?.dispatchNonce === 'number' ? currentRow.dispatchNonce : undefined;
|
|
1189
|
+
if (currentNonce !== undefined && startedNonce < currentNonce) {
|
|
1190
|
+
LOG.warn('MeshQueue', `Rejecting stale mesh dispatch: task ${startedTaskId} generating_started from session ${sessionId} `
|
|
1191
|
+
+ `(node ${nodeId ?? '?'}) carries dispatchNonce ${startedNonce} < current ${currentNonce} — the task was reclaimed and `
|
|
1192
|
+
+ `re-dispatched; stopping this worker to prevent duplicate execution.`);
|
|
1193
|
+
traceMeshEventDrop('stale_dispatch_nonce_rejected', {
|
|
1194
|
+
taskId: startedTaskId,
|
|
1195
|
+
sessionId,
|
|
1196
|
+
nodeId,
|
|
1197
|
+
meshId: args.meshId,
|
|
1198
|
+
event: 'agent:generating_started',
|
|
1199
|
+
}, `nonce ${startedNonce} < ${currentNonce}`);
|
|
1200
|
+
stopStaleMeshWorker(components, {
|
|
1201
|
+
meshId: args.meshId,
|
|
1202
|
+
sessionId,
|
|
1203
|
+
nodeId,
|
|
1204
|
+
providerType: readNonEmptyString(args.metadataEvent.providerType) || readNonEmptyString(args.metadataEvent.cliType),
|
|
1205
|
+
daemonId: readNonEmptyString(args.metadataEvent.sourceDaemonId) || readNonEmptyString(args.metadataEvent.daemonId),
|
|
1206
|
+
});
|
|
1207
|
+
return { success: true, forwarded: 0, suppressed: true, staleDispatchRejected: true };
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1118
1210
|
// WARMUPGAP: only ack a dispatch row when the event names its task, or the session
|
|
1119
1211
|
// currently holds an active assignment. A no-taskId generating_started from an
|
|
1120
1212
|
// unassigned session is a pre-assignment warmup — the session_id fallback would ack a
|
|
@@ -588,6 +588,10 @@ export function tryAssignQueueTask(
|
|
|
588
588
|
meshId,
|
|
589
589
|
nodeId,
|
|
590
590
|
taskId: task.id,
|
|
591
|
+
// REDRIVE-DUP: carry the current dispatch nonce so the worker can echo it
|
|
592
|
+
// back on generating_started; a reclaim bumps this row's nonce, making an
|
|
593
|
+
// already-in-flight stale inject rejectable on arrival.
|
|
594
|
+
...(typeof task.dispatchNonce === 'number' ? { dispatchNonce: task.dispatchNonce } : {}),
|
|
591
595
|
...(localDaemonIdForDispatch ? { coordinatorDaemonId: localDaemonIdForDispatch } : {}),
|
|
592
596
|
...(sourceCoordinatorSessionId ? { coordinatorSessionId: sourceCoordinatorSessionId } : {}),
|
|
593
597
|
},
|
|
@@ -668,6 +672,8 @@ export function tryAssignQueueTask(
|
|
|
668
672
|
meshId,
|
|
669
673
|
nodeId,
|
|
670
674
|
taskId: task.id,
|
|
675
|
+
// REDRIVE-DUP: carry the current dispatch nonce (see remote branch above).
|
|
676
|
+
...(typeof task.dispatchNonce === 'number' ? { dispatchNonce: task.dispatchNonce } : {}),
|
|
671
677
|
...(localCoordinatorDaemonId() ? { coordinatorDaemonId: localCoordinatorDaemonId() } : {}),
|
|
672
678
|
...(readNonEmptyString(task.sourceCoordinatorSessionId) ? { coordinatorSessionId: readNonEmptyString(task.sourceCoordinatorSessionId) } : {}),
|
|
673
679
|
},
|
|
@@ -1043,6 +1043,11 @@ export class MeshRuntimeStore {
|
|
|
1043
1043
|
entry.assignedSessionId = sessionId;
|
|
1044
1044
|
if (providerType) entry.assignedProviderType = providerType;
|
|
1045
1045
|
entry.dispatchTimestamp = now;
|
|
1046
|
+
// REDRIVE-DUP: bump the per-task dispatch nonce on every claim so this dispatch
|
|
1047
|
+
// carries a nonce strictly greater than any prior (reclaimed) dispatch of the same
|
|
1048
|
+
// task. The worker echoes it on agent:generating_started; the coordinator rejects a
|
|
1049
|
+
// stale-nonce ack so a reclaimed+re-dispatched task's original inject cannot execute.
|
|
1050
|
+
entry.dispatchNonce = (entry.dispatchNonce || 0) + 1;
|
|
1046
1051
|
entry.updatedAt = now;
|
|
1047
1052
|
|
|
1048
1053
|
this.db.prepare(`
|
|
@@ -644,6 +644,17 @@ export interface MeshWorkQueueEntry {
|
|
|
644
644
|
};
|
|
645
645
|
/** ISO timestamp when the task was dispatched (assigned) to a node/session. Used for precise matching on completion. */
|
|
646
646
|
dispatchTimestamp?: string;
|
|
647
|
+
/**
|
|
648
|
+
* REDRIVE-DUP: monotonic per-task dispatch nonce. Bumped on every (re)dispatch of
|
|
649
|
+
* this task (assignQueueTask) AND on every reclaim (reclaimStrandedAssignedTask), and
|
|
650
|
+
* carried to the worker in meshContext.dispatchNonce. The worker echoes it back on
|
|
651
|
+
* agent:generating_started (metadataEvent.dispatchNonce). When a delivered-not-consumed
|
|
652
|
+
* task is reclaimed and re-dispatched to a different node, the ORIGINAL inject to the
|
|
653
|
+
* first node still carries the now-stale nonce; the coordinator rejects that node's
|
|
654
|
+
* generating_started ack (and stops it) so the SAME taskId is never executed twice.
|
|
655
|
+
* Absent on legacy rows → the coordinator skips the stale-nonce guard (backward safe).
|
|
656
|
+
*/
|
|
657
|
+
dispatchNonce?: number;
|
|
647
658
|
/**
|
|
648
659
|
* (3) The ORIGINATING coordinator session that enqueued this task. Stamped onto the
|
|
649
660
|
* worker at dispatch (meshCoordinatorSessionId) so the task's completion routes back to
|
|
@@ -1386,6 +1397,13 @@ export function reclaimStrandedAssignedTask(
|
|
|
1386
1397
|
delete entry.assignedSessionId;
|
|
1387
1398
|
delete entry.assignedProviderType;
|
|
1388
1399
|
delete entry.dispatchTimestamp;
|
|
1400
|
+
// REDRIVE-DUP: bump the dispatch nonce so the ORIGINAL inject to prevNode/prevSession
|
|
1401
|
+
// (which is delivered-but-unconsumed and about to be re-dispatched elsewhere) now
|
|
1402
|
+
// carries a stale nonce. When that stranded inject finally fires and the worker emits
|
|
1403
|
+
// agent:generating_started echoing the old nonce, the coordinator's stale-nonce guard
|
|
1404
|
+
// rejects the ack and stops that worker — so the reclaimed+re-dispatched task is never
|
|
1405
|
+
// executed by the originally-assigned session (no duplicate execution).
|
|
1406
|
+
entry.dispatchNonce = (entry.dispatchNonce || 0) + 1;
|
|
1389
1407
|
entry.strandedReclaimCount = reclaims;
|
|
1390
1408
|
entry.updatedAt = now;
|
|
1391
1409
|
// The stranded assignment is being torn down (→ pending or failed); end its
|
|
@@ -180,13 +180,18 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
180
180
|
* INVARIANT (do not regress): must be STRICTLY GREATER than
|
|
181
181
|
* AUTO_APPROVE_FLAP_CONTINUITY_MS + max_busy_phase + AUTO_APPROVE_SETTLE_MS so
|
|
182
182
|
* that during a flap the settle clock (which FLAP_CONTINUITY keeps alive across
|
|
183
|
-
* each
|
|
184
|
-
*
|
|
185
|
-
* busy ~4.3–4.5s
|
|
183
|
+
* each busy phase) gets to accrue its 600ms on the RETURNING approval frame
|
|
184
|
+
* before this stall bound can trip. Observed geometry — worker: approval ~1.5s,
|
|
185
|
+
* busy ~4.3–4.5s; coordinator self-session: approval ~1.5s, busy ~2.85s. Both
|
|
186
|
+
* now use the extended window (isAutonomousMeshSession covers worker +
|
|
187
|
+
* meshCoordinatorFor). Worst case: CONTINUITY(6000) + busy(~4.5s) + SETTLE(600)
|
|
188
|
+
* = ~11100ms, so the stall bound must exceed that. 10500ms satisfies the invariant
|
|
189
|
+
* for coordinator (6000 + 2850 + 600 = 9450 < 10500) and was previously 9000ms
|
|
190
|
+
* (which failed for a worker busy phase of 4.5s: 6000+4500+600=11100 > 9000).
|
|
186
191
|
* the old 4500ms tripped inside the very first busy phase (while modal=none, so
|
|
187
192
|
* the nudge was NOT deferred) and leaked to the coordinator.
|
|
188
193
|
*/
|
|
189
|
-
private static readonly AUTO_APPROVE_MASK_STALL_MS =
|
|
194
|
+
private static readonly AUTO_APPROVE_MASK_STALL_MS = 10500;
|
|
190
195
|
|
|
191
196
|
private adapter: ProviderCliAdapter;
|
|
192
197
|
private context: InstanceContext | null = null;
|
|
@@ -825,7 +830,7 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
825
830
|
* completion events silently drop because the forwarder has nothing to
|
|
826
831
|
* match against.
|
|
827
832
|
*/
|
|
828
|
-
attachMeshAssignment(assignment: { meshId: string; nodeId?: string; taskId?: string; coordinatorDaemonId?: string; coordinatorSessionId?: string }): void {
|
|
833
|
+
attachMeshAssignment(assignment: { meshId: string; nodeId?: string; taskId?: string; dispatchNonce?: number; coordinatorDaemonId?: string; coordinatorSessionId?: string }): void {
|
|
829
834
|
if (!assignment?.meshId) return;
|
|
830
835
|
this.settings = {
|
|
831
836
|
...this.settings,
|
|
@@ -838,6 +843,10 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
838
843
|
// shares this daemon. See isMeshOwnedDelegateSession's post-detach gate.
|
|
839
844
|
...(assignment.nodeId ? { meshNodeId: assignment.nodeId, meshLastNodeId: assignment.nodeId } : {}),
|
|
840
845
|
...(assignment.taskId ? { meshActiveTaskId: assignment.taskId } : {}),
|
|
846
|
+
// REDRIVE-DUP: task-level dispatch nonce, echoed on generating_started so the
|
|
847
|
+
// coordinator can reject a stale (reclaimed) dispatch. Cleared with meshActiveTaskId
|
|
848
|
+
// on detach so a subsequent unrelated turn never re-echoes a prior task's nonce.
|
|
849
|
+
...(typeof assignment.dispatchNonce === 'number' ? { meshActiveDispatchNonce: assignment.dispatchNonce } : {}),
|
|
841
850
|
...(assignment.coordinatorDaemonId ? { meshCoordinatorDaemonId: assignment.coordinatorDaemonId } : {}),
|
|
842
851
|
// Session-level routing anchor: the originating coordinator session, so this
|
|
843
852
|
// worker's completion events route back to the exact session that dispatched it.
|
|
@@ -872,17 +881,18 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
872
881
|
*/
|
|
873
882
|
detachMeshAssignment(): void {
|
|
874
883
|
if (!this.settings.meshNodeFor && !this.settings.meshActiveTaskId && !this.settings.meshNodeId) return;
|
|
875
|
-
// Session-level member: keep membership, drop only the task-level
|
|
884
|
+
// Session-level member: keep membership, drop only the task-level markers.
|
|
876
885
|
if (this.settings.launchedByCoordinator === true) {
|
|
877
886
|
if (!this.settings.meshActiveTaskId) return;
|
|
878
|
-
|
|
879
|
-
|
|
887
|
+
// REDRIVE-DUP: clear the task-level dispatch nonce with the task marker.
|
|
888
|
+
const { meshActiveTaskId, meshActiveDispatchNonce, ...rest } = this.settings;
|
|
889
|
+
void meshActiveTaskId; void meshActiveDispatchNonce;
|
|
880
890
|
this.settings = rest;
|
|
881
891
|
this.adapter.updateRuntimeSettings?.(this.settings);
|
|
882
892
|
return;
|
|
883
893
|
}
|
|
884
|
-
const { meshNodeFor, meshNodeId, meshActiveTaskId, ...rest } = this.settings;
|
|
885
|
-
void meshNodeFor; void meshActiveTaskId;
|
|
894
|
+
const { meshNodeFor, meshNodeId, meshActiveTaskId, meshActiveDispatchNonce, ...rest } = this.settings;
|
|
895
|
+
void meshNodeFor; void meshActiveTaskId; void meshActiveDispatchNonce;
|
|
886
896
|
// WTCLAIM (A): clear the active binding but PRESERVE the last bound node id
|
|
887
897
|
// (meshLastNodeId) so a later sessionless dispatch can re-adopt this idle
|
|
888
898
|
// session ONLY for the node it last served. Carry the id being cleared, or
|
|
@@ -1783,7 +1793,7 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
1783
1793
|
* genuine resolution frees the gate promptly.
|
|
1784
1794
|
*/
|
|
1785
1795
|
private autoApproveContinuityWindowMs(): number {
|
|
1786
|
-
return this.autoApproveMaskSince > 0 && this.
|
|
1796
|
+
return this.autoApproveMaskSince > 0 && this.isAutonomousMeshSession()
|
|
1787
1797
|
? CliProviderInstance.AUTO_APPROVE_FLAP_CONTINUITY_MS
|
|
1788
1798
|
: CliProviderInstance.AUTO_APPROVE_GATE_HYSTERESIS_MS;
|
|
1789
1799
|
}
|
|
@@ -3189,6 +3199,12 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
3189
3199
|
const resolved = this.completingTurnTaskId();
|
|
3190
3200
|
if (resolved) enrichedEvent.taskId = resolved;
|
|
3191
3201
|
}
|
|
3202
|
+
// REDRIVE-DUP: echo the dispatch nonce this session's active task was stamped with
|
|
3203
|
+
// so the coordinator's generating_started handler can reject a stale (reclaimed)
|
|
3204
|
+
// dispatch and stop this worker before it double-executes the reclaimed task.
|
|
3205
|
+
if (enrichedEvent.dispatchNonce === undefined && typeof this.settings.meshActiveDispatchNonce === 'number') {
|
|
3206
|
+
enrichedEvent.dispatchNonce = this.settings.meshActiveDispatchNonce;
|
|
3207
|
+
}
|
|
3192
3208
|
}
|
|
3193
3209
|
if (this.context?.emitProviderEvent) {
|
|
3194
3210
|
this.context.emitProviderEvent(enrichedEvent);
|
|
@@ -312,7 +312,7 @@ export class ProviderInstanceManager {
|
|
|
312
312
|
* applied, or `{ stamped: false, reason }` when it was refused — the instance
|
|
313
313
|
* was missing / has no attach method, or the DOUBLE-DISPATCH idempotence guard
|
|
314
314
|
* fired (the same task is already running on another live session here). */
|
|
315
|
-
attachMeshAssignmentToInstance(instanceId: string, assignment: { meshId: string; nodeId?: string; taskId?: string; coordinatorDaemonId?: string; coordinatorSessionId?: string }): { stamped: boolean; reason?: string } {
|
|
315
|
+
attachMeshAssignmentToInstance(instanceId: string, assignment: { meshId: string; nodeId?: string; taskId?: string; dispatchNonce?: number; coordinatorDaemonId?: string; coordinatorSessionId?: string }): { stamped: boolean; reason?: string } {
|
|
316
316
|
const inst = this.instances.get(instanceId);
|
|
317
317
|
if (!inst || typeof inst.attachMeshAssignment !== 'function') {
|
|
318
318
|
LOG.warn('MeshDispatch', `attachMeshAssignment skipped: instance ${instanceId} ${inst ? 'has no attach method' : 'not found'}`);
|
|
@@ -220,7 +220,7 @@ export interface ProviderInstance {
|
|
|
220
220
|
/** Stamp a direct-dispatch mesh task assignment so generating_completed
|
|
221
221
|
* events route back to the originating coordinator. Cleared by
|
|
222
222
|
* detachMeshAssignment when the task reaches a terminal state. */
|
|
223
|
-
attachMeshAssignment?(assignment: { meshId: string; nodeId?: string; taskId?: string; coordinatorDaemonId?: string; coordinatorSessionId?: string }): void;
|
|
223
|
+
attachMeshAssignment?(assignment: { meshId: string; nodeId?: string; taskId?: string; dispatchNonce?: number; coordinatorDaemonId?: string; coordinatorSessionId?: string }): void;
|
|
224
224
|
detachMeshAssignment?(): void;
|
|
225
225
|
|
|
226
226
|
/** Refresh static provider definition/scripts without restarting the live runtime. */
|