@adhdev/daemon-core 0.9.82-rc.431 → 0.9.82-rc.433
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +455 -320
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +455 -320
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-host-ownership.d.ts +21 -1
- package/dist/providers/cli-provider-instance.d.ts +31 -0
- package/package.json +2 -2
- package/src/commands/high-family/mesh-status.ts +5 -1
- package/src/mesh/mesh-event-forwarding.ts +10 -1
- package/src/mesh/mesh-host-ownership.ts +41 -3
- package/src/mesh/mesh-reconcile-loop.ts +270 -53
- package/src/providers/cli-provider-instance.ts +44 -0
|
@@ -1,6 +1,26 @@
|
|
|
1
1
|
import type { RepoMeshDaemonRole, RepoMeshHostMetadata, RepoMeshHostStatus } from '../repo-mesh-types.js';
|
|
2
2
|
export declare function normalizeMeshDaemonRole(value: unknown): RepoMeshDaemonRole | undefined;
|
|
3
|
-
|
|
3
|
+
/**
|
|
4
|
+
* Options for resolveMeshHostStatus's read-side host-pin default.
|
|
5
|
+
*
|
|
6
|
+
* `localDaemonId` is the id of the daemon evaluating the mesh (typically
|
|
7
|
+
* `deps.statusInstanceId`). When the persisted `meshHost` declares `role:'host'`
|
|
8
|
+
* but carries NO `hostDaemonId` (the first-setup miss — a host mesh whose pin was
|
|
9
|
+
* never written to config), the host daemon IS this local daemon by definition, so
|
|
10
|
+
* we synthesize `hostDaemonId = localDaemonId` (and, when the mesh has a node
|
|
11
|
+
* representing this daemon, `hostNodeId`). This is the read-side default — the SSOT
|
|
12
|
+
* is computed from the role + the evaluating daemon's identity rather than requiring
|
|
13
|
+
* a config migration to backfill every already-created mesh.
|
|
14
|
+
*
|
|
15
|
+
* HARD guard: the synthesis fires ONLY for `role:'host'`. A `role:'member'` daemon
|
|
16
|
+
* must NEVER fill itself in as host — that would make a member falsely claim
|
|
17
|
+
* coordinator/queue ownership.
|
|
18
|
+
*/
|
|
19
|
+
export interface ResolveMeshHostOptions {
|
|
20
|
+
/** Id of the daemon evaluating this mesh (e.g. deps.statusInstanceId). */
|
|
21
|
+
localDaemonId?: string;
|
|
22
|
+
}
|
|
23
|
+
export declare function resolveMeshHostStatus(mesh: unknown, opts?: ResolveMeshHostOptions): RepoMeshHostStatus;
|
|
4
24
|
export declare function isMeshHostOwner(mesh: unknown): boolean;
|
|
5
25
|
export declare function buildMeshHostRequiredFailure(mesh: unknown, operation: string): Record<string, unknown>;
|
|
6
26
|
export declare function requireMeshHostQueueOwner(opts?: {
|
|
@@ -231,6 +231,37 @@ export declare class CliProviderInstance implements ProviderInstance {
|
|
|
231
231
|
private isTransientToolConsent;
|
|
232
232
|
/** True when this session is parked on a modal awaiting a human answer. */
|
|
233
233
|
isModalParked(): boolean;
|
|
234
|
+
/**
|
|
235
|
+
* PTY-OVERTRUST-DRAIN (Defect B). The deliverability/drain status the mesh
|
|
236
|
+
* reconcile loop must consult — the RAW adapter turn-state, with the
|
|
237
|
+
* auto-approve "hold-idle" visual mask STRIPPED.
|
|
238
|
+
*
|
|
239
|
+
* getState().status overlays `autoApproveHoldIdle`/`autoApproveActive` to paint a
|
|
240
|
+
* genuinely-idle adapter as `generating` (a UI-flicker suppression while an
|
|
241
|
+
* auto-approve key-press settles — see getState() ~:800). That mask is correct
|
|
242
|
+
* for the dashboard, but the reconcile loop trusts it as "the coordinator is
|
|
243
|
+
* busy" and therefore HOLDS a worker's completion under
|
|
244
|
+
* `generating_no_idle_coordinator` even though the coordinator's PTY is at a real
|
|
245
|
+
* turn end and would accept the inject as a turn — the completion is stranded.
|
|
246
|
+
*
|
|
247
|
+
* This accessor reports the drain truth instead:
|
|
248
|
+
* - 'modal_parked' — a GENUINE human-await modal (AskUserQuestion / a non-
|
|
249
|
+
* transient tool-consent). Still excluded from drain (a force-inject here
|
|
250
|
+
* writes raw keystrokes the modal eats → data corruption). Mirrors
|
|
251
|
+
* isModalParked(), evaluated first so a parked session never reads idle.
|
|
252
|
+
* - 'idle' — the RAW adapter is at a turn end (adapter.getStatus(allowParse:false)
|
|
253
|
+
* === 'idle') and the session is not modal-parked. Drain-eligible REGARDLESS
|
|
254
|
+
* of the auto-approve mask. This is the case the mask used to hide.
|
|
255
|
+
* - 'generating' — the raw adapter is genuinely mid-turn. Held (a raw PTY write
|
|
256
|
+
* into a generating claude-cli is not consumed as a turn → data loss). The
|
|
257
|
+
* intentional removal of force-inject-into-generating is preserved.
|
|
258
|
+
* - 'other' — any other raw status (error / starting / waiting_choice handled by
|
|
259
|
+
* modal-park above). Not a drain target.
|
|
260
|
+
*
|
|
261
|
+
* Uses allowParse:false (engine.activeModal only, side-effect-free) so it never
|
|
262
|
+
* mutates the very auto-approve mask state the diagnostics read.
|
|
263
|
+
*/
|
|
264
|
+
getDrainStatus(): 'idle' | 'generating' | 'modal_parked' | 'other';
|
|
234
265
|
onEvent(event: string, data?: any): void;
|
|
235
266
|
recordAcknowledgedUserInput(input: InputEnvelope | string): void;
|
|
236
267
|
/** Drop user-input ack entries older than the dedup window so the map can't grow unbounded. */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adhdev/daemon-core",
|
|
3
|
-
"version": "0.9.82-rc.
|
|
3
|
+
"version": "0.9.82-rc.433",
|
|
4
4
|
"description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
"author": "vilmire",
|
|
47
47
|
"license": "AGPL-3.0-or-later",
|
|
48
48
|
"dependencies": {
|
|
49
|
-
"@adhdev/mesh-shared": "0.9.82-rc.
|
|
49
|
+
"@adhdev/mesh-shared": "0.9.82-rc.433",
|
|
50
50
|
"@adhdev/session-host-core": "*",
|
|
51
51
|
"@agentclientprotocol/sdk": "^0.16.1",
|
|
52
52
|
"ajv": "^8.20.0",
|
|
@@ -65,7 +65,11 @@ export const meshStatusHandlers: Record<string, HighFamilyHandler> = {
|
|
|
65
65
|
const meshRecord = await ctx.getMeshForCommand(meshId, args?.inlineMesh, { preferInline: true });
|
|
66
66
|
const mesh = meshRecord?.mesh;
|
|
67
67
|
if (!mesh) return { success: false, error: 'Mesh not found' };
|
|
68
|
-
|
|
68
|
+
// Pass the evaluating daemon's id so a host mesh whose
|
|
69
|
+
// hostDaemonId was never persisted (HOST-MISSEED-FIRSTSETUP) gets
|
|
70
|
+
// pinned to THIS daemon — the dashboard then renders M4 as host
|
|
71
|
+
// instead of falling back to 'no host yet'.
|
|
72
|
+
const meshHost = resolveMeshHostStatus(mesh, { localDaemonId: ctx.deps.statusInstanceId });
|
|
69
73
|
|
|
70
74
|
const refreshRequested = args?.refresh === true || args?.forceRefresh === true;
|
|
71
75
|
// Compact (default) elides each mission's full goal text from the
|
|
@@ -1717,7 +1717,16 @@ export function flushPendingForMeshIdleCoordinators(components: DaemonComponents
|
|
|
1717
1717
|
const modalParked = typeof (inst as any).isModalParked === 'function'
|
|
1718
1718
|
? (inst as any).isModalParked() === true
|
|
1719
1719
|
: (status === 'waiting_choice' || status === 'waiting_approval');
|
|
1720
|
-
|
|
1720
|
+
// PTY-OVERTRUST-DRAIN (Defect B): decide idle on the RAW adapter turn-state
|
|
1721
|
+
// (getDrainStatus, mask-stripped) to match the reconcile loop — getState().status
|
|
1722
|
+
// overlays the auto-approve hold-idle mask that paints a genuinely-idle coordinator
|
|
1723
|
+
// `generating`, which would make this opportunistic flush skip a real drain target.
|
|
1724
|
+
// Fall back to the masked literal for any instance without getDrainStatus().
|
|
1725
|
+
const drainStatus: string | null = typeof (inst as any).getDrainStatus === 'function'
|
|
1726
|
+
? (inst as any).getDrainStatus()
|
|
1727
|
+
: null;
|
|
1728
|
+
const idle = drainStatus !== null ? drainStatus === 'idle' : (status === 'idle');
|
|
1729
|
+
if (idle && !modalParked) {
|
|
1721
1730
|
idleCoordinators.push({ instance: inst, sessionId: readNonEmptyString(state.instanceId) });
|
|
1722
1731
|
}
|
|
1723
1732
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { daemonIdsEquivalent } from '@adhdev/mesh-shared';
|
|
1
2
|
import type { RepoMeshDaemonRole, RepoMeshHostMetadata, RepoMeshHostStatus } from '../repo-mesh-types.js';
|
|
2
3
|
|
|
3
4
|
function readObject(value: unknown): Record<string, unknown> | null {
|
|
@@ -12,7 +13,28 @@ export function normalizeMeshDaemonRole(value: unknown): RepoMeshDaemonRole | un
|
|
|
12
13
|
return value === 'host' || value === 'member' ? value : undefined;
|
|
13
14
|
}
|
|
14
15
|
|
|
15
|
-
|
|
16
|
+
/**
|
|
17
|
+
* Options for resolveMeshHostStatus's read-side host-pin default.
|
|
18
|
+
*
|
|
19
|
+
* `localDaemonId` is the id of the daemon evaluating the mesh (typically
|
|
20
|
+
* `deps.statusInstanceId`). When the persisted `meshHost` declares `role:'host'`
|
|
21
|
+
* but carries NO `hostDaemonId` (the first-setup miss — a host mesh whose pin was
|
|
22
|
+
* never written to config), the host daemon IS this local daemon by definition, so
|
|
23
|
+
* we synthesize `hostDaemonId = localDaemonId` (and, when the mesh has a node
|
|
24
|
+
* representing this daemon, `hostNodeId`). This is the read-side default — the SSOT
|
|
25
|
+
* is computed from the role + the evaluating daemon's identity rather than requiring
|
|
26
|
+
* a config migration to backfill every already-created mesh.
|
|
27
|
+
*
|
|
28
|
+
* HARD guard: the synthesis fires ONLY for `role:'host'`. A `role:'member'` daemon
|
|
29
|
+
* must NEVER fill itself in as host — that would make a member falsely claim
|
|
30
|
+
* coordinator/queue ownership.
|
|
31
|
+
*/
|
|
32
|
+
export interface ResolveMeshHostOptions {
|
|
33
|
+
/** Id of the daemon evaluating this mesh (e.g. deps.statusInstanceId). */
|
|
34
|
+
localDaemonId?: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function resolveMeshHostStatus(mesh: unknown, opts?: ResolveMeshHostOptions): RepoMeshHostStatus {
|
|
16
38
|
const meshRecord = readObject(mesh);
|
|
17
39
|
const raw = readObject(meshRecord?.meshHost);
|
|
18
40
|
const role = normalizeMeshDaemonRole(raw?.role) ?? 'host';
|
|
@@ -23,9 +45,25 @@ export function resolveMeshHostStatus(mesh: unknown): RepoMeshHostStatus {
|
|
|
23
45
|
canOwnQueue: role === 'host',
|
|
24
46
|
defaulted: !raw,
|
|
25
47
|
};
|
|
26
|
-
|
|
27
|
-
|
|
48
|
+
let hostDaemonId = readString(raw?.hostDaemonId);
|
|
49
|
+
let hostNodeId = readString(raw?.hostNodeId);
|
|
28
50
|
const hostAddress = readString(raw?.hostAddress);
|
|
51
|
+
// HOST-MISSEED-FIRSTSETUP read-side default: a host mesh with no persisted
|
|
52
|
+
// hostDaemonId is hosted by THIS daemon (role:'host' is local-relative), so
|
|
53
|
+
// fill the pin from the evaluating daemon. Member daemons are never synthesized.
|
|
54
|
+
const localDaemonId = readString(opts?.localDaemonId);
|
|
55
|
+
if (role === 'host' && !hostDaemonId && localDaemonId) {
|
|
56
|
+
hostDaemonId = localDaemonId;
|
|
57
|
+
// Anchor hostNodeId to the node representing this local daemon, when present.
|
|
58
|
+
if (!hostNodeId && Array.isArray(meshRecord?.nodes)) {
|
|
59
|
+
const selfNode = (meshRecord!.nodes as unknown[]).find(n => {
|
|
60
|
+
const nodeDaemonId = readString(readObject(n)?.daemonId);
|
|
61
|
+
return nodeDaemonId ? daemonIdsEquivalent(nodeDaemonId, localDaemonId) : false;
|
|
62
|
+
});
|
|
63
|
+
const selfNodeId = readString(readObject(selfNode)?.id);
|
|
64
|
+
if (selfNodeId) hostNodeId = selfNodeId;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
29
67
|
if (hostDaemonId) normalized.hostDaemonId = hostDaemonId;
|
|
30
68
|
if (hostNodeId) normalized.hostNodeId = hostNodeId;
|
|
31
69
|
if (hostAddress) normalized.hostAddress = hostAddress;
|
|
@@ -88,6 +88,26 @@ function resolveAutoPruneMinAgeMs(): number {
|
|
|
88
88
|
return DEFAULT_AUTO_PRUNE_MIN_AGE_MS;
|
|
89
89
|
}
|
|
90
90
|
|
|
91
|
+
// PTY-OVERTRUST-DRAIN (Defect B, fix B). Age-based escape for the
|
|
92
|
+
// `generating_no_idle_coordinator` hold. Fix A makes the drain predicate read the RAW
|
|
93
|
+
// adapter (mask-stripped), so the common mask-driven false-busy is gone. But a hold can
|
|
94
|
+
// still arise from a genuine status-source desync that fix A does not reach (e.g. the
|
|
95
|
+
// adapter raw itself momentarily reads generating while the coordinator is actually at a
|
|
96
|
+
// turn end). This is a TIME-BASED BACKSTOP: when a mesh's pending terminal events have
|
|
97
|
+
// been held this long, re-confirm the coordinator's RAW adapter idle on the tick and, if
|
|
98
|
+
// it is genuinely idle, drain ONCE. It NEVER injects into a genuinely-generating PTY —
|
|
99
|
+
// the re-confirmation gates on raw adapter idle, so the intentional removal of
|
|
100
|
+
// force-inject-into-generating (data-loss) is preserved. Default 12s = 3 reconcile ticks
|
|
101
|
+
// at the 4s cadence: long enough that a normal mid-turn settle is not pre-empted, short
|
|
102
|
+
// enough that a desync-stranded completion is not held for minutes. Env-tunable.
|
|
103
|
+
const DEFAULT_PENDING_HELD_DRAIN_ESCALATE_MS = 12_000;
|
|
104
|
+
|
|
105
|
+
function resolvePendingHeldDrainEscalateMs(): number {
|
|
106
|
+
// Floor 4s (one tick) so a mis-set env cannot make the escape race a normal settle;
|
|
107
|
+
// ceiling 5min so it cannot be disabled into a permanent strand.
|
|
108
|
+
return resolveTunedReconcileMs('MESH_PENDING_HELD_DRAIN_ESCALATE_MS', DEFAULT_PENDING_HELD_DRAIN_ESCALATE_MS, 4_000, 5 * 60_000);
|
|
109
|
+
}
|
|
110
|
+
|
|
91
111
|
function resolveReconcileIntervalMs(): number {
|
|
92
112
|
const raw = readNonEmptyString(process.env.MESH_RECONCILE_INTERVAL_MS);
|
|
93
113
|
if (raw) {
|
|
@@ -163,12 +183,58 @@ function resolveAckedDeathDeadlineMs(): number {
|
|
|
163
183
|
return resolveTunedReconcileMs('MESH_INFLIGHT_ACKED_DEATH_DEADLINE_MS', 8 * 60_000, 0, 60 * 60_000);
|
|
164
184
|
}
|
|
165
185
|
|
|
186
|
+
// ACKED-HOLD-IDLE-OVERTRUST (transcript-completion fast-track). The indefinite acked-hold above is
|
|
187
|
+
// safe but SLOW: when the worker's real generating_completed emit is dropped/lost, the only thing
|
|
188
|
+
// that promotes the missing completion is the 8-min death backstop — even though the answer has been
|
|
189
|
+
// FULLY rendered in the transcript for minutes (read_chat reports idle WITH a final visible assistant
|
|
190
|
+
// message every ~4s). Observed live: completions surfaced 144s / 492s late, both incompatible with the
|
|
191
|
+
// provider's own emit ceiling (COMPLETED_FINALIZATION_MAX_WAIT_MS 30s + NATIVE_HISTORY_MESH_IDLE_SETTLE
|
|
192
|
+
// 4s ≈ 34s). That gap = a worker that finished, whose PTY generating→idle edge / real emit was lost,
|
|
193
|
+
// held hostage to the 8-min net.
|
|
194
|
+
//
|
|
195
|
+
// Fast-track: when an acked task reads idle AND a final visible assistant message is present (the same
|
|
196
|
+
// transcript-completion evidence PHASE 4 already requires to synth), and that idle-with-final-assistant
|
|
197
|
+
// state has PERSISTED for a short continuous grace, promote the synth EARLY — ahead of the 8-min
|
|
198
|
+
// backstop. The grace is the correctness gate: a SINGLE idle read could be a mid-turn blip (PTY
|
|
199
|
+
// inter-tool-call settle, or final text rendered while the next tool call is about to start), so we
|
|
200
|
+
// require the idle-with-final-assistant signal to hold continuously for the grace window before
|
|
201
|
+
// trusting it as a genuine turn-end. Any non-idle read (generating / waiting_approval), a read
|
|
202
|
+
// failure, or the disappearance of the final assistant message RESETS the streak — so an actively
|
|
203
|
+
// streaming worker that momentarily reads idle never crosses the grace.
|
|
204
|
+
//
|
|
205
|
+
// Safety: this only changes WHEN an acked synth fires (earlier), never WHETHER it is correct —
|
|
206
|
+
// reconcileDirectDispatchCompletionFromTranscript's hasTerminalLedgerAfterDispatch makes a real
|
|
207
|
+
// emit that lands later an idempotent no-op, exactly as the death-backstop synth relies on. The
|
|
208
|
+
// death backstop (8 min) is PRESERVED unchanged as the final net; the fast-track is a faster path in
|
|
209
|
+
// front of it. The grace is set ABOVE the provider's own emit ceiling (~34s) so a worker still inside
|
|
210
|
+
// its normal finalization window is never pre-empted — we only fast-track once enough continuous idle
|
|
211
|
+
// has elapsed that a live emit would already have arrived.
|
|
212
|
+
function resolveAckedTranscriptFastTrackGraceMs(): number {
|
|
213
|
+
// Default 40s — above the provider emit ceiling (30s COMPLETED_FINALIZATION_MAX_WAIT_MS + 4s
|
|
214
|
+
// NATIVE_HISTORY_MESH_IDLE_SETTLE ≈ 34s): a genuinely-live worker would have emitted its real
|
|
215
|
+
// terminal within that window, so 40s of CONTINUOUS idle-with-final-assistant means the emit was
|
|
216
|
+
// lost, not late. Far below the 8-min death backstop, so the fast-track is the dominant path for a
|
|
217
|
+
// lost emit while the backstop remains the last-resort net. Floor 0 lets tests force an immediate
|
|
218
|
+
// fast-track; ceiling 5min keeps a mis-set env from collapsing it into the death backstop.
|
|
219
|
+
return resolveTunedReconcileMs('MESH_INFLIGHT_ACKED_TRANSCRIPT_FASTTRACK_GRACE_MS', 40_000, 0, 5 * 60_000);
|
|
220
|
+
}
|
|
221
|
+
|
|
166
222
|
// Per-task in-flight hold state for an acked dispatch:
|
|
167
223
|
// - liveConfirmedSinceAck: we have seen at least one conclusive read (idle OR generating) since
|
|
168
224
|
// the ack — proves the session is reachable, so a later read FAILURE is a genuine liveness loss
|
|
169
225
|
// rather than a node that was never reachable.
|
|
170
226
|
// - consecutiveReadFailures: streak of inconclusive read_chat results (death backstop (a)).
|
|
171
|
-
|
|
227
|
+
// - transcriptIdleSinceMs: the timestamp of the FIRST tick in the current continuous run of
|
|
228
|
+
// idle-with-final-assistant reads (ACKED-HOLD-IDLE-OVERTRUST fast-track). Cleared to undefined
|
|
229
|
+
// whenever the signal breaks (non-idle read, read failure, or no final assistant message), so a
|
|
230
|
+
// mid-turn idle blip never accumulates grace. When `now - transcriptIdleSinceMs` exceeds the
|
|
231
|
+
// fast-track grace the synth is promoted ahead of the death backstop.
|
|
232
|
+
interface AckedHoldState {
|
|
233
|
+
liveConfirmedSinceAck: boolean;
|
|
234
|
+
consecutiveReadFailures: number;
|
|
235
|
+
transcriptIdleSinceMs?: number;
|
|
236
|
+
}
|
|
237
|
+
const inFlightAckedHoldState = new Map<string, AckedHoldState>();
|
|
172
238
|
|
|
173
239
|
function inFlightSynthKey(meshId: string, taskId: string): string {
|
|
174
240
|
return `${meshId}::${taskId}`;
|
|
@@ -187,6 +253,13 @@ interface LiveCoordinator {
|
|
|
187
253
|
// routes back to the exact originating coordinator session, not a sibling on the same
|
|
188
254
|
// daemon (the multi-coordinator misroute).
|
|
189
255
|
sessionId: string;
|
|
256
|
+
// PTY-OVERTRUST-DRAIN (Defect B): drain-eligibility, decided on the RAW adapter
|
|
257
|
+
// turn-state (mask-stripped) — NOT on getState().status, which overlays the
|
|
258
|
+
// auto-approve "hold-idle" visual mask that paints a genuinely-idle coordinator
|
|
259
|
+
// `generating` and so used to strand its worker's completion. True only when the
|
|
260
|
+
// raw adapter is at a real turn end AND the session is not modal-parked. When the
|
|
261
|
+
// instance does not expose getDrainStatus() (non-CLI / older), this falls back to
|
|
262
|
+
// the masked `status === 'idle'` (the pre-fix behaviour) so nothing regresses.
|
|
190
263
|
idle: boolean;
|
|
191
264
|
// True when the coordinator session is parked on a harness modal awaiting a
|
|
192
265
|
// human answer — claude-cli AskUserQuestion (waiting_choice) or a tool-consent
|
|
@@ -318,6 +391,18 @@ function findLiveCoordinators(components: DaemonComponents): LiveCoordinator[] {
|
|
|
318
391
|
const modalParked = typeof (inst as any).isModalParked === 'function'
|
|
319
392
|
? (inst as any).isModalParked() === true
|
|
320
393
|
: (status === 'waiting_choice' || status === 'waiting_approval');
|
|
394
|
+
// PTY-OVERTRUST-DRAIN (Defect B, fix A): drain-eligible idle is decided on the
|
|
395
|
+
// RAW adapter turn-state, not getState().status. getState() overlays the
|
|
396
|
+
// auto-approve hold-idle mask that paints a genuinely-idle coordinator
|
|
397
|
+
// `generating` (a UI-flicker suppressant), and the reconcile loop used to trust
|
|
398
|
+
// that mask and HOLD the worker's completion (generating_no_idle_coordinator)
|
|
399
|
+
// even though the PTY was at a real turn end. getDrainStatus() strips the mask
|
|
400
|
+
// (raw adapter idle, modal-park preserved). Fall back to the masked literal for
|
|
401
|
+
// any instance that does not expose it (non-CLI / older) — regression-0.
|
|
402
|
+
const drainStatus: string | null = typeof (inst as any).getDrainStatus === 'function'
|
|
403
|
+
? (inst as any).getDrainStatus()
|
|
404
|
+
: null;
|
|
405
|
+
const idle = drainStatus !== null ? drainStatus === 'idle' : (status === 'idle');
|
|
321
406
|
const sessionId = readNonEmptyString(state.instanceId);
|
|
322
407
|
// ── NOTIF (B) desync diagnostic (read-only, no behavior change) ───────────
|
|
323
408
|
// The confirmed (B) defect: a coordinator whose FSM is idle (status above ===
|
|
@@ -349,7 +434,10 @@ function findLiveCoordinators(components: DaemonComponents): LiveCoordinator[] {
|
|
|
349
434
|
const lastStatus = readNonEmptyString((inst as any).lastStatus) || '?';
|
|
350
435
|
const autoApproveBusy = (inst as any).autoApproveBusy;
|
|
351
436
|
const maskSince = (inst as any).autoApproveMaskSince;
|
|
352
|
-
|
|
437
|
+
// PTY-OVERTRUST-DRAIN: include the mask-stripped drainStatus next to the three
|
|
438
|
+
// legacy sources so the divergence (getState=generating while adapterRaw=idle =
|
|
439
|
+
// the mask) is directly readable, and confirm drain now follows adapterRaw.
|
|
440
|
+
LOG.debug('MeshReconcile', `coordDiag sess=${sessionId || '?'} mesh=${meshId} getState=${status || '?'} drainStatus=${drainStatus || 'n/a'} lastStatus=${lastStatus} adapterRaw=${adapterRaw} autoApproveBusy=${autoApproveBusy === true} maskSince=${maskSince || 0}`);
|
|
353
441
|
}
|
|
354
442
|
// Modal-park transition observability: a coordinator entering modal-park is what
|
|
355
443
|
// begins holding completion events under `modal_parked`; one leaving it is what
|
|
@@ -365,7 +453,7 @@ function findLiveCoordinators(components: DaemonComponents): LiveCoordinator[] {
|
|
|
365
453
|
LOG.info('MeshReconcile', `Coordinator ${sessionId || '?'} (mesh ${meshId}) left modal-park (status=${status}) — held events will drain on this/next tick`);
|
|
366
454
|
}
|
|
367
455
|
}
|
|
368
|
-
out.push({ meshId, instance: inst, sessionId, idle
|
|
456
|
+
out.push({ meshId, instance: inst, sessionId, idle, modalParked });
|
|
369
457
|
}
|
|
370
458
|
return out;
|
|
371
459
|
}
|
|
@@ -566,6 +654,107 @@ function recordHeldTerminalEventsToLedger(
|
|
|
566
654
|
}
|
|
567
655
|
}
|
|
568
656
|
|
|
657
|
+
// PTY-OVERTRUST-DRAIN (Defect B, fix B). Age of the OLDEST queued terminal/force-inject
|
|
658
|
+
// event for a mesh, in ms — the signal the generating-hold age-escape gates on. Returns 0
|
|
659
|
+
// when there is no held terminal event (no escape needed). Best-effort: a peek failure
|
|
660
|
+
// returns 0 (no escape this tick), never throws into the tick.
|
|
661
|
+
function oldestHeldTerminalEventAgeMs(meshId: string, drainDaemonIds: string[]): number {
|
|
662
|
+
let pending: readonly PendingMeshCoordinatorEvent[];
|
|
663
|
+
try {
|
|
664
|
+
pending = getPendingMeshCoordinatorEvents(meshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
|
|
665
|
+
} catch {
|
|
666
|
+
return 0;
|
|
667
|
+
}
|
|
668
|
+
const now = Date.now();
|
|
669
|
+
let maxAge = 0;
|
|
670
|
+
for (const event of pending) {
|
|
671
|
+
if (!shouldForceInjectMeshEvent(event.event)) continue; // only terminal events matter
|
|
672
|
+
const queuedAt = typeof event.queuedAt === 'number' ? event.queuedAt : now;
|
|
673
|
+
const age = now - queuedAt;
|
|
674
|
+
if (age > maxAge) maxAge = age;
|
|
675
|
+
}
|
|
676
|
+
return maxAge;
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
// PTY-OVERTRUST-DRAIN (Defect B, fix B). Re-confirm, on the RAW adapter (mask-stripped),
|
|
680
|
+
// which of the held-as-generating coordinators is GENUINELY idle right now. A coordinator
|
|
681
|
+
// whose getDrainStatus() reads 'idle' is a real drain target the time-based escape may
|
|
682
|
+
// deliver into. One that still reads 'generating'/'modal_parked'/'other' stays held — the
|
|
683
|
+
// escape NEVER injects into a genuinely-busy PTY (that is the data-loss force-inject path
|
|
684
|
+
// intentionally removed; re-confirmation is what keeps this safe). Falls back to the
|
|
685
|
+
// coordinator's already-computed `idle` flag when the instance does not expose
|
|
686
|
+
// getDrainStatus() (non-CLI / older) — that flag is itself raw-adapter-derived post-fix-A.
|
|
687
|
+
function reconfirmGenuinelyIdleCoordinators(generating: LiveCoordinator[]): LiveCoordinator[] {
|
|
688
|
+
const out: LiveCoordinator[] = [];
|
|
689
|
+
for (const c of generating) {
|
|
690
|
+
const inst = c.instance as any;
|
|
691
|
+
const drainStatus: string | null = typeof inst?.getDrainStatus === 'function'
|
|
692
|
+
? inst.getDrainStatus()
|
|
693
|
+
: null;
|
|
694
|
+
const genuinelyIdle = drainStatus !== null ? drainStatus === 'idle' : c.idle;
|
|
695
|
+
if (genuinelyIdle) out.push({ ...c, idle: true });
|
|
696
|
+
}
|
|
697
|
+
return out;
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
// Full-drain the local pending queue for a mesh and inject every event into the given
|
|
701
|
+
// IDLE target coordinators, honouring strict session routing. Shared by the normal idle
|
|
702
|
+
// delivery path and the Defect-B age-escape so both deliver identically (one drain, one
|
|
703
|
+
// inject-per-event, strict-route hold for an unmatched session). Returns the number of
|
|
704
|
+
// events drained (0 when the queue was empty / drain failed). Callers must have already
|
|
705
|
+
// confirmed the targets are genuinely idle.
|
|
706
|
+
function drainAndInjectIntoTargets(
|
|
707
|
+
meshId: string,
|
|
708
|
+
drainDaemonIds: string[],
|
|
709
|
+
localDaemonId: string | undefined,
|
|
710
|
+
targetCoordinators: LiveCoordinator[],
|
|
711
|
+
logLabel: string,
|
|
712
|
+
): number {
|
|
713
|
+
let pendingEvents: PendingMeshCoordinatorEvent[] = [];
|
|
714
|
+
try {
|
|
715
|
+
pendingEvents = drainPendingMeshCoordinatorEvents(
|
|
716
|
+
meshId,
|
|
717
|
+
drainDaemonIds.length > 0 ? drainDaemonIds : localDaemonId,
|
|
718
|
+
);
|
|
719
|
+
} catch (e: any) {
|
|
720
|
+
LOG.warn('MeshReconcile', `Drain failed for mesh ${meshId}: ${e?.message || e}`);
|
|
721
|
+
return 0;
|
|
722
|
+
}
|
|
723
|
+
if (pendingEvents.length === 0) return 0;
|
|
724
|
+
|
|
725
|
+
LOG.info('MeshReconcile', `Reconcile inject → ${logLabel}: ${pendingEvents.length} pending event(s) → ${targetCoordinators.length} coordinator(s) for mesh ${meshId}`);
|
|
726
|
+
for (const pending of pendingEvents) {
|
|
727
|
+
// Strict session routing (multi-coordinator): when the event names an
|
|
728
|
+
// originating coordinator session, deliver ONLY to the live coordinator whose
|
|
729
|
+
// session id matches — a sibling coordinator on the same daemon must NOT receive
|
|
730
|
+
// another coordinator's completion. When the event carries no session id (legacy /
|
|
731
|
+
// version-skewed / single-coordinator), fall back to the daemon-level set
|
|
732
|
+
// (unchanged behaviour — regression-0 for the common case).
|
|
733
|
+
const wantSession = readNonEmptyString(pending.targetCoordinatorSessionId);
|
|
734
|
+
if (wantSession) {
|
|
735
|
+
// SESSION-ID IS SINGLE-FORM: a coordinator session id is one canonical
|
|
736
|
+
// UUID (crypto.randomUUID), carried verbatim end-to-end — no node/daemon-id
|
|
737
|
+
// style serialization variants. Exact `===` is the correct match; unlike
|
|
738
|
+
// the daemon-level set below it needs no equivalence helper.
|
|
739
|
+
const matched = targetCoordinators.filter(c => c.sessionId === wantSession);
|
|
740
|
+
if (matched.length === 0) {
|
|
741
|
+
// The originating coordinator session is not deliverable on this daemon
|
|
742
|
+
// right now (gone, or modal-parked and excluded from targets). Strict mode
|
|
743
|
+
// does NOT broadcast to siblings — hold the event for a later tick, and
|
|
744
|
+
// ledger-expire it past a TTL so it can never wedge forever.
|
|
745
|
+
holdOrExpireStrictUnmatchedEvent(pending, wantSession, meshId);
|
|
746
|
+
continue;
|
|
747
|
+
}
|
|
748
|
+
for (const c of matched) injectPendingIntoCoordinator(c.instance, pending);
|
|
749
|
+
continue;
|
|
750
|
+
}
|
|
751
|
+
for (const c of targetCoordinators) {
|
|
752
|
+
injectPendingIntoCoordinator(c.instance, pending);
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
return pendingEvents.length;
|
|
756
|
+
}
|
|
757
|
+
|
|
569
758
|
// One reconcile tick. Two independent phases:
|
|
570
759
|
//
|
|
571
760
|
// PHASE 1 — Remote queue pull (the fix for remote worktree completions never
|
|
@@ -974,6 +1163,31 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
974
1163
|
try { hasPending = store.pendingEventCount(meshId) > 0; } catch { /* peek below */ }
|
|
975
1164
|
}
|
|
976
1165
|
if (hasPending) {
|
|
1166
|
+
// ── PTY-OVERTRUST-DRAIN (Defect B, fix B): age-based escape ───────────
|
|
1167
|
+
// Fix A already routes the common mask-driven false-busy to the idle path,
|
|
1168
|
+
// so reaching here means the coordinator's RAW adapter reads generating.
|
|
1169
|
+
// That is almost always genuine — but a status-source desync fix A does not
|
|
1170
|
+
// reach can momentarily make the raw adapter read generating while the PTY
|
|
1171
|
+
// is actually at a turn end, stranding the completion across many ticks. As a
|
|
1172
|
+
// TIME-BASED BACKSTOP, once the oldest held terminal event has aged past the
|
|
1173
|
+
// escalate threshold, RE-CONFIRM each held coordinator's raw adapter idle and,
|
|
1174
|
+
// if genuinely idle, drain ONCE into it. The re-confirmation gate is what makes
|
|
1175
|
+
// this safe: it NEVER injects into a genuinely-generating PTY (that is the
|
|
1176
|
+
// data-loss force-inject path intentionally removed). A coordinator still
|
|
1177
|
+
// genuinely generating stays held.
|
|
1178
|
+
const escalateMs = resolvePendingHeldDrainEscalateMs();
|
|
1179
|
+
const heldAgeMs = oldestHeldTerminalEventAgeMs(
|
|
1180
|
+
meshId,
|
|
1181
|
+
drainDaemonIds.length > 0 ? drainDaemonIds : (localDaemonId ? [localDaemonId] : []),
|
|
1182
|
+
);
|
|
1183
|
+
if (heldAgeMs >= escalateMs) {
|
|
1184
|
+
const escapeTargets = reconfirmGenuinelyIdleCoordinators(generatingCoordinators);
|
|
1185
|
+
if (escapeTargets.length > 0) {
|
|
1186
|
+
LOG.info('MeshReconcile', `Reconcile age-escape → generating-hold: held terminal event(s) for mesh ${meshId} aged ${Math.round(heldAgeMs / 1000)}s (≥ ${Math.round(escalateMs / 1000)}s) and ${escapeTargets.length} coordinator(s) re-confirmed genuinely idle on the raw adapter — draining once`);
|
|
1187
|
+
const drained = drainAndInjectIntoTargets(meshId, drainDaemonIds, localDaemonId, escapeTargets, 'age-escape');
|
|
1188
|
+
if (drained > 0) continue; // delivered → no hold this tick
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
977
1191
|
LOG.info('MeshReconcile', `Reconcile skip → generating: holding pending event(s) for mesh ${meshId} (${generatingCoordinators.length} coordinator(s) busy; events left queued for the next idle tick)`);
|
|
978
1192
|
// NOTIF (B) diagnostic: this is the hold that strands the completion. Name
|
|
979
1193
|
// the sessionId(s) the loop just classified non-idle/non-modal so the
|
|
@@ -1005,48 +1219,7 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
1005
1219
|
// queued event and deliver it to the idle input box as a real turn. The no-idle case
|
|
1006
1220
|
// (generating/modal-only) was already held above and never reaches here, so there is
|
|
1007
1221
|
// no force-drain-into-generating path left — the single delivery is the idle drain.
|
|
1008
|
-
|
|
1009
|
-
try {
|
|
1010
|
-
pendingEvents = drainPendingMeshCoordinatorEvents(
|
|
1011
|
-
meshId,
|
|
1012
|
-
drainDaemonIds.length > 0 ? drainDaemonIds : localDaemonId,
|
|
1013
|
-
);
|
|
1014
|
-
} catch (e: any) {
|
|
1015
|
-
LOG.warn('MeshReconcile', `Drain failed for mesh ${meshId}: ${e?.message || e}`);
|
|
1016
|
-
continue;
|
|
1017
|
-
}
|
|
1018
|
-
if (pendingEvents.length === 0) continue;
|
|
1019
|
-
|
|
1020
|
-
LOG.info('MeshReconcile', `Reconcile inject → idle: ${pendingEvents.length} pending event(s) → ${targetCoordinators.length} coordinator(s) for mesh ${meshId}`);
|
|
1021
|
-
for (const pending of pendingEvents) {
|
|
1022
|
-
// Strict session routing (multi-coordinator): when the event names an
|
|
1023
|
-
// originating coordinator session, deliver ONLY to the live coordinator whose
|
|
1024
|
-
// session id matches — a sibling coordinator on the same daemon must NOT receive
|
|
1025
|
-
// another coordinator's completion. When the event carries no session id (legacy /
|
|
1026
|
-
// version-skewed / single-coordinator), fall back to the daemon-level set
|
|
1027
|
-
// (unchanged behaviour — regression-0 for the common case).
|
|
1028
|
-
const wantSession = readNonEmptyString(pending.targetCoordinatorSessionId);
|
|
1029
|
-
if (wantSession) {
|
|
1030
|
-
// SESSION-ID IS SINGLE-FORM: a coordinator session id is one canonical
|
|
1031
|
-
// UUID (crypto.randomUUID), carried verbatim end-to-end — no node/daemon-id
|
|
1032
|
-
// style serialization variants. Exact `===` is the correct match; unlike
|
|
1033
|
-
// the daemon-level set below it needs no equivalence helper.
|
|
1034
|
-
const matched = targetCoordinators.filter(c => c.sessionId === wantSession);
|
|
1035
|
-
if (matched.length === 0) {
|
|
1036
|
-
// The originating coordinator session is not deliverable on this daemon
|
|
1037
|
-
// right now (gone, or modal-parked and excluded from targets). Strict mode
|
|
1038
|
-
// does NOT broadcast to siblings — hold the event for a later tick, and
|
|
1039
|
-
// ledger-expire it past a TTL so it can never wedge forever.
|
|
1040
|
-
holdOrExpireStrictUnmatchedEvent(pending, wantSession, meshId);
|
|
1041
|
-
continue;
|
|
1042
|
-
}
|
|
1043
|
-
for (const c of matched) injectPendingIntoCoordinator(c.instance, pending);
|
|
1044
|
-
continue;
|
|
1045
|
-
}
|
|
1046
|
-
for (const c of targetCoordinators) {
|
|
1047
|
-
injectPendingIntoCoordinator(c.instance, pending);
|
|
1048
|
-
}
|
|
1049
|
-
}
|
|
1222
|
+
drainAndInjectIntoTargets(meshId, drainDaemonIds, localDaemonId, targetCoordinators, 'idle');
|
|
1050
1223
|
}
|
|
1051
1224
|
}
|
|
1052
1225
|
|
|
@@ -1485,6 +1658,8 @@ async function reconcileUnterminatedDirectDispatches(
|
|
|
1485
1658
|
const prior = inFlightAckedHoldState.get(synthKey);
|
|
1486
1659
|
const failures = (prior?.consecutiveReadFailures ?? 0) + 1;
|
|
1487
1660
|
const liveConfirmedSinceAck = prior?.liveConfirmedSinceAck ?? false;
|
|
1661
|
+
// A read failure breaks the idle-with-final-assistant run → reset the fast-track streak
|
|
1662
|
+
// (transcriptIdleSinceMs cleared by omission) so it must re-accumulate from scratch.
|
|
1488
1663
|
inFlightAckedHoldState.set(synthKey, { liveConfirmedSinceAck, consecutiveReadFailures: failures });
|
|
1489
1664
|
if (liveConfirmedSinceAck && failures >= ACKED_DEATH_CONSECUTIVE_READ_FAILURES) {
|
|
1490
1665
|
LOG.warn('MeshReconcile', `Acked-hold death signal: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read_chat failed ${failures}x consecutively after a live-confirmed ack — worker session presumed gone mid-turn; releasing the indefinite synth hold to the stranded-reclaim / orphan-prune nets`);
|
|
@@ -1495,8 +1670,15 @@ async function reconcileUnterminatedDirectDispatches(
|
|
|
1495
1670
|
|
|
1496
1671
|
// Read succeeded (a conclusive idle/generating status) → the session is reachable: reset the
|
|
1497
1672
|
// failure streak and mark it live-confirmed-since-ack, so a LATER read failure is recognized
|
|
1498
|
-
// as a genuine liveness loss (backstop a) rather than a node that was never reachable.
|
|
1499
|
-
|
|
1673
|
+
// as a genuine liveness loss (backstop a) rather than a node that was never reachable. The
|
|
1674
|
+
// fast-track idle streak (transcriptIdleSinceMs) is PRESERVED across this reset — it is
|
|
1675
|
+
// managed below where the idle + final-assistant signal is actually evaluated.
|
|
1676
|
+
const priorHoldState = inFlightAckedHoldState.get(synthKey);
|
|
1677
|
+
inFlightAckedHoldState.set(synthKey, {
|
|
1678
|
+
liveConfirmedSinceAck: true,
|
|
1679
|
+
consecutiveReadFailures: 0,
|
|
1680
|
+
...(priorHoldState?.transcriptIdleSinceMs !== undefined ? { transcriptIdleSinceMs: priorHoldState.transcriptIdleSinceMs } : {}),
|
|
1681
|
+
});
|
|
1500
1682
|
|
|
1501
1683
|
// Only act on a session that has actually settled to idle. A generating /
|
|
1502
1684
|
// waiting_approval session is mid-turn — synthesizing a completion now would
|
|
@@ -1504,7 +1686,9 @@ async function reconcileUnterminatedDirectDispatches(
|
|
|
1504
1686
|
const nowMs = Date.now();
|
|
1505
1687
|
if (readChatPayloadStatus(payload) !== 'idle') {
|
|
1506
1688
|
// Not idle → the worker is genuinely mid-turn (a clear live signal). Keep the
|
|
1507
|
-
// live-confirmed flag set (above) but
|
|
1689
|
+
// live-confirmed flag set (above) but RESET the fast-track idle streak: a turn that
|
|
1690
|
+
// resumed generating proves the prior idle was a mid-turn blip, not a settled turn-end.
|
|
1691
|
+
inFlightAckedHoldState.set(synthKey, { liveConfirmedSinceAck: true, consecutiveReadFailures: 0 });
|
|
1508
1692
|
continue;
|
|
1509
1693
|
}
|
|
1510
1694
|
|
|
@@ -1525,15 +1709,50 @@ async function reconcileUnterminatedDirectDispatches(
|
|
|
1525
1709
|
// A never-acked dispatch (worker never started) is exempt — no in-flight generation to
|
|
1526
1710
|
// pre-empt; it keeps the first-idle-tick synth, with the downstream grace + stale-summary
|
|
1527
1711
|
// guards as its backstops.
|
|
1712
|
+
//
|
|
1713
|
+
// ACKED-HOLD-IDLE-OVERTRUST: the read is idle. Extract the final-assistant evidence NOW (the
|
|
1714
|
+
// same signal the synth below requires) so the fast-track can gate on idle-WITH-final-assistant
|
|
1715
|
+
// rather than bare idle. Only when a final visible assistant message is present do we treat
|
|
1716
|
+
// this tick as a candidate turn-end and accumulate the fast-track grace streak; a bare idle
|
|
1717
|
+
// with no assistant result is the worker still warming up and resets the streak.
|
|
1718
|
+
const messages = Array.isArray(payload.messages) ? payload.messages as ChatMessage[] : [];
|
|
1719
|
+
const evidence = extractFinalAssistantSummaryEvidence(messages);
|
|
1720
|
+
|
|
1528
1721
|
if (isAcked) {
|
|
1529
1722
|
const ackedAtMs = Date.parse(readNonEmptyString(dispatch.updatedAt));
|
|
1530
1723
|
const sinceAckMs = Number.isFinite(ackedAtMs) ? nowMs - ackedAtMs : Number.POSITIVE_INFINITY;
|
|
1531
1724
|
const deathDeadlineMs = resolveAckedDeathDeadlineMs();
|
|
1532
|
-
|
|
1533
|
-
|
|
1725
|
+
|
|
1726
|
+
// ACKED-HOLD-IDLE-OVERTRUST fast-track. Maintain the continuous idle-with-final-assistant
|
|
1727
|
+
// streak. The streak starts (or continues) only while a final visible assistant message is
|
|
1728
|
+
// present; a tick with idle-but-no-assistant breaks it (the answer is not yet rendered).
|
|
1729
|
+
const holdState = inFlightAckedHoldState.get(synthKey);
|
|
1730
|
+
let fastTrackReady = false;
|
|
1731
|
+
if (evidence.finalSummary) {
|
|
1732
|
+
const idleSinceMs = holdState?.transcriptIdleSinceMs ?? nowMs;
|
|
1733
|
+
if (holdState && holdState.transcriptIdleSinceMs === undefined) {
|
|
1734
|
+
inFlightAckedHoldState.set(synthKey, { ...holdState, transcriptIdleSinceMs: idleSinceMs });
|
|
1735
|
+
}
|
|
1736
|
+
const fastTrackGraceMs = resolveAckedTranscriptFastTrackGraceMs();
|
|
1737
|
+
const idleHeldMs = nowMs - idleSinceMs;
|
|
1738
|
+
if (idleHeldMs >= fastTrackGraceMs) {
|
|
1739
|
+
fastTrackReady = true;
|
|
1740
|
+
LOG.info('MeshReconcile', `Acked-hold transcript fast-track: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read idle WITH a final assistant message for ${Math.round(idleHeldMs / 1000)}s continuous (grace ${Math.round(fastTrackGraceMs / 1000)}s) — promoting the synth ahead of the ${Math.round(deathDeadlineMs / 1000)}s death backstop; the worker's real emit was lost/late and a later one no-ops idempotently.`);
|
|
1741
|
+
}
|
|
1742
|
+
} else if (holdState?.transcriptIdleSinceMs !== undefined) {
|
|
1743
|
+
// Idle but no final assistant yet → not a turn-end; reset the streak.
|
|
1744
|
+
inFlightAckedHoldState.set(synthKey, { ...holdState, transcriptIdleSinceMs: undefined });
|
|
1745
|
+
}
|
|
1746
|
+
|
|
1747
|
+
// Hold indefinitely UNLESS the fast-track grace was met OR the absolute death deadline is
|
|
1748
|
+
// reached. The fast-track is the new fast path in front of the (preserved) 8-min backstop.
|
|
1749
|
+
if (!fastTrackReady && sinceAckMs < deathDeadlineMs) {
|
|
1750
|
+
LOG.info('MeshReconcile', `Acked-hold: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read idle ${Number.isFinite(sinceAckMs) ? Math.round(sinceAckMs / 1000) + 's' : '∞'} since the generating_started ack — HOLDING synth (worker presumed alive; a later real emit is idempotent). Transcript fast-track promotes at ${Math.round(resolveAckedTranscriptFastTrackGraceMs() / 1000)}s continuous idle-with-final-assistant; death backstop at ${Math.round(deathDeadlineMs / 1000)}s or on consecutive read failures.`);
|
|
1534
1751
|
continue;
|
|
1535
1752
|
}
|
|
1536
|
-
|
|
1753
|
+
if (!fastTrackReady) {
|
|
1754
|
+
LOG.warn('MeshReconcile', `Acked-hold death deadline reached: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) still idle ${Math.round(sinceAckMs / 1000)}s after the ack (deadline ${Math.round(deathDeadlineMs / 1000)}s) — synthesizing the missing completion as a notification-loss net (a real emit, if it ever lands, no-ops idempotently).`);
|
|
1755
|
+
}
|
|
1537
1756
|
}
|
|
1538
1757
|
|
|
1539
1758
|
// R4f (auxiliary, was R4e fix 3) — worker-emit priority. Secondary check: if the worker's
|
|
@@ -1550,8 +1769,6 @@ async function reconcileUnterminatedDirectDispatches(
|
|
|
1550
1769
|
continue;
|
|
1551
1770
|
}
|
|
1552
1771
|
|
|
1553
|
-
const messages = Array.isArray(payload.messages) ? payload.messages as ChatMessage[] : [];
|
|
1554
|
-
const evidence = extractFinalAssistantSummaryEvidence(messages);
|
|
1555
1772
|
if (!evidence.finalSummary) continue; // no assistant result yet — nothing to attribute
|
|
1556
1773
|
|
|
1557
1774
|
// STALE-SUMMARY guard (modal-parked / reused-session misattribution): a direct
|