@adhdev/daemon-core 0.9.82-rc.463 → 0.9.82-rc.465

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,103 @@
1
+ // ---------------------------------------------------------------------------
2
+ // mesh-reconcile-identity — daemon-id / self-identity resolution for the reconcile loop
3
+ // ---------------------------------------------------------------------------
4
+ // Pure move out of mesh-reconcile-loop.ts (no behavior change). This is the single
5
+ // source of truth for "which id-forms does THIS daemon answer to?" across both the
6
+ // coordinator-daemon drain scope and the per-mesh host gate / remote pull filter.
7
+ // ---------------------------------------------------------------------------
8
+
9
+ import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
10
+ import type { LocalMeshEntry } from '../repo-mesh-types.js';
11
+ import { loadConfig } from '../config/config.js';
12
+ import { expandDaemonIdForms, daemonIdsEquivalent } from '@adhdev/mesh-shared';
13
+ import { readNonEmptyString } from './mesh-events-utils.js';
14
+
15
+ // The set of coordinator-daemon ids THIS daemon answers to when draining the
16
+ // pending-events queue. A unicast completion event is stamped with the worker's
17
+ // meshCoordinatorDaemonId, which can be either:
18
+ // - the daemon's canonical status id (`standalone_<machineId>` / `daemon_<machineId>`),
19
+ // stamped by the MCP layer via ctx.localDaemonId (= getStatus().status.instanceId), or
20
+ // - the bare machineId, stamped by the local queue-assignment path (loadConfig().machineId).
21
+ // - the config-form node daemonId (`daemon_<machineId>`), which the MCP layer's
22
+ // resolveCoordinatorDaemonId prefers and stamps onto direct-dispatch workers.
23
+ // Draining with only one of these silently misses events stamped with the other —
24
+ // the exact reason a generating coordinator never self-received local completions,
25
+ // and the base-node completion-surface bug (base completions land full-form
26
+ // `daemon_<machineId>` while a coordinator that only knows itself as bare
27
+ // `<machineId>` never matches them). We expand to EVERY equivalent form so the
28
+ // scope match (host gate, self-node detection, and the drain IN-filter downstream)
29
+ // succeeds regardless of which path stamped the event.
30
+ export function resolveCoordinatorDaemonIds(components: DaemonComponents): string[] {
31
+ const statusInstanceId = readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId);
32
+ const machineId = readNonEmptyString(loadConfig().machineId);
33
+ return expandDaemonIdForms([statusInstanceId, machineId]);
34
+ }
35
+
36
+ // Whether THIS daemon is the coordinator/host for a mesh — i.e. the daemon that
37
+ // owns coordinator ownership and must collect every worker node's completion
38
+ // events into its local queue. This is true regardless of whether a *live CLI*
39
+ // coordinator session currently exists: the coordinator is frequently a pure
40
+ // stdio MCP LLM (no live CLI session to inject into), and that LLM only sees the
41
+ // queue when it next calls a mesh tool. For it to see remote worker completions
42
+ // at all, the daemon must have already pulled them into the local queue on the
43
+ // timer — which is exactly what this predicate gates.
44
+ //
45
+ // Rule: this daemon hosts the mesh when meshHost.role is 'host' (the default for
46
+ // standalone-compat meshes with no host metadata) AND, when a hostDaemonId is
47
+ // pinned, it resolves to one of this daemon's ids. Member-only daemons return
48
+ // false — their own queue is pulled BY the host, not the other way around.
49
+ //
50
+ // `daemonIds` here is the EXPANDED self-identity set (runtime drain ids ∪ this
51
+ // daemon's mesh-config node id forms) — see resolveCoordinatorSelfIds. The
52
+ // pinned hostDaemonId is itself a config-form id and frequently does NOT equal a
53
+ // runtime id (bare machineId / status id), so gating on the runtime ids alone
54
+ // would wrongly classify the real host as a non-host and skip the remote pull
55
+ // entirely.
56
+ export function daemonHostsMesh(mesh: LocalMeshEntry, daemonIds: string[]): boolean {
57
+ const host = mesh.meshHost;
58
+ // No metadata → default host (standalone compatibility, see createDefaultMeshHostMetadata).
59
+ if (!host) return true;
60
+ if (host.role && host.role !== 'host') return false;
61
+ const hostDaemonId = readNonEmptyString(host.hostDaemonId);
62
+ // Host role but no pinned hostDaemonId → treat as host (single-daemon / legacy).
63
+ if (!hostDaemonId) return true;
64
+ return daemonIdListIncludes(daemonIds, hostDaemonId);
65
+ }
66
+
67
+ export function daemonIdListIncludes(ids: readonly string[], id: string | undefined): boolean {
68
+ if (!id) return false;
69
+ return ids.some(candidate => candidate === id || daemonIdsEquivalent(candidate, id));
70
+ }
71
+
72
+ // Resolve EVERY id-form this daemon answers to FOR A GIVEN MESH: the runtime drain
73
+ // ids (status id + bare machineId) unioned with this daemon's mesh-config identity
74
+ // forms — the self node's daemonId/machineId (the node whose daemonId/machineId
75
+ // matches a runtime id) and the pinned meshHost.hostDaemonId WHEN it is provably
76
+ // ours. This is the single source of truth for "is this id me?" across both the
77
+ // host gate and the remote pull filter; the worker's meshCoordinatorDaemonId stamp
78
+ // is guaranteed to be one of these forms (it comes from resolveCoordinatorDaemonId,
79
+ // which prefers the coordinator node's config-form daemonId over the runtime status id).
80
+ export function resolveCoordinatorSelfIds(mesh: LocalMeshEntry, drainDaemonIds: string[]): string[] {
81
+ const ids = new Set<string>(drainDaemonIds);
82
+ // Expand with the config-form id(s) of the self node — the mesh node whose
83
+ // daemonId/machineId matches a runtime id. Its config-form daemonId is exactly
84
+ // what resolveCoordinatorNode()→resolveCoordinatorDaemonId() stamps onto a worker.
85
+ for (const node of mesh.nodes) {
86
+ const nodeDaemonId = readNonEmptyString(node.daemonId);
87
+ const nodeMachineId = readNonEmptyString(node.machineId);
88
+ const isSelf = (nodeDaemonId && daemonIdListIncludes(drainDaemonIds, nodeDaemonId))
89
+ || (nodeMachineId && daemonIdListIncludes(drainDaemonIds, nodeMachineId));
90
+ if (!isSelf) continue;
91
+ if (nodeDaemonId) ids.add(nodeDaemonId);
92
+ if (nodeMachineId) ids.add(nodeMachineId);
93
+ }
94
+ // The pinned host id is included ONLY when it is provably one of THIS daemon's ids
95
+ // (it already matches a runtime id or a resolved self-node id). A hostDaemonId that
96
+ // names a DIFFERENT daemon must NOT be claimed — that would make a member-only
97
+ // daemon believe it is the host and pull queues it does not own. Having a node on
98
+ // this daemon does not make this daemon the host; daemonHostsMesh still honours a
99
+ // foreign hostDaemonId and rejects ownership.
100
+ const hostDaemonId = readNonEmptyString(mesh.meshHost?.hostDaemonId);
101
+ if (hostDaemonId && daemonIdListIncludes([...ids], hostDaemonId)) ids.add(hostDaemonId);
102
+ return [...ids];
103
+ }
@@ -67,6 +67,33 @@ import { pruneStaleDirectDispatches } from './mesh-active-work.js';
67
67
  import { findTerminalLedgerEvidenceForTask, reconcileDirectDispatchCompletionFromTranscript } from './mesh-events-stale.js';
68
68
  import { extractFinalAssistantSummaryEvidence } from '../providers/chat-message-normalization.js';
69
69
  import type { ChatMessage } from '../types.js';
70
+ import {
71
+ resolveCoordinatorDaemonIds,
72
+ daemonHostsMesh,
73
+ daemonIdListIncludes,
74
+ resolveCoordinatorSelfIds,
75
+ } from './mesh-reconcile-identity.js';
76
+ import {
77
+ getMeshV2BackstopCounters,
78
+ recordBackstopFire,
79
+ } from './mesh-reconcile-v2-backstop.js';
80
+ import {
81
+ ACKED_DEATH_CONSECUTIVE_READ_FAILURES,
82
+ resolveTunedReconcileMs,
83
+ resolveAckedDeathDeadlineMs,
84
+ resolveAckedTranscriptFastTrackGraceMs,
85
+ inFlightSynthKey,
86
+ getHoldState,
87
+ setHoldState,
88
+ deleteHoldState,
89
+ rehydrateAckedHoldsForMesh,
90
+ collectHeldSynthKeysForMesh,
91
+ } from './mesh-reconcile-acked-hold.js';
92
+
93
+ // Re-export the extracted public API so existing importers (mesh-events.ts barrel;
94
+ // the reconcile-loop test suite) keep their `from './mesh-reconcile-loop.js'` paths.
95
+ export { getMeshV2BackstopCounters, __resetMeshV2BackstopCountersForTests } from './mesh-reconcile-v2-backstop.js';
96
+ export { __resetReconcileInFlightSynthDebounceForTests } from './mesh-reconcile-acked-hold.js';
70
97
 
71
98
  // Default reconcile cadence. approval/completion notifications to a live CLI
72
99
  // coordinator land within at most one interval. Overridable via env for tuning.
@@ -120,296 +147,6 @@ function resolveReconcileIntervalMs(): number {
120
147
  return DEFAULT_RECONCILE_INTERVAL_MS;
121
148
  }
122
149
 
123
- // R4f (GENERATING-BOUNDARY, acked-hold redesign). PHASE 4 only synthesizes a missing completion
124
- // when the worker session reads `idle`. But a worker that is GENUINELY generating (it emitted
125
- // agent:generating_started — the dispatch row is 'acked' — and has not yet completed) can
126
- // momentarily read `idle` mid-turn (a CLI PTY inter-tool-call settle, or the final assistant text
127
- // already rendered while the turn's generating_completed lifecycle close still lags). A premature
128
- // synth writes a terminal that then masks the worker's REAL completion when it lands seconds later
129
- // (drop:duplicate_completion_terminal_ledger; the observed 71s task a250fb44 lost its [System]
130
- // notification this way; the R4e 53s task synth fired 16s BEFORE the worker's real emit).
131
- //
132
- // R4 → R4e used FINITE timers (consecutive ticks / MIN_IDLE_SETTLE / ACKED_TURN_SETTLE) to delay the
133
- // synth. That class of fix is fundamentally a RACE: the worker's real emit latency is variable and
134
- // unbounded (win32 idle reads can flip before the emit arrives), so ANY finite timer eventually
135
- // loses to a slow-enough turn — and the synth pre-empts the real completion. R4e live-FAILED for
136
- // exactly this reason.
137
- //
138
- // R4f redesign (direction B). An `acked` task means the worker ECHOED generating_started (the
139
- // taskId flip) — it is alive and mid-turn, so it WILL eventually emit a real terminal. We therefore
140
- // HOLD the synth INDEFINITELY for an acked task. This is safe against the emit actually arriving:
141
- // when the worker's real generating_completed lands, it writes a terminal ledger, and
142
- // reconcileDirectDispatchCompletionFromTranscript's hasTerminalLedgerAfterDispatch check makes any
143
- // later synth an idempotent no-op (alreadyTerminal). So the hold never costs a missed notification —
144
- // the real emit always wins, no matter how late.
145
- //
146
- // The indefinite hold is released ONLY by a genuine-DEATH / emit-loss BACKSTOP — never a finite
147
- // timer that races normal lag:
148
- // (a) liveness failure — read_chat reports the session is gone, OR N consecutive read failures
149
- // accumulate (a transport/session-gone signal, counted as death rather than swallowed via
150
- // `continue`). A worker that died mid-turn will never emit, so the synth must eventually fire.
151
- // (b) an absolute LONG death-deadline — time since the generating_started ack exceeds
152
- // ACKED_DEATH_DEADLINE_MS, a backstop set FAR above any observed emit latency (default 8 min)
153
- // so it does not race a normal slow turn; it only catches a worker that is genuinely wedged or
154
- // whose emit was permanently lost. This is a notification-loss net, not a completion timer.
155
- //
156
- // A dispatch that was never acked (worker never started) is NOT held here: there is no in-flight
157
- // generation to protect, so it keeps the existing first-idle-tick synth behavior (its lost-dispatch
158
- // case is covered by the downstream grace + stale-summary guards). The map is pruned each PHASE-4
159
- // pass to the set of currently active dispatches, so a completed/pruned task's state is dropped (no
160
- // unbounded growth). Keyed by `${meshId}::${taskId}`.
161
-
162
- // R4f backstop (a): how many CONSECUTIVE read_chat failures (transport error / success:false /
163
- // no payload) for an acked task are treated as a death signal that releases the indefinite hold.
164
- // A single failed read is a transient probe blip; a session that genuinely died reads-fail every
165
- // tick, so a small streak distinguishes the two without racing a live-but-slow worker.
166
- const ACKED_DEATH_CONSECUTIVE_READ_FAILURES = 3;
167
-
168
- // R4f backstop (b): the absolute death-deadline. An acked task is held indefinitely until this much
169
- // time has elapsed since its generating_started ack (dispatch.updatedAt); past it, a persistently
170
- // idle session is synthesized as a notification-loss net. This is set FAR above any observed emit
171
- // latency (R4e's worst case was ~16s) so it does NOT race a normal slow turn — it only catches a
172
- // genuinely wedged worker or a permanently-lost emit. Read at call time so tests can tune it.
173
- function resolveTunedReconcileMs(envName: string, def: number, min: number, max: number): number {
174
- const raw = readNonEmptyString(process.env[envName]);
175
- if (raw) {
176
- const parsed = Number.parseInt(raw, 10);
177
- if (Number.isFinite(parsed) && parsed >= min && parsed <= max) return parsed;
178
- }
179
- return def;
180
- }
181
- function resolveAckedDeathDeadlineMs(): number {
182
- // Default 8 min — FAR above the variable emit latency the finite R4..R4e timers raced (R4e's
183
- // worst case was ~16s); by the time this fires a live worker would long since have emitted its
184
- // real terminal. The env-override floor is 0 so tests can force the deadline (production never
185
- // sets it that low); the ceiling is 60min so a mis-set env cannot disable the loss-net forever.
186
- return resolveTunedReconcileMs('MESH_INFLIGHT_ACKED_DEATH_DEADLINE_MS', 8 * 60_000, 0, 60 * 60_000);
187
- }
188
-
189
- // ACKED-HOLD-IDLE-OVERTRUST (transcript-completion fast-track). The indefinite acked-hold above is
190
- // safe but SLOW: when the worker's real generating_completed emit is dropped/lost, the only thing
191
- // that promotes the missing completion is the 8-min death backstop — even though the answer has been
192
- // FULLY rendered in the transcript for minutes (read_chat reports idle WITH a final visible assistant
193
- // message every ~4s). Observed live: completions surfaced 144s / 492s late, both incompatible with the
194
- // provider's own emit ceiling (COMPLETED_FINALIZATION_MAX_WAIT_MS 30s + NATIVE_HISTORY_MESH_IDLE_SETTLE
195
- // 4s ≈ 34s). That gap = a worker that finished, whose PTY generating→idle edge / real emit was lost,
196
- // held hostage to the 8-min net.
197
- //
198
- // Fast-track: when an acked task reads idle AND a final visible assistant message is present (the same
199
- // transcript-completion evidence PHASE 4 already requires to synth), and that idle-with-final-assistant
200
- // state has PERSISTED for a short continuous grace, promote the synth EARLY — ahead of the 8-min
201
- // backstop. The grace is the correctness gate: a SINGLE idle read could be a mid-turn blip (PTY
202
- // inter-tool-call settle, or final text rendered while the next tool call is about to start), so we
203
- // require the idle-with-final-assistant signal to hold continuously for the grace window before
204
- // trusting it as a genuine turn-end. Any non-idle read (generating / waiting_approval), a read
205
- // failure, or the disappearance of the final assistant message RESETS the streak — so an actively
206
- // streaming worker that momentarily reads idle never crosses the grace.
207
- //
208
- // Safety: this only changes WHEN an acked synth fires (earlier), never WHETHER it is correct —
209
- // reconcileDirectDispatchCompletionFromTranscript's hasTerminalLedgerAfterDispatch makes a real
210
- // emit that lands later an idempotent no-op, exactly as the death-backstop synth relies on. The
211
- // death backstop (8 min) is PRESERVED unchanged as the final net; the fast-track is a faster path in
212
- // front of it. The grace is set ABOVE the provider's own emit ceiling (~34s) so a worker still inside
213
- // its normal finalization window is never pre-empted — we only fast-track once enough continuous idle
214
- // has elapsed that a live emit would already have arrived.
215
- function resolveAckedTranscriptFastTrackGraceMs(): number {
216
- // Default 40s — above the provider emit ceiling (30s COMPLETED_FINALIZATION_MAX_WAIT_MS + 4s
217
- // NATIVE_HISTORY_MESH_IDLE_SETTLE ≈ 34s): a genuinely-live worker would have emitted its real
218
- // terminal within that window, so 40s of CONTINUOUS idle-with-final-assistant means the emit was
219
- // lost, not late. Far below the 8-min death backstop, so the fast-track is the dominant path for a
220
- // lost emit while the backstop remains the last-resort net. Floor 0 lets tests force an immediate
221
- // fast-track; ceiling 5min keeps a mis-set env from collapsing it into the death backstop.
222
- return resolveTunedReconcileMs('MESH_INFLIGHT_ACKED_TRANSCRIPT_FASTTRACK_GRACE_MS', 40_000, 0, 5 * 60_000);
223
- }
224
-
225
- // Per-task in-flight hold state for an acked dispatch:
226
- // - liveConfirmedSinceAck: we have seen at least one conclusive read (idle OR generating) since
227
- // the ack — proves the session is reachable, so a later read FAILURE is a genuine liveness loss
228
- // rather than a node that was never reachable.
229
- // - consecutiveReadFailures: streak of inconclusive read_chat results (death backstop (a)).
230
- // - transcriptIdleSinceMs: the timestamp of the FIRST tick in the current continuous run of
231
- // idle-with-final-assistant reads (ACKED-HOLD-IDLE-OVERTRUST fast-track). Cleared to undefined
232
- // whenever the signal breaks (non-idle read, read failure, or no final assistant message), so a
233
- // mid-turn idle blip never accumulates grace. When `now - transcriptIdleSinceMs` exceeds the
234
- // fast-track grace the synth is promoted ahead of the death backstop.
235
- interface AckedHoldState {
236
- liveConfirmedSinceAck: boolean;
237
- consecutiveReadFailures: number;
238
- transcriptIdleSinceMs?: number;
239
- }
240
-
241
- // T2 (B2b): acked-hold state persistence. The Map below is a process-local CACHE;
242
- // the SSOT is the mesh_inflight_hold table in MeshRuntimeStore. Every read goes
243
- // read-through (Map miss → load from store, then cache), every mutation goes
244
- // write-through (Map set → store upsert; Map delete → store delete). On daemon
245
- // boot the reconcile loop rehydrates the Map from the store per-mesh the first
246
- // time it touches that mesh (rehydrateAckedHoldsForMesh), so a hold established
247
- // before a restart survives it — closing the duplicate-emit / drop window the
248
- // PHASE-4 transcript synth backstop otherwise had to correct after the fact.
249
- //
250
- // Store row ↔ AckedHoldState mapping:
251
- // hold_reason 'live'|'unconfirmed' ↔ liveConfirmedSinceAck (boolean)
252
- // read_failure_count ↔ consecutiveReadFailures
253
- // first_idle_since_ack ↔ transcriptIdleSinceMs (undefined ⇒ NULL)
254
- // mesh_id = the owning mesh (for listByMesh / prune)
255
- // held_at = ms the hold was first created (store-managed)
256
- const inFlightAckedHoldState = new Map<string, AckedHoldState>();
257
- // Meshes whose store rows have already been rehydrated into the Map this process.
258
- // A restart resets this set, so the first touch of each mesh reloads from disk.
259
- const rehydratedHoldMeshes = new Set<string>();
260
-
261
- // ─── T6 (B3c): PHASE-4 synthesis + acked-hold fast-track demoted to last-resort ──
262
- //
263
- // Under mesh-protocol-v2 enforce, the completion contract is explicit: a worker's
264
- // terminal emit is a v2 unicast event drained straight to the coordinator. The
265
- // PHASE-4 transcript-synthesis backstop and the acked-hold fast-track exist to
266
- // paper over a LOST emit — they should NEVER fire once v2 delivery is healthy. So
267
- // their firing is now a demoted last-resort signal: every fire bumps a counter, and
268
- // under enforce a fire additionally emits a WARN naming it a v2-contract violation
269
- // (a real emit was expected but never arrived). Target = 0 fires in steady state.
270
- //
271
- // The code is NOT removed — it stays as the correctness net for a genuinely lost
272
- // emit (rollout plan §B3c: "코드 삭제는 하지 않고 관측 후 다음 사이클에 판단"). Process-
273
- // lifetime totals; read by tests + surfaced in mesh_status.
274
- const meshV2BackstopCounters = {
275
- /** PHASE-4 transcript synthesis actually reconciled a missing completion. */
276
- phase4SynthesisFired: 0,
277
- /** Acked-hold transcript fast-track promoted a synth ahead of the death deadline. */
278
- ackedHoldFastTrackFired: 0,
279
- /** Acked-hold death-deadline backstop released a held synth (the 8-min net). */
280
- ackedHoldDeathDeadlineFired: 0,
281
- };
282
-
283
- /** Test/observability accessor for the v2 last-resort backstop counters (snapshot). */
284
- export function getMeshV2BackstopCounters(): Readonly<typeof meshV2BackstopCounters> {
285
- return { ...meshV2BackstopCounters };
286
- }
287
-
288
- /** Test helper: zero the backstop counters so a case starts from a clean slate. */
289
- export function __resetMeshV2BackstopCountersForTests(): void {
290
- for (const k of Object.keys(meshV2BackstopCounters) as Array<keyof typeof meshV2BackstopCounters>) {
291
- meshV2BackstopCounters[k] = 0;
292
- }
293
- }
294
-
295
- /** Enforce switch mirror (see isMeshProtocolV2EnforceEnabled in mesh-events-pending);
296
- * re-read here (not imported) to keep the reconcile loop free of a cross-file coupling
297
- * and to read env at fire time. Same truthy vocabulary. */
298
- function meshProtocolV2EnforceOn(): boolean {
299
- const raw = process.env.MESH_PROTOCOL_V2_ENFORCE;
300
- if (typeof raw !== 'string') return false;
301
- const v = raw.trim().toLowerCase();
302
- return v === '1' || v === 'true' || v === 'on' || v === 'yes';
303
- }
304
-
305
- /** Bump a backstop counter and, under enforce, WARN that a last-resort net fired —
306
- * which under a healthy v2 contract should not happen (the real emit was lost). */
307
- function recordBackstopFire(kind: keyof typeof meshV2BackstopCounters, detail: string): void {
308
- meshV2BackstopCounters[kind]++;
309
- if (meshProtocolV2EnforceOn()) {
310
- LOG.warn('MeshReconcileV2', `v2 ENFORCE last-resort backstop fired (${kind}): ${detail}. Under a healthy v2 completion contract this should be 0 — a worker's real terminal emit was lost/late.`);
311
- }
312
- }
313
-
314
- function inFlightSynthKey(meshId: string, taskId: string): string {
315
- return `${meshId}::${taskId}`;
316
- }
317
-
318
- // Extract the taskId back out of a `${meshId}::${taskId}` synth key. The meshId
319
- // prefix can itself contain '::' only if the caller passed one (mesh ids are
320
- // config-derived and never do), so split on the FIRST '::' and treat the remainder
321
- // as the taskId.
322
- function taskIdFromSynthKey(meshId: string, synthKey: string): string {
323
- const prefix = `${meshId}::`;
324
- return synthKey.startsWith(prefix) ? synthKey.slice(prefix.length) : synthKey;
325
- }
326
-
327
- function holdStore(): MeshRuntimeStore | undefined {
328
- try { return MeshRuntimeStore.getInstance(); } catch { return undefined; }
329
- }
330
-
331
- // Read-through: Map hit returns the cached state; a miss consults the store and,
332
- // when a row exists, hydrates the Map from it before returning. A store failure
333
- // degrades to Map-only (returns undefined on a miss) — identical to the pre-T2
334
- // in-memory behavior, never worse.
335
- function getHoldState(synthKey: string, meshId: string): AckedHoldState | undefined {
336
- const cached = inFlightAckedHoldState.get(synthKey);
337
- if (cached) return cached;
338
- const store = holdStore();
339
- if (!store) return undefined;
340
- let row;
341
- try { row = store.getInflightHold(taskIdFromSynthKey(meshId, synthKey)); } catch { return undefined; }
342
- if (!row) return undefined;
343
- const state: AckedHoldState = {
344
- liveConfirmedSinceAck: row.holdReason === 'live',
345
- consecutiveReadFailures: row.readFailureCount ?? 0,
346
- ...(row.firstIdleSinceAck !== null && row.firstIdleSinceAck !== undefined
347
- ? { transcriptIdleSinceMs: row.firstIdleSinceAck }
348
- : {}),
349
- };
350
- inFlightAckedHoldState.set(synthKey, state);
351
- return state;
352
- }
353
-
354
- // Write-through: update the Map cache AND the store row. A store failure leaves the
355
- // Map authoritative for this process (degrade, never crash the tick).
356
- function setHoldState(synthKey: string, meshId: string, state: AckedHoldState): void {
357
- inFlightAckedHoldState.set(synthKey, state);
358
- const store = holdStore();
359
- if (!store) return;
360
- try {
361
- store.upsertInflightHold({
362
- taskId: taskIdFromSynthKey(meshId, synthKey),
363
- meshId,
364
- holdReason: state.liveConfirmedSinceAck ? 'live' : 'unconfirmed',
365
- firstIdleSinceAck: state.transcriptIdleSinceMs ?? null,
366
- readFailureCount: state.consecutiveReadFailures,
367
- });
368
- } catch { /* degrade to Map-only */ }
369
- }
370
-
371
- // Write-through delete: drop the Map entry AND the store row.
372
- function deleteHoldState(synthKey: string, meshId: string): void {
373
- inFlightAckedHoldState.delete(synthKey);
374
- const store = holdStore();
375
- if (!store) return;
376
- try { store.deleteInflightHold(taskIdFromSynthKey(meshId, synthKey)); } catch { /* degrade */ }
377
- }
378
-
379
- // Restart rehydration: on the first touch of a mesh this process, pull its persisted
380
- // acked-hold rows from the store into the Map cache so a hold that outlived a daemon
381
- // restart is honored again. Idempotent per process via rehydratedHoldMeshes. A store
382
- // failure just skips rehydration (Map starts empty for the mesh — pre-T2 behavior).
383
- function rehydrateAckedHoldsForMesh(meshId: string): void {
384
- if (rehydratedHoldMeshes.has(meshId)) return;
385
- rehydratedHoldMeshes.add(meshId);
386
- const store = holdStore();
387
- if (!store) return;
388
- let rows;
389
- try { rows = store.listInflightHoldsByMesh(meshId); } catch { return; }
390
- for (const row of rows) {
391
- const synthKey = inFlightSynthKey(meshId, row.taskId);
392
- if (inFlightAckedHoldState.has(synthKey)) continue; // a live tick already set fresher state
393
- inFlightAckedHoldState.set(synthKey, {
394
- liveConfirmedSinceAck: row.holdReason === 'live',
395
- consecutiveReadFailures: row.readFailureCount ?? 0,
396
- ...(row.firstIdleSinceAck !== null && row.firstIdleSinceAck !== undefined
397
- ? { transcriptIdleSinceMs: row.firstIdleSinceAck }
398
- : {}),
399
- });
400
- }
401
- if (rows.length > 0) {
402
- LOG.info('MeshReconcile', `Rehydrated ${rows.length} persisted acked-hold row(s) for mesh ${meshId} after (re)start`);
403
- }
404
- }
405
-
406
- // Test hook: clear the in-flight acked-hold state between cases (both the Map cache
407
- // and the per-mesh rehydrate guard, so each case starts from a clean read-through).
408
- export function __resetReconcileInFlightSynthDebounceForTests(): void {
409
- inFlightAckedHoldState.clear();
410
- rehydratedHoldMeshes.clear();
411
- }
412
-
413
150
  interface LiveCoordinator {
414
151
  meshId: string;
415
152
  instance: ReturnType<DaemonComponents['instanceManager']['getInstance']>;
@@ -435,96 +172,6 @@ interface LiveCoordinator {
435
172
  modalParked: boolean;
436
173
  }
437
174
 
438
- // The set of coordinator-daemon ids THIS daemon answers to when draining the
439
- // pending-events queue. A unicast completion event is stamped with the worker's
440
- // meshCoordinatorDaemonId, which can be either:
441
- // - the daemon's canonical status id (`standalone_<machineId>` / `daemon_<machineId>`),
442
- // stamped by the MCP layer via ctx.localDaemonId (= getStatus().status.instanceId), or
443
- // - the bare machineId, stamped by the local queue-assignment path (loadConfig().machineId).
444
- // - the config-form node daemonId (`daemon_<machineId>`), which the MCP layer's
445
- // resolveCoordinatorDaemonId prefers and stamps onto direct-dispatch workers.
446
- // Draining with only one of these silently misses events stamped with the other —
447
- // the exact reason a generating coordinator never self-received local completions,
448
- // and the base-node completion-surface bug (base completions land full-form
449
- // `daemon_<machineId>` while a coordinator that only knows itself as bare
450
- // `<machineId>` never matches them). We expand to EVERY equivalent form so the
451
- // scope match (host gate, self-node detection, and the drain IN-filter downstream)
452
- // succeeds regardless of which path stamped the event.
453
- function resolveCoordinatorDaemonIds(components: DaemonComponents): string[] {
454
- const statusInstanceId = readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId);
455
- const machineId = readNonEmptyString(loadConfig().machineId);
456
- return expandDaemonIdForms([statusInstanceId, machineId]);
457
- }
458
-
459
- // Whether THIS daemon is the coordinator/host for a mesh — i.e. the daemon that
460
- // owns coordinator ownership and must collect every worker node's completion
461
- // events into its local queue. This is true regardless of whether a *live CLI*
462
- // coordinator session currently exists: the coordinator is frequently a pure
463
- // stdio MCP LLM (no live CLI session to inject into), and that LLM only sees the
464
- // queue when it next calls a mesh tool. For it to see remote worker completions
465
- // at all, the daemon must have already pulled them into the local queue on the
466
- // timer — which is exactly what this predicate gates.
467
- //
468
- // Rule: this daemon hosts the mesh when meshHost.role is 'host' (the default for
469
- // standalone-compat meshes with no host metadata) AND, when a hostDaemonId is
470
- // pinned, it resolves to one of this daemon's ids. Member-only daemons return
471
- // false — their own queue is pulled BY the host, not the other way around.
472
- //
473
- // `daemonIds` here is the EXPANDED self-identity set (runtime drain ids ∪ this
474
- // daemon's mesh-config node id forms) — see resolveCoordinatorSelfIds. The
475
- // pinned hostDaemonId is itself a config-form id and frequently does NOT equal a
476
- // runtime id (bare machineId / status id), so gating on the runtime ids alone
477
- // would wrongly classify the real host as a non-host and skip the remote pull
478
- // entirely.
479
- function daemonHostsMesh(mesh: LocalMeshEntry, daemonIds: string[]): boolean {
480
- const host = mesh.meshHost;
481
- // No metadata → default host (standalone compatibility, see createDefaultMeshHostMetadata).
482
- if (!host) return true;
483
- if (host.role && host.role !== 'host') return false;
484
- const hostDaemonId = readNonEmptyString(host.hostDaemonId);
485
- // Host role but no pinned hostDaemonId → treat as host (single-daemon / legacy).
486
- if (!hostDaemonId) return true;
487
- return daemonIdListIncludes(daemonIds, hostDaemonId);
488
- }
489
-
490
- function daemonIdListIncludes(ids: readonly string[], id: string | undefined): boolean {
491
- if (!id) return false;
492
- return ids.some(candidate => candidate === id || daemonIdsEquivalent(candidate, id));
493
- }
494
-
495
- // Resolve EVERY id-form this daemon answers to FOR A GIVEN MESH: the runtime drain
496
- // ids (status id + bare machineId) unioned with this daemon's mesh-config identity
497
- // forms — the self node's daemonId/machineId (the node whose daemonId/machineId
498
- // matches a runtime id) and the pinned meshHost.hostDaemonId WHEN it is provably
499
- // ours. This is the single source of truth for "is this id me?" across both the
500
- // host gate and the remote pull filter; the worker's meshCoordinatorDaemonId stamp
501
- // is guaranteed to be one of these forms (it comes from resolveCoordinatorDaemonId,
502
- // which prefers the coordinator node's config-form daemonId over the runtime status id).
503
- function resolveCoordinatorSelfIds(mesh: LocalMeshEntry, drainDaemonIds: string[]): string[] {
504
- const ids = new Set<string>(drainDaemonIds);
505
- // Expand with the config-form id(s) of the self node — the mesh node whose
506
- // daemonId/machineId matches a runtime id. Its config-form daemonId is exactly
507
- // what resolveCoordinatorNode()→resolveCoordinatorDaemonId() stamps onto a worker.
508
- for (const node of mesh.nodes) {
509
- const nodeDaemonId = readNonEmptyString(node.daemonId);
510
- const nodeMachineId = readNonEmptyString(node.machineId);
511
- const isSelf = (nodeDaemonId && daemonIdListIncludes(drainDaemonIds, nodeDaemonId))
512
- || (nodeMachineId && daemonIdListIncludes(drainDaemonIds, nodeMachineId));
513
- if (!isSelf) continue;
514
- if (nodeDaemonId) ids.add(nodeDaemonId);
515
- if (nodeMachineId) ids.add(nodeMachineId);
516
- }
517
- // The pinned host id is included ONLY when it is provably one of THIS daemon's ids
518
- // (it already matches a runtime id or a resolved self-node id). A hostDaemonId that
519
- // names a DIFFERENT daemon must NOT be claimed — that would make a member-only
520
- // daemon believe it is the host and pull queues it does not own. Having a node on
521
- // this daemon does not make this daemon the host; daemonHostsMesh still honours a
522
- // foreign hostDaemonId and rejects ownership.
523
- const hostDaemonId = readNonEmptyString(mesh.meshHost?.hostDaemonId);
524
- if (hostDaemonId && daemonIdListIncludes([...ids], hostDaemonId)) ids.add(hostDaemonId);
525
- return [...ids];
526
- }
527
-
528
175
  // Observability: last-seen modal-park state per coordinator session, so we LOG.info
529
176
  // only on a TRANSITION (clear → parked, parked → cleared) instead of every 4s tick.
530
177
  // Per-process; a restart re-logs the first observation, which is desirable — it
@@ -2017,18 +1664,7 @@ async function reconcileUnterminatedDirectDispatches(
2017
1664
  .filter(Boolean)
2018
1665
  .map(taskId => inFlightSynthKey(mesh.id, taskId)),
2019
1666
  );
2020
- const heldKeys = new Set<string>();
2021
- for (const key of inFlightAckedHoldState.keys()) {
2022
- if (key.startsWith(`${mesh.id}::`)) heldKeys.add(key);
2023
- }
2024
- const store = holdStore();
2025
- if (store) {
2026
- try {
2027
- for (const row of store.listInflightHoldsByMesh(mesh.id)) {
2028
- heldKeys.add(inFlightSynthKey(mesh.id, row.taskId));
2029
- }
2030
- } catch { /* degrade — prune only what's in the Map */ }
2031
- }
1667
+ const heldKeys = collectHeldSynthKeysForMesh(mesh.id);
2032
1668
  for (const key of heldKeys) {
2033
1669
  if (!activeTaskKeys.has(key)) deleteHoldState(key, mesh.id);
2034
1670
  }
@@ -0,0 +1,62 @@
1
+ // ---------------------------------------------------------------------------
2
+ // mesh-reconcile-v2-backstop — last-resort completion backstop counters
3
+ // ---------------------------------------------------------------------------
4
+ // Pure move out of mesh-reconcile-loop.ts (no behavior change).
5
+ //
6
+ // T6 (B3c): PHASE-4 synthesis + acked-hold fast-track demoted to last-resort.
7
+ //
8
+ // Under mesh-protocol-v2 enforce, the completion contract is explicit: a worker's
9
+ // terminal emit is a v2 unicast event drained straight to the coordinator. The
10
+ // PHASE-4 transcript-synthesis backstop and the acked-hold fast-track exist to
11
+ // paper over a LOST emit — they should NEVER fire once v2 delivery is healthy. So
12
+ // their firing is now a demoted last-resort signal: every fire bumps a counter, and
13
+ // under enforce a fire additionally emits a WARN naming it a v2-contract violation
14
+ // (a real emit was expected but never arrived). Target = 0 fires in steady state.
15
+ //
16
+ // The code is NOT removed — it stays as the correctness net for a genuinely lost
17
+ // emit (rollout plan §B3c: "코드 삭제는 하지 않고 관측 후 다음 사이클에 판단"). Process-
18
+ // lifetime totals; read by tests + surfaced in mesh_status.
19
+ // ---------------------------------------------------------------------------
20
+
21
+ import { LOG } from '../logging/logger.js';
22
+
23
+ const meshV2BackstopCounters = {
24
+ /** PHASE-4 transcript synthesis actually reconciled a missing completion. */
25
+ phase4SynthesisFired: 0,
26
+ /** Acked-hold transcript fast-track promoted a synth ahead of the death deadline. */
27
+ ackedHoldFastTrackFired: 0,
28
+ /** Acked-hold death-deadline backstop released a held synth (the 8-min net). */
29
+ ackedHoldDeathDeadlineFired: 0,
30
+ };
31
+
32
+ /** Test/observability accessor for the v2 last-resort backstop counters (snapshot). */
33
+ export function getMeshV2BackstopCounters(): Readonly<typeof meshV2BackstopCounters> {
34
+ return { ...meshV2BackstopCounters };
35
+ }
36
+
37
+ /** Test helper: zero the backstop counters so a case starts from a clean slate. */
38
+ export function __resetMeshV2BackstopCountersForTests(): void {
39
+ for (const k of Object.keys(meshV2BackstopCounters) as Array<keyof typeof meshV2BackstopCounters>) {
40
+ meshV2BackstopCounters[k] = 0;
41
+ }
42
+ }
43
+
44
+ /** Enforce switch mirror (see isMeshProtocolV2EnforceEnabled in mesh-events-pending);
45
+ * re-read here (not imported) to keep the reconcile loop free of a cross-file coupling
46
+ * and to read env at fire time. On by default; set MESH_PROTOCOL_V2_ENFORCE=0/false/
47
+ * off/no to disable. Same vocabulary as the source of truth. */
48
+ function meshProtocolV2EnforceOn(): boolean {
49
+ const raw = process.env.MESH_PROTOCOL_V2_ENFORCE;
50
+ if (typeof raw !== 'string' || !raw.trim()) return true; // unset/blank = default ON
51
+ const v = raw.trim().toLowerCase();
52
+ return !(v === '0' || v === 'false' || v === 'off' || v === 'no');
53
+ }
54
+
55
+ /** Bump a backstop counter and, under enforce, WARN that a last-resort net fired —
56
+ * which under a healthy v2 contract should not happen (the real emit was lost). */
57
+ export function recordBackstopFire(kind: keyof typeof meshV2BackstopCounters, detail: string): void {
58
+ meshV2BackstopCounters[kind]++;
59
+ if (meshProtocolV2EnforceOn()) {
60
+ LOG.warn('MeshReconcileV2', `v2 ENFORCE last-resort backstop fired (${kind}): ${detail}. Under a healthy v2 completion contract this should be 0 — a worker's real terminal emit was lost/late.`);
61
+ }
62
+ }