@adhdev/daemon-core 0.9.82-rc.464 → 0.9.82-rc.466

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -67,6 +67,33 @@ import { pruneStaleDirectDispatches } from './mesh-active-work.js';
67
67
  import { findTerminalLedgerEvidenceForTask, reconcileDirectDispatchCompletionFromTranscript } from './mesh-events-stale.js';
68
68
  import { extractFinalAssistantSummaryEvidence } from '../providers/chat-message-normalization.js';
69
69
  import type { ChatMessage } from '../types.js';
70
+ import {
71
+ resolveCoordinatorDaemonIds,
72
+ daemonHostsMesh,
73
+ daemonIdListIncludes,
74
+ resolveCoordinatorSelfIds,
75
+ } from './mesh-reconcile-identity.js';
76
+ import {
77
+ getMeshV2BackstopCounters,
78
+ recordBackstopFire,
79
+ } from './mesh-reconcile-v2-backstop.js';
80
+ import {
81
+ ACKED_DEATH_CONSECUTIVE_READ_FAILURES,
82
+ resolveTunedReconcileMs,
83
+ resolveAckedDeathDeadlineMs,
84
+ resolveAckedTranscriptFastTrackGraceMs,
85
+ inFlightSynthKey,
86
+ getHoldState,
87
+ setHoldState,
88
+ deleteHoldState,
89
+ rehydrateAckedHoldsForMesh,
90
+ collectHeldSynthKeysForMesh,
91
+ } from './mesh-reconcile-acked-hold.js';
92
+
93
+ // Re-export the extracted public API so existing importers (mesh-events.ts barrel;
94
+ // the reconcile-loop test suite) keep their `from './mesh-reconcile-loop.js'` paths.
95
+ export { getMeshV2BackstopCounters, __resetMeshV2BackstopCountersForTests } from './mesh-reconcile-v2-backstop.js';
96
+ export { __resetReconcileInFlightSynthDebounceForTests } from './mesh-reconcile-acked-hold.js';
70
97
 
71
98
  // Default reconcile cadence. approval/completion notifications to a live CLI
72
99
  // coordinator land within at most one interval. Overridable via env for tuning.
@@ -120,297 +147,6 @@ function resolveReconcileIntervalMs(): number {
120
147
  return DEFAULT_RECONCILE_INTERVAL_MS;
121
148
  }
122
149
 
123
- // R4f (GENERATING-BOUNDARY, acked-hold redesign). PHASE 4 only synthesizes a missing completion
124
- // when the worker session reads `idle`. But a worker that is GENUINELY generating (it emitted
125
- // agent:generating_started — the dispatch row is 'acked' — and has not yet completed) can
126
- // momentarily read `idle` mid-turn (a CLI PTY inter-tool-call settle, or the final assistant text
127
- // already rendered while the turn's generating_completed lifecycle close still lags). A premature
128
- // synth writes a terminal that then masks the worker's REAL completion when it lands seconds later
129
- // (drop:duplicate_completion_terminal_ledger; the observed 71s task a250fb44 lost its [System]
130
- // notification this way; the R4e 53s task synth fired 16s BEFORE the worker's real emit).
131
- //
132
- // R4 → R4e used FINITE timers (consecutive ticks / MIN_IDLE_SETTLE / ACKED_TURN_SETTLE) to delay the
133
- // synth. That class of fix is fundamentally a RACE: the worker's real emit latency is variable and
134
- // unbounded (win32 idle reads can flip before the emit arrives), so ANY finite timer eventually
135
- // loses to a slow-enough turn — and the synth pre-empts the real completion. R4e live-FAILED for
136
- // exactly this reason.
137
- //
138
- // R4f redesign (direction B). An `acked` task means the worker ECHOED generating_started (the
139
- // taskId flip) — it is alive and mid-turn, so it WILL eventually emit a real terminal. We therefore
140
- // HOLD the synth INDEFINITELY for an acked task. This is safe against the emit actually arriving:
141
- // when the worker's real generating_completed lands, it writes a terminal ledger, and
142
- // reconcileDirectDispatchCompletionFromTranscript's hasTerminalLedgerAfterDispatch check makes any
143
- // later synth an idempotent no-op (alreadyTerminal). So the hold never costs a missed notification —
144
- // the real emit always wins, no matter how late.
145
- //
146
- // The indefinite hold is released ONLY by a genuine-DEATH / emit-loss BACKSTOP — never a finite
147
- // timer that races normal lag:
148
- // (a) liveness failure — read_chat reports the session is gone, OR N consecutive read failures
149
- // accumulate (a transport/session-gone signal, counted as death rather than swallowed via
150
- // `continue`). A worker that died mid-turn will never emit, so the synth must eventually fire.
151
- // (b) an absolute LONG death-deadline — time since the generating_started ack exceeds
152
- // ACKED_DEATH_DEADLINE_MS, a backstop set FAR above any observed emit latency (default 8 min)
153
- // so it does not race a normal slow turn; it only catches a worker that is genuinely wedged or
154
- // whose emit was permanently lost. This is a notification-loss net, not a completion timer.
155
- //
156
- // A dispatch that was never acked (worker never started) is NOT held here: there is no in-flight
157
- // generation to protect, so it keeps the existing first-idle-tick synth behavior (its lost-dispatch
158
- // case is covered by the downstream grace + stale-summary guards). The map is pruned each PHASE-4
159
- // pass to the set of currently active dispatches, so a completed/pruned task's state is dropped (no
160
- // unbounded growth). Keyed by `${meshId}::${taskId}`.
161
-
162
- // R4f backstop (a): how many CONSECUTIVE read_chat failures (transport error / success:false /
163
- // no payload) for an acked task are treated as a death signal that releases the indefinite hold.
164
- // A single failed read is a transient probe blip; a session that genuinely died reads-fail every
165
- // tick, so a small streak distinguishes the two without racing a live-but-slow worker.
166
- const ACKED_DEATH_CONSECUTIVE_READ_FAILURES = 3;
167
-
168
- // R4f backstop (b): the absolute death-deadline. An acked task is held indefinitely until this much
169
- // time has elapsed since its generating_started ack (dispatch.updatedAt); past it, a persistently
170
- // idle session is synthesized as a notification-loss net. This is set FAR above any observed emit
171
- // latency (R4e's worst case was ~16s) so it does NOT race a normal slow turn — it only catches a
172
- // genuinely wedged worker or a permanently-lost emit. Read at call time so tests can tune it.
173
- function resolveTunedReconcileMs(envName: string, def: number, min: number, max: number): number {
174
- const raw = readNonEmptyString(process.env[envName]);
175
- if (raw) {
176
- const parsed = Number.parseInt(raw, 10);
177
- if (Number.isFinite(parsed) && parsed >= min && parsed <= max) return parsed;
178
- }
179
- return def;
180
- }
181
- function resolveAckedDeathDeadlineMs(): number {
182
- // Default 8 min — FAR above the variable emit latency the finite R4..R4e timers raced (R4e's
183
- // worst case was ~16s); by the time this fires a live worker would long since have emitted its
184
- // real terminal. The env-override floor is 0 so tests can force the deadline (production never
185
- // sets it that low); the ceiling is 60min so a mis-set env cannot disable the loss-net forever.
186
- return resolveTunedReconcileMs('MESH_INFLIGHT_ACKED_DEATH_DEADLINE_MS', 8 * 60_000, 0, 60 * 60_000);
187
- }
188
-
189
- // ACKED-HOLD-IDLE-OVERTRUST (transcript-completion fast-track). The indefinite acked-hold above is
190
- // safe but SLOW: when the worker's real generating_completed emit is dropped/lost, the only thing
191
- // that promotes the missing completion is the 8-min death backstop — even though the answer has been
192
- // FULLY rendered in the transcript for minutes (read_chat reports idle WITH a final visible assistant
193
- // message every ~4s). Observed live: completions surfaced 144s / 492s late, both incompatible with the
194
- // provider's own emit ceiling (COMPLETED_FINALIZATION_MAX_WAIT_MS 30s + NATIVE_HISTORY_MESH_IDLE_SETTLE
195
- // 4s ≈ 34s). That gap = a worker that finished, whose PTY generating→idle edge / real emit was lost,
196
- // held hostage to the 8-min net.
197
- //
198
- // Fast-track: when an acked task reads idle AND a final visible assistant message is present (the same
199
- // transcript-completion evidence PHASE 4 already requires to synth), and that idle-with-final-assistant
200
- // state has PERSISTED for a short continuous grace, promote the synth EARLY — ahead of the 8-min
201
- // backstop. The grace is the correctness gate: a SINGLE idle read could be a mid-turn blip (PTY
202
- // inter-tool-call settle, or final text rendered while the next tool call is about to start), so we
203
- // require the idle-with-final-assistant signal to hold continuously for the grace window before
204
- // trusting it as a genuine turn-end. Any non-idle read (generating / waiting_approval), a read
205
- // failure, or the disappearance of the final assistant message RESETS the streak — so an actively
206
- // streaming worker that momentarily reads idle never crosses the grace.
207
- //
208
- // Safety: this only changes WHEN an acked synth fires (earlier), never WHETHER it is correct —
209
- // reconcileDirectDispatchCompletionFromTranscript's hasTerminalLedgerAfterDispatch makes a real
210
- // emit that lands later an idempotent no-op, exactly as the death-backstop synth relies on. The
211
- // death backstop (8 min) is PRESERVED unchanged as the final net; the fast-track is a faster path in
212
- // front of it. The grace is set ABOVE the provider's own emit ceiling (~34s) so a worker still inside
213
- // its normal finalization window is never pre-empted — we only fast-track once enough continuous idle
214
- // has elapsed that a live emit would already have arrived.
215
- function resolveAckedTranscriptFastTrackGraceMs(): number {
216
- // Default 40s — above the provider emit ceiling (30s COMPLETED_FINALIZATION_MAX_WAIT_MS + 4s
217
- // NATIVE_HISTORY_MESH_IDLE_SETTLE ≈ 34s): a genuinely-live worker would have emitted its real
218
- // terminal within that window, so 40s of CONTINUOUS idle-with-final-assistant means the emit was
219
- // lost, not late. Far below the 8-min death backstop, so the fast-track is the dominant path for a
220
- // lost emit while the backstop remains the last-resort net. Floor 0 lets tests force an immediate
221
- // fast-track; ceiling 5min keeps a mis-set env from collapsing it into the death backstop.
222
- return resolveTunedReconcileMs('MESH_INFLIGHT_ACKED_TRANSCRIPT_FASTTRACK_GRACE_MS', 40_000, 0, 5 * 60_000);
223
- }
224
-
225
- // Per-task in-flight hold state for an acked dispatch:
226
- // - liveConfirmedSinceAck: we have seen at least one conclusive read (idle OR generating) since
227
- // the ack — proves the session is reachable, so a later read FAILURE is a genuine liveness loss
228
- // rather than a node that was never reachable.
229
- // - consecutiveReadFailures: streak of inconclusive read_chat results (death backstop (a)).
230
- // - transcriptIdleSinceMs: the timestamp of the FIRST tick in the current continuous run of
231
- // idle-with-final-assistant reads (ACKED-HOLD-IDLE-OVERTRUST fast-track). Cleared to undefined
232
- // whenever the signal breaks (non-idle read, read failure, or no final assistant message), so a
233
- // mid-turn idle blip never accumulates grace. When `now - transcriptIdleSinceMs` exceeds the
234
- // fast-track grace the synth is promoted ahead of the death backstop.
235
- interface AckedHoldState {
236
- liveConfirmedSinceAck: boolean;
237
- consecutiveReadFailures: number;
238
- transcriptIdleSinceMs?: number;
239
- }
240
-
241
- // T2 (B2b): acked-hold state persistence. The Map below is a process-local CACHE;
242
- // the SSOT is the mesh_inflight_hold table in MeshRuntimeStore. Every read goes
243
- // read-through (Map miss → load from store, then cache), every mutation goes
244
- // write-through (Map set → store upsert; Map delete → store delete). On daemon
245
- // boot the reconcile loop rehydrates the Map from the store per-mesh the first
246
- // time it touches that mesh (rehydrateAckedHoldsForMesh), so a hold established
247
- // before a restart survives it — closing the duplicate-emit / drop window the
248
- // PHASE-4 transcript synth backstop otherwise had to correct after the fact.
249
- //
250
- // Store row ↔ AckedHoldState mapping:
251
- // hold_reason 'live'|'unconfirmed' ↔ liveConfirmedSinceAck (boolean)
252
- // read_failure_count ↔ consecutiveReadFailures
253
- // first_idle_since_ack ↔ transcriptIdleSinceMs (undefined ⇒ NULL)
254
- // mesh_id = the owning mesh (for listByMesh / prune)
255
- // held_at = ms the hold was first created (store-managed)
256
- const inFlightAckedHoldState = new Map<string, AckedHoldState>();
257
- // Meshes whose store rows have already been rehydrated into the Map this process.
258
- // A restart resets this set, so the first touch of each mesh reloads from disk.
259
- const rehydratedHoldMeshes = new Set<string>();
260
-
261
- // ─── T6 (B3c): PHASE-4 synthesis + acked-hold fast-track demoted to last-resort ──
262
- //
263
- // Under mesh-protocol-v2 enforce, the completion contract is explicit: a worker's
264
- // terminal emit is a v2 unicast event drained straight to the coordinator. The
265
- // PHASE-4 transcript-synthesis backstop and the acked-hold fast-track exist to
266
- // paper over a LOST emit — they should NEVER fire once v2 delivery is healthy. So
267
- // their firing is now a demoted last-resort signal: every fire bumps a counter, and
268
- // under enforce a fire additionally emits a WARN naming it a v2-contract violation
269
- // (a real emit was expected but never arrived). Target = 0 fires in steady state.
270
- //
271
- // The code is NOT removed — it stays as the correctness net for a genuinely lost
272
- // emit (rollout plan §B3c: "코드 삭제는 하지 않고 관측 후 다음 사이클에 판단"). Process-
273
- // lifetime totals; read by tests + surfaced in mesh_status.
274
- const meshV2BackstopCounters = {
275
- /** PHASE-4 transcript synthesis actually reconciled a missing completion. */
276
- phase4SynthesisFired: 0,
277
- /** Acked-hold transcript fast-track promoted a synth ahead of the death deadline. */
278
- ackedHoldFastTrackFired: 0,
279
- /** Acked-hold death-deadline backstop released a held synth (the 8-min net). */
280
- ackedHoldDeathDeadlineFired: 0,
281
- };
282
-
283
- /** Test/observability accessor for the v2 last-resort backstop counters (snapshot). */
284
- export function getMeshV2BackstopCounters(): Readonly<typeof meshV2BackstopCounters> {
285
- return { ...meshV2BackstopCounters };
286
- }
287
-
288
- /** Test helper: zero the backstop counters so a case starts from a clean slate. */
289
- export function __resetMeshV2BackstopCountersForTests(): void {
290
- for (const k of Object.keys(meshV2BackstopCounters) as Array<keyof typeof meshV2BackstopCounters>) {
291
- meshV2BackstopCounters[k] = 0;
292
- }
293
- }
294
-
295
- /** Enforce switch mirror (see isMeshProtocolV2EnforceEnabled in mesh-events-pending);
296
- * re-read here (not imported) to keep the reconcile loop free of a cross-file coupling
297
- * and to read env at fire time. On by default; set MESH_PROTOCOL_V2_ENFORCE=0/false/
298
- * off/no to disable. Same vocabulary as the source of truth. */
299
- function meshProtocolV2EnforceOn(): boolean {
300
- const raw = process.env.MESH_PROTOCOL_V2_ENFORCE;
301
- if (typeof raw !== 'string' || !raw.trim()) return true; // unset/blank = default ON
302
- const v = raw.trim().toLowerCase();
303
- return !(v === '0' || v === 'false' || v === 'off' || v === 'no');
304
- }
305
-
306
- /** Bump a backstop counter and, under enforce, WARN that a last-resort net fired —
307
- * which under a healthy v2 contract should not happen (the real emit was lost). */
308
- function recordBackstopFire(kind: keyof typeof meshV2BackstopCounters, detail: string): void {
309
- meshV2BackstopCounters[kind]++;
310
- if (meshProtocolV2EnforceOn()) {
311
- LOG.warn('MeshReconcileV2', `v2 ENFORCE last-resort backstop fired (${kind}): ${detail}. Under a healthy v2 completion contract this should be 0 — a worker's real terminal emit was lost/late.`);
312
- }
313
- }
314
-
315
- function inFlightSynthKey(meshId: string, taskId: string): string {
316
- return `${meshId}::${taskId}`;
317
- }
318
-
319
- // Extract the taskId back out of a `${meshId}::${taskId}` synth key. The meshId
320
- // prefix can itself contain '::' only if the caller passed one (mesh ids are
321
- // config-derived and never do), so split on the FIRST '::' and treat the remainder
322
- // as the taskId.
323
- function taskIdFromSynthKey(meshId: string, synthKey: string): string {
324
- const prefix = `${meshId}::`;
325
- return synthKey.startsWith(prefix) ? synthKey.slice(prefix.length) : synthKey;
326
- }
327
-
328
- function holdStore(): MeshRuntimeStore | undefined {
329
- try { return MeshRuntimeStore.getInstance(); } catch { return undefined; }
330
- }
331
-
332
- // Read-through: Map hit returns the cached state; a miss consults the store and,
333
- // when a row exists, hydrates the Map from it before returning. A store failure
334
- // degrades to Map-only (returns undefined on a miss) — identical to the pre-T2
335
- // in-memory behavior, never worse.
336
- function getHoldState(synthKey: string, meshId: string): AckedHoldState | undefined {
337
- const cached = inFlightAckedHoldState.get(synthKey);
338
- if (cached) return cached;
339
- const store = holdStore();
340
- if (!store) return undefined;
341
- let row;
342
- try { row = store.getInflightHold(taskIdFromSynthKey(meshId, synthKey)); } catch { return undefined; }
343
- if (!row) return undefined;
344
- const state: AckedHoldState = {
345
- liveConfirmedSinceAck: row.holdReason === 'live',
346
- consecutiveReadFailures: row.readFailureCount ?? 0,
347
- ...(row.firstIdleSinceAck !== null && row.firstIdleSinceAck !== undefined
348
- ? { transcriptIdleSinceMs: row.firstIdleSinceAck }
349
- : {}),
350
- };
351
- inFlightAckedHoldState.set(synthKey, state);
352
- return state;
353
- }
354
-
355
- // Write-through: update the Map cache AND the store row. A store failure leaves the
356
- // Map authoritative for this process (degrade, never crash the tick).
357
- function setHoldState(synthKey: string, meshId: string, state: AckedHoldState): void {
358
- inFlightAckedHoldState.set(synthKey, state);
359
- const store = holdStore();
360
- if (!store) return;
361
- try {
362
- store.upsertInflightHold({
363
- taskId: taskIdFromSynthKey(meshId, synthKey),
364
- meshId,
365
- holdReason: state.liveConfirmedSinceAck ? 'live' : 'unconfirmed',
366
- firstIdleSinceAck: state.transcriptIdleSinceMs ?? null,
367
- readFailureCount: state.consecutiveReadFailures,
368
- });
369
- } catch { /* degrade to Map-only */ }
370
- }
371
-
372
- // Write-through delete: drop the Map entry AND the store row.
373
- function deleteHoldState(synthKey: string, meshId: string): void {
374
- inFlightAckedHoldState.delete(synthKey);
375
- const store = holdStore();
376
- if (!store) return;
377
- try { store.deleteInflightHold(taskIdFromSynthKey(meshId, synthKey)); } catch { /* degrade */ }
378
- }
379
-
380
- // Restart rehydration: on the first touch of a mesh this process, pull its persisted
381
- // acked-hold rows from the store into the Map cache so a hold that outlived a daemon
382
- // restart is honored again. Idempotent per process via rehydratedHoldMeshes. A store
383
- // failure just skips rehydration (Map starts empty for the mesh — pre-T2 behavior).
384
- function rehydrateAckedHoldsForMesh(meshId: string): void {
385
- if (rehydratedHoldMeshes.has(meshId)) return;
386
- rehydratedHoldMeshes.add(meshId);
387
- const store = holdStore();
388
- if (!store) return;
389
- let rows;
390
- try { rows = store.listInflightHoldsByMesh(meshId); } catch { return; }
391
- for (const row of rows) {
392
- const synthKey = inFlightSynthKey(meshId, row.taskId);
393
- if (inFlightAckedHoldState.has(synthKey)) continue; // a live tick already set fresher state
394
- inFlightAckedHoldState.set(synthKey, {
395
- liveConfirmedSinceAck: row.holdReason === 'live',
396
- consecutiveReadFailures: row.readFailureCount ?? 0,
397
- ...(row.firstIdleSinceAck !== null && row.firstIdleSinceAck !== undefined
398
- ? { transcriptIdleSinceMs: row.firstIdleSinceAck }
399
- : {}),
400
- });
401
- }
402
- if (rows.length > 0) {
403
- LOG.info('MeshReconcile', `Rehydrated ${rows.length} persisted acked-hold row(s) for mesh ${meshId} after (re)start`);
404
- }
405
- }
406
-
407
- // Test hook: clear the in-flight acked-hold state between cases (both the Map cache
408
- // and the per-mesh rehydrate guard, so each case starts from a clean read-through).
409
- export function __resetReconcileInFlightSynthDebounceForTests(): void {
410
- inFlightAckedHoldState.clear();
411
- rehydratedHoldMeshes.clear();
412
- }
413
-
414
150
  interface LiveCoordinator {
415
151
  meshId: string;
416
152
  instance: ReturnType<DaemonComponents['instanceManager']['getInstance']>;
@@ -436,96 +172,6 @@ interface LiveCoordinator {
436
172
  modalParked: boolean;
437
173
  }
438
174
 
439
- // The set of coordinator-daemon ids THIS daemon answers to when draining the
440
- // pending-events queue. A unicast completion event is stamped with the worker's
441
- // meshCoordinatorDaemonId, which can be either:
442
- // - the daemon's canonical status id (`standalone_<machineId>` / `daemon_<machineId>`),
443
- // stamped by the MCP layer via ctx.localDaemonId (= getStatus().status.instanceId), or
444
- // - the bare machineId, stamped by the local queue-assignment path (loadConfig().machineId).
445
- // - the config-form node daemonId (`daemon_<machineId>`), which the MCP layer's
446
- // resolveCoordinatorDaemonId prefers and stamps onto direct-dispatch workers.
447
- // Draining with only one of these silently misses events stamped with the other —
448
- // the exact reason a generating coordinator never self-received local completions,
449
- // and the base-node completion-surface bug (base completions land full-form
450
- // `daemon_<machineId>` while a coordinator that only knows itself as bare
451
- // `<machineId>` never matches them). We expand to EVERY equivalent form so the
452
- // scope match (host gate, self-node detection, and the drain IN-filter downstream)
453
- // succeeds regardless of which path stamped the event.
454
- function resolveCoordinatorDaemonIds(components: DaemonComponents): string[] {
455
- const statusInstanceId = readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId);
456
- const machineId = readNonEmptyString(loadConfig().machineId);
457
- return expandDaemonIdForms([statusInstanceId, machineId]);
458
- }
459
-
460
- // Whether THIS daemon is the coordinator/host for a mesh — i.e. the daemon that
461
- // owns coordinator ownership and must collect every worker node's completion
462
- // events into its local queue. This is true regardless of whether a *live CLI*
463
- // coordinator session currently exists: the coordinator is frequently a pure
464
- // stdio MCP LLM (no live CLI session to inject into), and that LLM only sees the
465
- // queue when it next calls a mesh tool. For it to see remote worker completions
466
- // at all, the daemon must have already pulled them into the local queue on the
467
- // timer — which is exactly what this predicate gates.
468
- //
469
- // Rule: this daemon hosts the mesh when meshHost.role is 'host' (the default for
470
- // standalone-compat meshes with no host metadata) AND, when a hostDaemonId is
471
- // pinned, it resolves to one of this daemon's ids. Member-only daemons return
472
- // false — their own queue is pulled BY the host, not the other way around.
473
- //
474
- // `daemonIds` here is the EXPANDED self-identity set (runtime drain ids ∪ this
475
- // daemon's mesh-config node id forms) — see resolveCoordinatorSelfIds. The
476
- // pinned hostDaemonId is itself a config-form id and frequently does NOT equal a
477
- // runtime id (bare machineId / status id), so gating on the runtime ids alone
478
- // would wrongly classify the real host as a non-host and skip the remote pull
479
- // entirely.
480
- function daemonHostsMesh(mesh: LocalMeshEntry, daemonIds: string[]): boolean {
481
- const host = mesh.meshHost;
482
- // No metadata → default host (standalone compatibility, see createDefaultMeshHostMetadata).
483
- if (!host) return true;
484
- if (host.role && host.role !== 'host') return false;
485
- const hostDaemonId = readNonEmptyString(host.hostDaemonId);
486
- // Host role but no pinned hostDaemonId → treat as host (single-daemon / legacy).
487
- if (!hostDaemonId) return true;
488
- return daemonIdListIncludes(daemonIds, hostDaemonId);
489
- }
490
-
491
- function daemonIdListIncludes(ids: readonly string[], id: string | undefined): boolean {
492
- if (!id) return false;
493
- return ids.some(candidate => candidate === id || daemonIdsEquivalent(candidate, id));
494
- }
495
-
496
- // Resolve EVERY id-form this daemon answers to FOR A GIVEN MESH: the runtime drain
497
- // ids (status id + bare machineId) unioned with this daemon's mesh-config identity
498
- // forms — the self node's daemonId/machineId (the node whose daemonId/machineId
499
- // matches a runtime id) and the pinned meshHost.hostDaemonId WHEN it is provably
500
- // ours. This is the single source of truth for "is this id me?" across both the
501
- // host gate and the remote pull filter; the worker's meshCoordinatorDaemonId stamp
502
- // is guaranteed to be one of these forms (it comes from resolveCoordinatorDaemonId,
503
- // which prefers the coordinator node's config-form daemonId over the runtime status id).
504
- function resolveCoordinatorSelfIds(mesh: LocalMeshEntry, drainDaemonIds: string[]): string[] {
505
- const ids = new Set<string>(drainDaemonIds);
506
- // Expand with the config-form id(s) of the self node — the mesh node whose
507
- // daemonId/machineId matches a runtime id. Its config-form daemonId is exactly
508
- // what resolveCoordinatorNode()→resolveCoordinatorDaemonId() stamps onto a worker.
509
- for (const node of mesh.nodes) {
510
- const nodeDaemonId = readNonEmptyString(node.daemonId);
511
- const nodeMachineId = readNonEmptyString(node.machineId);
512
- const isSelf = (nodeDaemonId && daemonIdListIncludes(drainDaemonIds, nodeDaemonId))
513
- || (nodeMachineId && daemonIdListIncludes(drainDaemonIds, nodeMachineId));
514
- if (!isSelf) continue;
515
- if (nodeDaemonId) ids.add(nodeDaemonId);
516
- if (nodeMachineId) ids.add(nodeMachineId);
517
- }
518
- // The pinned host id is included ONLY when it is provably one of THIS daemon's ids
519
- // (it already matches a runtime id or a resolved self-node id). A hostDaemonId that
520
- // names a DIFFERENT daemon must NOT be claimed — that would make a member-only
521
- // daemon believe it is the host and pull queues it does not own. Having a node on
522
- // this daemon does not make this daemon the host; daemonHostsMesh still honours a
523
- // foreign hostDaemonId and rejects ownership.
524
- const hostDaemonId = readNonEmptyString(mesh.meshHost?.hostDaemonId);
525
- if (hostDaemonId && daemonIdListIncludes([...ids], hostDaemonId)) ids.add(hostDaemonId);
526
- return [...ids];
527
- }
528
-
529
175
  // Observability: last-seen modal-park state per coordinator session, so we LOG.info
530
176
  // only on a TRANSITION (clear → parked, parked → cleared) instead of every 4s tick.
531
177
  // Per-process; a restart re-logs the first observation, which is desirable — it
@@ -2018,18 +1664,7 @@ async function reconcileUnterminatedDirectDispatches(
2018
1664
  .filter(Boolean)
2019
1665
  .map(taskId => inFlightSynthKey(mesh.id, taskId)),
2020
1666
  );
2021
- const heldKeys = new Set<string>();
2022
- for (const key of inFlightAckedHoldState.keys()) {
2023
- if (key.startsWith(`${mesh.id}::`)) heldKeys.add(key);
2024
- }
2025
- const store = holdStore();
2026
- if (store) {
2027
- try {
2028
- for (const row of store.listInflightHoldsByMesh(mesh.id)) {
2029
- heldKeys.add(inFlightSynthKey(mesh.id, row.taskId));
2030
- }
2031
- } catch { /* degrade — prune only what's in the Map */ }
2032
- }
1667
+ const heldKeys = collectHeldSynthKeysForMesh(mesh.id);
2033
1668
  for (const key of heldKeys) {
2034
1669
  if (!activeTaskKeys.has(key)) deleteHoldState(key, mesh.id);
2035
1670
  }
@@ -0,0 +1,62 @@
1
+ // ---------------------------------------------------------------------------
2
+ // mesh-reconcile-v2-backstop — last-resort completion backstop counters
3
+ // ---------------------------------------------------------------------------
4
+ // Pure move out of mesh-reconcile-loop.ts (no behavior change).
5
+ //
6
+ // T6 (B3c): PHASE-4 synthesis + acked-hold fast-track demoted to last-resort.
7
+ //
8
+ // Under mesh-protocol-v2 enforce, the completion contract is explicit: a worker's
9
+ // terminal emit is a v2 unicast event drained straight to the coordinator. The
10
+ // PHASE-4 transcript-synthesis backstop and the acked-hold fast-track exist to
11
+ // paper over a LOST emit — they should NEVER fire once v2 delivery is healthy. So
12
+ // their firing is now a demoted last-resort signal: every fire bumps a counter, and
13
+ // under enforce a fire additionally emits a WARN naming it a v2-contract violation
14
+ // (a real emit was expected but never arrived). Target = 0 fires in steady state.
15
+ //
16
+ // The code is NOT removed — it stays as the correctness net for a genuinely lost
17
+ // emit (rollout plan §B3c: "코드 삭제는 하지 않고 관측 후 다음 사이클에 판단"). Process-
18
+ // lifetime totals; read by tests + surfaced in mesh_status.
19
+ // ---------------------------------------------------------------------------
20
+
21
+ import { LOG } from '../logging/logger.js';
22
+
23
+ const meshV2BackstopCounters = {
24
+ /** PHASE-4 transcript synthesis actually reconciled a missing completion. */
25
+ phase4SynthesisFired: 0,
26
+ /** Acked-hold transcript fast-track promoted a synth ahead of the death deadline. */
27
+ ackedHoldFastTrackFired: 0,
28
+ /** Acked-hold death-deadline backstop released a held synth (the 8-min net). */
29
+ ackedHoldDeathDeadlineFired: 0,
30
+ };
31
+
32
+ /** Test/observability accessor for the v2 last-resort backstop counters (snapshot). */
33
+ export function getMeshV2BackstopCounters(): Readonly<typeof meshV2BackstopCounters> {
34
+ return { ...meshV2BackstopCounters };
35
+ }
36
+
37
+ /** Test helper: zero the backstop counters so a case starts from a clean slate. */
38
+ export function __resetMeshV2BackstopCountersForTests(): void {
39
+ for (const k of Object.keys(meshV2BackstopCounters) as Array<keyof typeof meshV2BackstopCounters>) {
40
+ meshV2BackstopCounters[k] = 0;
41
+ }
42
+ }
43
+
44
+ /** Enforce switch mirror (see isMeshProtocolV2EnforceEnabled in mesh-events-pending);
45
+ * re-read here (not imported) to keep the reconcile loop free of a cross-file coupling
46
+ * and to read env at fire time. On by default; set MESH_PROTOCOL_V2_ENFORCE=0/false/
47
+ * off/no to disable. Same vocabulary as the source of truth. */
48
+ function meshProtocolV2EnforceOn(): boolean {
49
+ const raw = process.env.MESH_PROTOCOL_V2_ENFORCE;
50
+ if (typeof raw !== 'string' || !raw.trim()) return true; // unset/blank = default ON
51
+ const v = raw.trim().toLowerCase();
52
+ return !(v === '0' || v === 'false' || v === 'off' || v === 'no');
53
+ }
54
+
55
+ /** Bump a backstop counter and, under enforce, WARN that a last-resort net fired —
56
+ * which under a healthy v2 contract should not happen (the real emit was lost). */
57
+ export function recordBackstopFire(kind: keyof typeof meshV2BackstopCounters, detail: string): void {
58
+ meshV2BackstopCounters[kind]++;
59
+ if (meshProtocolV2EnforceOn()) {
60
+ LOG.warn('MeshReconcileV2', `v2 ENFORCE last-resort backstop fired (${kind}): ${detail}. Under a healthy v2 completion contract this should be 0 — a worker's real terminal emit was lost/late.`);
61
+ }
62
+ }
@@ -1551,6 +1551,18 @@ export class MeshRuntimeStore {
1551
1551
  providerType?: string | null;
1552
1552
  payload?: unknown;
1553
1553
  }): void {
1554
+ // Ledger `kind` is a mandatory schema invariant (mesh_event_ledger.kind is
1555
+ // NOT NULL; every MeshLedgerKind is a non-empty tag). A blank kind would be a
1556
+ // structurally-broken entry — reject it here rather than write an unqueryable
1557
+ // row. NOTE: pending-event JSONL files (`*.pending-events.jsonl`) are a
1558
+ // SEPARATE shape that intentionally has NO `kind` field (they key off `.event`);
1559
+ // a generic audit that scans the whole ledger DIRECTORY and reads `.kind` off
1560
+ // those rows sees "kind=None", which is an artifact of mixing the two files, not
1561
+ // a ledger defect. This guard makes the ledger-side invariant explicit.
1562
+ if (!entry.kind || !String(entry.kind).trim()) {
1563
+ LOG.warn('MeshRuntimeStore', `Refusing to append ledger entry with empty kind for mesh ${entry.meshId} (id ${entry.id})`);
1564
+ return;
1565
+ }
1554
1566
  this.db.prepare(
1555
1567
  `INSERT OR IGNORE INTO mesh_event_ledger
1556
1568
  (id, mesh_id, timestamp, kind, node_id, session_id, provider_type, payload)
@@ -1693,6 +1705,11 @@ export class MeshRuntimeStore {
1693
1705
  );
1694
1706
  this.db.transaction(() => {
1695
1707
  for (const e of entries) {
1708
+ // Skip structurally-broken entries with a blank kind (see appendLedgerEntry):
1709
+ // mesh_event_ledger.kind is NOT NULL and every kind is a non-empty tag, so an
1710
+ // empty-kind row is unqueryable noise. Mirrors readLedgerFile's `entry.id && entry.kind`
1711
+ // JSONL guard, keeping the import path from re-introducing what the read path filters.
1712
+ if (!e.kind || !String(e.kind).trim()) continue;
1696
1713
  const result = stmt.run(
1697
1714
  e.id, e.meshId, e.timestamp, e.kind,
1698
1715
  e.nodeId ?? null, e.sessionId ?? null, e.providerType ?? null,
@@ -2053,4 +2070,40 @@ export class MeshRuntimeStore {
2053
2070
  `DELETE FROM mesh_pending_events WHERE id IN (${idList.map(() => '?').join(',')})`
2054
2071
  ).run(...idList).changes;
2055
2072
  }
2073
+
2074
+ /**
2075
+ * Retention prune for mesh_pending_events. This table has no lifecycle GC of its
2076
+ * own: a drained row is soft-marked (drained=1) and RETAINED — deliberately, so
2077
+ * drainedEventIdsForMesh() has a durable v2-eventId dedup baseline — and an
2078
+ * undrained row queued for a coordinator that never returned (a dead/evicted
2079
+ * coordinator identity) stays drained=0 forever. Both accumulate without bound
2080
+ * (observed: tens of thousands of rows, mostly stale). This is the missing
2081
+ * retention step. Two independent windows:
2082
+ *
2083
+ * - drained rows older than `drainedOlderThanMs`: the coordinator consumed them
2084
+ * long ago; the only thing they still back is the eventId re-delivery guard,
2085
+ * which is only meaningful for the recent past (a re-delivery of a week-old
2086
+ * event cannot occur — its producer session is long gone). Safe to delete.
2087
+ * - UNDRAINED rows older than `undrainedOlderThanMs` (a much wider window):
2088
+ * these are orphaned events for a coordinator identity that never drained
2089
+ * them. Kept wide so a genuinely-offline-but-returning coordinator still
2090
+ * receives its backlog; only genuinely unrecoverable orphans are swept.
2091
+ *
2092
+ * Both windows key off `queued_at` (always present) — `drained_at` can be NULL on
2093
+ * legacy rows. Returns the number of rows deleted. Best-effort / idempotent:
2094
+ * running it repeatedly with nothing to prune is a cheap no-op.
2095
+ */
2096
+ prunePendingEvents(opts: { drainedOlderThanMs: number; undrainedOlderThanMs: number }): number {
2097
+ const now = Date.now();
2098
+ const drainedCutoff = now - Math.max(0, opts.drainedOlderThanMs);
2099
+ const undrainedCutoff = now - Math.max(0, opts.undrainedOlderThanMs);
2100
+ let removed = 0;
2101
+ removed += this.db.prepare(
2102
+ 'DELETE FROM mesh_pending_events WHERE drained = 1 AND queued_at < ?'
2103
+ ).run(drainedCutoff).changes;
2104
+ removed += this.db.prepare(
2105
+ 'DELETE FROM mesh_pending_events WHERE drained = 0 AND queued_at < ?'
2106
+ ).run(undrainedCutoff).changes;
2107
+ return removed;
2108
+ }
2056
2109
  }
@@ -0,0 +1,75 @@
1
+ /**
2
+ * CLI provider persisted-history dedup — incremental append computation.
3
+ *
4
+ * Pure move out of cli-provider-instance.ts (no behavior change): the
5
+ * shared-prefix diff that turns a full parsed transcript into the newly-added
6
+ * tail to append to the persisted chat history. cli-provider-instance
7
+ * re-exports buildIncrementalHistoryAppendMessages so existing importers/tests
8
+ * keep their path.
9
+ */
10
+
11
+ import { flattenContent } from './contracts.js';
12
+
13
+ export type PersistableCliHistoryMessage = {
14
+ role: string;
15
+ content: string;
16
+ kind?: string;
17
+ senderName?: string;
18
+ receivedAt?: number;
19
+ };
20
+
21
+ function normalizePersistableCliHistoryContent(content: unknown): string {
22
+ return flattenContent(content as any).replace(/\s+/g, ' ').trim();
23
+ }
24
+
25
+ function buildPersistableCliHistorySignature(message: PersistableCliHistoryMessage): string {
26
+ return [
27
+ String(message.role || ''),
28
+ String(message.kind || ''),
29
+ String(message.senderName || ''),
30
+ normalizePersistableCliHistoryContent(message.content),
31
+ ].join('|');
32
+ }
33
+
34
+ function hasSamePersistableCliHistoryIdentity(a: PersistableCliHistoryMessage, b: PersistableCliHistoryMessage): boolean {
35
+ return String(a?.role || '') === String(b?.role || '')
36
+ && String(a?.kind || '') === String(b?.kind || '')
37
+ && String(a?.senderName || '') === String(b?.senderName || '')
38
+ && String(a?.content || '') === String(b?.content || '');
39
+ }
40
+
41
+ export function buildIncrementalHistoryAppendMessages(
42
+ previousMessages: PersistableCliHistoryMessage[],
43
+ currentMessages: PersistableCliHistoryMessage[],
44
+ ): PersistableCliHistoryMessage[] {
45
+ if (!Array.isArray(currentMessages) || currentMessages.length === 0) return [];
46
+ if (!Array.isArray(previousMessages) || previousMessages.length === 0) return currentMessages;
47
+
48
+ const comparableLength = Math.min(previousMessages.length, currentMessages.length);
49
+ let sharedPrefixLength = 0;
50
+ while (
51
+ sharedPrefixLength < comparableLength
52
+ && hasSamePersistableCliHistoryIdentity(previousMessages[sharedPrefixLength], currentMessages[sharedPrefixLength])
53
+ ) {
54
+ sharedPrefixLength += 1;
55
+ }
56
+
57
+ if (sharedPrefixLength === currentMessages.length) return [];
58
+ if (sharedPrefixLength === previousMessages.length) return currentMessages.slice(sharedPrefixLength);
59
+
60
+ // Rare fallback: preserve the older whitespace-normalized behavior only when
61
+ // the cheap identity check detects a changed prefix. Recomputing normalized
62
+ // signatures for the full transcript on every idle status poll was a CPU
63
+ // hot path for long CLI sessions.
64
+ while (
65
+ sharedPrefixLength < comparableLength
66
+ && buildPersistableCliHistorySignature(previousMessages[sharedPrefixLength])
67
+ === buildPersistableCliHistorySignature(currentMessages[sharedPrefixLength])
68
+ ) {
69
+ sharedPrefixLength += 1;
70
+ }
71
+
72
+ if (sharedPrefixLength === currentMessages.length) return [];
73
+ if (sharedPrefixLength === previousMessages.length) return currentMessages.slice(sharedPrefixLength);
74
+ return currentMessages;
75
+ }