@adhdev/daemon-core 0.9.82-rc.460 → 0.9.82-rc.462

package/src/mesh/mesh-reconcile-loop.ts

@@ -45,7 +45,7 @@ import type { LocalMeshEntry } from '../repo-mesh-types.js';
 import { loadConfig } from '../config/config.js';
 import { listMeshes } from '../config/mesh-config.js';
 import { LOG, getLogLevel } from '../logging/logger.js';
-import { drainPendingMeshCoordinatorEvents, getPendingMeshCoordinatorEvents, buildPendingEventFingerprint, queuePendingMeshCoordinatorEvent } from './mesh-events-pending.js';
+import { drainPendingMeshCoordinatorEvents, getPendingMeshCoordinatorEvents, buildPendingEventFingerprint, queuePendingMeshCoordinatorEvent, serializeV2EnvelopeToWire } from './mesh-events-pending.js';
 import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
 import { appendLedgerEntry } from './mesh-ledger.js';
 import { MeshRuntimeStore } from './mesh-runtime-store.js';
@@ -258,6 +258,59 @@ const inFlightAckedHoldState = new Map<string, AckedHoldState>();
 // A restart resets this set, so the first touch of each mesh reloads from disk.
 const rehydratedHoldMeshes = new Set<string>();
 
+// ─── T6 (B3c): PHASE-4 synthesis + acked-hold fast-track demoted to last-resort ──
+//
+// Under mesh-protocol-v2 enforce, the completion contract is explicit: a worker's
+// terminal emit is a v2 unicast event drained straight to the coordinator. The
+// PHASE-4 transcript-synthesis backstop and the acked-hold fast-track exist to
+// paper over a LOST emit — they should NEVER fire once v2 delivery is healthy. So
+// their firing is now a demoted last-resort signal: every fire bumps a counter, and
+// under enforce a fire additionally emits a WARN naming it a v2-contract violation
+// (a real emit was expected but never arrived). Target = 0 fires in steady state.
+//
+// The code is NOT removed — it stays as the correctness net for a genuinely lost
+// emit (rollout plan §B3c: "코드 삭제는 하지 않고 관측 후 다음 사이클에 판단"). Process-
+// lifetime totals; read by tests + surfaced in mesh_status.
+const meshV2BackstopCounters = {
+    /** PHASE-4 transcript synthesis actually reconciled a missing completion. */
+    phase4SynthesisFired: 0,
+    /** Acked-hold transcript fast-track promoted a synth ahead of the death deadline. */
+    ackedHoldFastTrackFired: 0,
+    /** Acked-hold death-deadline backstop released a held synth (the 8-min net). */
+    ackedHoldDeathDeadlineFired: 0,
+};
+
+/** Test/observability accessor for the v2 last-resort backstop counters (snapshot). */
+export function getMeshV2BackstopCounters(): Readonly<typeof meshV2BackstopCounters> {
+    return { ...meshV2BackstopCounters };
+}
+
+/** Test helper: zero the backstop counters so a case starts from a clean slate. */
+export function __resetMeshV2BackstopCountersForTests(): void {
+    for (const k of Object.keys(meshV2BackstopCounters) as Array<keyof typeof meshV2BackstopCounters>) {
+        meshV2BackstopCounters[k] = 0;
+    }
+}
+
+/** Enforce switch mirror (see isMeshProtocolV2EnforceEnabled in mesh-events-pending);
+ *  re-read here (not imported) to keep the reconcile loop free of a cross-file coupling
+ *  and to read env at fire time. Same truthy vocabulary. */
+function meshProtocolV2EnforceOn(): boolean {
+    const raw = process.env.MESH_PROTOCOL_V2_ENFORCE;
+    if (typeof raw !== 'string') return false;
+    const v = raw.trim().toLowerCase();
+    return v === '1' || v === 'true' || v === 'on' || v === 'yes';
+}
+
+/** Bump a backstop counter and, under enforce, WARN that a last-resort net fired —
+ *  which under a healthy v2 contract should not happen (the real emit was lost). */
+function recordBackstopFire(kind: keyof typeof meshV2BackstopCounters, detail: string): void {
+    meshV2BackstopCounters[kind]++;
+    if (meshProtocolV2EnforceOn()) {
+        LOG.warn('MeshReconcileV2', `v2 ENFORCE last-resort backstop fired (${kind}): ${detail}. Under a healthy v2 completion contract this should be 0 — a worker's real terminal emit was lost/late.`);
+    }
+}
+
 function inFlightSynthKey(meshId: string, taskId: string): string {
     return `${meshId}::${taskId}`;
 }
@@ -2011,6 +2064,12 @@ async function reconcileUnterminatedDirectDispatches(
 
         const synthKey = inFlightSynthKey(mesh.id, taskId);
         const isAcked = dispatch.status === 'acked';
+        // T6: which last-resort backstop (if any) drove this synth. Set when the
+        // acked-hold fast-track / death-deadline promotes the synth; the counter is
+        // bumped only if the synth actually COMMITS (result.reconciled), so a
+        // deferred/re-probed-away synth is not miscounted. A never-acked dispatch
+        // that reaches the commit is a plain PHASE-4 transcript synthesis.
+        let backstopKind: keyof ReturnType<typeof getMeshV2BackstopCounters> | undefined;
 
         // R4f: read the worker session. A FAILED read (transport error / success:false / no payload)
         // is no longer silently swallowed for an acked task — it is the liveness side of the
@@ -2130,6 +2189,7 @@ async function reconcileUnterminatedDirectDispatches(
                 const idleHeldMs = nowMs - idleSinceMs;
                 if (idleHeldMs >= fastTrackGraceMs) {
                     fastTrackReady = true;
+                    backstopKind = 'ackedHoldFastTrackFired';
                     LOG.info('MeshReconcile', `Acked-hold transcript fast-track: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read idle WITH a final assistant message for ${Math.round(idleHeldMs / 1000)}s continuous (grace ${Math.round(fastTrackGraceMs / 1000)}s) — promoting the synth ahead of the ${Math.round(deathDeadlineMs / 1000)}s death backstop; the worker's real emit was lost/late and a later one no-ops idempotently.`);
                 }
             } else if (holdState?.transcriptIdleSinceMs !== undefined) {
@@ -2144,6 +2204,7 @@ async function reconcileUnterminatedDirectDispatches(
                 continue;
             }
             if (!fastTrackReady) {
+                backstopKind = 'ackedHoldDeathDeadlineFired';
                 LOG.warn('MeshReconcile', `Acked-hold death deadline reached: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) still idle ${Math.round(sinceAckMs / 1000)}s after the ack (deadline ${Math.round(deathDeadlineMs / 1000)}s) — synthesizing the missing completion as a notification-loss net (a real emit, if it ever lands, no-ops idempotently).`);
             }
         }
@@ -2221,6 +2282,11 @@ async function reconcileUnterminatedDirectDispatches(
                 source: 'daemon_reconcile_transcript_completion',
             });
             if (result.reconciled) {
+                // T6: this synth actually committed → count the last-resort backstop fire.
+                // An acked hold routes to the fast-track / death-deadline kind captured
+                // above; a never-acked dispatch is a plain PHASE-4 transcript synthesis.
+                // Under enforce, recordBackstopFire additionally WARNs (target = 0 fires).
+                recordBackstopFire(backstopKind ?? 'phase4SynthesisFired', `task ${taskId} on node ${nodeId} (mesh ${mesh.id}), kind=${result.kind}`);
                 LOG.info('MeshReconcile', `Synthesized missing completion (${result.kind}) for task ${taskId} on node ${nodeId} (mesh ${mesh.id})`);
             }
         } catch (e: any) {
@@ -2360,6 +2426,13 @@ function buildForwardPayloadFromPending(event: any): Record<string, unknown> {
             const tid = readNonEmptyString(metadata.taskId) || readNonEmptyString(metadata.meshActiveTaskId);
             return tid ? { taskId: tid } : {};
         })(),
+        // T4 (B3b): carry the v2 envelope (protocolVersion/eventId/scope/dispatchedBy/
+        // intendedFor) across the P2P relay boundary at the TOP LEVEL. These live on the
+        // pending event itself, not inside metadataEvent, so without this the remote pull
+        // re-queue would re-stamp a fresh eventId — breaking cross-machine idempotency and
+        // downgrading the relayed completion to v1 broadcast routing. Spread LAST so the
+        // authoritative envelope always wins over any stale key the metadata spread carried.
+        ...serializeV2EnvelopeToWire(event as PendingMeshCoordinatorEvent),
     };
 }
 

package/src/mesh/mesh-runtime-store.ts

@@ -1917,6 +1917,36 @@ export class MeshRuntimeStore {
         return row !== undefined;
     }
 
+    /**
+     * B3a — v2 eventId idempotency. Returns true when a row with this event_id has
+     * ALREADY been drained (drained = 1) for the mesh. Drained rows are retained
+     * (soft-marked, not deleted until mesh deletion), so this is a durable, restart-
+     * surviving dedup: a v2 event whose eventId was already consumed is skipped on
+     * re-delivery even when its content fingerprint differs. Scoped by mesh_id +
+     * the partial event_id index (idx_mesh_pending_events_event_id).
+     */
+    hasDrainedEventId(meshId: string, eventId: string): boolean {
+        if (!eventId) return false;
+        const row = this.db.prepare(
+            'SELECT 1 FROM mesh_pending_events WHERE mesh_id = ? AND event_id = ? AND drained = 1 LIMIT 1'
+        ).get(meshId, eventId);
+        return row !== undefined;
+    }
+
+    /**
+     * B3a — snapshot of the v2 event_ids ALREADY drained (drained = 1) for the mesh.
+     * Taken BEFORE a drain call marks the current batch drained=1, so the resulting
+     * set names only PRIOR drains — the re-delivery dedup baseline. (Reading it after
+     * the drain would self-match the batch's own freshly-drained rows.) Non-v2 rows
+     * have a NULL event_id and are excluded by the index/WHERE.
+     */
+    drainedEventIdsForMesh(meshId: string): Set<string> {
+        const rows = this.db.prepare(
+            'SELECT DISTINCT event_id FROM mesh_pending_events WHERE mesh_id = ? AND drained = 1 AND event_id IS NOT NULL'
+        ).all(meshId) as Array<{ event_id: string }>;
+        return new Set(rows.map(r => r.event_id));
+    }
+
     // ── M3: Mission Records ─────────────────────────────────────────────────
 
     upsertMission(mission: {

package/src/providers/approval-utils.ts

@@ -16,7 +16,7 @@ const DEFAULT_APPROVAL_POSITIVE_HINTS = [
     'always allow',
 ];
 
-function normalizeApprovalLabel(value: string): string {
+export function normalizeApprovalLabel(value: string): string {
     return String(value || '')
         .toLowerCase()
         .replace(/^[\s\[(<{]*\d+(?:\s*[.)\]}>:-]|\s)+/, '')

package/src/providers/cli-provider-instance.ts

@@ -27,7 +27,7 @@ import { shouldCollectTraceCategory } from '../logging/debug-config.js';
 import { traceMeshEventStage, traceMeshEventDrop } from '../mesh/mesh-event-trace.js';
 import type { ChatMessage } from '../types.js';
 import { buildPersistedProviderEffectMessage, normalizeProviderEffects } from './control-effects.js';
-import { formatAutoApprovalMessage, pickApprovalButton, hasNegativeApprovalOption, hasReliableApprovalAffirmative, looksLikeActiveApprovalPromptText } from './approval-utils.js';
+import { formatAutoApprovalMessage, pickApprovalButton, hasNegativeApprovalOption, hasReliableApprovalAffirmative, looksLikeActiveApprovalPromptText, normalizeApprovalLabel } from './approval-utils.js';
 import { getCliScriptCommand, parseCliScriptResult } from './cli-script-results.js';
 import { mergeProviderPatchState, resolveProviderStateSurface } from './provider-patch-state.js';
 import { normalizeProviderSessionId } from './provider-session-id.js';
@@ -2317,19 +2317,31 @@ export class CliProviderInstance implements ProviderInstance {
             // kind gate). Surface the modal so the user decides; never pick blindly.
             return autoApproveActive;
         }
-        // Modal *identity* signature — the question/button set only, NO volatile
-        // counters. This is what the settle gate tracks: the FSM bumps
-        // approvalEntrySeq on every fresh waiting_approval entry, and a
-        // modal→generating→modal flap (the question line scrolled out of the
-        // captured frame while the button block stays) re-enters and bumps it
-        // again. Folding that seq into the settle signature made the 600ms
-        // settle clock restart on every flap, so the modal never stayed stable
-        // long enough to fire — the gate was never satisfied. Identity excludes
-        // the seq so button/seq flap of the SAME modal keeps one settle clock.
+        // Modal *identity* signature — the question plus the STABLE affirmative
+        // anchor only, NO volatile counters and NO raw button set. This is what
+        // the settle gate tracks: the FSM bumps approvalEntrySeq on every fresh
+        // waiting_approval entry, and a modal→generating→modal flap (the question
+        // line scrolled out of the captured frame while the button block stays)
+        // re-enters and bumps it again. Folding that seq into the settle signature
+        // made the 600ms settle clock restart on every flap, so the modal never
+        // stayed stable long enough to fire — the gate was never satisfied.
+        // Identity excludes the seq so seq flap of the SAME modal keeps one clock.
+        //
+        // The raw button set is also excluded: on a TALL Write/Edit diff claude's
+        // TUI repaints the button block 3↔5↔none between frames (buttons scroll in
+        // and out of the captured region), which flipped both buttons.join('|') and
+        // the positional buttonIndex every frame → signature flap → the settle
+        // clock reset 4–9s and only mask-stalled episodes leaked to the coordinator
+        // (AUTOAPPROVE-SETTLE-FLAP). The affirmative the auto-approve will actually
+        // press is the invariant across those repaints, so we anchor on its
+        // NORMALIZED label (numbers/bullets/punctuation stripped, so "1. Yes" and
+        // "3. Yes" collapse to "yes"). Message + affirmative label uniquely
+        // identifies the consent question without tracking the volatile button
+        // positions.
+        const affirmativeAnchor = normalizeApprovalLabel(buttonLabel);
         const modalSignature = [
             typeof modal?.message === 'string' ? modal.message.trim() : '',
-            buttons.join('|'),
-            buttonIndex,
+            affirmativeAnchor,
         ].join('::');
         // Busy-window re-entry guard still needs the seq: two DISTINCT
         // back-to-back approvals can carry identical message/buttons (common

package/src/repo-mesh-types.ts

@@ -703,6 +703,23 @@ export interface RepoMeshNodeCapabilities {
     canPush?: boolean;
     readOnly?: boolean;
     userLabels?: string[];
+    /**
+     * Detected provider CLI/ACP versions on this node, keyed by provider id
+     * (e.g. `{ 'claude-cli': '1.2.3', 'codex-cli': '0.9.0' }`). Populated from the
+     * same CLI detection pass that feeds providerPriority (see buildProviderVersions
+     * over detectCLIs' CLIInfo[]). Absent/undefined when detection has not run or a
+     * daemon predates the exposure — never a hard signal, purely observability so a
+     * coordinator can spot a provider-version skew across nodes before dispatch.
+     * Additive: existing status consumers ignore it.
+     */
+    providerVersions?: Record<string, string>;
+    /**
+     * The daemon build version (package.json version baked into the running bundle,
+     * see getDaemonBuildInfo().version) that detected the above providerVersions.
+     * Complements the commit-level daemonBuild stamp with a human-readable version
+     * for node-card rendering. Absent when the build define was not injected.
+     */
+    daemonBuildVersion?: string;
 }
 
 export interface DetectedCommand {
@@ -849,6 +866,19 @@ export interface LocalMeshNodeEntry {
      */
     reportedPlatform?: string;
     reportedArch?: string;
+    /**
+     * Live, self-healed provider CLI/ACP versions reported by the daemon that owns
+     * this node's workspace, carried on the git_status envelope (reporterProviderVersions)
+     * and persisted by the coordinator on each direct git probe — mirrors the
+     * reportedPlatform/reportedArch self-heal pattern. Auto-detected truth, overwritten
+     * by the next report so a stale value never sticks. Absent until the first probe
+     * carrying versions succeeds. Surfaced as RepoMeshNodeStatus.providerVersions.
+     */
+    reportedProviderVersions?: Record<string, string>;
+    /** Live, self-healed daemon build version (getDaemonBuildInfo().version) of the
+     *  owning daemon, carried on the git_status envelope (reporterDaemonBuildVersion)
+     *  alongside the provider versions. Absent until first reported. */
+    reportedDaemonBuildVersion?: string;
     /**
      * The operator-set machine nickname (config.machineNickname) of the daemon
      * that owns this node's workspace. The local coordinator stamps its own
@@ -974,6 +1004,76 @@ export interface RepoMeshStatus {
      * optional. Mirrors the MCP `mesh_status` tool's `magiActivity` field.
      */
     magiActivity?: MeshMagiActivitySummary[];
+    /**
+     * T7 (visibility 7-2b): provider CLI/ACP version skew across nodes. Each entry
+     * names a provider running ≥2 distinct versions across the nodes that reported
+     * it, with the node ids per version. Observational only — never a dispatch
+     * blocker. Omitted when every reported provider is uniform (or none reported).
+     * Mirrors the MCP `mesh_status` tool's `providerVersionSkew` field.
+     */
+    providerVersionSkew?: MeshProviderVersionSkew[];
+    /** Human-readable companion warning to providerVersionSkew. Omitted when no skew. */
+    providerVersionSkewWarning?: string;
+    /**
+     * T7 (B4): mesh-protocol-v2 adoption metrics for the batch of pending events
+     * surfaced in the drain backing this status. Snapshot, not a durable counter.
+     * Omitted when nothing was drained. Mirrors the MCP tool's meshProtocolMetrics.
+     */
+    meshProtocolMetrics?: MeshProtocolMetrics;
+    /**
+     * T6 (B3c): live process-lifetime mesh-protocol-v2 enforce counters from THIS
+     * daemon — the enforce flag state, drain-routing tallies (deliver / route-away /
+     * dedup / quarantine), and the last-resort backstop fire counts (PHASE-4 synth,
+     * acked-hold fast-track / death-deadline). Diagnostic-only and never cached (a
+     * live snapshot). Under enforce, non-zero quarantine or backstop counts are the
+     * rollout-health signal (target 0). Omitted when unavailable.
+     */
+    meshProtocolV2Counters?: MeshProtocolV2Counters;
+}
+
+/** T6 (B3c) live v2 enforce/observability counters (see RepoMeshStatus.meshProtocolV2Counters). */
+export interface MeshProtocolV2Counters {
+    /** True when MESH_PROTOCOL_V2_ENFORCE is active on this daemon. */
+    enforce: boolean;
+    /** Drain-path routing tallies (accept + enforce). Process-lifetime totals. */
+    drain: {
+        v2Delivered: number;
+        v2RoutedAway: number;
+        v2DedupSkipped: number;
+        v2ValidationFailedAccepted: number;
+        v2ReattributedToDrainer: number;
+        v1BroadcastAccepted: number;
+        v2ValidationFailedQuarantined: number;
+        v1UnversionedQuarantined: number;
+    };
+    /** Last-resort backstop fire counts. Target 0 under a healthy v2 contract. */
+    backstop: {
+        phase4SynthesisFired: number;
+        ackedHoldFastTrackFired: number;
+        ackedHoldDeathDeadlineFired: number;
+    };
+}
+
+/** One provider's version skew across mesh nodes (see RepoMeshStatus.providerVersionSkew). */
+export interface MeshProviderVersionSkew {
+    /** Provider id (e.g. 'claude-cli'). */
+    provider: string;
+    /** Each distinct detected version and the node ids running it. */
+    versions: Array<{ version: string; nodeIds: string[] }>;
+}
+
+/** Mesh-protocol-v2 adoption snapshot over one drain (see RepoMeshStatus.meshProtocolMetrics). */
+export interface MeshProtocolMetrics {
+    /** Total pending events surfaced in the drain. */
+    total: number;
+    /** Count carrying a v2 envelope (protocolVersion '2.0'). */
+    v2: number;
+    /** Count still on v1 (unstamped). */
+    v1: number;
+    /** v2/total, rounded to 2 decimals (0 when total is 0). */
+    v2Ratio: number;
+    /** Scope breakdown of the v2 events (unicast/broadcast/system/unspecified → count). */
+    scopes: Record<string, number>;
 }
 
 // RepoMeshSessionStatus shape now lives in @adhdev/mesh-shared (shared with
@@ -1026,6 +1126,17 @@ export interface RepoMeshNodeStatus {
      */
     gitProbePending?: boolean;
     providers: string[];
+    /**
+     * Detected provider CLI/ACP versions on this node, keyed by provider id. Mirrors
+     * RepoMeshNodeCapabilities.providerVersions onto the status snapshot so the mesh
+     * UI / coordinator prompt can render per-provider versions and flag a version
+     * skew across nodes. Optional — omitted by daemons predating the exposure or when
+     * detection has not run. Additive; existing consumers ignore it. */
+    providerVersions?: Record<string, string>;
+    /** Human-readable daemon build version (getDaemonBuildInfo().version) of the
+     *  daemon that owns this node. Complements the per-daemon commit stamp
+     *  (daemonBuilds) for node-card display. Omitted when unknown. */
+    daemonBuildVersion?: string;
     activeSessions: string[];
     activeSessionDetails?: RepoMeshSessionStatus[];
     providerPriority?: string[];