@adhdev/daemon-core 0.9.82-rc.48 → 0.9.82-rc.481

package/src/mesh/mesh-reconcile-acked-hold.ts

@@ -0,0 +1,230 @@
+// ---------------------------------------------------------------------------
+// mesh-reconcile-acked-hold — in-flight acked-hold state (persistence + tuning)
+// ---------------------------------------------------------------------------
+// Pure move out of mesh-reconcile-loop.ts (no behavior change). This is the
+// read-through/write-through cache over the mesh_inflight_hold table plus the
+// env-tunable timers (death deadline, transcript fast-track grace) that govern
+// when an acked dispatch's indefinite synth hold is released. See the R4f and
+// ACKED-HOLD-IDLE-OVERTRUST design commentary inline.
+// ---------------------------------------------------------------------------
+
+import { LOG } from '../logging/logger.js';
+import { MeshRuntimeStore } from './mesh-runtime-store.js';
+import { readNonEmptyString } from './mesh-events-utils.js';
+
+// R4f backstop (a): how many CONSECUTIVE read_chat failures (transport error / success:false /
+// no payload) for an acked task are treated as a death signal that releases the indefinite hold.
+// A single failed read is a transient probe blip; a session that genuinely died reads-fail every
+// tick, so a small streak distinguishes the two without racing a live-but-slow worker.
+export const ACKED_DEATH_CONSECUTIVE_READ_FAILURES = 3;
+
+// R4f backstop (b): the absolute death-deadline. An acked task is held indefinitely until this much
+// time has elapsed since its generating_started ack (dispatch.updatedAt); past it, a persistently
+// idle session is synthesized as a notification-loss net. This is set FAR above any observed emit
+// latency (R4e's worst case was ~16s) so it does NOT race a normal slow turn — it only catches a
+// genuinely wedged worker or a permanently-lost emit. Read at call time so tests can tune it.
+export function resolveTunedReconcileMs(envName: string, def: number, min: number, max: number): number {
+    const raw = readNonEmptyString(process.env[envName]);
+    if (raw) {
+        const parsed = Number.parseInt(raw, 10);
+        if (Number.isFinite(parsed) && parsed >= min && parsed <= max) return parsed;
+    }
+    return def;
+}
+export function resolveAckedDeathDeadlineMs(): number {
+    // Default 8 min — FAR above the variable emit latency the finite R4..R4e timers raced (R4e's
+    // worst case was ~16s); by the time this fires a live worker would long since have emitted its
+    // real terminal. The env-override floor is 0 so tests can force the deadline (production never
+    // sets it that low); the ceiling is 60min so a mis-set env cannot disable the loss-net forever.
+    return resolveTunedReconcileMs('MESH_INFLIGHT_ACKED_DEATH_DEADLINE_MS', 8 * 60_000, 0, 60 * 60_000);
+}
+
+// ACKED-HOLD-IDLE-OVERTRUST (transcript-completion fast-track). The indefinite acked-hold above is
+// safe but SLOW: when the worker's real generating_completed emit is dropped/lost, the only thing
+// that promotes the missing completion is the 8-min death backstop — even though the answer has been
+// FULLY rendered in the transcript for minutes (read_chat reports idle WITH a final visible assistant
+// message every ~4s). Observed live: completions surfaced 144s / 492s late, both incompatible with the
+// provider's own emit ceiling (COMPLETED_FINALIZATION_MAX_WAIT_MS 30s + NATIVE_HISTORY_MESH_IDLE_SETTLE
+// 4s ≈ 34s). That gap = a worker that finished, whose PTY generating→idle edge / real emit was lost,
+// held hostage to the 8-min net.
+//
+// Fast-track: when an acked task reads idle AND a final visible assistant message is present (the same
+// transcript-completion evidence PHASE 4 already requires to synth), and that idle-with-final-assistant
+// state has PERSISTED for a short continuous grace, promote the synth EARLY — ahead of the 8-min
+// backstop. The grace is the correctness gate: a SINGLE idle read could be a mid-turn blip (PTY
+// inter-tool-call settle, or final text rendered while the next tool call is about to start), so we
+// require the idle-with-final-assistant signal to hold continuously for the grace window before
+// trusting it as a genuine turn-end. Any non-idle read (generating / waiting_approval), a read
+// failure, or the disappearance of the final assistant message RESETS the streak — so an actively
+// streaming worker that momentarily reads idle never crosses the grace.
+//
+// Safety: this only changes WHEN an acked synth fires (earlier), never WHETHER it is correct —
+// reconcileDirectDispatchCompletionFromTranscript's hasTerminalLedgerAfterDispatch makes a real
+// emit that lands later an idempotent no-op, exactly as the death-backstop synth relies on. The
+// death backstop (8 min) is PRESERVED unchanged as the final net; the fast-track is a faster path in
+// front of it. The grace is set ABOVE the provider's own emit ceiling (~34s) so a worker still inside
+// its normal finalization window is never pre-empted — we only fast-track once enough continuous idle
+// has elapsed that a live emit would already have arrived.
+export function resolveAckedTranscriptFastTrackGraceMs(): number {
+    // Default 40s — above the provider emit ceiling (30s COMPLETED_FINALIZATION_MAX_WAIT_MS + 4s
+    // NATIVE_HISTORY_MESH_IDLE_SETTLE ≈ 34s): a genuinely-live worker would have emitted its real
+    // terminal within that window, so 40s of CONTINUOUS idle-with-final-assistant means the emit was
+    // lost, not late. Far below the 8-min death backstop, so the fast-track is the dominant path for a
+    // lost emit while the backstop remains the last-resort net. Floor 0 lets tests force an immediate
+    // fast-track; ceiling 5min keeps a mis-set env from collapsing it into the death backstop.
+    return resolveTunedReconcileMs('MESH_INFLIGHT_ACKED_TRANSCRIPT_FASTTRACK_GRACE_MS', 40_000, 0, 5 * 60_000);
+}
+
+// Per-task in-flight hold state for an acked dispatch:
+//   - liveConfirmedSinceAck: we have seen at least one conclusive read (idle OR generating) since
+//     the ack — proves the session is reachable, so a later read FAILURE is a genuine liveness loss
+//     rather than a node that was never reachable.
+//   - consecutiveReadFailures: streak of inconclusive read_chat results (death backstop (a)).
+//   - transcriptIdleSinceMs: the timestamp of the FIRST tick in the current continuous run of
+//     idle-with-final-assistant reads (ACKED-HOLD-IDLE-OVERTRUST fast-track). Cleared to undefined
+//     whenever the signal breaks (non-idle read, read failure, or no final assistant message), so a
+//     mid-turn idle blip never accumulates grace. When `now - transcriptIdleSinceMs` exceeds the
+//     fast-track grace the synth is promoted ahead of the death backstop.
+export interface AckedHoldState {
+    liveConfirmedSinceAck: boolean;
+    consecutiveReadFailures: number;
+    transcriptIdleSinceMs?: number;
+}
+
+// T2 (B2b): acked-hold state persistence. The Map below is a process-local CACHE;
+// the SSOT is the mesh_inflight_hold table in MeshRuntimeStore. Every read goes
+// read-through (Map miss → load from store, then cache), every mutation goes
+// write-through (Map set → store upsert; Map delete → store delete). On daemon
+// boot the reconcile loop rehydrates the Map from the store per-mesh the first
+// time it touches that mesh (rehydrateAckedHoldsForMesh), so a hold established
+// before a restart survives it — closing the duplicate-emit / drop window the
+// PHASE-4 transcript synth backstop otherwise had to correct after the fact.
+//
+// Store row ↔ AckedHoldState mapping:
+//   hold_reason 'live'|'unconfirmed'  ↔ liveConfirmedSinceAck (boolean)
+//   read_failure_count                ↔ consecutiveReadFailures
+//   first_idle_since_ack              ↔ transcriptIdleSinceMs (undefined ⇒ NULL)
+//   mesh_id                            = the owning mesh (for listByMesh / prune)
+//   held_at                            = ms the hold was first created (store-managed)
+export const inFlightAckedHoldState = new Map<string, AckedHoldState>();
+// Meshes whose store rows have already been rehydrated into the Map this process.
+// A restart resets this set, so the first touch of each mesh reloads from disk.
+const rehydratedHoldMeshes = new Set<string>();
+
+export function inFlightSynthKey(meshId: string, taskId: string): string {
+    return `${meshId}::${taskId}`;
+}
+
+// Extract the taskId back out of a `${meshId}::${taskId}` synth key. The meshId
+// prefix can itself contain '::' only if the caller passed one (mesh ids are
+// config-derived and never do), so split on the FIRST '::' and treat the remainder
+// as the taskId.
+function taskIdFromSynthKey(meshId: string, synthKey: string): string {
+    const prefix = `${meshId}::`;
+    return synthKey.startsWith(prefix) ? synthKey.slice(prefix.length) : synthKey;
+}
+
+function holdStore(): MeshRuntimeStore | undefined {
+    try { return MeshRuntimeStore.getInstance(); } catch { return undefined; }
+}
+
+// Read-through: Map hit returns the cached state; a miss consults the store and,
+// when a row exists, hydrates the Map from it before returning. A store failure
+// degrades to Map-only (returns undefined on a miss) — identical to the pre-T2
+// in-memory behavior, never worse.
+export function getHoldState(synthKey: string, meshId: string): AckedHoldState | undefined {
+    const cached = inFlightAckedHoldState.get(synthKey);
+    if (cached) return cached;
+    const store = holdStore();
+    if (!store) return undefined;
+    let row;
+    try { row = store.getInflightHold(taskIdFromSynthKey(meshId, synthKey)); } catch { return undefined; }
+    if (!row) return undefined;
+    const state: AckedHoldState = {
+        liveConfirmedSinceAck: row.holdReason === 'live',
+        consecutiveReadFailures: row.readFailureCount ?? 0,
+        ...(row.firstIdleSinceAck !== null && row.firstIdleSinceAck !== undefined
+            ? { transcriptIdleSinceMs: row.firstIdleSinceAck }
+            : {}),
+    };
+    inFlightAckedHoldState.set(synthKey, state);
+    return state;
+}
+
+// Write-through: update the Map cache AND the store row. A store failure leaves the
+// Map authoritative for this process (degrade, never crash the tick).
+export function setHoldState(synthKey: string, meshId: string, state: AckedHoldState): void {
+    inFlightAckedHoldState.set(synthKey, state);
+    const store = holdStore();
+    if (!store) return;
+    try {
+        store.upsertInflightHold({
+            taskId: taskIdFromSynthKey(meshId, synthKey),
+            meshId,
+            holdReason: state.liveConfirmedSinceAck ? 'live' : 'unconfirmed',
+            firstIdleSinceAck: state.transcriptIdleSinceMs ?? null,
+            readFailureCount: state.consecutiveReadFailures,
+        });
+    } catch { /* degrade to Map-only */ }
+}
+
+// Write-through delete: drop the Map entry AND the store row.
+export function deleteHoldState(synthKey: string, meshId: string): void {
+    inFlightAckedHoldState.delete(synthKey);
+    const store = holdStore();
+    if (!store) return;
+    try { store.deleteInflightHold(taskIdFromSynthKey(meshId, synthKey)); } catch { /* degrade */ }
+}
+
+// Restart rehydration: on the first touch of a mesh this process, pull its persisted
+// acked-hold rows from the store into the Map cache so a hold that outlived a daemon
+// restart is honored again. Idempotent per process via rehydratedHoldMeshes. A store
+// failure just skips rehydration (Map starts empty for the mesh — pre-T2 behavior).
+export function rehydrateAckedHoldsForMesh(meshId: string): void {
+    if (rehydratedHoldMeshes.has(meshId)) return;
+    rehydratedHoldMeshes.add(meshId);
+    const store = holdStore();
+    if (!store) return;
+    let rows;
+    try { rows = store.listInflightHoldsByMesh(meshId); } catch { return; }
+    for (const row of rows) {
+        const synthKey = inFlightSynthKey(meshId, row.taskId);
+        if (inFlightAckedHoldState.has(synthKey)) continue; // a live tick already set fresher state
+        inFlightAckedHoldState.set(synthKey, {
+            liveConfirmedSinceAck: row.holdReason === 'live',
+            consecutiveReadFailures: row.readFailureCount ?? 0,
+            ...(row.firstIdleSinceAck !== null && row.firstIdleSinceAck !== undefined
+                ? { transcriptIdleSinceMs: row.firstIdleSinceAck }
+                : {}),
+        });
+    }
+    if (rows.length > 0) {
+        LOG.info('MeshReconcile', `Rehydrated ${rows.length} persisted acked-hold row(s) for mesh ${meshId} after (re)start`);
+    }
+}
+
+// Collect the union of held synth keys for a mesh — the Map cache entries plus every
+// persisted store row — so the PHASE-4 prune can drop a hold that exists ONLY on disk
+// (not yet cached). A store failure degrades to the Map-only key set (never throws).
+export function collectHeldSynthKeysForMesh(meshId: string): Set<string> {
+    const heldKeys = new Set<string>();
+    for (const key of inFlightAckedHoldState.keys()) {
+        if (key.startsWith(`${meshId}::`)) heldKeys.add(key);
+    }
+    const store = holdStore();
+    if (store) {
+        try {
+            for (const row of store.listInflightHoldsByMesh(meshId)) {
+                heldKeys.add(inFlightSynthKey(meshId, row.taskId));
+            }
+        } catch { /* degrade — prune only what's in the Map */ }
+    }
+    return heldKeys;
+}
+
+// Test hook: clear the in-flight acked-hold state between cases (both the Map cache
+// and the per-mesh rehydrate guard, so each case starts from a clean read-through).
+export function __resetReconcileInFlightSynthDebounceForTests(): void {
+    inFlightAckedHoldState.clear();
+    rehydratedHoldMeshes.clear();
+}

package/src/mesh/mesh-reconcile-config.ts

@@ -0,0 +1,66 @@
+// ---------------------------------------------------------------------------
+// mesh-reconcile-config — reconcile-loop timing tunables + env resolvers
+// ---------------------------------------------------------------------------
+// Extracted from mesh-reconcile-loop.ts (A-3 god-module decomposition, pure move,
+// no behavior change). Holds the loop-cadence tunables and their env-override
+// resolvers. Each resolver reads MESH_*_MS from the environment and clamps the
+// value so a mis-set env cannot make the loop pathological. Single-consumer
+// deadline constants that live next to their sole reader (e.g.
+// ASSIGNED_STRANDED_DEADLINE_MS, STRICT_SESSION_MATCH_TTL_MS) intentionally stay
+// in mesh-reconcile-loop.ts — only the shared loop-cadence tunables move here.
+// ---------------------------------------------------------------------------
+
+import { readNonEmptyString } from './mesh-events-utils.js';
+import { resolveTunedReconcileMs } from './mesh-reconcile-acked-hold.js';
+
+// Default reconcile cadence. approval/completion notifications to a live CLI
+// coordinator land within at most one interval. Overridable via env for tuning.
+export const DEFAULT_RECONCILE_INTERVAL_MS = 4_000;
+
+// PHASE 5 (auto-prune) conservative age gate. A direct dispatch whose node/session is
+// orphaned (no longer in the live mesh) is only auto-pruned once it is at least this old,
+// measured from its dispatch time. This protects against a node/session that is only
+// *transiently* invisible (a momentary probe failure, a daemon restart) being pruned the
+// instant it disappears. The MANUAL prune (mesh_prune_stale_direct) has no age gate — an
+// operator pruning explicitly wants the orphan gone now. Overridable via env for tuning.
+export const DEFAULT_AUTO_PRUNE_MIN_AGE_MS = 24 * 60 * 60_000; // 24h
+
+export function resolveAutoPruneMinAgeMs(): number {
+    const raw = readNonEmptyString(process.env.MESH_AUTO_PRUNE_MIN_AGE_MS);
+    if (raw) {
+        const parsed = Number.parseInt(raw, 10);
+        // Clamp to [1h, 30d] so a mis-set env can't make the gate pathologically aggressive
+        // (prune the moment something blinks) or effectively disable it forever.
+        if (Number.isFinite(parsed) && parsed >= 60 * 60_000 && parsed <= 30 * 24 * 60 * 60_000) return parsed;
+    }
+    return DEFAULT_AUTO_PRUNE_MIN_AGE_MS;
+}
+
+// PTY-OVERTRUST-DRAIN (Defect B, fix B). Age-based escape for the
+// `generating_no_idle_coordinator` hold. Fix A makes the drain predicate read the RAW
+// adapter (mask-stripped), so the common mask-driven false-busy is gone. But a hold can
+// still arise from a genuine status-source desync that fix A does not reach (e.g. the
+// adapter raw itself momentarily reads generating while the coordinator is actually at a
+// turn end). This is a TIME-BASED BACKSTOP: when a mesh's pending terminal events have
+// been held this long, re-confirm the coordinator's RAW adapter idle on the tick and, if
+// it is genuinely idle, drain ONCE. It NEVER injects into a genuinely-generating PTY —
+// the re-confirmation gates on raw adapter idle, so the intentional removal of
+// force-inject-into-generating (data-loss) is preserved. Default 12s = 3 reconcile ticks
+// at the 4s cadence: long enough that a normal mid-turn settle is not pre-empted, short
+// enough that a desync-stranded completion is not held for minutes. Env-tunable.
+export const DEFAULT_PENDING_HELD_DRAIN_ESCALATE_MS = 12_000;
+
+export function resolvePendingHeldDrainEscalateMs(): number {
+    // Floor 4s (one tick) so a mis-set env cannot make the escape race a normal settle;
+    // ceiling 5min so it cannot be disabled into a permanent strand.
+    return resolveTunedReconcileMs('MESH_PENDING_HELD_DRAIN_ESCALATE_MS', DEFAULT_PENDING_HELD_DRAIN_ESCALATE_MS, 4_000, 5 * 60_000);
+}
+
+export function resolveReconcileIntervalMs(): number {
+    const raw = readNonEmptyString(process.env.MESH_RECONCILE_INTERVAL_MS);
+    if (raw) {
+        const parsed = Number.parseInt(raw, 10);
+        if (Number.isFinite(parsed) && parsed >= 1_000 && parsed <= 60_000) return parsed;
+    }
+    return DEFAULT_RECONCILE_INTERVAL_MS;
+}

package/src/mesh/mesh-reconcile-identity.ts

@@ -0,0 +1,103 @@
+// ---------------------------------------------------------------------------
+// mesh-reconcile-identity — daemon-id / self-identity resolution for the reconcile loop
+// ---------------------------------------------------------------------------
+// Pure move out of mesh-reconcile-loop.ts (no behavior change). This is the single
+// source of truth for "which id-forms does THIS daemon answer to?" across both the
+// coordinator-daemon drain scope and the per-mesh host gate / remote pull filter.
+// ---------------------------------------------------------------------------
+
+import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
+import type { LocalMeshEntry } from '../repo-mesh-types.js';
+import { loadConfig } from '../config/config.js';
+import { expandDaemonIdForms, daemonIdsEquivalent } from '@adhdev/mesh-shared';
+import { readNonEmptyString } from './mesh-events-utils.js';
+
+// The set of coordinator-daemon ids THIS daemon answers to when draining the
+// pending-events queue. A unicast completion event is stamped with the worker's
+// meshCoordinatorDaemonId, which can be either:
+//   - the daemon's canonical status id (`standalone_<machineId>` / `daemon_<machineId>`),
+//     stamped by the MCP layer via ctx.localDaemonId (= getStatus().status.instanceId), or
+//   - the bare machineId, stamped by the local queue-assignment path (loadConfig().machineId).
+//   - the config-form node daemonId (`daemon_<machineId>`), which the MCP layer's
+//     resolveCoordinatorDaemonId prefers and stamps onto direct-dispatch workers.
+// Draining with only one of these silently misses events stamped with the other —
+// the exact reason a generating coordinator never self-received local completions,
+// and the base-node completion-surface bug (base completions land full-form
+// `daemon_<machineId>` while a coordinator that only knows itself as bare
+// `<machineId>` never matches them). We expand to EVERY equivalent form so the
+// scope match (host gate, self-node detection, and the drain IN-filter downstream)
+// succeeds regardless of which path stamped the event.
+export function resolveCoordinatorDaemonIds(components: DaemonComponents): string[] {
+    const statusInstanceId = readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId);
+    const machineId = readNonEmptyString(loadConfig().machineId);
+    return expandDaemonIdForms([statusInstanceId, machineId]);
+}
+
+// Whether THIS daemon is the coordinator/host for a mesh — i.e. the daemon that
+// owns coordinator ownership and must collect every worker node's completion
+// events into its local queue. This is true regardless of whether a *live CLI*
+// coordinator session currently exists: the coordinator is frequently a pure
+// stdio MCP LLM (no live CLI session to inject into), and that LLM only sees the
+// queue when it next calls a mesh tool. For it to see remote worker completions
+// at all, the daemon must have already pulled them into the local queue on the
+// timer — which is exactly what this predicate gates.
+//
+// Rule: this daemon hosts the mesh when meshHost.role is 'host' (the default for
+// standalone-compat meshes with no host metadata) AND, when a hostDaemonId is
+// pinned, it resolves to one of this daemon's ids. Member-only daemons return
+// false — their own queue is pulled BY the host, not the other way around.
+//
+// `daemonIds` here is the EXPANDED self-identity set (runtime drain ids ∪ this
+// daemon's mesh-config node id forms) — see resolveCoordinatorSelfIds. The
+// pinned hostDaemonId is itself a config-form id and frequently does NOT equal a
+// runtime id (bare machineId / status id), so gating on the runtime ids alone
+// would wrongly classify the real host as a non-host and skip the remote pull
+// entirely.
+export function daemonHostsMesh(mesh: LocalMeshEntry, daemonIds: string[]): boolean {
+    const host = mesh.meshHost;
+    // No metadata → default host (standalone compatibility, see createDefaultMeshHostMetadata).
+    if (!host) return true;
+    if (host.role && host.role !== 'host') return false;
+    const hostDaemonId = readNonEmptyString(host.hostDaemonId);
+    // Host role but no pinned hostDaemonId → treat as host (single-daemon / legacy).
+    if (!hostDaemonId) return true;
+    return daemonIdListIncludes(daemonIds, hostDaemonId);
+}
+
+export function daemonIdListIncludes(ids: readonly string[], id: string | undefined): boolean {
+    if (!id) return false;
+    return ids.some(candidate => candidate === id || daemonIdsEquivalent(candidate, id));
+}
+
+// Resolve EVERY id-form this daemon answers to FOR A GIVEN MESH: the runtime drain
+// ids (status id + bare machineId) unioned with this daemon's mesh-config identity
+// forms — the self node's daemonId/machineId (the node whose daemonId/machineId
+// matches a runtime id) and the pinned meshHost.hostDaemonId WHEN it is provably
+// ours. This is the single source of truth for "is this id me?" across both the
+// host gate and the remote pull filter; the worker's meshCoordinatorDaemonId stamp
+// is guaranteed to be one of these forms (it comes from resolveCoordinatorDaemonId,
+// which prefers the coordinator node's config-form daemonId over the runtime status id).
+export function resolveCoordinatorSelfIds(mesh: LocalMeshEntry, drainDaemonIds: string[]): string[] {
+    const ids = new Set<string>(drainDaemonIds);
+    // Expand with the config-form id(s) of the self node — the mesh node whose
+    // daemonId/machineId matches a runtime id. Its config-form daemonId is exactly
+    // what resolveCoordinatorNode()→resolveCoordinatorDaemonId() stamps onto a worker.
+    for (const node of mesh.nodes) {
+        const nodeDaemonId = readNonEmptyString(node.daemonId);
+        const nodeMachineId = readNonEmptyString(node.machineId);
+        const isSelf = (nodeDaemonId && daemonIdListIncludes(drainDaemonIds, nodeDaemonId))
+            || (nodeMachineId && daemonIdListIncludes(drainDaemonIds, nodeMachineId));
+        if (!isSelf) continue;
+        if (nodeDaemonId) ids.add(nodeDaemonId);
+        if (nodeMachineId) ids.add(nodeMachineId);
+    }
+    // The pinned host id is included ONLY when it is provably one of THIS daemon's ids
+    // (it already matches a runtime id or a resolved self-node id). A hostDaemonId that
+    // names a DIFFERENT daemon must NOT be claimed — that would make a member-only
+    // daemon believe it is the host and pull queues it does not own. Having a node on
+    // this daemon does not make this daemon the host; daemonHostsMesh still honours a
+    // foreign hostDaemonId and rejects ownership.
+    const hostDaemonId = readNonEmptyString(mesh.meshHost?.hostDaemonId);
+    if (hostDaemonId && daemonIdListIncludes([...ids], hostDaemonId)) ids.add(hostDaemonId);
+    return [...ids];
+}