@adhdev/daemon-core 0.9.82-rc.481 → 0.9.82-rc.482

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -195,7 +195,6 @@ export type { SavedProviderSessionEntry } from './config/saved-sessions.js';
195
195
  export {
196
196
  listMeshes, getMesh, getMeshByRepo, createMesh, updateMesh, deleteMesh,
197
197
  addNode, removeNode, updateNode, normalizeRepoIdentity,
198
- listMagiPanels, getMagiPanel, upsertMagiPanel, removeMagiPanel, normalizeMagiPanel,
199
198
  listMagiKindPanels, getMagiKindPanel, setMagiKindPanel, removeMagiKindPanel, normalizeMagiSlots,
200
199
  } from './config/mesh-config.js';
201
200
  export type { CreateMeshOptions, UpdateMeshOptions, AddNodeOptions } from './config/mesh-config.js';
@@ -203,7 +202,7 @@ export type { CreateMeshOptions, UpdateMeshOptions, AddNodeOptions } from './con
203
202
  // leaf so the mcp-server — which depends only on @adhdev/daemon-core — can consume
204
203
  // them without taking a direct @adhdev/mesh-shared dependency).
205
204
  export type {
206
- MagiPanel, MagiPanelMember, MagiPanelMap, MagiMode, MagiTaskKind, MagiPanelDefaultKind,
205
+ MagiMode, MagiTaskKind,
207
206
  MagiSlot, MagiKindPanelMap,
208
207
  MagiClaim, MagiClaimStance, MagiAgentResponse,
209
208
  MagiResponseSource, MagiReplicaGitRef, MagiGitSkew, MagiSynthesizedResponse,
@@ -368,25 +368,31 @@ const TERMINAL_TASK_EVENTS: ReadonlySet<string> = new Set([
368
368
  ]);
369
369
 
370
370
  /**
371
- * Infrastructure/system events that no coordinator should surface — ledger
372
- * consistency and dispatch-plane signals. Delivered to the daemon-level handler
373
- * only (scope 'system').
371
+ * Coordinator-addressed dispatch-plane alerts. `mesh:dispatch_blocked` is the
372
+ * Fix (1) actionable dispatch-skip notification: it exists precisely to page the
373
+ * ORIGINATING coordinator (it carries a why+how coordinatorMessage and is
374
+ * targetCoordinator*-addressed by its producer), so it routes unicast exactly
375
+ * like a terminal task event. B2a originally classed it 'system' — that made it
376
+ * a dead letter: no daemon-level system drain exists, so the blocker never
377
+ * reached any coordinator and the task sat silently undispatched.
374
378
  */
375
- const SYSTEM_EVENTS: ReadonlySet<string> = new Set([
379
+ const COORDINATOR_ALERT_EVENTS: ReadonlySet<string> = new Set([
376
380
  'mesh:dispatch_blocked',
377
381
  ]);
378
382
 
379
383
  /**
380
384
  * Default the v2 scope for an event by its producer event name (design decision
381
- * §3). Terminal task events → unicast (routed to the originating coordinator).
382
- * Ledger-consistency / dispatch-plane events → system. Everything else — node
383
- * lifecycle and progress signals — → broadcast, which also matches v1's
384
- * implicit "deliver to any coordinator" behaviour, so an unstamped v1 event and
385
- * a v2-stamped-as-broadcast event route identically during rollout.
385
+ * §3). Terminal task events and coordinator-addressed alerts → unicast (routed
386
+ * to the originating coordinator). Everything else — node lifecycle and
387
+ * progress signals — → broadcast, which also matches v1's implicit "deliver to
388
+ * any coordinator" behaviour, so an unstamped v1 event and a v2-stamped-as-
389
+ * broadcast event route identically during rollout. No event currently defaults
390
+ * to 'system'; the scope remains in MESH_EVENT_SCOPES for wire compatibility
391
+ * (an already-queued or version-skewed 'system' event still routes away from
392
+ * coordinators).
386
393
  */
387
394
  export function defaultScopeForEvent(eventName: string): MeshEventScope {
388
- if (SYSTEM_EVENTS.has(eventName)) return 'system';
389
- if (TERMINAL_TASK_EVENTS.has(eventName)) return 'unicast';
395
+ if (TERMINAL_TASK_EVENTS.has(eventName) || COORDINATOR_ALERT_EVENTS.has(eventName)) return 'unicast';
390
396
  return 'broadcast';
391
397
  }
392
398
 
@@ -649,9 +649,7 @@ const TOOLS_SECTION = `## Available Tools
649
649
  | \`mesh_write_mesh_json_config\` | Gated write of \`.adhdev/mesh.json\` (repo coordinator-prompt config) from the mesh entry — dry-run/overwrite like mesh_init |
650
650
  | \`mesh_magi_review\` | Cross-verify a read-only investigation across a standing panel of independent mesh agents (different machines/providers) instead of a single worker |
651
651
  | \`mesh_magi_collect\` | Collect + synthesize a previously dispatched MAGI fan-out by its consensus group id (async companion to mesh_magi_review wait:false) |
652
- | \`mesh_magi_panel_set\` | Upsert a named MAGI panel (standing set of independent node×provider members) into machine-local config |
653
- | \`mesh_magi_panel_list\` | List configured MAGI panels and resolve each member's availability against the current mesh (read-only) |
654
- | \`mesh_magi_kind_panel_set\` | Bind a task_kind → MAGI kind-panel slots (machine-local, wholesale replacement — approve current-vs-new first) |
652
+ | \`mesh_magi_kind_panel_set\` | Bind a task_kind MAGI kind-panel slots (the SOLE MAGI panel-resolution surface; machine-local, wholesale replacement — approve current-vs-new first) |
655
653
  | \`mesh_magi_kind_panel_list\` | List configured task_kind → MAGI kind-panel slot bindings (machine-local, read-only) |`;
656
654
 
657
655
  const TOOL_EXPOSURE_PREFLIGHT_SECTION = `## Tool Exposure Preflight
@@ -694,7 +692,7 @@ When the user asks to **set up / configure / onboard** this repo for Repo Mesh (
694
692
 
695
693
  **Save scopes — label every draft with its scope before asking for approval:**
696
694
  - **repo-file (commit target)** — \`.adhdev/refine.json\`, \`.adhdev/worktree_bootstrap.json\`, \`.adhdev/change-impact.json\`, \`.adhdev/mesh.json\`. These are committed to the repository and shared with every machine/contributor.
697
- - **machine-local** — MAGI kind→panel bindings and named MAGI panels, node providerPriority (\`~/.adhdev/meshes.json\`). These stay on this machine and are NOT committed.
695
+ - **machine-local** — MAGI kind→panel bindings, node providerPriority (\`~/.adhdev/meshes.json\`). These stay on this machine and are NOT committed.
698
696
 
699
697
  **Guided sequence:**
700
698
  1. **Scan (dry-run)** — Call \`mesh_init\` (write=false, the default). It returns per-domain suggested configs for refine / worktree_bootstrap / change-impact, a recommended providerPriority, AND \`currentConfig\` — the currently-saved config per domain (repo files + machine-local \`magiKindPanels\`). Nothing is written.
@@ -702,8 +700,7 @@ When the user asks to **set up / configure / onboard** this repo for Repo Mesh (
702
700
  3. **Approve → gated write** — Only after the user approves, call the matching gated-write tool:
703
701
  - repo \`.adhdev/*\` config files → \`mesh_init\` with \`write=true\` (and \`overwrite=true\` ONLY for domains the user approved replacing).
704
702
  - \`.adhdev/mesh.json\` (coordinator prompt / operating notes) → \`mesh_write_mesh_json_config\` (write=true, overwrite only if approved).
705
- - machine-local MAGI kind→panel slots → \`mesh_magi_kind_panel_set\` (write=true). NOTE: a kind binding is a **wholesale replacement** of that kind's slot list — present the current-vs-new slots first.
706
- - machine-local named MAGI panels → \`mesh_magi_panel_set\`. providerPriority → apply via node policy update.
703
+ - machine-local MAGI kind→panel slots → \`mesh_magi_kind_panel_set\` (write=true). NOTE: a kind binding is a **wholesale replacement** of that kind's slot list — present the current-vs-new slots first. providerPriority → apply via node policy update.
707
704
 
708
705
  **init vs reinit:**
709
706
  - **\`mesh_init\`** — for a fresh, never-onboarded repo. Existing config files are kept (existing-wins) unless the user explicitly approves overwrite. Use for first-time setup.
@@ -6,12 +6,12 @@ import { appendLedgerEntry, buildTaskCompletionEvidence, getSessionRecoveryConte
6
6
  import type { SessionRecoveryContext } from './mesh-ledger.js';
7
7
  import { updateSessionTaskStatus, enqueueTask, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, getActiveDirectDispatches, hasPendingDependents, getQueue } from './mesh-work-queue.js';
8
8
  import { markSessionDeliveriesTerminal, updateSessionDeliveryStatus } from './mesh-delivery-policy.js';
9
- import { MeshRuntimeStore } from './mesh-runtime-store.js';
9
+ import { MeshRuntimeStore, pruneMeshRuntimeRetention } from './mesh-runtime-store.js';
10
10
  import { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents, prunePendingMeshCoordinatorEventsRetention, readV2EnvelopeFromWire, type PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
11
11
  import type { ProviderInstance } from '../providers/provider-instance.js';
12
12
  import { resolveWorkerDelegateRouting, recordUnroutableDelegateEvent, isUnroutableDelegateRejection } from './mesh-routing.js';
13
13
  import { resolveMeshHostStatus } from './mesh-host-ownership.js';
14
- import { enqueueUnresolvedDelegateForward, peekUnresolvedDelegateForwards, ackUnresolvedDelegateForward } from './mesh-unresolved-forward-outbox.js';
14
+ import { enqueueUnresolvedDelegateForward, nudgeUnresolvedForwardRetry } from './mesh-unresolved-forward-outbox.js';
15
15
  import { traceMeshEventStage, traceMeshEventDrop } from './mesh-event-trace.js';
16
16
  import { getLastDisplayMessage } from '../status/snapshot.js';
17
17
  import { resolveDelegatedWorkerAutoApprove } from '../repo-mesh-types.js';
@@ -189,13 +189,16 @@ function sweepExpiredRemoteIdleSessions(): void {
189
189
  try {
190
190
  MeshRuntimeStore.getInstance().pruneExpiredRemoteIdleSessions();
191
191
  } catch { /* best-effort */ }
192
- // Piggyback the pending-event retention prune on the same periodic sweep, but
193
- // hourly — this is the maintenance hook that keeps mesh_pending_events from
194
- // accumulating stale drained/orphaned rows without bound.
192
+ // Piggyback the retention prunes on the same periodic sweep, but hourly — this
193
+ // is the maintenance hook that keeps mesh-runtime.db from accumulating stale
194
+ // rows without bound: mesh_pending_events (drained/orphaned rows) plus, on the
195
+ // SAME cadence (SoT 1-11 (b)), the event ledger / tool-call log / terminal
196
+ // queue retention in pruneMeshRuntimeRetention.
195
197
  const now = Date.now();
196
198
  if (now - lastPendingEventsPruneAt >= PENDING_EVENTS_PRUNE_INTERVAL_MS) {
197
199
  lastPendingEventsPruneAt = now;
198
200
  prunePendingMeshCoordinatorEventsRetention();
201
+ pruneMeshRuntimeRetention();
199
202
  }
200
203
  }
201
204
 
@@ -1591,43 +1594,6 @@ export function handleMeshForwardEvent(components: DaemonComponents, payload: Re
1591
1594
  });
1592
1595
  }
1593
1596
 
1594
- // ---------------------------------------------------------------------------
1595
- // Per-coordinator forward serialization (P2P send-backpressure relief).
1596
- //
1597
- // When several workers finish at once, each completion runs forwardUnresolvedDelegate
1598
- // Event and fires its own `mesh_forward_event` push. Firing the whole burst
1599
- // concurrently dumps it into the single per-peer P2P DataChannel buffer in one tick,
1600
- // which starves the rpc_ack/rpc_res replies the same channel must carry — a
1601
- // coordinator's inbound `git_status` then times out even though the worker's own
1602
- // forward acks return in ~1s. To cap the concurrent burst we serialize the immediate
1603
- // pushes per coordinator: at most one push is in flight to a given coordinator at a
1604
- // time, the rest run in arrival order behind it. A lone event (idle lane) still
1605
- // dispatches immediately — only a genuine burst is paced. Durability is unchanged:
1606
- // every event is already persisted to the outbox before the push runs, so serializing
1607
- // only delays the best-effort fast path; PHASE 0 retry still covers any gap. This pairs
1608
- // with the DataChannel send-buffer gate in daemon-cloud's mesh manager (writeRequest),
1609
- // which is the hard guarantee; this throttle keeps the burst from piling up there.
1610
- interface CoordinatorForwardLane { tail: Promise<unknown>; depth: number; }
1611
- const coordinatorForwardLanes = new Map<string, CoordinatorForwardLane>();
1612
- function enqueueCoordinatorForwardPush(coordinatorDaemonId: string, run: () => Promise<unknown>): void {
1613
- let lane = coordinatorForwardLanes.get(coordinatorDaemonId);
1614
- if (!lane) { lane = { tail: Promise.resolve(), depth: 0 }; coordinatorForwardLanes.set(coordinatorDaemonId, lane); }
1615
- const wasIdle = lane.depth === 0;
1616
- lane.depth += 1;
1617
- const dec = (): void => { lane!.depth -= 1; };
1618
- if (wasIdle) {
1619
- // Idle lane → dispatch synchronously, so a lone completion (the common case) has
1620
- // ZERO added latency and the push call happens in-line. Only a genuine burst —
1621
- // events arriving while a push is still in flight — is paced (else branch).
1622
- lane.tail = Promise.resolve(run()).catch(() => {}).then(dec, dec);
1623
- } else {
1624
- // Burst: queue behind the in-flight push(es) in arrival order so the whole burst
1625
- // is not dumped into the shared DataChannel buffer at once. The tail is guarded
1626
- // so one rejecting push never wedges the lane for the next.
1627
- lane.tail = lane.tail.then(() => run()).catch(() => {}).then(dec, dec);
1628
- }
1629
- }
1630
-
1631
1597
  // ---------------------------------------------------------------------------
1632
1598
  // Worker-side fallback forward for unresolved-mesh delegates.
1633
1599
  //
@@ -1641,8 +1607,9 @@ function enqueueCoordinatorForwardPush(coordinatorDaemonId: string, run: () => P
1641
1607
  // the worker's queue — which it can't, because the worker never queued an unroutable
1642
1608
  // event. Live symptom: `WARN [MeshEvents] delivery_unroutable: ... mesh unresolved`.
1643
1609
  //
1644
- // The fix: the routing object still carries coordinatorDaemonId. Forward the raw event
1645
- // straight to that coordinator daemon over P2P (mesh_forward_event). The coordinator
1610
+ // The fix: the routing object still carries coordinatorDaemonId. Persist the raw event
1611
+ // to the durable worker-side outbox addressed to that coordinator daemon; the reconcile
1612
+ // loop's PHASE 0 delivers it (mesh_forward_event, acked, retry-capped). The coordinator
1646
1613
  // hosts the mesh, so it recovers the mesh id by workspace in handleMeshForwardEvent and
1647
1614
  // injects/queues it normally. meshId is intentionally omitted from the payload (the
1648
1615
  // worker has none); workspace is the routing anchor the coordinator resolves from.
@@ -1656,14 +1623,17 @@ function enqueueCoordinatorForwardPush(coordinatorDaemonId: string, run: () => P
1656
1623
  //
1657
1624
  // Returns true when the event was durably accepted for delivery to the coordinator
1658
1625
  // daemon (so the caller skips the delivery_unroutable diagnostic); false when no
1659
- // fallback was possible (no coordinator anchor / no dispatch transport).
1626
+ // fallback was possible (no coordinator anchor / no dispatch transport) OR the durable
1627
+ // enqueue itself failed — the delivery_unroutable diagnostic is thereby narrowed to
1628
+ // "could not even persist to the queue" (a real potential loss), per the polling-
1629
+ // single-model design (docs/refactoring/2026-06-16-mesh-completion-polling-single-model.md §2.1/§2.6).
1660
1630
  //
1661
- // Durability: the directed push to the coordinator is the ONLY delivery route for an
1662
- // unresolved-mesh worker (it is in no mesh.node the coordinator can pull). So instead
1663
- // of a fire-and-forget push that drops on one transient P2P failure, the event is
1664
- // persisted to the worker-side outbox FIRST and only acked after a successful push.
1665
- // A best-effort immediate push keeps latency low on the happy path; a failed or
1666
- // un-acked push leaves the durable row for setupMeshReconcileLoop's PHASE 0 to retry.
1631
+ // Single delivery path (polling single-model): the spontaneous best-effort immediate
1632
+ // push that used to run here was REMOVED. Every unresolved-delegate event is persisted
1633
+ // to the outbox and delivered ONLY by setupMeshReconcileLoop's PHASE 0 retry (acked;
1634
+ // a failed push leaves the row queued). For happy-path latency the enqueue emits a
1635
+ // data-free reconcile NUDGE (nudgeUnresolvedForwardRetry) asking the loop to run the
1636
+ // retry soon; a lost nudge costs at most one reconcile interval, never the event.
1667
1637
  function forwardUnresolvedDelegateEvent(
1668
1638
  components: DaemonComponents,
1669
1639
  routing: ReturnType<typeof resolveWorkerDelegateRouting>,
@@ -1718,10 +1688,11 @@ function forwardUnresolvedDelegateEvent(
1718
1688
  return true;
1719
1689
  }
1720
1690
 
1721
- // 1) Persist durably FIRST. Idempotent on fingerprint, so a re-fired completion
1722
- // does not duplicate the outbox row. If persistence fails we still attempt the
1723
- // push below (degrades to the old at-most-once behaviour rather than dropping
1724
- // the chance entirely).
1691
+ // Persist durably. Idempotent on fingerprint, so a re-fired completion does not
1692
+ // duplicate the outbox row. The outbox is the ONLY delivery route now (no
1693
+ // spontaneous push), so a hard persistence failure means the event has nowhere to
1694
+ // live — return false so the caller records the delivery_unroutable diagnostic,
1695
+ // which is thereby narrowed to exactly this "could not even enqueue" real-loss case.
1725
1696
  const persisted = enqueueUnresolvedDelegateForward(coordinatorDaemonId, eventName, payload);
1726
1697
  // EVTTRACE: unresolved-mesh worker persisted its completion to the outbox (no meshId
1727
1698
  // available locally; coordinator will recover it on receive).
@@ -1731,56 +1702,21 @@ function forwardUnresolvedDelegateEvent(
1731
1702
  nodeId: readNonEmptyString(routing.nodeId) || readNonEmptyString(event.meshNodeId),
1732
1703
  event: eventName,
1733
1704
  };
1734
- traceMeshEventStage('outbox_enqueue', fwdTraceCtx, `coordinatorDaemon=${coordinatorDaemonId} meshId=absent`);
1735
-
1736
- // 2) Best-effort immediate push for low latency. On success, ack the outbox row so
1737
- // the retry loop won't re-send it. On failure, leave it queued — PHASE 0 retries.
1738
- traceMeshEventStage('forward_send', fwdTraceCtx, 'immediate push');
1739
- // Serialize per coordinator so a multi-worker completion burst is paced rather than
1740
- // dumped concurrently into the shared P2P DataChannel buffer (see coordinator
1741
- // ForwardLanes). dispatchMeshCommand was null-checked above; capture it for the
1742
- // deferred closure.
1743
- const dispatchMeshCommand = components.dispatchMeshCommand;
1744
- enqueueCoordinatorForwardPush(coordinatorDaemonId, () =>
1745
- Promise.resolve(dispatchMeshCommand(coordinatorDaemonId, 'mesh_forward_event', payload))
1746
- .then((result: any) => {
1747
- if (result && result.success === false) {
1748
- LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} rejected (${readNonEmptyString(result.error) || 'no reason'}) — left queued for retry`);
1749
- traceMeshEventDrop('immediate_forward_rejected', fwdTraceCtx, readNonEmptyString(result.error) || 'no reason');
1750
- return;
1751
- }
1752
- // Acked. Mark the durable copy delivered so the retry loop skips it.
1753
- if (persisted) ackUnresolvedDelegateForwardByFingerprint(coordinatorDaemonId, eventName, payload);
1754
- })
1755
- .catch((e: any) => {
1756
- // Coordinator momentarily unreachable; the durable row stays queued and the
1757
- // reconcile loop retries it. Trace so the relay attempt is visible.
1758
- LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} failed: ${e?.message || e} — left queued for retry`);
1759
- }));
1760
- LOG.info('MeshEvents', `Durably forwarded ${eventName} for unresolved-mesh worker at ${routing.workspace || '(no workspace)'} to coordinator daemon ${coordinatorDaemonId}`);
1705
+ if (!persisted) {
1706
+ traceMeshEventDrop('outbox_enqueue_failed', fwdTraceCtx, `coordinatorDaemon=${coordinatorDaemonId}`);
1707
+ return false;
1708
+ }
1709
+ traceMeshEventStage('outbox_enqueue', fwdTraceCtx, `coordinatorDaemon=${coordinatorDaemonId} meshId=${readNonEmptyString(payload.meshId) || 'absent'}`);
1710
+
1711
+ // Data-free reconcile nudge (polling single-model §2.1 (B)): ask the reconcile
1712
+ // loop to run its PHASE 0 outbox retry soon instead of pushing the payload here.
1713
+ // Delivery itself stays on the single acked PHASE 0 path; losing the nudge costs
1714
+ // at most one reconcile interval of latency, never the event.
1715
+ nudgeUnresolvedForwardRetry();
1716
+ LOG.info('MeshEvents', `Durably queued ${eventName} for unresolved-mesh worker at ${routing.workspace || '(no workspace)'} to coordinator daemon ${coordinatorDaemonId} (reconcile PHASE 0 delivers)`);
1761
1717
  return true;
1762
1718
  }
1763
1719
 
1764
- // Ack a just-pushed outbox entry by re-deriving its row from the same coordinator +
1765
- // event + payload. We don't thread the row id back from enqueue (the immediate push is
1766
- // fire-then-ack), so locate it among the undrained entries by matching coordinator and
1767
- // the flat payload's forward identity. A miss is harmless — the retry loop's own
1768
- // receiver-side dedup suppresses a duplicate delivery.
1769
- function ackUnresolvedDelegateForwardByFingerprint(
1770
- coordinatorDaemonId: string,
1771
- eventName: string,
1772
- payload: Record<string, unknown>,
1773
- ): void {
1774
- const match = peekUnresolvedDelegateForwards().find(entry =>
1775
- daemonIdsEquivalent(entry.coordinatorDaemonId, coordinatorDaemonId)
1776
- && readNonEmptyString(entry.payload.event) === eventName
1777
- && readNonEmptyString(entry.payload.targetSessionId || entry.payload.sessionId || entry.payload.instanceId)
1778
- === readNonEmptyString(payload.targetSessionId || payload.sessionId || payload.instanceId)
1779
- && readNonEmptyString(entry.payload.workspace) === readNonEmptyString(payload.workspace),
1780
- );
1781
- if (match) ackUnresolvedDelegateForward(match.id);
1782
- }
1783
-
1784
1720
  /**
1785
1721
  * NOTIF-HELD-DRAIN (Fix 2): event-driven coordinator drain. The reconcile loop delivers a
1786
1722
  * worker's queued completion to an IDLE local coordinator only on its periodic poll. When a
@@ -54,10 +54,11 @@ import {
54
54
  peekUnresolvedDelegateForwards,
55
55
  ackUnresolvedDelegateForward,
56
56
  expireStaleUnresolvedDelegateForwards,
57
+ registerUnresolvedForwardRetryNudge,
57
58
  } from './mesh-unresolved-forward-outbox.js';
58
59
  import { readNonEmptyString, readMeshCompletionSummary, buildMeshSystemMessage } from './mesh-events-utils.js';
59
60
  import { traceMeshEventStage, traceMeshEventDrop } from './mesh-event-trace.js';
60
- import { expandDaemonIdForms, daemonIdsEquivalent, sessionIdsEquivalent } from '@adhdev/mesh-shared';
61
+ import { expandDaemonIdForms, daemonIdsEquivalent, sessionIdsEquivalent, meshNodeIdMatches } from '@adhdev/mesh-shared';
61
62
  import { getQueue, reclaimStrandedAssignedTask, updateTaskStatus } from './mesh-work-queue.js';
62
63
  import { resolveSessionBusyVerdict } from './mesh-queue-assignment.js';
63
64
  import { readLedgerEntries } from './mesh-ledger.js';
@@ -841,6 +842,107 @@ function recoverStrandedAssignedDispatches(components: DaemonComponents, meshId:
841
842
  }
842
843
  }
843
844
 
845
+ // ── PHASE 2.6: assigned-zombie sweep (runtime-store GC, SoT 1-11 (a)) ─────────
846
+ // recoverStrandedAssignedDispatches (PHASE 2.5) can only age a row by its
847
+ // dispatchTimestamp — a row that never got one (a legacy claim, a crashed claim
848
+ // path, a row whose payload drifted) is invisible to it FOREVER: it contributes 0
849
+ // pending (PHASE 3 skips), holds the node-busy gate (hasActiveNodeAssignment), and
850
+ // nothing ever transitions it. This sweep is that missing terminal net, scoped
851
+ // PRECISELY to the rows PHASE 2.5 can never touch (no parseable dispatchTimestamp)
852
+ // so the two nets never race each other over the same row.
853
+ //
854
+ // Conservative by construction:
855
+ // - age-gated on updatedAt/createdAt (>= ZOMBIE_ASSIGNED_MIN_AGE_MS) so a freshly
856
+ // claimed row mid-launch is never touched;
857
+ // - terminal ledger evidence wins first (row flips to the evidenced terminal,
858
+ // mirroring PHASE 2.5's terminal branch);
859
+ // - only fails a row whose assigned session is POSITIVELY absent on the daemon
860
+ // that owns the assigned node — a locally-present session (idle or generating)
861
+ // is skipped, and a REMOTE node's session (not locally observable) is skipped
862
+ // entirely rather than guessed dead;
863
+ // - the failure reason is explicit in both the queue mutation trace and a
864
+ // task_failed ledger entry, so the transition is auditable, never silent.
865
+ const ZOMBIE_ASSIGNED_MIN_AGE_MS = 30 * 60 * 1000; // 30 min — generous vs. session launch/restart races
866
+
867
+ export function reconcileZombieAssignedTasks(
868
+ components: DaemonComponents,
869
+ mesh: { id: string; nodes?: unknown[] },
870
+ selfIds: string[],
871
+ ): void {
872
+ const meshId = mesh.id;
873
+ const assigned = getQueue(meshId, { status: ['assigned'] });
874
+ if (!assigned.length) return;
875
+ const nowMs = Date.now();
876
+
877
+ // True when THIS daemon is authoritative for the row's assigned node — the only
878
+ // case where "no local instance" positively means "session no longer exists".
879
+ // Accepts a daemon-id form match against selfIds, or a mesh-node whose daemonId
880
+ // resolves to this daemon. Absent assignedNodeId → local (nothing remote to defer to).
881
+ const assignedNodeIsLocal = (assignedNodeId?: string): boolean => {
882
+ if (!assignedNodeId) return true;
883
+ if (selfIds.some(id => daemonIdsEquivalent(id, assignedNodeId))) return true;
884
+ const nodes = Array.isArray(mesh.nodes) ? mesh.nodes : [];
885
+ const node = nodes.find(n => meshNodeIdMatches(n as never, assignedNodeId)) as { daemonId?: unknown } | undefined;
886
+ const nodeDaemonId = readNonEmptyString(node?.daemonId);
887
+ return !!nodeDaemonId && selfIds.some(id => daemonIdsEquivalent(id, nodeDaemonId));
888
+ };
889
+
890
+ for (const row of assigned) {
891
+ // Rows WITH a parseable dispatchTimestamp belong to PHASE 2.5 — never double-handle.
892
+ if (Number.isFinite(Date.parse(row.dispatchTimestamp ?? ''))) continue;
893
+ const updatedMs = Date.parse(row.updatedAt ?? '');
894
+ const createdMs = Date.parse(row.createdAt ?? '');
895
+ const anchorMs = Number.isFinite(updatedMs) ? updatedMs : createdMs;
896
+ if (!Number.isFinite(anchorMs)) continue; // cannot age it → leave untouched
897
+ if (nowMs - anchorMs < ZOMBIE_ASSIGNED_MIN_AGE_MS) continue;
898
+
899
+ // A terminal already evidenced in the ledger → flip the row to that terminal
900
+ // (the completion arrived but the queue flip was lost), same as PHASE 2.5.
901
+ const terminal = findTerminalLedgerEvidenceForTask({ meshId, taskId: row.id });
902
+ if (terminal) {
903
+ const status = terminal.kind === 'task_completed' ? 'completed' : 'failed';
904
+ updateTaskStatus(meshId, row.id, status);
905
+ LOG.warn('MeshReconcile', `Zombie assigned task ${row.id} on mesh ${meshId} had ${terminal.kind} ledger evidence — flipped to ${status}`);
906
+ continue;
907
+ }
908
+
909
+ if (!assignedNodeIsLocal(row.assignedNodeId)) continue; // remote session not locally observable — never guess
910
+ if (row.assignedSessionId) {
911
+ const verdict = resolveSessionBusyVerdict(components, row.assignedSessionId);
912
+ if (verdict !== 'UNKNOWN') continue; // session exists locally (idle or busy) → not a zombie
913
+ }
914
+
915
+ const reason = row.assignedSessionId
916
+ ? 'assigned_zombie_session_missing'
917
+ : 'assigned_zombie_no_session_bound';
918
+ const failed = updateTaskStatus(meshId, row.id, 'failed');
919
+ if (!failed) continue;
920
+ try {
921
+ appendLedgerEntry(meshId, {
922
+ kind: 'task_failed',
923
+ nodeId: row.assignedNodeId,
924
+ sessionId: row.assignedSessionId,
925
+ payload: {
926
+ taskId: row.id,
927
+ reason,
928
+ source: 'reconcile_zombie_assigned_sweep',
929
+ ageMs: nowMs - anchorMs,
930
+ },
931
+ });
932
+ } catch { /* ledger write is best-effort */ }
933
+ LOG.warn('MeshReconcile', `Failed zombie assigned task ${row.id} on mesh ${meshId} `
934
+ + `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}, no dispatchTimestamp, `
935
+ + `stale ${Math.round((nowMs - anchorMs) / 60000)}m, ${reason})`);
936
+ traceMeshEventDrop('assigned_zombie_failed', {
937
+ taskId: row.id,
938
+ sessionId: row.assignedSessionId,
939
+ nodeId: row.assignedNodeId,
940
+ meshId,
941
+ event: 'agent:generating_completed',
942
+ }, `${reason} stale=${Math.round((nowMs - anchorMs) / 60000)}m`);
943
+ }
944
+ }
945
+
844
946
  export async function runMeshReconcileTick(components: DaemonComponents): Promise<void> {
845
947
  const localDaemonId = readNonEmptyString(loadConfig().machineId) || undefined;
846
948
  // The id-set used to scope the local queue drain (status id + machineId). See
@@ -857,7 +959,11 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
857
959
  // coordinator's mesh cannot be reached by the coordinator's PHASE 1 pull (it is
858
960
  // in no mesh.node), so its completion must be PUSHED to the coordinator. This
859
961
  // drains the durable outbox enqueued by forwardUnresolvedDelegateEvent and retries
860
- // any push that has not yet been acked. See mesh-unresolved-forward-outbox.ts.
962
+ // any push that has not yet been acked. Since the spontaneous immediate push was
963
+ // removed (polling single-model §2.1), this PHASE 0 retry is the ONLY delivery
964
+ // path for unresolved-delegate events; the enqueue site nudges an early run of it
965
+ // (scheduleUnresolvedForwardNudge) so happy-path latency stays sub-interval.
966
+ // See mesh-unresolved-forward-outbox.ts.
861
967
  if (dispatchMeshCommand) {
862
968
  try {
863
969
  await retryUnresolvedDelegateForwards(components);
@@ -897,6 +1003,13 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
897
1003
  } catch (e: any) {
898
1004
  LOG.warn('MeshReconcile', `Assigned-stranded watchdog failed for mesh ${mesh.id}: ${e?.message || e}`);
899
1005
  }
1006
+ // PHASE 2.6 — assigned-zombie sweep: terminal-fails the rows PHASE 2.5
1007
+ // can never age (no dispatchTimestamp) whose session is positively gone.
1008
+ try {
1009
+ reconcileZombieAssignedTasks(components, mesh, selfIds);
1010
+ } catch (e: any) {
1011
+ LOG.warn('MeshReconcile', `Assigned-zombie sweep failed for mesh ${mesh.id}: ${e?.message || e}`);
1012
+ }
900
1013
  }
901
1014
  }
902
1015
 
@@ -1328,6 +1441,42 @@ export function __resetUnresolvedForwardRejectionCountsForTests(): void {
1328
1441
  unresolvedForwardRejectionCounts.clear();
1329
1442
  }
1330
1443
 
1444
+ // ── Unresolved-forward reconcile nudge (polling single-model §2.1 (B)) ────────
1445
+ // forwardUnresolvedDelegateEvent no longer pushes the event itself — it only
1446
+ // persists to the durable outbox and fires a data-free nudge asking THIS loop to
1447
+ // run the PHASE 0 retry soon. The nudge is:
1448
+ // - coalesced: one pending timer at a time, so a completion burst schedules a
1449
+ // single early retry pass instead of one per event;
1450
+ // - non-overlapping: skipped while a nudged pass is still in flight (the
1451
+ // periodic tick remains the backstop);
1452
+ // - loss-tolerant: an unregistered/cleared/failed nudge merely means delivery
1453
+ // waits for the next periodic tick (≤ one reconcile interval) — never a loss.
1454
+ const UNRESOLVED_FORWARD_NUDGE_DELAY_MS = 250;
1455
+ let unresolvedForwardNudgeTimer: NodeJS.Timeout | undefined;
1456
+ let unresolvedForwardNudgeRunning = false;
1457
+
1458
+ function scheduleUnresolvedForwardNudge(components: DaemonComponents): void {
1459
+ if (!components.dispatchMeshCommand) return; // no transport → periodic tick handles/no-ops
1460
+ if (unresolvedForwardNudgeTimer) return; // coalesce a burst into one early pass
1461
+ unresolvedForwardNudgeTimer = setTimeout(() => {
1462
+ unresolvedForwardNudgeTimer = undefined;
1463
+ if (unresolvedForwardNudgeRunning) return; // an earlier pass is in flight — tick covers
1464
+ unresolvedForwardNudgeRunning = true;
1465
+ void retryUnresolvedDelegateForwards(components)
1466
+ .catch((e: any) => LOG.warn('MeshReconcile', `Nudged unresolved-forward retry failed: ${e?.message || e}`))
1467
+ .finally(() => { unresolvedForwardNudgeRunning = false; });
1468
+ }, UNRESOLVED_FORWARD_NUDGE_DELAY_MS);
1469
+ // Never keep the process alive solely for a pending nudge.
1470
+ if (typeof unresolvedForwardNudgeTimer.unref === 'function') unresolvedForwardNudgeTimer.unref();
1471
+ }
1472
+
1473
+ function clearUnresolvedForwardNudge(): void {
1474
+ if (unresolvedForwardNudgeTimer) {
1475
+ clearTimeout(unresolvedForwardNudgeTimer);
1476
+ unresolvedForwardNudgeTimer = undefined;
1477
+ }
1478
+ }
1479
+
1331
1480
  async function retryUnresolvedDelegateForwards(components: DaemonComponents): Promise<void> {
1332
1481
  const dispatchMeshCommand = components.dispatchMeshCommand;
1333
1482
  if (!dispatchMeshCommand) return;
@@ -1463,10 +1612,16 @@ export function setupMeshReconcileLoop(components: DaemonComponents): ReconcileL
1463
1612
  }, intervalMs);
1464
1613
  // Don't keep the process alive solely for this timer.
1465
1614
  if (typeof timer.unref === 'function') timer.unref();
1615
+ // Register the unresolved-forward nudge handler: the enqueue site
1616
+ // (forwardUnresolvedDelegateEvent) fires it after persisting an outbox row so
1617
+ // the PHASE 0 retry runs early instead of waiting for the next periodic tick.
1618
+ registerUnresolvedForwardRetryNudge(() => scheduleUnresolvedForwardNudge(components));
1466
1619
  LOG.info('MeshReconcile', `Mesh reconcile loop started (interval ${intervalMs}ms)`);
1467
1620
  return {
1468
1621
  stop() {
1469
1622
  clearInterval(timer);
1623
+ registerUnresolvedForwardRetryNudge(undefined);
1624
+ clearUnresolvedForwardNudge();
1470
1625
  LOG.info('MeshReconcile', 'Mesh reconcile loop stopped');
1471
1626
  },
1472
1627
  };
@@ -1557,10 +1557,82 @@ export class MeshRuntimeStore {
1557
1557
 
1558
1558
  /**
1559
1559
  * Prune tool call log entries older than the given age in ms.
1560
- * Exposed for testing.
1560
+ * Returns the number of rows deleted. Also used by the periodic retention
1561
+ * sweep (pruneMeshRuntimeRetention) — the in-write sweep in recordMeshToolCall
1562
+ * only fires every 200 calls and only covers the rate-limit window, so a
1563
+ * quiet mesh otherwise accumulates rows indefinitely.
1561
1564
  */
1562
- pruneToolCallLog(olderThanMs: number): void {
1563
- this.db.prepare('DELETE FROM mesh_tool_call_log WHERE called_at < ?').run(Date.now() - olderThanMs);
1565
+ pruneToolCallLog(olderThanMs: number): number {
1566
+ return this.db.prepare('DELETE FROM mesh_tool_call_log WHERE called_at < ?').run(Date.now() - olderThanMs).changes;
1567
+ }
1568
+
1569
+ /**
1570
+ * Retention prune for mesh_event_ledger (SoT 1-11 (b)). The ledger is append-only
1571
+ * with NO lifecycle GC of its own, so lifecycle events accumulate without bound
1572
+ * (the dominant mesh-runtime.db growth). Every production reader is bounded to a
1573
+ * recent window (readLedgerEntries tail/limit ≤ a few hundred; task-stats /
1574
+ * terminal-evidence scans look at recent tasks), so rows past a generous age only
1575
+ * cost space. Excluded from deletion — retained forever:
1576
+ * - coordinator_operating_note / _tombstone: runtime-accumulated lessons whose
1577
+ * whole point is surviving restarts; a tombstone must also outlive the notes
1578
+ * it retracts.
1579
+ * Timestamps are ISO-8601 TEXT, so the lexicographic `<` cutoff is a correct time
1580
+ * comparison; a malformed timestamp compares greater than any ISO date and is
1581
+ * conservatively retained. Returns rows deleted.
1582
+ */
1583
+ pruneEventLedger(olderThanMs: number): number {
1584
+ const cutoffIso = new Date(Date.now() - Math.max(0, olderThanMs)).toISOString();
1585
+ return this.db.prepare(
1586
+ `DELETE FROM mesh_event_ledger
1587
+ WHERE timestamp < ?
1588
+ AND kind NOT IN ('coordinator_operating_note', 'coordinator_operating_note_tombstone')`
1589
+ ).run(cutoffIso).changes;
1590
+ }
1591
+
1592
+ /**
1593
+ * Retention prune for TERMINAL (completed/cancelled/failed) mesh_queue rows
1594
+ * (SoT 1-11 (b)). Terminal rows are kept as recent history (mesh_task_history,
1595
+ * completion-dedup taskId lookups) but nothing ever deletes them, so the queue
1596
+ * table grows monotonically. Rows past the retention window serve no reader —
1597
+ * every dedup/attribution path operates on recent tasks — EXCEPT as a dependency
1598
+ * anchor: taskDependenciesSatisfied resolves dependsOn by id and treats a MISSING
1599
+ * row as not-completed, so deleting a completed row that a still-live
1600
+ * (pending/assigned) row depends on would permanently strand the dependent.
1601
+ * Those ids are collected first and excluded. Returns rows deleted.
1602
+ */
1603
+ pruneTerminalQueueEntries(olderThanMs: number): number {
1604
+ const cutoffIso = new Date(Date.now() - Math.max(0, olderThanMs)).toISOString();
1605
+ return this.transaction(() => {
1606
+ // Dependency guard: protect every id a live row still depends on.
1607
+ const liveRows = this.db.prepare(
1608
+ `SELECT payload FROM mesh_queue WHERE status IN ('pending', 'assigned')`
1609
+ ).all() as Array<{ payload: string }>;
1610
+ const protectedIds = new Set<string>();
1611
+ for (const row of liveRows) {
1612
+ try {
1613
+ const entry = JSON.parse(row.payload) as MeshWorkQueueEntry;
1614
+ if (Array.isArray(entry.dependsOn)) {
1615
+ for (const dep of entry.dependsOn) {
1616
+ if (typeof dep === 'string' && dep) protectedIds.add(dep);
1617
+ }
1618
+ }
1619
+ } catch { /* unparsable payload → nothing to protect */ }
1620
+ }
1621
+ const candidates = this.db.prepare(
1622
+ `SELECT id FROM mesh_queue
1623
+ WHERE status IN ('completed', 'cancelled', 'failed') AND updated_at < ?`
1624
+ ).all(cutoffIso) as Array<{ id: string }>;
1625
+ const deletable = candidates.map(r => r.id).filter(id => !protectedIds.has(id));
1626
+ let removed = 0;
1627
+ // Chunk the DELETE to stay well under SQLite's bind-parameter limit.
1628
+ for (let i = 0; i < deletable.length; i += 500) {
1629
+ const chunk = deletable.slice(i, i + 500);
1630
+ removed += this.db.prepare(
1631
+ `DELETE FROM mesh_queue WHERE id IN (${chunk.map(() => '?').join(',')})`
1632
+ ).run(...chunk).changes;
1633
+ }
1634
+ return removed;
1635
+ });
1564
1636
  }
1565
1637
 
1566
1638
  // ── G2: Event Ledger ────────────────────────────────────────────────────
@@ -2148,3 +2220,46 @@ export class MeshRuntimeStore {
2148
2220
  return removed;
2149
2221
  }
2150
2222
  }
2223
+
2224
+ // ─── Mesh runtime retention windows (SoT 1-11 (b) / gap I-10) ────────────────
2225
+ // mesh-runtime.db had lifecycle GC only for mesh_pending_events (prunePendingEvents,
2226
+ // hourly via the mesh-event maintenance sweep) and fingerprints/tool-call windows;
2227
+ // mesh_event_ledger and terminal mesh_queue rows grew without bound. These windows
2228
+ // are deliberately CONSERVATIVE — every production reader operates on a recent
2229
+ // window far narrower than these, so the deletes trade only dead space:
2230
+ // - Event ledger 30 days: readers are tail/limit-bounded (≤ a few hundred rows) or
2231
+ // recent-task scoped; 30d comfortably exceeds any reconcile/stat/audit horizon.
2232
+ // Operating notes are exempted inside pruneEventLedger (retained forever).
2233
+ // - Tool-call log 14 days: it backs a seconds-scale rate-limit window; 14d keeps a
2234
+ // generous debugging horizon at trivial cost.
2235
+ // - Terminal queue rows 30 days: mesh_task_history / completion-dedup lookups are
2236
+ // recent-task scoped; live dependsOn anchors are exempted inside
2237
+ // pruneTerminalQueueEntries.
2238
+ // No VACUUM here by design: reclaiming file pages is not worth stalling the daemon's
2239
+ // single writer; freed pages are reused by future inserts.
2240
+ export const MESH_EVENT_LEDGER_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
2241
+ export const MESH_TOOL_CALL_LOG_RETENTION_MS = 14 * 24 * 60 * 60 * 1000; // 14 days
2242
+ export const MESH_TERMINAL_QUEUE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
2243
+
2244
+ /**
2245
+ * Periodic retention sweep for the mesh-runtime.db tables that previously had no
2246
+ * lifecycle GC (event ledger, tool-call log, terminal queue rows). Runs on the SAME
2247
+ * cadence as the pending-events retention prune (the hourly mesh-event maintenance
2248
+ * sweep in mesh-event-forwarding.ts). Best-effort and idempotent: a store failure
2249
+ * degrades to a no-op with one warn; an empty table costs three cheap DELETEs.
2250
+ */
2251
+ export function pruneMeshRuntimeRetention(): { ledger: number; toolCalls: number; terminalQueue: number } {
2252
+ try {
2253
+ const store = MeshRuntimeStore.getInstance();
2254
+ const ledger = store.pruneEventLedger(MESH_EVENT_LEDGER_RETENTION_MS);
2255
+ const toolCalls = store.pruneToolCallLog(MESH_TOOL_CALL_LOG_RETENTION_MS);
2256
+ const terminalQueue = store.pruneTerminalQueueEntries(MESH_TERMINAL_QUEUE_RETENTION_MS);
2257
+ if (ledger + toolCalls + terminalQueue > 0) {
2258
+ LOG.info('MeshRuntimeStore', `Retention prune removed ${ledger} ledger / ${toolCalls} tool-call / ${terminalQueue} terminal-queue row(s)`);
2259
+ }
2260
+ return { ledger, toolCalls, terminalQueue };
2261
+ } catch (e: any) {
2262
+ LOG.warn('MeshRuntimeStore', `Runtime retention prune failed: ${e?.message || e}`);
2263
+ return { ledger: 0, toolCalls: 0, terminalQueue: 0 };
2264
+ }
2265
+ }