@adhdev/daemon-core 0.9.82-rc.481 → 0.9.82-rc.482
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/mesh-config.d.ts +5 -28
- package/dist/index.d.ts +2 -2
- package/dist/index.js +243 -246
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +243 -241
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/contracts.d.ts +8 -5
- package/dist/mesh/mesh-reconcile-loop.d.ts +4 -0
- package/dist/mesh/mesh-runtime-store.d.ts +47 -2
- package/dist/mesh/mesh-unresolved-forward-outbox.d.ts +4 -0
- package/dist/repo-mesh-types.d.ts +7 -13
- package/dist/types.d.ts +15 -6
- package/package.json +3 -3
- package/src/commands/chat-commands-read.ts +81 -60
- package/src/commands/med-family/mesh-crud.ts +8 -62
- package/src/config/mesh-config.ts +9 -140
- package/src/index.ts +1 -2
- package/src/mesh/contracts.ts +17 -11
- package/src/mesh/coordinator-prompt.ts +3 -6
- package/src/mesh/mesh-event-forwarding.ts +38 -102
- package/src/mesh/mesh-reconcile-loop.ts +157 -2
- package/src/mesh/mesh-runtime-store.ts +118 -3
- package/src/mesh/mesh-unresolved-forward-outbox.ts +30 -0
- package/src/repo-mesh-types.ts +7 -13
- package/src/types.ts +21 -6
package/src/index.ts
CHANGED
|
@@ -195,7 +195,6 @@ export type { SavedProviderSessionEntry } from './config/saved-sessions.js';
|
|
|
195
195
|
export {
|
|
196
196
|
listMeshes, getMesh, getMeshByRepo, createMesh, updateMesh, deleteMesh,
|
|
197
197
|
addNode, removeNode, updateNode, normalizeRepoIdentity,
|
|
198
|
-
listMagiPanels, getMagiPanel, upsertMagiPanel, removeMagiPanel, normalizeMagiPanel,
|
|
199
198
|
listMagiKindPanels, getMagiKindPanel, setMagiKindPanel, removeMagiKindPanel, normalizeMagiSlots,
|
|
200
199
|
} from './config/mesh-config.js';
|
|
201
200
|
export type { CreateMeshOptions, UpdateMeshOptions, AddNodeOptions } from './config/mesh-config.js';
|
|
@@ -203,7 +202,7 @@ export type { CreateMeshOptions, UpdateMeshOptions, AddNodeOptions } from './con
|
|
|
203
202
|
// leaf so the mcp-server — which depends only on @adhdev/daemon-core — can consume
|
|
204
203
|
// them without taking a direct @adhdev/mesh-shared dependency).
|
|
205
204
|
export type {
|
|
206
|
-
|
|
205
|
+
MagiMode, MagiTaskKind,
|
|
207
206
|
MagiSlot, MagiKindPanelMap,
|
|
208
207
|
MagiClaim, MagiClaimStance, MagiAgentResponse,
|
|
209
208
|
MagiResponseSource, MagiReplicaGitRef, MagiGitSkew, MagiSynthesizedResponse,
|
package/src/mesh/contracts.ts
CHANGED
|
@@ -368,25 +368,31 @@ const TERMINAL_TASK_EVENTS: ReadonlySet<string> = new Set([
|
|
|
368
368
|
]);
|
|
369
369
|
|
|
370
370
|
/**
|
|
371
|
-
*
|
|
372
|
-
*
|
|
373
|
-
*
|
|
371
|
+
* Coordinator-addressed dispatch-plane alerts. `mesh:dispatch_blocked` is the
|
|
372
|
+
* Fix (1) actionable dispatch-skip notification: it exists precisely to page the
|
|
373
|
+
* ORIGINATING coordinator (it carries a why+how coordinatorMessage and is
|
|
374
|
+
* targetCoordinator*-addressed by its producer), so it routes unicast exactly
|
|
375
|
+
* like a terminal task event. B2a originally classed it 'system' — that made it
|
|
376
|
+
* a dead letter: no daemon-level system drain exists, so the blocker never
|
|
377
|
+
* reached any coordinator and the task sat silently undispatched.
|
|
374
378
|
*/
|
|
375
|
-
const
|
|
379
|
+
const COORDINATOR_ALERT_EVENTS: ReadonlySet<string> = new Set([
|
|
376
380
|
'mesh:dispatch_blocked',
|
|
377
381
|
]);
|
|
378
382
|
|
|
379
383
|
/**
|
|
380
384
|
* Default the v2 scope for an event by its producer event name (design decision
|
|
381
|
-
* §3). Terminal task events → unicast (routed
|
|
382
|
-
*
|
|
383
|
-
*
|
|
384
|
-
*
|
|
385
|
-
*
|
|
385
|
+
* §3). Terminal task events and coordinator-addressed alerts → unicast (routed
|
|
386
|
+
* to the originating coordinator). Everything else — node lifecycle and
|
|
387
|
+
* progress signals — → broadcast, which also matches v1's implicit "deliver to
|
|
388
|
+
* any coordinator" behaviour, so an unstamped v1 event and a v2-stamped-as-
|
|
389
|
+
* broadcast event route identically during rollout. No event currently defaults
|
|
390
|
+
* to 'system'; the scope remains in MESH_EVENT_SCOPES for wire compatibility
|
|
391
|
+
* (an already-queued or version-skewed 'system' event still routes away from
|
|
392
|
+
* coordinators).
|
|
386
393
|
*/
|
|
387
394
|
export function defaultScopeForEvent(eventName: string): MeshEventScope {
|
|
388
|
-
if (
|
|
389
|
-
if (TERMINAL_TASK_EVENTS.has(eventName)) return 'unicast';
|
|
395
|
+
if (TERMINAL_TASK_EVENTS.has(eventName) || COORDINATOR_ALERT_EVENTS.has(eventName)) return 'unicast';
|
|
390
396
|
return 'broadcast';
|
|
391
397
|
}
|
|
392
398
|
|
|
@@ -649,9 +649,7 @@ const TOOLS_SECTION = `## Available Tools
|
|
|
649
649
|
| \`mesh_write_mesh_json_config\` | Gated write of \`.adhdev/mesh.json\` (repo coordinator-prompt config) from the mesh entry — dry-run/overwrite like mesh_init |
|
|
650
650
|
| \`mesh_magi_review\` | Cross-verify a read-only investigation across a standing panel of independent mesh agents (different machines/providers) instead of a single worker |
|
|
651
651
|
| \`mesh_magi_collect\` | Collect + synthesize a previously dispatched MAGI fan-out by its consensus group id (async companion to mesh_magi_review wait:false) |
|
|
652
|
-
| \`
|
|
653
|
-
| \`mesh_magi_panel_list\` | List configured MAGI panels and resolve each member's availability against the current mesh (read-only) |
|
|
654
|
-
| \`mesh_magi_kind_panel_set\` | Bind a task_kind → MAGI kind-panel slots (machine-local, wholesale replacement — approve current-vs-new first) |
|
|
652
|
+
| \`mesh_magi_kind_panel_set\` | Bind a task_kind → MAGI kind-panel slots (the SOLE MAGI panel-resolution surface; machine-local, wholesale replacement — approve current-vs-new first) |
|
|
655
653
|
| \`mesh_magi_kind_panel_list\` | List configured task_kind → MAGI kind-panel slot bindings (machine-local, read-only) |`;
|
|
656
654
|
|
|
657
655
|
const TOOL_EXPOSURE_PREFLIGHT_SECTION = `## Tool Exposure Preflight
|
|
@@ -694,7 +692,7 @@ When the user asks to **set up / configure / onboard** this repo for Repo Mesh (
|
|
|
694
692
|
|
|
695
693
|
**Save scopes — label every draft with its scope before asking for approval:**
|
|
696
694
|
- **repo-file (commit target)** — \`.adhdev/refine.json\`, \`.adhdev/worktree_bootstrap.json\`, \`.adhdev/change-impact.json\`, \`.adhdev/mesh.json\`. These are committed to the repository and shared with every machine/contributor.
|
|
697
|
-
- **machine-local** — MAGI kind→panel bindings
|
|
695
|
+
- **machine-local** — MAGI kind→panel bindings, node providerPriority (\`~/.adhdev/meshes.json\`). These stay on this machine and are NOT committed.
|
|
698
696
|
|
|
699
697
|
**Guided sequence:**
|
|
700
698
|
1. **Scan (dry-run)** — Call \`mesh_init\` (write=false, the default). It returns per-domain suggested configs for refine / worktree_bootstrap / change-impact, a recommended providerPriority, AND \`currentConfig\` — the currently-saved config per domain (repo files + machine-local \`magiKindPanels\`). Nothing is written.
|
|
@@ -702,8 +700,7 @@ When the user asks to **set up / configure / onboard** this repo for Repo Mesh (
|
|
|
702
700
|
3. **Approve → gated write** — Only after the user approves, call the matching gated-write tool:
|
|
703
701
|
- repo \`.adhdev/*\` config files → \`mesh_init\` with \`write=true\` (and \`overwrite=true\` ONLY for domains the user approved replacing).
|
|
704
702
|
- \`.adhdev/mesh.json\` (coordinator prompt / operating notes) → \`mesh_write_mesh_json_config\` (write=true, overwrite only if approved).
|
|
705
|
-
- machine-local MAGI kind→panel slots → \`mesh_magi_kind_panel_set\` (write=true). NOTE: a kind binding is a **wholesale replacement** of that kind's slot list — present the current-vs-new slots first.
|
|
706
|
-
- machine-local named MAGI panels → \`mesh_magi_panel_set\`. providerPriority → apply via node policy update.
|
|
703
|
+
- machine-local MAGI kind→panel slots → \`mesh_magi_kind_panel_set\` (write=true). NOTE: a kind binding is a **wholesale replacement** of that kind's slot list — present the current-vs-new slots first. providerPriority → apply via node policy update.
|
|
707
704
|
|
|
708
705
|
**init vs reinit:**
|
|
709
706
|
- **\`mesh_init\`** — for a fresh, never-onboarded repo. Existing config files are kept (existing-wins) unless the user explicitly approves overwrite. Use for first-time setup.
|
|
@@ -6,12 +6,12 @@ import { appendLedgerEntry, buildTaskCompletionEvidence, getSessionRecoveryConte
|
|
|
6
6
|
import type { SessionRecoveryContext } from './mesh-ledger.js';
|
|
7
7
|
import { updateSessionTaskStatus, enqueueTask, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, getActiveDirectDispatches, hasPendingDependents, getQueue } from './mesh-work-queue.js';
|
|
8
8
|
import { markSessionDeliveriesTerminal, updateSessionDeliveryStatus } from './mesh-delivery-policy.js';
|
|
9
|
-
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
9
|
+
import { MeshRuntimeStore, pruneMeshRuntimeRetention } from './mesh-runtime-store.js';
|
|
10
10
|
import { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents, prunePendingMeshCoordinatorEventsRetention, readV2EnvelopeFromWire, type PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
11
11
|
import type { ProviderInstance } from '../providers/provider-instance.js';
|
|
12
12
|
import { resolveWorkerDelegateRouting, recordUnroutableDelegateEvent, isUnroutableDelegateRejection } from './mesh-routing.js';
|
|
13
13
|
import { resolveMeshHostStatus } from './mesh-host-ownership.js';
|
|
14
|
-
import { enqueueUnresolvedDelegateForward,
|
|
14
|
+
import { enqueueUnresolvedDelegateForward, nudgeUnresolvedForwardRetry } from './mesh-unresolved-forward-outbox.js';
|
|
15
15
|
import { traceMeshEventStage, traceMeshEventDrop } from './mesh-event-trace.js';
|
|
16
16
|
import { getLastDisplayMessage } from '../status/snapshot.js';
|
|
17
17
|
import { resolveDelegatedWorkerAutoApprove } from '../repo-mesh-types.js';
|
|
@@ -189,13 +189,16 @@ function sweepExpiredRemoteIdleSessions(): void {
|
|
|
189
189
|
try {
|
|
190
190
|
MeshRuntimeStore.getInstance().pruneExpiredRemoteIdleSessions();
|
|
191
191
|
} catch { /* best-effort */ }
|
|
192
|
-
// Piggyback the
|
|
193
|
-
//
|
|
194
|
-
//
|
|
192
|
+
// Piggyback the retention prunes on the same periodic sweep, but hourly — this
|
|
193
|
+
// is the maintenance hook that keeps mesh-runtime.db from accumulating stale
|
|
194
|
+
// rows without bound: mesh_pending_events (drained/orphaned rows) plus, on the
|
|
195
|
+
// SAME cadence (SoT 1-11 (b)), the event ledger / tool-call log / terminal
|
|
196
|
+
// queue retention in pruneMeshRuntimeRetention.
|
|
195
197
|
const now = Date.now();
|
|
196
198
|
if (now - lastPendingEventsPruneAt >= PENDING_EVENTS_PRUNE_INTERVAL_MS) {
|
|
197
199
|
lastPendingEventsPruneAt = now;
|
|
198
200
|
prunePendingMeshCoordinatorEventsRetention();
|
|
201
|
+
pruneMeshRuntimeRetention();
|
|
199
202
|
}
|
|
200
203
|
}
|
|
201
204
|
|
|
@@ -1591,43 +1594,6 @@ export function handleMeshForwardEvent(components: DaemonComponents, payload: Re
|
|
|
1591
1594
|
});
|
|
1592
1595
|
}
|
|
1593
1596
|
|
|
1594
|
-
// ---------------------------------------------------------------------------
|
|
1595
|
-
// Per-coordinator forward serialization (P2P send-backpressure relief).
|
|
1596
|
-
//
|
|
1597
|
-
// When several workers finish at once, each completion runs forwardUnresolvedDelegate
|
|
1598
|
-
// Event and fires its own `mesh_forward_event` push. Firing the whole burst
|
|
1599
|
-
// concurrently dumps it into the single per-peer P2P DataChannel buffer in one tick,
|
|
1600
|
-
// which starves the rpc_ack/rpc_res replies the same channel must carry — a
|
|
1601
|
-
// coordinator's inbound `git_status` then times out even though the worker's own
|
|
1602
|
-
// forward acks return in ~1s. To cap the concurrent burst we serialize the immediate
|
|
1603
|
-
// pushes per coordinator: at most one push is in flight to a given coordinator at a
|
|
1604
|
-
// time, the rest run in arrival order behind it. A lone event (idle lane) still
|
|
1605
|
-
// dispatches immediately — only a genuine burst is paced. Durability is unchanged:
|
|
1606
|
-
// every event is already persisted to the outbox before the push runs, so serializing
|
|
1607
|
-
// only delays the best-effort fast path; PHASE 0 retry still covers any gap. This pairs
|
|
1608
|
-
// with the DataChannel send-buffer gate in daemon-cloud's mesh manager (writeRequest),
|
|
1609
|
-
// which is the hard guarantee; this throttle keeps the burst from piling up there.
|
|
1610
|
-
interface CoordinatorForwardLane { tail: Promise<unknown>; depth: number; }
|
|
1611
|
-
const coordinatorForwardLanes = new Map<string, CoordinatorForwardLane>();
|
|
1612
|
-
function enqueueCoordinatorForwardPush(coordinatorDaemonId: string, run: () => Promise<unknown>): void {
|
|
1613
|
-
let lane = coordinatorForwardLanes.get(coordinatorDaemonId);
|
|
1614
|
-
if (!lane) { lane = { tail: Promise.resolve(), depth: 0 }; coordinatorForwardLanes.set(coordinatorDaemonId, lane); }
|
|
1615
|
-
const wasIdle = lane.depth === 0;
|
|
1616
|
-
lane.depth += 1;
|
|
1617
|
-
const dec = (): void => { lane!.depth -= 1; };
|
|
1618
|
-
if (wasIdle) {
|
|
1619
|
-
// Idle lane → dispatch synchronously, so a lone completion (the common case) has
|
|
1620
|
-
// ZERO added latency and the push call happens in-line. Only a genuine burst —
|
|
1621
|
-
// events arriving while a push is still in flight — is paced (else branch).
|
|
1622
|
-
lane.tail = Promise.resolve(run()).catch(() => {}).then(dec, dec);
|
|
1623
|
-
} else {
|
|
1624
|
-
// Burst: queue behind the in-flight push(es) in arrival order so the whole burst
|
|
1625
|
-
// is not dumped into the shared DataChannel buffer at once. The tail is guarded
|
|
1626
|
-
// so one rejecting push never wedges the lane for the next.
|
|
1627
|
-
lane.tail = lane.tail.then(() => run()).catch(() => {}).then(dec, dec);
|
|
1628
|
-
}
|
|
1629
|
-
}
|
|
1630
|
-
|
|
1631
1597
|
// ---------------------------------------------------------------------------
|
|
1632
1598
|
// Worker-side fallback forward for unresolved-mesh delegates.
|
|
1633
1599
|
//
|
|
@@ -1641,8 +1607,9 @@ function enqueueCoordinatorForwardPush(coordinatorDaemonId: string, run: () => P
|
|
|
1641
1607
|
// the worker's queue — which it can't, because the worker never queued an unroutable
|
|
1642
1608
|
// event. Live symptom: `WARN [MeshEvents] delivery_unroutable: ... mesh unresolved`.
|
|
1643
1609
|
//
|
|
1644
|
-
// The fix: the routing object still carries coordinatorDaemonId.
|
|
1645
|
-
//
|
|
1610
|
+
// The fix: the routing object still carries coordinatorDaemonId. Persist the raw event
|
|
1611
|
+
// to the durable worker-side outbox addressed to that coordinator daemon; the reconcile
|
|
1612
|
+
// loop's PHASE 0 delivers it (mesh_forward_event, acked, retry-capped). The coordinator
|
|
1646
1613
|
// hosts the mesh, so it recovers the mesh id by workspace in handleMeshForwardEvent and
|
|
1647
1614
|
// injects/queues it normally. meshId is intentionally omitted from the payload (the
|
|
1648
1615
|
// worker has none); workspace is the routing anchor the coordinator resolves from.
|
|
@@ -1656,14 +1623,17 @@ function enqueueCoordinatorForwardPush(coordinatorDaemonId: string, run: () => P
|
|
|
1656
1623
|
//
|
|
1657
1624
|
// Returns true when the event was durably accepted for delivery to the coordinator
|
|
1658
1625
|
// daemon (so the caller skips the delivery_unroutable diagnostic); false when no
|
|
1659
|
-
// fallback was possible (no coordinator anchor / no dispatch transport)
|
|
1626
|
+
// fallback was possible (no coordinator anchor / no dispatch transport) OR the durable
|
|
1627
|
+
// enqueue itself failed — the delivery_unroutable diagnostic is thereby narrowed to
|
|
1628
|
+
// "could not even persist to the queue" (a real potential loss), per the polling-
|
|
1629
|
+
// single-model design (docs/refactoring/2026-06-16-mesh-completion-polling-single-model.md §2.1/§2.6).
|
|
1660
1630
|
//
|
|
1661
|
-
//
|
|
1662
|
-
//
|
|
1663
|
-
//
|
|
1664
|
-
//
|
|
1665
|
-
//
|
|
1666
|
-
//
|
|
1631
|
+
// Single delivery path (polling single-model): the spontaneous best-effort immediate
|
|
1632
|
+
// push that used to run here was REMOVED. Every unresolved-delegate event is persisted
|
|
1633
|
+
// to the outbox and delivered ONLY by setupMeshReconcileLoop's PHASE 0 retry (acked;
|
|
1634
|
+
// a failed push leaves the row queued). For happy-path latency the enqueue emits a
|
|
1635
|
+
// data-free reconcile NUDGE (nudgeUnresolvedForwardRetry) asking the loop to run the
|
|
1636
|
+
// retry soon; a lost nudge costs at most one reconcile interval, never the event.
|
|
1667
1637
|
function forwardUnresolvedDelegateEvent(
|
|
1668
1638
|
components: DaemonComponents,
|
|
1669
1639
|
routing: ReturnType<typeof resolveWorkerDelegateRouting>,
|
|
@@ -1718,10 +1688,11 @@ function forwardUnresolvedDelegateEvent(
|
|
|
1718
1688
|
return true;
|
|
1719
1689
|
}
|
|
1720
1690
|
|
|
1721
|
-
//
|
|
1722
|
-
//
|
|
1723
|
-
//
|
|
1724
|
-
//
|
|
1691
|
+
// Persist durably. Idempotent on fingerprint, so a re-fired completion does not
|
|
1692
|
+
// duplicate the outbox row. The outbox is the ONLY delivery route now (no
|
|
1693
|
+
// spontaneous push), so a hard persistence failure means the event has nowhere to
|
|
1694
|
+
// live — return false so the caller records the delivery_unroutable diagnostic,
|
|
1695
|
+
// which is thereby narrowed to exactly this "could not even enqueue" real-loss case.
|
|
1725
1696
|
const persisted = enqueueUnresolvedDelegateForward(coordinatorDaemonId, eventName, payload);
|
|
1726
1697
|
// EVTTRACE: unresolved-mesh worker persisted its completion to the outbox (no meshId
|
|
1727
1698
|
// available locally; coordinator will recover it on receive).
|
|
@@ -1731,56 +1702,21 @@ function forwardUnresolvedDelegateEvent(
|
|
|
1731
1702
|
nodeId: readNonEmptyString(routing.nodeId) || readNonEmptyString(event.meshNodeId),
|
|
1732
1703
|
event: eventName,
|
|
1733
1704
|
};
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
traceMeshEventStage('
|
|
1739
|
-
|
|
1740
|
-
//
|
|
1741
|
-
//
|
|
1742
|
-
//
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
.then((result: any) => {
|
|
1747
|
-
if (result && result.success === false) {
|
|
1748
|
-
LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} rejected (${readNonEmptyString(result.error) || 'no reason'}) — left queued for retry`);
|
|
1749
|
-
traceMeshEventDrop('immediate_forward_rejected', fwdTraceCtx, readNonEmptyString(result.error) || 'no reason');
|
|
1750
|
-
return;
|
|
1751
|
-
}
|
|
1752
|
-
// Acked. Mark the durable copy delivered so the retry loop skips it.
|
|
1753
|
-
if (persisted) ackUnresolvedDelegateForwardByFingerprint(coordinatorDaemonId, eventName, payload);
|
|
1754
|
-
})
|
|
1755
|
-
.catch((e: any) => {
|
|
1756
|
-
// Coordinator momentarily unreachable; the durable row stays queued and the
|
|
1757
|
-
// reconcile loop retries it. Trace so the relay attempt is visible.
|
|
1758
|
-
LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} failed: ${e?.message || e} — left queued for retry`);
|
|
1759
|
-
}));
|
|
1760
|
-
LOG.info('MeshEvents', `Durably forwarded ${eventName} for unresolved-mesh worker at ${routing.workspace || '(no workspace)'} to coordinator daemon ${coordinatorDaemonId}`);
|
|
1705
|
+
if (!persisted) {
|
|
1706
|
+
traceMeshEventDrop('outbox_enqueue_failed', fwdTraceCtx, `coordinatorDaemon=${coordinatorDaemonId}`);
|
|
1707
|
+
return false;
|
|
1708
|
+
}
|
|
1709
|
+
traceMeshEventStage('outbox_enqueue', fwdTraceCtx, `coordinatorDaemon=${coordinatorDaemonId} meshId=${readNonEmptyString(payload.meshId) || 'absent'}`);
|
|
1710
|
+
|
|
1711
|
+
// Data-free reconcile nudge (polling single-model §2.1 (B)): ask the reconcile
|
|
1712
|
+
// loop to run its PHASE 0 outbox retry soon instead of pushing the payload here.
|
|
1713
|
+
// Delivery itself stays on the single acked PHASE 0 path; losing the nudge costs
|
|
1714
|
+
// at most one reconcile interval of latency, never the event.
|
|
1715
|
+
nudgeUnresolvedForwardRetry();
|
|
1716
|
+
LOG.info('MeshEvents', `Durably queued ${eventName} for unresolved-mesh worker at ${routing.workspace || '(no workspace)'} to coordinator daemon ${coordinatorDaemonId} (reconcile PHASE 0 delivers)`);
|
|
1761
1717
|
return true;
|
|
1762
1718
|
}
|
|
1763
1719
|
|
|
1764
|
-
// Ack a just-pushed outbox entry by re-deriving its row from the same coordinator +
|
|
1765
|
-
// event + payload. We don't thread the row id back from enqueue (the immediate push is
|
|
1766
|
-
// fire-then-ack), so locate it among the undrained entries by matching coordinator and
|
|
1767
|
-
// the flat payload's forward identity. A miss is harmless — the retry loop's own
|
|
1768
|
-
// receiver-side dedup suppresses a duplicate delivery.
|
|
1769
|
-
function ackUnresolvedDelegateForwardByFingerprint(
|
|
1770
|
-
coordinatorDaemonId: string,
|
|
1771
|
-
eventName: string,
|
|
1772
|
-
payload: Record<string, unknown>,
|
|
1773
|
-
): void {
|
|
1774
|
-
const match = peekUnresolvedDelegateForwards().find(entry =>
|
|
1775
|
-
daemonIdsEquivalent(entry.coordinatorDaemonId, coordinatorDaemonId)
|
|
1776
|
-
&& readNonEmptyString(entry.payload.event) === eventName
|
|
1777
|
-
&& readNonEmptyString(entry.payload.targetSessionId || entry.payload.sessionId || entry.payload.instanceId)
|
|
1778
|
-
=== readNonEmptyString(payload.targetSessionId || payload.sessionId || payload.instanceId)
|
|
1779
|
-
&& readNonEmptyString(entry.payload.workspace) === readNonEmptyString(payload.workspace),
|
|
1780
|
-
);
|
|
1781
|
-
if (match) ackUnresolvedDelegateForward(match.id);
|
|
1782
|
-
}
|
|
1783
|
-
|
|
1784
1720
|
/**
|
|
1785
1721
|
* NOTIF-HELD-DRAIN (Fix 2): event-driven coordinator drain. The reconcile loop delivers a
|
|
1786
1722
|
* worker's queued completion to an IDLE local coordinator only on its periodic poll. When a
|
|
@@ -54,10 +54,11 @@ import {
|
|
|
54
54
|
peekUnresolvedDelegateForwards,
|
|
55
55
|
ackUnresolvedDelegateForward,
|
|
56
56
|
expireStaleUnresolvedDelegateForwards,
|
|
57
|
+
registerUnresolvedForwardRetryNudge,
|
|
57
58
|
} from './mesh-unresolved-forward-outbox.js';
|
|
58
59
|
import { readNonEmptyString, readMeshCompletionSummary, buildMeshSystemMessage } from './mesh-events-utils.js';
|
|
59
60
|
import { traceMeshEventStage, traceMeshEventDrop } from './mesh-event-trace.js';
|
|
60
|
-
import { expandDaemonIdForms, daemonIdsEquivalent, sessionIdsEquivalent } from '@adhdev/mesh-shared';
|
|
61
|
+
import { expandDaemonIdForms, daemonIdsEquivalent, sessionIdsEquivalent, meshNodeIdMatches } from '@adhdev/mesh-shared';
|
|
61
62
|
import { getQueue, reclaimStrandedAssignedTask, updateTaskStatus } from './mesh-work-queue.js';
|
|
62
63
|
import { resolveSessionBusyVerdict } from './mesh-queue-assignment.js';
|
|
63
64
|
import { readLedgerEntries } from './mesh-ledger.js';
|
|
@@ -841,6 +842,107 @@ function recoverStrandedAssignedDispatches(components: DaemonComponents, meshId:
|
|
|
841
842
|
}
|
|
842
843
|
}
|
|
843
844
|
|
|
845
|
+
// ── PHASE 2.6: assigned-zombie sweep (runtime-store GC, SoT 1-11 (a)) ─────────
|
|
846
|
+
// recoverStrandedAssignedDispatches (PHASE 2.5) can only age a row by its
|
|
847
|
+
// dispatchTimestamp — a row that never got one (a legacy claim, a crashed claim
|
|
848
|
+
// path, a row whose payload drifted) is invisible to it FOREVER: it contributes 0
|
|
849
|
+
// pending (PHASE 3 skips), holds the node-busy gate (hasActiveNodeAssignment), and
|
|
850
|
+
// nothing ever transitions it. This sweep is that missing terminal net, scoped
|
|
851
|
+
// PRECISELY to the rows PHASE 2.5 can never touch (no parseable dispatchTimestamp)
|
|
852
|
+
// so the two nets never race each other over the same row.
|
|
853
|
+
//
|
|
854
|
+
// Conservative by construction:
|
|
855
|
+
// - age-gated on updatedAt/createdAt (>= ZOMBIE_ASSIGNED_MIN_AGE_MS) so a freshly
|
|
856
|
+
// claimed row mid-launch is never touched;
|
|
857
|
+
// - terminal ledger evidence wins first (row flips to the evidenced terminal,
|
|
858
|
+
// mirroring PHASE 2.5's terminal branch);
|
|
859
|
+
// - only fails a row whose assigned session is POSITIVELY absent on the daemon
|
|
860
|
+
// that owns the assigned node — a locally-present session (idle or generating)
|
|
861
|
+
// is skipped, and a REMOTE node's session (not locally observable) is skipped
|
|
862
|
+
// entirely rather than guessed dead;
|
|
863
|
+
// - the failure reason is explicit in both the queue mutation trace and a
|
|
864
|
+
// task_failed ledger entry, so the transition is auditable, never silent.
|
|
865
|
+
const ZOMBIE_ASSIGNED_MIN_AGE_MS = 30 * 60 * 1000; // 30 min — generous vs. session launch/restart races
|
|
866
|
+
|
|
867
|
+
export function reconcileZombieAssignedTasks(
|
|
868
|
+
components: DaemonComponents,
|
|
869
|
+
mesh: { id: string; nodes?: unknown[] },
|
|
870
|
+
selfIds: string[],
|
|
871
|
+
): void {
|
|
872
|
+
const meshId = mesh.id;
|
|
873
|
+
const assigned = getQueue(meshId, { status: ['assigned'] });
|
|
874
|
+
if (!assigned.length) return;
|
|
875
|
+
const nowMs = Date.now();
|
|
876
|
+
|
|
877
|
+
// True when THIS daemon is authoritative for the row's assigned node — the only
|
|
878
|
+
// case where "no local instance" positively means "session no longer exists".
|
|
879
|
+
// Accepts a daemon-id form match against selfIds, or a mesh-node whose daemonId
|
|
880
|
+
// resolves to this daemon. Absent assignedNodeId → local (nothing remote to defer to).
|
|
881
|
+
const assignedNodeIsLocal = (assignedNodeId?: string): boolean => {
|
|
882
|
+
if (!assignedNodeId) return true;
|
|
883
|
+
if (selfIds.some(id => daemonIdsEquivalent(id, assignedNodeId))) return true;
|
|
884
|
+
const nodes = Array.isArray(mesh.nodes) ? mesh.nodes : [];
|
|
885
|
+
const node = nodes.find(n => meshNodeIdMatches(n as never, assignedNodeId)) as { daemonId?: unknown } | undefined;
|
|
886
|
+
const nodeDaemonId = readNonEmptyString(node?.daemonId);
|
|
887
|
+
return !!nodeDaemonId && selfIds.some(id => daemonIdsEquivalent(id, nodeDaemonId));
|
|
888
|
+
};
|
|
889
|
+
|
|
890
|
+
for (const row of assigned) {
|
|
891
|
+
// Rows WITH a parseable dispatchTimestamp belong to PHASE 2.5 — never double-handle.
|
|
892
|
+
if (Number.isFinite(Date.parse(row.dispatchTimestamp ?? ''))) continue;
|
|
893
|
+
const updatedMs = Date.parse(row.updatedAt ?? '');
|
|
894
|
+
const createdMs = Date.parse(row.createdAt ?? '');
|
|
895
|
+
const anchorMs = Number.isFinite(updatedMs) ? updatedMs : createdMs;
|
|
896
|
+
if (!Number.isFinite(anchorMs)) continue; // cannot age it → leave untouched
|
|
897
|
+
if (nowMs - anchorMs < ZOMBIE_ASSIGNED_MIN_AGE_MS) continue;
|
|
898
|
+
|
|
899
|
+
// A terminal already evidenced in the ledger → flip the row to that terminal
|
|
900
|
+
// (the completion arrived but the queue flip was lost), same as PHASE 2.5.
|
|
901
|
+
const terminal = findTerminalLedgerEvidenceForTask({ meshId, taskId: row.id });
|
|
902
|
+
if (terminal) {
|
|
903
|
+
const status = terminal.kind === 'task_completed' ? 'completed' : 'failed';
|
|
904
|
+
updateTaskStatus(meshId, row.id, status);
|
|
905
|
+
LOG.warn('MeshReconcile', `Zombie assigned task ${row.id} on mesh ${meshId} had ${terminal.kind} ledger evidence — flipped to ${status}`);
|
|
906
|
+
continue;
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
if (!assignedNodeIsLocal(row.assignedNodeId)) continue; // remote session not locally observable — never guess
|
|
910
|
+
if (row.assignedSessionId) {
|
|
911
|
+
const verdict = resolveSessionBusyVerdict(components, row.assignedSessionId);
|
|
912
|
+
if (verdict !== 'UNKNOWN') continue; // session exists locally (idle or busy) → not a zombie
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
const reason = row.assignedSessionId
|
|
916
|
+
? 'assigned_zombie_session_missing'
|
|
917
|
+
: 'assigned_zombie_no_session_bound';
|
|
918
|
+
const failed = updateTaskStatus(meshId, row.id, 'failed');
|
|
919
|
+
if (!failed) continue;
|
|
920
|
+
try {
|
|
921
|
+
appendLedgerEntry(meshId, {
|
|
922
|
+
kind: 'task_failed',
|
|
923
|
+
nodeId: row.assignedNodeId,
|
|
924
|
+
sessionId: row.assignedSessionId,
|
|
925
|
+
payload: {
|
|
926
|
+
taskId: row.id,
|
|
927
|
+
reason,
|
|
928
|
+
source: 'reconcile_zombie_assigned_sweep',
|
|
929
|
+
ageMs: nowMs - anchorMs,
|
|
930
|
+
},
|
|
931
|
+
});
|
|
932
|
+
} catch { /* ledger write is best-effort */ }
|
|
933
|
+
LOG.warn('MeshReconcile', `Failed zombie assigned task ${row.id} on mesh ${meshId} `
|
|
934
|
+
+ `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}, no dispatchTimestamp, `
|
|
935
|
+
+ `stale ${Math.round((nowMs - anchorMs) / 60000)}m, ${reason})`);
|
|
936
|
+
traceMeshEventDrop('assigned_zombie_failed', {
|
|
937
|
+
taskId: row.id,
|
|
938
|
+
sessionId: row.assignedSessionId,
|
|
939
|
+
nodeId: row.assignedNodeId,
|
|
940
|
+
meshId,
|
|
941
|
+
event: 'agent:generating_completed',
|
|
942
|
+
}, `${reason} stale=${Math.round((nowMs - anchorMs) / 60000)}m`);
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
|
|
844
946
|
export async function runMeshReconcileTick(components: DaemonComponents): Promise<void> {
|
|
845
947
|
const localDaemonId = readNonEmptyString(loadConfig().machineId) || undefined;
|
|
846
948
|
// The id-set used to scope the local queue drain (status id + machineId). See
|
|
@@ -857,7 +959,11 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
857
959
|
// coordinator's mesh cannot be reached by the coordinator's PHASE 1 pull (it is
|
|
858
960
|
// in no mesh.node), so its completion must be PUSHED to the coordinator. This
|
|
859
961
|
// drains the durable outbox enqueued by forwardUnresolvedDelegateEvent and retries
|
|
860
|
-
// any push that has not yet been acked.
|
|
962
|
+
// any push that has not yet been acked. Since the spontaneous immediate push was
|
|
963
|
+
// removed (polling single-model §2.1), this PHASE 0 retry is the ONLY delivery
|
|
964
|
+
// path for unresolved-delegate events; the enqueue site nudges an early run of it
|
|
965
|
+
// (scheduleUnresolvedForwardNudge) so happy-path latency stays sub-interval.
|
|
966
|
+
// See mesh-unresolved-forward-outbox.ts.
|
|
861
967
|
if (dispatchMeshCommand) {
|
|
862
968
|
try {
|
|
863
969
|
await retryUnresolvedDelegateForwards(components);
|
|
@@ -897,6 +1003,13 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
897
1003
|
} catch (e: any) {
|
|
898
1004
|
LOG.warn('MeshReconcile', `Assigned-stranded watchdog failed for mesh ${mesh.id}: ${e?.message || e}`);
|
|
899
1005
|
}
|
|
1006
|
+
// PHASE 2.6 — assigned-zombie sweep: terminal-fails the rows PHASE 2.5
|
|
1007
|
+
// can never age (no dispatchTimestamp) whose session is positively gone.
|
|
1008
|
+
try {
|
|
1009
|
+
reconcileZombieAssignedTasks(components, mesh, selfIds);
|
|
1010
|
+
} catch (e: any) {
|
|
1011
|
+
LOG.warn('MeshReconcile', `Assigned-zombie sweep failed for mesh ${mesh.id}: ${e?.message || e}`);
|
|
1012
|
+
}
|
|
900
1013
|
}
|
|
901
1014
|
}
|
|
902
1015
|
|
|
@@ -1328,6 +1441,42 @@ export function __resetUnresolvedForwardRejectionCountsForTests(): void {
|
|
|
1328
1441
|
unresolvedForwardRejectionCounts.clear();
|
|
1329
1442
|
}
|
|
1330
1443
|
|
|
1444
|
+
// ── Unresolved-forward reconcile nudge (polling single-model §2.1 (B)) ────────
|
|
1445
|
+
// forwardUnresolvedDelegateEvent no longer pushes the event itself — it only
|
|
1446
|
+
// persists to the durable outbox and fires a data-free nudge asking THIS loop to
|
|
1447
|
+
// run the PHASE 0 retry soon. The nudge is:
|
|
1448
|
+
// - coalesced: one pending timer at a time, so a completion burst schedules a
|
|
1449
|
+
// single early retry pass instead of one per event;
|
|
1450
|
+
// - non-overlapping: skipped while a nudged pass is still in flight (the
|
|
1451
|
+
// periodic tick remains the backstop);
|
|
1452
|
+
// - loss-tolerant: an unregistered/cleared/failed nudge merely means delivery
|
|
1453
|
+
// waits for the next periodic tick (≤ one reconcile interval) — never a loss.
|
|
1454
|
+
const UNRESOLVED_FORWARD_NUDGE_DELAY_MS = 250;
|
|
1455
|
+
let unresolvedForwardNudgeTimer: NodeJS.Timeout | undefined;
|
|
1456
|
+
let unresolvedForwardNudgeRunning = false;
|
|
1457
|
+
|
|
1458
|
+
function scheduleUnresolvedForwardNudge(components: DaemonComponents): void {
|
|
1459
|
+
if (!components.dispatchMeshCommand) return; // no transport → periodic tick handles/no-ops
|
|
1460
|
+
if (unresolvedForwardNudgeTimer) return; // coalesce a burst into one early pass
|
|
1461
|
+
unresolvedForwardNudgeTimer = setTimeout(() => {
|
|
1462
|
+
unresolvedForwardNudgeTimer = undefined;
|
|
1463
|
+
if (unresolvedForwardNudgeRunning) return; // an earlier pass is in flight — tick covers
|
|
1464
|
+
unresolvedForwardNudgeRunning = true;
|
|
1465
|
+
void retryUnresolvedDelegateForwards(components)
|
|
1466
|
+
.catch((e: any) => LOG.warn('MeshReconcile', `Nudged unresolved-forward retry failed: ${e?.message || e}`))
|
|
1467
|
+
.finally(() => { unresolvedForwardNudgeRunning = false; });
|
|
1468
|
+
}, UNRESOLVED_FORWARD_NUDGE_DELAY_MS);
|
|
1469
|
+
// Never keep the process alive solely for a pending nudge.
|
|
1470
|
+
if (typeof unresolvedForwardNudgeTimer.unref === 'function') unresolvedForwardNudgeTimer.unref();
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1473
|
+
function clearUnresolvedForwardNudge(): void {
|
|
1474
|
+
if (unresolvedForwardNudgeTimer) {
|
|
1475
|
+
clearTimeout(unresolvedForwardNudgeTimer);
|
|
1476
|
+
unresolvedForwardNudgeTimer = undefined;
|
|
1477
|
+
}
|
|
1478
|
+
}
|
|
1479
|
+
|
|
1331
1480
|
async function retryUnresolvedDelegateForwards(components: DaemonComponents): Promise<void> {
|
|
1332
1481
|
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
1333
1482
|
if (!dispatchMeshCommand) return;
|
|
@@ -1463,10 +1612,16 @@ export function setupMeshReconcileLoop(components: DaemonComponents): ReconcileL
|
|
|
1463
1612
|
}, intervalMs);
|
|
1464
1613
|
// Don't keep the process alive solely for this timer.
|
|
1465
1614
|
if (typeof timer.unref === 'function') timer.unref();
|
|
1615
|
+
// Register the unresolved-forward nudge handler: the enqueue site
|
|
1616
|
+
// (forwardUnresolvedDelegateEvent) fires it after persisting an outbox row so
|
|
1617
|
+
// the PHASE 0 retry runs early instead of waiting for the next periodic tick.
|
|
1618
|
+
registerUnresolvedForwardRetryNudge(() => scheduleUnresolvedForwardNudge(components));
|
|
1466
1619
|
LOG.info('MeshReconcile', `Mesh reconcile loop started (interval ${intervalMs}ms)`);
|
|
1467
1620
|
return {
|
|
1468
1621
|
stop() {
|
|
1469
1622
|
clearInterval(timer);
|
|
1623
|
+
registerUnresolvedForwardRetryNudge(undefined);
|
|
1624
|
+
clearUnresolvedForwardNudge();
|
|
1470
1625
|
LOG.info('MeshReconcile', 'Mesh reconcile loop stopped');
|
|
1471
1626
|
},
|
|
1472
1627
|
};
|
|
@@ -1557,10 +1557,82 @@ export class MeshRuntimeStore {
|
|
|
1557
1557
|
|
|
1558
1558
|
/**
|
|
1559
1559
|
* Prune tool call log entries older than the given age in ms.
|
|
1560
|
-
*
|
|
1560
|
+
* Returns the number of rows deleted. Also used by the periodic retention
|
|
1561
|
+
* sweep (pruneMeshRuntimeRetention) — the in-write sweep in recordMeshToolCall
|
|
1562
|
+
* only fires every 200 calls and only covers the rate-limit window, so a
|
|
1563
|
+
* quiet mesh otherwise accumulates rows indefinitely.
|
|
1561
1564
|
*/
|
|
1562
|
-
pruneToolCallLog(olderThanMs: number):
|
|
1563
|
-
this.db.prepare('DELETE FROM mesh_tool_call_log WHERE called_at < ?').run(Date.now() - olderThanMs);
|
|
1565
|
+
pruneToolCallLog(olderThanMs: number): number {
|
|
1566
|
+
return this.db.prepare('DELETE FROM mesh_tool_call_log WHERE called_at < ?').run(Date.now() - olderThanMs).changes;
|
|
1567
|
+
}
|
|
1568
|
+
|
|
1569
|
+
/**
|
|
1570
|
+
* Retention prune for mesh_event_ledger (SoT 1-11 (b)). The ledger is append-only
|
|
1571
|
+
* with NO lifecycle GC of its own, so lifecycle events accumulate without bound
|
|
1572
|
+
* (the dominant mesh-runtime.db growth). Every production reader is bounded to a
|
|
1573
|
+
* recent window (readLedgerEntries tail/limit ≤ a few hundred; task-stats /
|
|
1574
|
+
* terminal-evidence scans look at recent tasks), so rows past a generous age only
|
|
1575
|
+
* cost space. Excluded from deletion — retained forever:
|
|
1576
|
+
* - coordinator_operating_note / _tombstone: runtime-accumulated lessons whose
|
|
1577
|
+
* whole point is surviving restarts; a tombstone must also outlive the notes
|
|
1578
|
+
* it retracts.
|
|
1579
|
+
* Timestamps are ISO-8601 TEXT, so the lexicographic `<` cutoff is a correct time
|
|
1580
|
+
* comparison; a malformed timestamp compares greater than any ISO date and is
|
|
1581
|
+
* conservatively retained. Returns rows deleted.
|
|
1582
|
+
*/
|
|
1583
|
+
pruneEventLedger(olderThanMs: number): number {
|
|
1584
|
+
const cutoffIso = new Date(Date.now() - Math.max(0, olderThanMs)).toISOString();
|
|
1585
|
+
return this.db.prepare(
|
|
1586
|
+
`DELETE FROM mesh_event_ledger
|
|
1587
|
+
WHERE timestamp < ?
|
|
1588
|
+
AND kind NOT IN ('coordinator_operating_note', 'coordinator_operating_note_tombstone')`
|
|
1589
|
+
).run(cutoffIso).changes;
|
|
1590
|
+
}
|
|
1591
|
+
|
|
1592
|
+
/**
|
|
1593
|
+
* Retention prune for TERMINAL (completed/cancelled/failed) mesh_queue rows
|
|
1594
|
+
* (SoT 1-11 (b)). Terminal rows are kept as recent history (mesh_task_history,
|
|
1595
|
+
* completion-dedup taskId lookups) but nothing ever deletes them, so the queue
|
|
1596
|
+
* table grows monotonically. Rows past the retention window serve no reader —
|
|
1597
|
+
* every dedup/attribution path operates on recent tasks — EXCEPT as a dependency
|
|
1598
|
+
* anchor: taskDependenciesSatisfied resolves dependsOn by id and treats a MISSING
|
|
1599
|
+
* row as not-completed, so deleting a completed row that a still-live
|
|
1600
|
+
* (pending/assigned) row depends on would permanently strand the dependent.
|
|
1601
|
+
* Those ids are collected first and excluded. Returns rows deleted.
|
|
1602
|
+
*/
|
|
1603
|
+
pruneTerminalQueueEntries(olderThanMs: number): number {
|
|
1604
|
+
const cutoffIso = new Date(Date.now() - Math.max(0, olderThanMs)).toISOString();
|
|
1605
|
+
return this.transaction(() => {
|
|
1606
|
+
// Dependency guard: protect every id a live row still depends on.
|
|
1607
|
+
const liveRows = this.db.prepare(
|
|
1608
|
+
`SELECT payload FROM mesh_queue WHERE status IN ('pending', 'assigned')`
|
|
1609
|
+
).all() as Array<{ payload: string }>;
|
|
1610
|
+
const protectedIds = new Set<string>();
|
|
1611
|
+
for (const row of liveRows) {
|
|
1612
|
+
try {
|
|
1613
|
+
const entry = JSON.parse(row.payload) as MeshWorkQueueEntry;
|
|
1614
|
+
if (Array.isArray(entry.dependsOn)) {
|
|
1615
|
+
for (const dep of entry.dependsOn) {
|
|
1616
|
+
if (typeof dep === 'string' && dep) protectedIds.add(dep);
|
|
1617
|
+
}
|
|
1618
|
+
}
|
|
1619
|
+
} catch { /* unparsable payload → nothing to protect */ }
|
|
1620
|
+
}
|
|
1621
|
+
const candidates = this.db.prepare(
|
|
1622
|
+
`SELECT id FROM mesh_queue
|
|
1623
|
+
WHERE status IN ('completed', 'cancelled', 'failed') AND updated_at < ?`
|
|
1624
|
+
).all(cutoffIso) as Array<{ id: string }>;
|
|
1625
|
+
const deletable = candidates.map(r => r.id).filter(id => !protectedIds.has(id));
|
|
1626
|
+
let removed = 0;
|
|
1627
|
+
// Chunk the DELETE to stay well under SQLite's bind-parameter limit.
|
|
1628
|
+
for (let i = 0; i < deletable.length; i += 500) {
|
|
1629
|
+
const chunk = deletable.slice(i, i + 500);
|
|
1630
|
+
removed += this.db.prepare(
|
|
1631
|
+
`DELETE FROM mesh_queue WHERE id IN (${chunk.map(() => '?').join(',')})`
|
|
1632
|
+
).run(...chunk).changes;
|
|
1633
|
+
}
|
|
1634
|
+
return removed;
|
|
1635
|
+
});
|
|
1564
1636
|
}
|
|
1565
1637
|
|
|
1566
1638
|
// ── G2: Event Ledger ────────────────────────────────────────────────────
|
|
@@ -2148,3 +2220,46 @@ export class MeshRuntimeStore {
|
|
|
2148
2220
|
return removed;
|
|
2149
2221
|
}
|
|
2150
2222
|
}
|
|
2223
|
+
|
|
2224
|
+
// ─── Mesh runtime retention windows (SoT 1-11 (b) / gap I-10) ────────────────
|
|
2225
|
+
// mesh-runtime.db had lifecycle GC only for mesh_pending_events (prunePendingEvents,
|
|
2226
|
+
// hourly via the mesh-event maintenance sweep) and fingerprints/tool-call windows;
|
|
2227
|
+
// mesh_event_ledger and terminal mesh_queue rows grew without bound. These windows
|
|
2228
|
+
// are deliberately CONSERVATIVE — every production reader operates on a recent
|
|
2229
|
+
// window far narrower than these, so the deletes trade only dead space:
|
|
2230
|
+
// - Event ledger 30 days: readers are tail/limit-bounded (≤ a few hundred rows) or
|
|
2231
|
+
// recent-task scoped; 30d comfortably exceeds any reconcile/stat/audit horizon.
|
|
2232
|
+
// Operating notes are exempted inside pruneEventLedger (retained forever).
|
|
2233
|
+
// - Tool-call log 14 days: it backs a seconds-scale rate-limit window; 14d keeps a
|
|
2234
|
+
// generous debugging horizon at trivial cost.
|
|
2235
|
+
// - Terminal queue rows 30 days: mesh_task_history / completion-dedup lookups are
|
|
2236
|
+
// recent-task scoped; live dependsOn anchors are exempted inside
|
|
2237
|
+
// pruneTerminalQueueEntries.
|
|
2238
|
+
// No VACUUM here by design: reclaiming file pages is not worth stalling the daemon's
|
|
2239
|
+
// single writer; freed pages are reused by future inserts.
|
|
2240
|
+
export const MESH_EVENT_LEDGER_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
|
|
2241
|
+
export const MESH_TOOL_CALL_LOG_RETENTION_MS = 14 * 24 * 60 * 60 * 1000; // 14 days
|
|
2242
|
+
export const MESH_TERMINAL_QUEUE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
|
|
2243
|
+
|
|
2244
|
+
/**
|
|
2245
|
+
* Periodic retention sweep for the mesh-runtime.db tables that previously had no
|
|
2246
|
+
* lifecycle GC (event ledger, tool-call log, terminal queue rows). Runs on the SAME
|
|
2247
|
+
* cadence as the pending-events retention prune (the hourly mesh-event maintenance
|
|
2248
|
+
* sweep in mesh-event-forwarding.ts). Best-effort and idempotent: a store failure
|
|
2249
|
+
* degrades to a no-op with one warn; an empty table costs three cheap DELETEs.
|
|
2250
|
+
*/
|
|
2251
|
+
export function pruneMeshRuntimeRetention(): { ledger: number; toolCalls: number; terminalQueue: number } {
|
|
2252
|
+
try {
|
|
2253
|
+
const store = MeshRuntimeStore.getInstance();
|
|
2254
|
+
const ledger = store.pruneEventLedger(MESH_EVENT_LEDGER_RETENTION_MS);
|
|
2255
|
+
const toolCalls = store.pruneToolCallLog(MESH_TOOL_CALL_LOG_RETENTION_MS);
|
|
2256
|
+
const terminalQueue = store.pruneTerminalQueueEntries(MESH_TERMINAL_QUEUE_RETENTION_MS);
|
|
2257
|
+
if (ledger + toolCalls + terminalQueue > 0) {
|
|
2258
|
+
LOG.info('MeshRuntimeStore', `Retention prune removed ${ledger} ledger / ${toolCalls} tool-call / ${terminalQueue} terminal-queue row(s)`);
|
|
2259
|
+
}
|
|
2260
|
+
return { ledger, toolCalls, terminalQueue };
|
|
2261
|
+
} catch (e: any) {
|
|
2262
|
+
LOG.warn('MeshRuntimeStore', `Runtime retention prune failed: ${e?.message || e}`);
|
|
2263
|
+
return { ledger: 0, toolCalls: 0, terminalQueue: 0 };
|
|
2264
|
+
}
|
|
2265
|
+
}
|