@adhdev/daemon-core 0.9.82-rc.292 → 0.9.82-rc.294
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +418 -55
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +420 -57
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-missions.d.ts +25 -1
- package/dist/mesh/mesh-runtime-store.d.ts +15 -0
- package/dist/mesh/mesh-unresolved-forward-outbox.d.ts +30 -0
- package/dist/providers/spec/cli-adapter.d.ts +14 -0
- package/dist/providers/types/interactive-prompt.d.ts +19 -0
- package/package.json +2 -2
- package/src/commands/router.ts +15 -3
- package/src/config/chat-history.ts +255 -10
- package/src/git/git-status.ts +46 -11
- package/src/mesh/coordinator-prompt.ts +3 -2
- package/src/mesh/mesh-events-coordinator.ts +51 -6
- package/src/mesh/mesh-missions.ts +41 -3
- package/src/mesh/mesh-reconcile-loop.ts +55 -0
- package/src/mesh/mesh-runtime-store.ts +30 -0
- package/src/mesh/mesh-unresolved-forward-outbox.ts +185 -0
- package/src/providers/cli-provider-instance.ts +57 -30
- package/src/providers/extension-provider-instance.ts +5 -1
- package/src/providers/ide-provider-instance.ts +6 -1
- package/src/providers/spec/cli-adapter.ts +40 -0
- package/src/providers/types/interactive-prompt.ts +1 -1
|
@@ -13,6 +13,7 @@ import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
|
13
13
|
import { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents } from './mesh-events-pending.js';
|
|
14
14
|
import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
15
15
|
import { resolveWorkerDelegateRouting, recordUnroutableDelegateEvent, isUnroutableDelegateRejection } from './mesh-routing.js';
|
|
16
|
+
import { enqueueUnresolvedDelegateForward, peekUnresolvedDelegateForwards, ackUnresolvedDelegateForward } from './mesh-unresolved-forward-outbox.js';
|
|
16
17
|
import { resolveDelegatedWorkerAutoApprove } from '../repo-mesh-types.js';
|
|
17
18
|
import { normalizeMeshNodeId, meshNodeIdMatches } from '@adhdev/mesh-shared';
|
|
18
19
|
import {
|
|
@@ -1539,8 +1540,16 @@ export function handleMeshForwardEvent(components: DaemonComponents, payload: Re
|
|
|
1539
1540
|
// - It fires only when the normal queue path did NOT run (isDelegate=false), so the
|
|
1540
1541
|
// event is never both queued locally and forwarded.
|
|
1541
1542
|
//
|
|
1542
|
-
// Returns true when the event was
|
|
1543
|
-
// skips the delivery_unroutable diagnostic); false when no
|
|
1543
|
+
// Returns true when the event was durably accepted for delivery to the coordinator
|
|
1544
|
+
// daemon (so the caller skips the delivery_unroutable diagnostic); false when no
|
|
1545
|
+
// fallback was possible (no coordinator anchor / no dispatch transport).
|
|
1546
|
+
//
|
|
1547
|
+
// Durability: the directed push to the coordinator is the ONLY delivery route for an
|
|
1548
|
+
// unresolved-mesh worker (it is in no mesh.node the coordinator can pull). So instead
|
|
1549
|
+
// of a fire-and-forget push that drops on one transient P2P failure, the event is
|
|
1550
|
+
// persisted to the worker-side outbox FIRST and only acked after a successful push.
|
|
1551
|
+
// A best-effort immediate push keeps latency low on the happy path; a failed or
|
|
1552
|
+
// un-acked push leaves the durable row for setupMeshReconcileLoop's PHASE 0 to retry.
|
|
1544
1553
|
function forwardUnresolvedDelegateEvent(
|
|
1545
1554
|
components: DaemonComponents,
|
|
1546
1555
|
routing: ReturnType<typeof resolveWorkerDelegateRouting>,
|
|
@@ -1564,16 +1573,52 @@ function forwardUnresolvedDelegateEvent(
|
|
|
1564
1573
|
workspace: readNonEmptyString(routing.workspace) || readNonEmptyString(event.workspace) || undefined,
|
|
1565
1574
|
};
|
|
1566
1575
|
|
|
1576
|
+
// 1) Persist durably FIRST. Idempotent on fingerprint, so a re-fired completion
|
|
1577
|
+
// does not duplicate the outbox row. If persistence fails we still attempt the
|
|
1578
|
+
// push below (degrades to the old at-most-once behaviour rather than dropping
|
|
1579
|
+
// the chance entirely).
|
|
1580
|
+
const persisted = enqueueUnresolvedDelegateForward(coordinatorDaemonId, eventName, payload);
|
|
1581
|
+
|
|
1582
|
+
// 2) Best-effort immediate push for low latency. On success, ack the outbox row so
|
|
1583
|
+
// the retry loop won't re-send it. On failure, leave it queued — PHASE 0 retries.
|
|
1567
1584
|
Promise.resolve(components.dispatchMeshCommand(coordinatorDaemonId, 'mesh_forward_event', payload))
|
|
1585
|
+
.then((result: any) => {
|
|
1586
|
+
if (result && result.success === false) {
|
|
1587
|
+
LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} rejected (${readNonEmptyString(result.error) || 'no reason'}) — left queued for retry`);
|
|
1588
|
+
return;
|
|
1589
|
+
}
|
|
1590
|
+
// Acked. Mark the durable copy delivered so the retry loop skips it.
|
|
1591
|
+
if (persisted) ackUnresolvedDelegateForwardByFingerprint(coordinatorDaemonId, eventName, payload);
|
|
1592
|
+
})
|
|
1568
1593
|
.catch((e: any) => {
|
|
1569
|
-
//
|
|
1570
|
-
//
|
|
1571
|
-
LOG.warn('MeshEvents', `
|
|
1594
|
+
// Coordinator momentarily unreachable; the durable row stays queued and the
|
|
1595
|
+
// reconcile loop retries it. Trace so the relay attempt is visible.
|
|
1596
|
+
LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} failed: ${e?.message || e} — left queued for retry`);
|
|
1572
1597
|
});
|
|
1573
|
-
LOG.info('MeshEvents', `
|
|
1598
|
+
LOG.info('MeshEvents', `Durably forwarded ${eventName} for unresolved-mesh worker at ${routing.workspace || '(no workspace)'} to coordinator daemon ${coordinatorDaemonId}`);
|
|
1574
1599
|
return true;
|
|
1575
1600
|
}
|
|
1576
1601
|
|
|
1602
|
+
// Ack a just-pushed outbox entry by re-deriving its row from the same coordinator +
|
|
1603
|
+
// event + payload. We don't thread the row id back from enqueue (the immediate push is
|
|
1604
|
+
// fire-then-ack), so locate it among the undrained entries by matching coordinator and
|
|
1605
|
+
// the flat payload's forward identity. A miss is harmless — the retry loop's own
|
|
1606
|
+
// receiver-side dedup suppresses a duplicate delivery.
|
|
1607
|
+
function ackUnresolvedDelegateForwardByFingerprint(
|
|
1608
|
+
coordinatorDaemonId: string,
|
|
1609
|
+
eventName: string,
|
|
1610
|
+
payload: Record<string, unknown>,
|
|
1611
|
+
): void {
|
|
1612
|
+
const match = peekUnresolvedDelegateForwards().find(entry =>
|
|
1613
|
+
entry.coordinatorDaemonId === coordinatorDaemonId
|
|
1614
|
+
&& readNonEmptyString(entry.payload.event) === eventName
|
|
1615
|
+
&& readNonEmptyString(entry.payload.targetSessionId || entry.payload.sessionId || entry.payload.instanceId)
|
|
1616
|
+
=== readNonEmptyString(payload.targetSessionId || payload.sessionId || payload.instanceId)
|
|
1617
|
+
&& readNonEmptyString(entry.payload.workspace) === readNonEmptyString(payload.workspace),
|
|
1618
|
+
);
|
|
1619
|
+
if (match) ackUnresolvedDelegateForward(match.id);
|
|
1620
|
+
}
|
|
1621
|
+
|
|
1577
1622
|
export function setupMeshEventForwarding(components: DaemonComponents) {
|
|
1578
1623
|
components.instanceManager.onEvent((event) => {
|
|
1579
1624
|
// --- Coordinator idle auto-flush (fast path) ---
|
|
@@ -47,6 +47,25 @@ export interface MeshMissionSummary extends MeshMissionRecord {
|
|
|
47
47
|
tasks: MeshMissionTaskAggregate;
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
+
/**
|
|
51
|
+
* Slim mission summary for the mesh_status compact (default) surface. Drops the
|
|
52
|
+
* full `goal` text — which can be hundreds of chars per mission and is repeated
|
|
53
|
+
* for every mission on every status call — keeping only a short preview plus a
|
|
54
|
+
* `goalTruncated` flag when the original was longer. The stored goal is never
|
|
55
|
+
* mutated; this is an output-only projection. Coordinators that need the full
|
|
56
|
+
* goal call mesh_status with verbose=true, or read the mission directly via
|
|
57
|
+
* mesh_mission_upsert / getMeshMission.
|
|
58
|
+
*/
|
|
59
|
+
export interface MeshMissionSlimSummary extends Omit<MeshMissionSummary, 'goal'> {
|
|
60
|
+
/** Short preview of the goal (≤ GOAL_PREVIEW_MAX chars), '' when goal empty. */
|
|
61
|
+
goalPreview: string;
|
|
62
|
+
/** True when the stored goal was longer than the preview (full text elided). */
|
|
63
|
+
goalTruncated: boolean;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Max chars of goal text retained in the slim (compact) mission summary. */
|
|
67
|
+
export const GOAL_PREVIEW_MAX = 120;
|
|
68
|
+
|
|
50
69
|
function normalizeMissionStatus(value: unknown): MeshMissionStatus {
|
|
51
70
|
return MESH_MISSION_STATUSES.includes(value as MeshMissionStatus)
|
|
52
71
|
? value as MeshMissionStatus
|
|
@@ -125,17 +144,35 @@ export function getActiveMeshMissionSummaries(meshId: string): MeshMissionSummar
|
|
|
125
144
|
return getMeshMissions(meshId, ['active']).map(mission => summarizeMeshMission(meshId, mission));
|
|
126
145
|
}
|
|
127
146
|
|
|
147
|
+
/** Project a full mission summary down to the slim (goal-elided) shape. */
|
|
148
|
+
function slimMissionSummary(summary: MeshMissionSummary): MeshMissionSlimSummary {
|
|
149
|
+
const goal = typeof summary.goal === 'string' ? summary.goal : '';
|
|
150
|
+
const goalTruncated = goal.length > GOAL_PREVIEW_MAX;
|
|
151
|
+
const { goal: _omitGoal, ...rest } = summary;
|
|
152
|
+
return {
|
|
153
|
+
...rest,
|
|
154
|
+
goalPreview: goalTruncated ? goal.slice(0, GOAL_PREVIEW_MAX) : goal,
|
|
155
|
+
goalTruncated,
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
|
|
128
159
|
/**
|
|
129
160
|
* Mission summaries for the mesh_status dashboard surface: every active/paused
|
|
130
161
|
* mission plus a capped, newest-first slice of completed/abandoned history so
|
|
131
162
|
* the dashboard can render a collapsible "history" section without unbounded
|
|
132
163
|
* payload growth. Returned newest-first within each group (active/paused first,
|
|
133
164
|
* then history), so the frontend can split on `status` directly.
|
|
165
|
+
*
|
|
166
|
+
* Compact mode (the default) elides each mission's full `goal` text — which is
|
|
167
|
+
* repeated verbatim on every status poll and dominates the payload when a mesh
|
|
168
|
+
* has many missions — returning only a short `goalPreview` + `goalTruncated`
|
|
169
|
+
* flag. Pass `verbose: true` to get the full `goal` text per mission. The stored
|
|
170
|
+
* goal is untouched in both modes; this is an output-only projection.
|
|
134
171
|
*/
|
|
135
172
|
export function getMeshStatusMissionSummaries(
|
|
136
173
|
meshId: string,
|
|
137
|
-
options?: { historyLimit?: number },
|
|
138
|
-
): MeshMissionSummary[] {
|
|
174
|
+
options?: { historyLimit?: number; verbose?: boolean },
|
|
175
|
+
): MeshMissionSummary[] | MeshMissionSlimSummary[] {
|
|
139
176
|
const historyLimit = Math.max(0, options?.historyLimit ?? 10);
|
|
140
177
|
const all = getMeshMissions(meshId);
|
|
141
178
|
const live = all.filter(m => m.status === 'active' || m.status === 'paused');
|
|
@@ -143,7 +180,8 @@ export function getMeshStatusMissionSummaries(
|
|
|
143
180
|
.filter(m => m.status === 'completed' || m.status === 'abandoned')
|
|
144
181
|
.sort((a, b) => (b.updatedAt || '').localeCompare(a.updatedAt || ''))
|
|
145
182
|
.slice(0, historyLimit);
|
|
146
|
-
|
|
183
|
+
const full = [...live, ...history].map(mission => summarizeMeshMission(meshId, mission));
|
|
184
|
+
return options?.verbose ? full : full.map(slimMissionSummary);
|
|
147
185
|
}
|
|
148
186
|
|
|
149
187
|
/**
|
|
@@ -49,6 +49,11 @@ import { drainPendingMeshCoordinatorEvents } from './mesh-events-pending.js';
|
|
|
49
49
|
import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
50
50
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
51
51
|
import { handleMeshForwardEvent, shouldForceInjectMeshEvent, MESH_FORCE_INJECT_EVENTS } from './mesh-events-coordinator.js';
|
|
52
|
+
import {
|
|
53
|
+
peekUnresolvedDelegateForwards,
|
|
54
|
+
ackUnresolvedDelegateForward,
|
|
55
|
+
expireStaleUnresolvedDelegateForwards,
|
|
56
|
+
} from './mesh-unresolved-forward-outbox.js';
|
|
52
57
|
import { readNonEmptyString } from './mesh-events-utils.js';
|
|
53
58
|
|
|
54
59
|
// Default reconcile cadence. approval/completion notifications to a live CLI
|
|
@@ -213,6 +218,20 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
213
218
|
try { return MeshRuntimeStore.getInstance(); } catch { return undefined; }
|
|
214
219
|
})();
|
|
215
220
|
|
|
221
|
+
// ── PHASE 0: retry the worker-side unresolved-delegate forward outbox ──────
|
|
222
|
+
// Cloud-only (needs dispatchMeshCommand). A worker that is NOT a member of the
|
|
223
|
+
// coordinator's mesh cannot be reached by the coordinator's PHASE 1 pull (it is
|
|
224
|
+
// in no mesh.node), so its completion must be PUSHED to the coordinator. This
|
|
225
|
+
// drains the durable outbox enqueued by forwardUnresolvedDelegateEvent and retries
|
|
226
|
+
// any push that has not yet been acked. See mesh-unresolved-forward-outbox.ts.
|
|
227
|
+
if (dispatchMeshCommand) {
|
|
228
|
+
try {
|
|
229
|
+
await retryUnresolvedDelegateForwards(components);
|
|
230
|
+
} catch (e: any) {
|
|
231
|
+
LOG.warn('MeshReconcile', `Unresolved-delegate forward retry failed: ${e?.message || e}`);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
216
235
|
// ── PHASE 1: pull remote node queues for every mesh this daemon hosts ──────
|
|
217
236
|
// Cloud-only (dispatchMeshCommand present). Runs whether or not a live CLI
|
|
218
237
|
// coordinator exists — this is what lets an MCP/LLM coordinator ever see a
|
|
@@ -295,6 +314,42 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
295
314
|
}
|
|
296
315
|
}
|
|
297
316
|
|
|
317
|
+
// Cloud-only: retry the worker-side unresolved-delegate forward outbox. For each
|
|
318
|
+
// durably-queued entry, push it to its coordinator daemon over P2P (mesh_forward_event)
|
|
319
|
+
// and ack (mark drained) ONLY on a successful, non-rejected response. A failed or
|
|
320
|
+
// rejected push leaves the entry queued for the next tick — at-least-once delivery.
|
|
321
|
+
// Stale entries (coordinator unreachable past the max age) are expired first so the
|
|
322
|
+
// outbox can't grow without bound. The coordinator dedups duplicate deliveries on its
|
|
323
|
+
// own fingerprint, so a retry that races the original immediate push is harmless.
|
|
324
|
+
async function retryUnresolvedDelegateForwards(components: DaemonComponents): Promise<void> {
|
|
325
|
+
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
326
|
+
if (!dispatchMeshCommand) return;
|
|
327
|
+
|
|
328
|
+
// Drop entries that have exhausted their retry budget (fail-loud inside).
|
|
329
|
+
expireStaleUnresolvedDelegateForwards();
|
|
330
|
+
|
|
331
|
+
const entries = peekUnresolvedDelegateForwards();
|
|
332
|
+
if (entries.length === 0) return;
|
|
333
|
+
|
|
334
|
+
for (const entry of entries) {
|
|
335
|
+
let result: any;
|
|
336
|
+
try {
|
|
337
|
+
result = await dispatchMeshCommand(entry.coordinatorDaemonId, 'mesh_forward_event', entry.payload);
|
|
338
|
+
} catch (e: any) {
|
|
339
|
+
// Coordinator unreachable — keep the entry queued and try again next tick.
|
|
340
|
+
LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} failed: ${e?.message || e} — left queued`);
|
|
341
|
+
continue;
|
|
342
|
+
}
|
|
343
|
+
if (result && result.success === false) {
|
|
344
|
+
LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} rejected (${readNonEmptyString(result.error) || 'no reason'}) — left queued`);
|
|
345
|
+
continue;
|
|
346
|
+
}
|
|
347
|
+
// Acked — mark the durable copy delivered.
|
|
348
|
+
ackUnresolvedDelegateForward(entry.id);
|
|
349
|
+
LOG.info('MeshReconcile', `Retried+delivered unresolved-delegate ${readNonEmptyString(entry.payload.event)} to coordinator ${entry.coordinatorDaemonId}`);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
|
|
298
353
|
// Cloud-only: poll each remote worker node daemon for pending coordinator events
|
|
299
354
|
// and re-inject them locally via handleMeshForwardEvent (which re-queues +
|
|
300
355
|
// surfaces to the live coordinator on the next tick / immediately if idle).
|
|
@@ -1396,4 +1396,34 @@ export class MeshRuntimeStore {
|
|
|
1396
1396
|
).get(meshId) as { cnt: number } | undefined;
|
|
1397
1397
|
return row?.cnt ?? 0;
|
|
1398
1398
|
}
|
|
1399
|
+
|
|
1400
|
+
/**
|
|
1401
|
+
* Mark specific pending-event rows drained by id (ack). Used by the
|
|
1402
|
+
* unresolved-delegate durable-forward outbox: an event is peeked (not drained)
|
|
1403
|
+
* while its push to the coordinator is unconfirmed, then marked drained ONLY
|
|
1404
|
+
* after the push is acked. A failed push leaves the row undrained so the next
|
|
1405
|
+
* reconcile tick retries it. Returns the number of rows newly marked drained.
|
|
1406
|
+
*/
|
|
1407
|
+
markPendingEventsDrainedById(ids: ReadonlyArray<string>): number {
|
|
1408
|
+
const idList = ids.filter((id): id is string => typeof id === 'string' && id.length > 0);
|
|
1409
|
+
if (idList.length === 0) return 0;
|
|
1410
|
+
const now = Date.now();
|
|
1411
|
+
return this.db.prepare(
|
|
1412
|
+
`UPDATE mesh_pending_events SET drained = 1, drained_at = ? WHERE drained = 0 AND id IN (${idList.map(() => '?').join(',')})`
|
|
1413
|
+
).run(now, ...idList).changes;
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1416
|
+
/**
|
|
1417
|
+
* Hard-delete pending-event rows by id (including the dedup fingerprint history).
|
|
1418
|
+
* Used to expire an unresolved-delegate outbox entry that has exhausted its retry
|
|
1419
|
+
* budget — fully removing it frees the fingerprint so a genuinely new completion
|
|
1420
|
+
* for the same task could be re-queued later. Returns the number of rows deleted.
|
|
1421
|
+
*/
|
|
1422
|
+
deletePendingEventsById(ids: ReadonlyArray<string>): number {
|
|
1423
|
+
const idList = ids.filter((id): id is string => typeof id === 'string' && id.length > 0);
|
|
1424
|
+
if (idList.length === 0) return 0;
|
|
1425
|
+
return this.db.prepare(
|
|
1426
|
+
`DELETE FROM mesh_pending_events WHERE id IN (${idList.map(() => '?').join(',')})`
|
|
1427
|
+
).run(...idList).changes;
|
|
1428
|
+
}
|
|
1399
1429
|
}
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// mesh-unresolved-forward-outbox — durable retry for unresolved-delegate forwards
|
|
3
|
+
// ---------------------------------------------------------------------------
|
|
4
|
+
// A REMOTE worker daemon that is P2P-remote-controlled by a coordinator is NOT a
|
|
5
|
+
// member of the coordinator's mesh — it has no local mesh record. So when its
|
|
6
|
+
// completion event reaches setupMeshEventForwarding, resolveWorkerDelegateRouting
|
|
7
|
+
// returns mesh_unresolved: the worker carries the coordinator daemon anchor but no
|
|
8
|
+
// resolvable mesh id, so the normal queue-then-coordinator-pull path can't be used
|
|
9
|
+
// (the coordinator's reconcile PHASE 1 only pulls mesh.nodes, and this worker is in
|
|
10
|
+
// none of them — see docs/refactoring/2026-06-16-mesh-completion-polling-single-model.md).
|
|
11
|
+
//
|
|
12
|
+
// The ONLY delivery route for this case is a directed push to the coordinator
|
|
13
|
+
// daemon (mesh_forward_event over P2P). Previously that push was fire-and-forget:
|
|
14
|
+
// a single dispatchMeshCommand whose rejection was only logged, so one transient
|
|
15
|
+
// P2P failure dropped the completion forever (delivery_unroutable).
|
|
16
|
+
//
|
|
17
|
+
// This outbox makes that push DURABLE without inventing a new persistence layer:
|
|
18
|
+
// it reuses the existing mesh_pending_events SQLite table (the same store every
|
|
19
|
+
// other mesh coordinator event is persisted to), under a synthetic, reserved
|
|
20
|
+
// "mesh id" namespace so it never collides with a real mesh's queue. The worker's
|
|
21
|
+
// reconcile tick (PHASE 0) peeks the outbox, pushes each entry to its coordinator,
|
|
22
|
+
// and marks it drained (acked) ONLY on a successful push — a failed push leaves the
|
|
23
|
+
// row undrained for the next tick. Entries that exceed a max age are expired so the
|
|
24
|
+
// outbox can't grow unbounded when a coordinator stays permanently unreachable.
|
|
25
|
+
//
|
|
26
|
+
// Idempotency:
|
|
27
|
+
// - Enqueue is idempotent on the event fingerprint (mesh_pending_events has a
|
|
28
|
+
// UNIQUE (mesh_id, fingerprint) index + INSERT OR IGNORE), so the same
|
|
29
|
+
// completion fired twice on the worker queues once.
|
|
30
|
+
// - The receiver dedups independently: handleMeshForwardEvent → injectMeshSystemMessage
|
|
31
|
+
// → queuePendingMeshCoordinatorEvent recomputes the fingerprint and suppresses a
|
|
32
|
+
// duplicate, so an at-least-once retry that double-delivers is harmless.
|
|
33
|
+
// ---------------------------------------------------------------------------
|
|
34
|
+
|
|
35
|
+
import { randomUUID } from 'crypto';
|
|
36
|
+
import { LOG } from '../logging/logger.js';
|
|
37
|
+
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
38
|
+
import { buildPendingEventFingerprint, type PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
39
|
+
import { readNonEmptyString } from './mesh-events-utils.js';
|
|
40
|
+
|
|
41
|
+
// Reserved synthetic mesh id for the worker-side unresolved-forward outbox. The `::`
|
|
42
|
+
// and leading `__` make it impossible to collide with a real mesh id (which are
|
|
43
|
+
// `mesh_*` / `mreg_*` slugs). Scoping rows by coordinator_daemon_id inside this one
|
|
44
|
+
// namespace lets a single worker forward to multiple coordinators.
|
|
45
|
+
export const UNRESOLVED_FORWARD_OUTBOX_MESH_ID = '__unresolved_forward_outbox__';
|
|
46
|
+
|
|
47
|
+
// Entries older than this are expired (dropped) rather than retried forever — a
|
|
48
|
+
// coordinator that has been unreachable this long is treated as gone. Generous
|
|
49
|
+
// enough to ride out a long coordinator outage / restart, bounded enough that a
|
|
50
|
+
// permanently-dead coordinator can't accumulate outbox rows indefinitely.
|
|
51
|
+
const UNRESOLVED_FORWARD_MAX_AGE_MS = 30 * 60 * 1000; // 30 minutes
|
|
52
|
+
|
|
53
|
+
export interface UnresolvedForwardEntry {
|
|
54
|
+
/** Row id in mesh_pending_events; pass back to ack/expire after a push attempt. */
|
|
55
|
+
id: string;
|
|
56
|
+
/** The coordinator daemon to push this event to (mesh_forward_event target). */
|
|
57
|
+
coordinatorDaemonId: string;
|
|
58
|
+
/** The flat payload to forward (already shaped for handleMeshForwardEvent). */
|
|
59
|
+
payload: Record<string, unknown>;
|
|
60
|
+
/** When the entry was first enqueued (epoch ms) — used for age-based expiry. */
|
|
61
|
+
queuedAt: number;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function getStore(): MeshRuntimeStore | undefined {
|
|
65
|
+
try { return MeshRuntimeStore.getInstance(); } catch { return undefined; }
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Durably enqueue an unresolved-delegate forward for a coordinator daemon. The
|
|
70
|
+
* `forwardPayload` is the flat shape handleMeshForwardEvent reads on the coordinator.
|
|
71
|
+
* Idempotent on the event fingerprint. Returns true when a new row was written (or a
|
|
72
|
+
* duplicate was harmlessly ignored), false on a hard persistence failure.
|
|
73
|
+
*/
|
|
74
|
+
export function enqueueUnresolvedDelegateForward(
|
|
75
|
+
coordinatorDaemonId: string,
|
|
76
|
+
eventName: string,
|
|
77
|
+
forwardPayload: Record<string, unknown>,
|
|
78
|
+
): boolean {
|
|
79
|
+
const target = readNonEmptyString(coordinatorDaemonId);
|
|
80
|
+
const event = readNonEmptyString(eventName);
|
|
81
|
+
if (!target || !event) return false;
|
|
82
|
+
const store = getStore();
|
|
83
|
+
if (!store) return false;
|
|
84
|
+
|
|
85
|
+
const queuedAt = Date.now();
|
|
86
|
+
// Reuse the standard pending-event fingerprint so enqueue is idempotent AND the
|
|
87
|
+
// receiver's own dedup keys on a comparable identity. We synthesise the minimal
|
|
88
|
+
// PendingMeshCoordinatorEvent shape buildPendingEventFingerprint needs.
|
|
89
|
+
const fingerprintSource: PendingMeshCoordinatorEvent = {
|
|
90
|
+
event,
|
|
91
|
+
meshId: UNRESOLVED_FORWARD_OUTBOX_MESH_ID,
|
|
92
|
+
nodeLabel: readNonEmptyString(forwardPayload.nodeId) || readNonEmptyString(forwardPayload.workspace) || 'unresolved-delegate',
|
|
93
|
+
nodeId: readNonEmptyString(forwardPayload.nodeId) || undefined,
|
|
94
|
+
workspace: readNonEmptyString(forwardPayload.workspace) || undefined,
|
|
95
|
+
metadataEvent: forwardPayload,
|
|
96
|
+
queuedAt,
|
|
97
|
+
targetCoordinatorDaemonId: target,
|
|
98
|
+
};
|
|
99
|
+
// Scope the fingerprint to the coordinator so two coordinators awaiting the same
|
|
100
|
+
// worker session don't collapse into one outbox row.
|
|
101
|
+
const fingerprint = `${target}::${buildPendingEventFingerprint(fingerprintSource)}`;
|
|
102
|
+
|
|
103
|
+
try {
|
|
104
|
+
const inserted = store.insertPendingEvent({
|
|
105
|
+
id: randomUUID(),
|
|
106
|
+
meshId: UNRESOLVED_FORWARD_OUTBOX_MESH_ID,
|
|
107
|
+
coordinatorDaemonId: target,
|
|
108
|
+
event,
|
|
109
|
+
// Store the flat forward payload + the queue timestamp so the retry tick can
|
|
110
|
+
// rebuild the push args and apply age-based expiry without a schema change.
|
|
111
|
+
payload: { forwardPayload, coordinatorDaemonId: target, queuedAt },
|
|
112
|
+
fingerprint,
|
|
113
|
+
queuedAt,
|
|
114
|
+
});
|
|
115
|
+
if (inserted) {
|
|
116
|
+
LOG.info('MeshEvents', `Durably queued unresolved-delegate ${event} for coordinator ${target} (outbox)`);
|
|
117
|
+
}
|
|
118
|
+
return true;
|
|
119
|
+
} catch (e: any) {
|
|
120
|
+
LOG.warn('MeshEvents', `Failed to persist unresolved-delegate forward to outbox: ${e?.message || e}`);
|
|
121
|
+
return false;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** Peek (non-destructive) every undrained outbox entry across all coordinators. */
|
|
126
|
+
export function peekUnresolvedDelegateForwards(): UnresolvedForwardEntry[] {
|
|
127
|
+
const store = getStore();
|
|
128
|
+
if (!store) return [];
|
|
129
|
+
let rows: Array<{ id: string; event: string; payload: unknown }>;
|
|
130
|
+
try {
|
|
131
|
+
rows = store.peekPendingEvents(UNRESOLVED_FORWARD_OUTBOX_MESH_ID);
|
|
132
|
+
} catch {
|
|
133
|
+
return [];
|
|
134
|
+
}
|
|
135
|
+
const out: UnresolvedForwardEntry[] = [];
|
|
136
|
+
for (const row of rows) {
|
|
137
|
+
const stored = row.payload && typeof row.payload === 'object' ? row.payload as Record<string, unknown> : {};
|
|
138
|
+
const coordinatorDaemonId = readNonEmptyString(stored.coordinatorDaemonId);
|
|
139
|
+
const forwardPayload = stored.forwardPayload && typeof stored.forwardPayload === 'object'
|
|
140
|
+
? stored.forwardPayload as Record<string, unknown>
|
|
141
|
+
: undefined;
|
|
142
|
+
if (!coordinatorDaemonId || !forwardPayload) continue;
|
|
143
|
+
const queuedAt = typeof stored.queuedAt === 'number' ? stored.queuedAt : 0;
|
|
144
|
+
out.push({ id: row.id, coordinatorDaemonId, payload: forwardPayload, queuedAt });
|
|
145
|
+
}
|
|
146
|
+
return out;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** Mark an outbox entry delivered (acked) after a successful push. */
|
|
150
|
+
export function ackUnresolvedDelegateForward(id: string): void {
|
|
151
|
+
const store = getStore();
|
|
152
|
+
if (!store) return;
|
|
153
|
+
try { store.markPendingEventsDrainedById([id]); } catch { /* best-effort */ }
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Drop outbox entries that have exceeded the max retry age. Returns the count
|
|
158
|
+
* expired so the caller can log a fail-loud trace (a dropped completion is a real
|
|
159
|
+
* loss; it must be visible, not silent).
|
|
160
|
+
*/
|
|
161
|
+
export function expireStaleUnresolvedDelegateForwards(nowMs: number = Date.now()): number {
|
|
162
|
+
const entries = peekUnresolvedDelegateForwards();
|
|
163
|
+
const staleIds = entries
|
|
164
|
+
.filter(e => e.queuedAt > 0 && nowMs - e.queuedAt >= UNRESOLVED_FORWARD_MAX_AGE_MS)
|
|
165
|
+
.map(e => e.id);
|
|
166
|
+
if (staleIds.length === 0) return 0;
|
|
167
|
+
const store = getStore();
|
|
168
|
+
if (!store) return 0;
|
|
169
|
+
try {
|
|
170
|
+
const removed = store.deletePendingEventsById(staleIds);
|
|
171
|
+
if (removed > 0) {
|
|
172
|
+
LOG.warn('MeshEvents', `Expired ${removed} unresolved-delegate forward(s) after ${Math.round(UNRESOLVED_FORWARD_MAX_AGE_MS / 60000)}m of failed retries — coordinator unreachable, completion dropped`);
|
|
173
|
+
}
|
|
174
|
+
return removed;
|
|
175
|
+
} catch {
|
|
176
|
+
return 0;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** Test helper: purge the entire outbox. */
|
|
181
|
+
export function __clearUnresolvedDelegateForwardOutboxForTests(): void {
|
|
182
|
+
const store = getStore();
|
|
183
|
+
if (!store) return;
|
|
184
|
+
try { store.clearPendingEventsForMesh(UNRESOLVED_FORWARD_OUTBOX_MESH_ID); } catch { /* nothing to clear */ }
|
|
185
|
+
}
|
|
@@ -37,6 +37,18 @@ type PersistableCliHistoryMessage = {
|
|
|
37
37
|
receivedAt?: number;
|
|
38
38
|
};
|
|
39
39
|
|
|
40
|
+
// Status snapshots only ever surface the newest messages: the cloud 'live'
|
|
41
|
+
// profile drops chat messages entirely (loaded lazily via read_chat on
|
|
42
|
+
// subscribe) and the 'full' profile caps activeChat.messages to the last 60
|
|
43
|
+
// (see status/normalize.ts). Unread/completion markers walk only the tail.
|
|
44
|
+
// So getState()'s saved-history hydration — which runs once per resume/manual
|
|
45
|
+
// CLI session on every status report — must read only a bounded tail, not the
|
|
46
|
+
// entire transcript. A full MAX_SAFE_INTEGER read here makes the initial
|
|
47
|
+
// status report O(transcript) × N(sessions), which is the real cold first-
|
|
48
|
+
// connection bottleneck on chat-heavy machines. The window comfortably exceeds
|
|
49
|
+
// the 60-message snapshot cap so dedup/collapse at the boundary stays stable.
|
|
50
|
+
const STATUS_HYDRATION_TAIL_LIMIT = 200;
|
|
51
|
+
|
|
40
52
|
type CompletedDebouncePending = {
|
|
41
53
|
chatTitle: string;
|
|
42
54
|
duration: number;
|
|
@@ -829,22 +841,27 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
829
841
|
}
|
|
830
842
|
|
|
831
843
|
updateSettings(newSettings: Record<string, any>): void {
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
844
|
+
// Merge semantics: a key omitted from newSettings preserves its existing
|
|
845
|
+
// value, a key present in newSettings (even as false) overrides it.
|
|
846
|
+
//
|
|
847
|
+
// This is required because updateSettings has two callers with opposite
|
|
848
|
+
// intent:
|
|
849
|
+
// 1. Full re-injection — the dashboard toggle path (handleSetProviderSetting
|
|
850
|
+
// → getSettings → updateInstanceSettings) sends the COMPLETE settings
|
|
851
|
+
// object, so an explicit autoApprove:false must win.
|
|
852
|
+
// 2. Partial stamp — the mesh relay-safety stamp (router.ts agent_command,
|
|
853
|
+
// buildMeshWorkerRelayStamp) sends ONLY {meshNodeFor, meshNodeId,
|
|
854
|
+
// meshCoordinatorDaemonId, launchedByCoordinator} on every coordinator
|
|
855
|
+
// re-dispatch. It carries no autoApprove, so a full replacement would
|
|
856
|
+
// wipe the launch-time autoApprove:true the worker was started with,
|
|
857
|
+
// silently dropping every later approval to a manual gate until the
|
|
858
|
+
// machine-page toggle re-injected the full settings.
|
|
859
|
+
//
|
|
860
|
+
// A plain merge satisfies both: undefined keys fall through to the existing
|
|
861
|
+
// value (preserving launch-stamp settings like autoApprove + the mesh routing
|
|
862
|
+
// keys), explicit keys override. This subsumes the previous mesh-key preserve
|
|
863
|
+
// list, which only protected the routing keys and not autoApprove.
|
|
864
|
+
this.settings = { ...this.settings, ...newSettings };
|
|
848
865
|
this.adapter.updateRuntimeSettings?.(this.settings);
|
|
849
866
|
this.monitor.updateConfig({
|
|
850
867
|
approvalAlert: this.settings.approvalAlert !== false,
|
|
@@ -2185,13 +2202,21 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
2185
2202
|
return newestMessageAt === 0;
|
|
2186
2203
|
}
|
|
2187
2204
|
|
|
2188
|
-
private syncCanonicalSavedHistoryIfNeeded(): boolean {
|
|
2205
|
+
private syncCanonicalSavedHistoryIfNeeded(options: { full?: boolean } = {}): boolean {
|
|
2189
2206
|
if (!this.providerSessionId) return false;
|
|
2190
2207
|
const canonicalHistory = this.provider.nativeHistory;
|
|
2191
2208
|
if (!canonicalHistory) return false;
|
|
2192
2209
|
|
|
2210
|
+
// Per-status-report hydration reads only a bounded tail (snapshot needs at
|
|
2211
|
+
// most the newest 60). The once-per-resume restore path passes full:true
|
|
2212
|
+
// because seedSessionHistory needs the COMPLETE transcript to seed dedup
|
|
2213
|
+
// state. The read-cache key encodes the window so the bounded and full
|
|
2214
|
+
// reads don't share/clobber each other's 2s cache entry.
|
|
2215
|
+
const limit = options.full ? Number.MAX_SAFE_INTEGER : STATUS_HYDRATION_TAIL_LIMIT;
|
|
2216
|
+
const windowTag = options.full ? 'full' : `tail:${STATUS_HYDRATION_TAIL_LIMIT}`;
|
|
2217
|
+
|
|
2193
2218
|
if (isNativeSourceCanonicalHistory(canonicalHistory)) {
|
|
2194
|
-
const cacheKey = [this.type, this.providerSessionId, this.workingDir].join('\0');
|
|
2219
|
+
const cacheKey = [this.type, this.providerSessionId, this.workingDir, windowTag].join('\0');
|
|
2195
2220
|
const now = Date.now();
|
|
2196
2221
|
if (cacheKey === this.lastNativeSourceCanonicalCacheKey && now - this.lastNativeSourceCanonicalCheckAt < 2_000) {
|
|
2197
2222
|
return true;
|
|
@@ -2204,7 +2229,7 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
2204
2229
|
historySessionId: this.providerSessionId,
|
|
2205
2230
|
workspace: this.workingDir,
|
|
2206
2231
|
offset: 0,
|
|
2207
|
-
limit
|
|
2232
|
+
limit,
|
|
2208
2233
|
historyBehavior: this.provider.historyBehavior,
|
|
2209
2234
|
scripts: this.provider.scripts as any,
|
|
2210
2235
|
});
|
|
@@ -2221,7 +2246,7 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
2221
2246
|
}
|
|
2222
2247
|
|
|
2223
2248
|
try {
|
|
2224
|
-
const cacheKey = [this.type, this.providerSessionId, this.workingDir, canonicalHistory.mode || 'materialized-mirror'].join('\0');
|
|
2249
|
+
const cacheKey = [this.type, this.providerSessionId, this.workingDir, canonicalHistory.mode || 'materialized-mirror', windowTag].join('\0');
|
|
2225
2250
|
const now = Date.now();
|
|
2226
2251
|
if (cacheKey === this.lastNativeSourceCanonicalCacheKey && now - this.lastNativeSourceCanonicalCheckAt < 2_000) {
|
|
2227
2252
|
return true;
|
|
@@ -2232,15 +2257,14 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
2232
2257
|
if (!materializeProviderNativeHistory(this.type, canonicalHistory, this.providerSessionId, this.workingDir, this.provider.scripts as any)) {
|
|
2233
2258
|
return false;
|
|
2234
2259
|
}
|
|
2235
|
-
//
|
|
2236
|
-
//
|
|
2237
|
-
//
|
|
2238
|
-
//
|
|
2239
|
-
//
|
|
2240
|
-
//
|
|
2241
|
-
//
|
|
2242
|
-
|
|
2243
|
-
const restoredHistory = readChatHistory(this.type, 0, Number.MAX_SAFE_INTEGER, this.providerSessionId, 0, this.provider.historyBehavior);
|
|
2260
|
+
// Bounded by default: the per-status-report path only needs the newest
|
|
2261
|
+
// STATUS_HYDRATION_TAIL_LIMIT messages because the snapshot caps
|
|
2262
|
+
// activeChat.messages to the last 60 (status/normalize.ts) and loads
|
|
2263
|
+
// the rest lazily via read_chat on subscribe. The once-per-resume
|
|
2264
|
+
// restore path passes full:true so seedSessionHistory still sees the
|
|
2265
|
+
// COMPLETE transcript for prefix-dedup seeding. readChatHistory serves
|
|
2266
|
+
// a bounded limit as an O(tail) read.
|
|
2267
|
+
const restoredHistory = readChatHistory(this.type, 0, limit, this.providerSessionId, 0, this.provider.historyBehavior);
|
|
2244
2268
|
this.lastPersistedHistoryMessages = restoredHistory.messages.map((message) => ({
|
|
2245
2269
|
role: message.role,
|
|
2246
2270
|
content: message.content,
|
|
@@ -2256,7 +2280,10 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
2256
2280
|
|
|
2257
2281
|
private restorePersistedHistoryFromCurrentSession(): void {
|
|
2258
2282
|
if (!this.providerSessionId) return;
|
|
2259
|
-
|
|
2283
|
+
// Restore is the once-per-resume seeding path: it needs the COMPLETE
|
|
2284
|
+
// transcript so seedSessionHistory can prime dedup state. Pass full so the
|
|
2285
|
+
// hydration read is unbounded here (and only here).
|
|
2286
|
+
this.syncCanonicalSavedHistoryIfNeeded({ full: true });
|
|
2260
2287
|
const restoredHistory = isNativeSourceCanonicalHistory(this.provider.nativeHistory)
|
|
2261
2288
|
? readProviderChatHistory(this.type, {
|
|
2262
2289
|
canonicalHistory: this.provider.nativeHistory,
|
|
@@ -171,7 +171,11 @@ export class ExtensionProviderInstance implements ProviderInstance {
|
|
|
171
171
|
}
|
|
172
172
|
|
|
173
173
|
updateSettings(newSettings: Record<string, any>): void {
|
|
174
|
-
|
|
174
|
+
// Merge (not replace): a key omitted from newSettings preserves its existing
|
|
175
|
+
// value, a key present (even false) overrides. Mirrors cli/ide-provider-instance
|
|
176
|
+
// so partial updates do not wipe launch-time settings (e.g. autoApprove) while a
|
|
177
|
+
// full settings object still applies explicit values.
|
|
178
|
+
this.settings = { ...this.settings, ...newSettings };
|
|
175
179
|
this.monitor.updateConfig({
|
|
176
180
|
approvalAlert: this.settings.approvalAlert !== false,
|
|
177
181
|
longGeneratingAlert: this.settings.longGeneratingAlert !== false,
|
|
@@ -259,7 +259,12 @@ export class IdeProviderInstance implements ProviderInstance {
|
|
|
259
259
|
}
|
|
260
260
|
|
|
261
261
|
updateSettings(newSettings: Record<string, any>): void {
|
|
262
|
-
|
|
262
|
+
// Merge (not replace): a key omitted from newSettings preserves its existing
|
|
263
|
+
// value, a key present (even false) overrides. Mirrors cli-provider-instance —
|
|
264
|
+
// a partial mesh relay-stamp must not wipe launch-time settings such as
|
|
265
|
+
// autoApprove, while the dashboard toggle's full settings object still applies
|
|
266
|
+
// an explicit autoApprove:false. See cli-provider-instance.updateSettings.
|
|
267
|
+
this.settings = { ...this.settings, ...newSettings };
|
|
263
268
|
this.monitor.updateConfig({
|
|
264
269
|
approvalAlert: this.settings.approvalAlert !== false,
|
|
265
270
|
longGeneratingAlert: this.settings.longGeneratingAlert !== false,
|