@adhdev/daemon-core 0.9.76 → 0.9.77-rc.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,9 @@
1
1
  import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
2
2
  import { getMesh, getMeshByRepo } from '../config/mesh-config.js';
3
3
  import { LOG } from '../logging/logger.js';
4
+ import { appendLedgerEntry, getSessionRecoveryContext } from './mesh-ledger.js';
5
+ import type { MeshLedgerKind, SessionRecoveryContext } from './mesh-ledger.js';
6
+ import { claimNextTask, updateSessionTaskStatus, enqueueTask } from './mesh-work-queue.js';
4
7
 
5
8
  // ---------------------------------------------------------------------------
6
9
  // MCP coordinator pending-event queue
@@ -37,6 +40,13 @@ const MESH_COORDINATOR_EVENTS = new Set([
37
40
  'monitor:long_generating',
38
41
  ]);
39
42
 
43
+ const EVENT_TO_LEDGER_KIND: Record<string, MeshLedgerKind> = {
44
+ 'agent:generating_completed': 'task_completed',
45
+ 'agent:waiting_approval': 'task_approval_needed',
46
+ 'agent:stopped': 'task_failed',
47
+ 'monitor:long_generating': 'task_stalled',
48
+ };
49
+
40
50
  function isMeshCoordinatorEvent(eventName: unknown): eventName is string {
41
51
  return typeof eventName === 'string' && MESH_COORDINATOR_EVENTS.has(eventName);
42
52
  }
@@ -50,10 +60,68 @@ function formatCompletionMetadata(event: Record<string, unknown>): string {
50
60
  return parts.length > 0 ? ` (${parts.join('; ')})` : '';
51
61
  }
52
62
 
63
+ export function tryAssignQueueTask(
64
+ components: { cliManager: any },
65
+ meshId: string,
66
+ nodeId: string,
67
+ sessionId: string,
68
+ providerType: string
69
+ ): boolean {
70
+ const task = claimNextTask(meshId, nodeId, sessionId);
71
+ if (!task) return false;
72
+
73
+ LOG.info('MeshQueue', `Node ${nodeId} (${sessionId}) pulled task ${task.id}`);
74
+
75
+ components.cliManager.handleCliCommand('agent_command', {
76
+ targetSessionId: sessionId,
77
+ cliType: providerType,
78
+ action: 'send_chat',
79
+ input: task.message,
80
+ }).catch((e: any) => {
81
+ LOG.error('MeshQueue', `Failed to dispatch task to node ${nodeId}: ${e?.message}`);
82
+ });
83
+
84
+ return true;
85
+ }
86
+
87
+ /**
88
+ * Triggers a queue check for all nodes in the mesh.
89
+ * Called when a new task is enqueued, in case nodes are already idle.
90
+ */
91
+ export function triggerMeshQueue(components: { instanceManager: any; cliManager: any }, meshId: string) {
92
+ const mesh = getMesh(meshId);
93
+ if (!mesh) return;
94
+
95
+ // Find all CLI instances that belong to this mesh and are idle
96
+ const cliInstances = components.instanceManager.getByCategory('cli');
97
+ for (const inst of cliInstances) {
98
+ const state = inst.getState();
99
+ const settings = state.settings as Record<string, unknown> || {};
100
+
101
+ const instMeshId = readNonEmptyString(settings.meshNodeFor);
102
+ if (instMeshId !== meshId && !settings.launchedByCoordinator) continue;
103
+
104
+ const nodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
105
+ if (!nodeId) continue;
106
+
107
+ // Is it idle? (online and waiting for input)
108
+ if (state.status !== 'idle' && state.status !== 'stopped' && state.activeChat?.status !== 'waiting_input') continue;
109
+
110
+ const sessionId = state.instanceId;
111
+ const providerType = state.type || readNonEmptyString(settings.providerType);
112
+
113
+ if (providerType) {
114
+ // Try to assign a task to this idle node
115
+ tryAssignQueueTask(components, meshId, nodeId, sessionId, providerType);
116
+ }
117
+ }
118
+ }
119
+
53
120
  function buildMeshSystemMessage(args: {
54
121
  event: string;
55
122
  nodeLabel: string;
56
123
  metadataEvent: Record<string, unknown>;
124
+ recoveryContext?: SessionRecoveryContext | null;
57
125
  }): string {
58
126
  const metadata = formatCompletionMetadata(args.metadataEvent);
59
127
  if (args.event === 'agent:generating_completed') {
@@ -63,6 +131,28 @@ function buildMeshSystemMessage(args: {
63
131
  return `[System] ${args.nodeLabel} is waiting for approval to proceed${metadata}. You may use mesh_read_chat and mesh_approve to handle it.`;
64
132
  }
65
133
  if (args.event === 'agent:stopped') {
134
+ const rc = args.recoveryContext;
135
+ if (rc && rc.consecutiveNodeFailures > 0) {
136
+ const parts = [
137
+ `[System] ${args.nodeLabel} has stopped unexpectedly${metadata}.`,
138
+ `\n\n**Recovery Context:**`,
139
+ `- Consecutive failures on this node: ${rc.consecutiveNodeFailures}`,
140
+ rc.taskAttemptCount > 0 ? `- This task has been attempted ${rc.taskAttemptCount} time(s)` : '',
141
+ `- Recommendation: ${rc.advice}`,
142
+ ];
143
+ if (rc.retryRecommended && rc.lastTaskMessage) {
144
+ parts.push(
145
+ `\n\n**Original task to retry:**`,
146
+ `> ${rc.lastTaskMessage.length > 300 ? rc.lastTaskMessage.slice(0, 300) + '...' : rc.lastTaskMessage}`,
147
+ `\nTo retry: call \`mesh_launch_session\` for this node, then \`mesh_send_task\` with the original task.`,
148
+ );
149
+ } else if (!rc.retryRecommended) {
150
+ parts.push(
151
+ `\nDo NOT retry on this node. Consider reassigning to a different node or asking the user for guidance.`,
152
+ );
153
+ }
154
+ return parts.filter(Boolean).join('\n');
155
+ }
66
156
  return `[System] ${args.nodeLabel} has stopped${metadata}. Use mesh_read_chat once if you need to inspect its last output.`;
67
157
  }
68
158
  if (args.event === 'monitor:long_generating') {
@@ -78,6 +168,111 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
78
168
  event: string;
79
169
  metadataEvent: Record<string, unknown>;
80
170
  }) {
171
+ // ── Task Queue & Ledger ──
172
+ if (args.event === 'agent:generating_completed') {
173
+ const sessionId = readNonEmptyString(args.metadataEvent.targetSessionId);
174
+ const nodeId = readNonEmptyString(args.metadataEvent.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
175
+ const providerType = readNonEmptyString(args.metadataEvent.providerType);
176
+
177
+ if (sessionId) {
178
+ updateSessionTaskStatus(args.meshId, sessionId, 'completed');
179
+ if (nodeId && providerType) {
180
+ // Short delay to allow completion event to propagate before pulling next
181
+ setTimeout(() => {
182
+ tryAssignQueueTask(components, args.meshId, nodeId, sessionId, providerType);
183
+ }, 500);
184
+ }
185
+ }
186
+ } else if (args.event === 'agent:stopped') {
187
+ const sessionId = readNonEmptyString(args.metadataEvent.targetSessionId);
188
+ if (sessionId) {
189
+ updateSessionTaskStatus(args.meshId, sessionId, 'failed');
190
+ }
191
+ }
192
+
193
+ const ledgerKind = EVENT_TO_LEDGER_KIND[args.event];
194
+ if (ledgerKind) {
195
+ try {
196
+ appendLedgerEntry(args.meshId, {
197
+ kind: ledgerKind,
198
+ nodeId: readNonEmptyString(args.metadataEvent.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
199
+ sessionId: readNonEmptyString(args.metadataEvent.targetSessionId) || undefined,
200
+ providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
201
+ payload: {
202
+ event: args.event,
203
+ nodeLabel: args.nodeLabel,
204
+ providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
205
+ },
206
+ });
207
+ } catch (e: any) {
208
+ LOG.warn('MeshLedger', `Failed to record ${ledgerKind}: ${e?.message || e}`);
209
+ }
210
+ }
211
+
212
+ // ── Recovery Context: enrich agent:stopped with retry intelligence ──
213
+ let recoveryContext: SessionRecoveryContext | null = null;
214
+ if (args.event === 'agent:stopped') {
215
+ try {
216
+ // Resolve maxTaskRetries from mesh policy
217
+ const mesh = getMesh(args.meshId);
218
+ const maxRetries = mesh?.policy?.maxTaskRetries ?? 1;
219
+
220
+ recoveryContext = getSessionRecoveryContext(args.meshId, {
221
+ sessionId: readNonEmptyString(args.metadataEvent.targetSessionId) || undefined,
222
+ nodeId: readNonEmptyString(args.metadataEvent.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
223
+ maxRetries,
224
+ });
225
+ recoveryContext.failedProviderType = readNonEmptyString(args.metadataEvent.providerType) || null;
226
+
227
+ // Record recovery_attempted if retry is recommended
228
+ if (recoveryContext.retryRecommended && recoveryContext.consecutiveNodeFailures > 0) {
229
+ appendLedgerEntry(args.meshId, {
230
+ kind: 'recovery_attempted',
231
+ nodeId: recoveryContext.failedNodeId || undefined,
232
+ sessionId: recoveryContext.failedSessionId || undefined,
233
+ providerType: recoveryContext.failedProviderType || undefined,
234
+ payload: {
235
+ consecutiveFailures: recoveryContext.consecutiveNodeFailures,
236
+ taskAttemptCount: recoveryContext.taskAttemptCount,
237
+ retryRecommended: recoveryContext.retryRecommended,
238
+ advice: recoveryContext.advice,
239
+ },
240
+ });
241
+
242
+ // Auto-Recovery (Phase 5): Automatically re-enqueue the task and re-launch the session
243
+ if (recoveryContext.lastTaskMessage && recoveryContext.failedNodeId && recoveryContext.failedProviderType) {
244
+ const autoNodeId = recoveryContext.failedNodeId;
245
+ try {
246
+ const task = enqueueTask(args.meshId, recoveryContext.lastTaskMessage, {
247
+ targetNodeId: autoNodeId
248
+ });
249
+ LOG.info('MeshRecovery', `Auto-requeued failed task: ${task.id} for node ${autoNodeId}`);
250
+
251
+ const node = mesh?.nodes.find(n => n.id === autoNodeId);
252
+ if (node) {
253
+ components.cliManager.handleCliCommand('launch_cli', {
254
+ cliType: recoveryContext.failedProviderType,
255
+ dir: node.workspace,
256
+ settings: {
257
+ meshNodeFor: args.meshId,
258
+ meshNodeId: node.id,
259
+ spawnedSessionVisibility: mesh?.policy?.spawnedSessionVisibility || 'hidden',
260
+ launchedByCoordinator: true,
261
+ }
262
+ }).catch((e: any) => LOG.error('MeshRecovery', `Failed to auto-relaunch session for ${node.id}: ${e?.message}`));
263
+ }
264
+ } catch (e: any) {
265
+ LOG.warn('MeshRecovery', `Failed to execute auto-recovery: ${e?.message}`);
266
+ }
267
+ }
268
+ }
269
+
270
+ LOG.info('MeshRecovery', `Recovery context for ${args.nodeLabel}: ${recoveryContext.advice}`);
271
+ } catch (e: any) {
272
+ LOG.warn('MeshRecovery', `Failed to build recovery context: ${e?.message || e}`);
273
+ }
274
+ }
275
+
81
276
  const coordinatorInstances = components.instanceManager.getByCategory('cli').filter((inst) => {
82
277
  const instState = inst.getState();
83
278
  if (instState.settings?.meshCoordinatorFor !== args.meshId) return false;
@@ -92,7 +287,10 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
92
287
  event: args.event,
93
288
  meshId: args.meshId,
94
289
  nodeLabel: args.nodeLabel,
95
- metadataEvent: args.metadataEvent,
290
+ metadataEvent: {
291
+ ...args.metadataEvent,
292
+ ...(recoveryContext ? { recoveryContext } : {}),
293
+ },
96
294
  queuedAt: Date.now(),
97
295
  });
98
296
  LOG.info('MeshEvents', `Queued ${args.event} for MCP coordinator (mesh ${args.meshId})`);
@@ -104,6 +302,7 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
104
302
  event: args.event,
105
303
  nodeLabel: args.nodeLabel,
106
304
  metadataEvent: args.metadataEvent,
305
+ recoveryContext,
107
306
  });
108
307
  if (!messageText) return { success: false, error: 'unsupported mesh event' };
109
308
 
@@ -0,0 +1,378 @@
1
+ /**
2
+ * Mesh Task Ledger — GasTown-inspired append-only JSONL task history
3
+ *
4
+ * Records all mesh orchestration events (task dispatch, completion, failure,
5
+ * checkpoint, node lifecycle) as an append-only JSONL file per mesh.
6
+ *
7
+ * Inspired by GasTown's "Beads" pattern: every action is a versioned record
8
+ * that persists across agent sessions, enabling recovery, auditing, and
9
+ * continuity when individual sessions fail or context windows are exhausted.
10
+ *
11
+ * Storage: ~/.adhdev/mesh-ledger/<meshId>.jsonl
12
+ * Format: One JSON object per line, newest entries appended at end
13
+ * Safety: mode 0o600, atomic append via appendFileSync
14
+ */
15
+
16
+ import { existsSync, mkdirSync, readFileSync, appendFileSync, statSync, renameSync } from 'fs';
17
+ import { join } from 'path';
18
+ import { randomUUID } from 'crypto';
19
+ import { getConfigDir } from '../config/config.js';
20
+ import { EventEmitter } from 'events';
21
+ // ─── Types ──────────────────────────────────────
22
+
23
+ export type MeshLedgerKind =
24
+ | 'task_dispatched'
25
+ | 'task_completed'
26
+ | 'task_failed'
27
+ | 'task_stalled'
28
+ | 'task_approval_needed'
29
+ | 'session_launched'
30
+ | 'session_stopped'
31
+ | 'checkpoint_created'
32
+ | 'node_cloned'
33
+ | 'node_removed'
34
+ | 'coordinator_started'
35
+ | 'recovery_attempted'
36
+ ;
37
+
38
+ export interface MeshLedgerEntry {
39
+ id: string;
40
+ meshId: string;
41
+ timestamp: string;
42
+ kind: MeshLedgerKind;
43
+ nodeId?: string;
44
+ sessionId?: string;
45
+ providerType?: string;
46
+ payload: Record<string, unknown>;
47
+ }
48
+
49
+ export interface MeshLedgerSummary {
50
+ meshId: string;
51
+ totalEntries: number;
52
+ taskDispatched: number;
53
+ taskCompleted: number;
54
+ taskFailed: number;
55
+ taskStalled: number;
56
+ sessionLaunched: number;
57
+ checkpointCreated: number;
58
+ lastActivityAt: string | null;
59
+ recentFailures: number; // failures in last 30 minutes
60
+ }
61
+
62
+ export interface ReadLedgerOptions {
63
+ tail?: number;
64
+ since?: string;
65
+ kind?: MeshLedgerKind[];
66
+ }
67
+
68
+ // ─── Constants ──────────────────────────────────
69
+
70
+ const LEDGER_DIR_NAME = 'mesh-ledger';
71
+ const MAX_FILE_SIZE_BYTES = 10 * 1024 * 1024; // 10 MB
72
+ const RECENT_FAILURE_WINDOW_MS = 30 * 60 * 1000; // 30 minutes
73
+
74
+ // ─── Path Helpers ───────────────────────────────
75
+
76
+ export function getLedgerDir(): string {
77
+ const dir = join(getConfigDir(), LEDGER_DIR_NAME);
78
+ if (!existsSync(dir)) {
79
+ mkdirSync(dir, { recursive: true, mode: 0o700 });
80
+ }
81
+ return dir;
82
+ }
83
+
84
+ function getLedgerPath(meshId: string): string {
85
+ // Sanitize meshId to prevent path traversal
86
+ const safe = meshId.replace(/[^a-zA-Z0-9_-]/g, '_');
87
+ return join(getLedgerDir(), `${safe}.jsonl`);
88
+ }
89
+
90
+ function getRotatedPath(meshId: string, index: number): string {
91
+ const safe = meshId.replace(/[^a-zA-Z0-9_-]/g, '_');
92
+ return join(getLedgerDir(), `${safe}.${index}.jsonl`);
93
+ }
94
+
95
+ // ─── Core API ───────────────────────────────────
96
+
97
+ /**
98
+ * Append a new entry to the mesh ledger.
99
+ * Handles file creation, rotation on size overflow, and atomic writes.
100
+ */
101
+ export const meshLedgerEvents = new EventEmitter();
102
+
103
+ export function appendLedgerEntry(
104
+ meshId: string,
105
+ partial: Omit<MeshLedgerEntry, 'id' | 'meshId' | 'timestamp'>,
106
+ ): MeshLedgerEntry {
107
+ const entry: MeshLedgerEntry = {
108
+ id: randomUUID(),
109
+ meshId,
110
+ timestamp: new Date().toISOString(),
111
+ ...partial,
112
+ };
113
+
114
+ const filePath = getLedgerPath(meshId);
115
+
116
+ // Rotate if file exceeds max size
117
+ if (existsSync(filePath)) {
118
+ try {
119
+ const stat = statSync(filePath);
120
+ if (stat.size >= MAX_FILE_SIZE_BYTES) {
121
+ rotateLedgerFile(meshId, filePath);
122
+ }
123
+ } catch {
124
+ // stat failed — proceed with append anyway
125
+ }
126
+ }
127
+
128
+ try {
129
+ const line = JSON.stringify(entry) + '\n';
130
+ appendFileSync(filePath, line, { encoding: 'utf-8', mode: 0o600 });
131
+ meshLedgerEvents.emit('append', meshId, entry);
132
+ return entry;
133
+ } catch (e: any) {
134
+ throw new Error(`Failed to append to ledger for mesh ${meshId}: ${e.message}`);
135
+ }
136
+ }
137
+
138
+ /**
139
+ * Append entries received from the cloud to the local ledger.
140
+ * This skips deduplicated entries and just writes new ones.
141
+ */
142
+ export function appendRemoteLedgerEntries(meshId: string, entries: MeshLedgerEntry[]): void {
143
+ if (entries.length === 0) return;
144
+ const ledgerPath = getLedgerPath(meshId);
145
+
146
+ // Read existing to deduplicate by ID
147
+ const existing = new Set(readLedgerEntries(meshId).map(e => e.id));
148
+ const newEntries = entries.filter(e => !existing.has(e.id));
149
+
150
+ if (newEntries.length === 0) return;
151
+
152
+ try {
153
+ const lines = newEntries.map(e => JSON.stringify(e)).join('\n') + '\n';
154
+ appendFileSync(ledgerPath, lines, { encoding: 'utf-8', mode: 0o600 });
155
+ } catch (e: any) {
156
+ throw new Error(`Failed to append remote ledger entries for mesh ${meshId}: ${e.message}`);
157
+ }
158
+ }
159
+
160
+ /**
161
+ * Read ledger entries with optional filtering.
162
+ */
163
+ export function readLedgerEntries(meshId: string, opts?: ReadLedgerOptions): MeshLedgerEntry[] {
164
+ const filePath = getLedgerPath(meshId);
165
+ if (!existsSync(filePath)) return [];
166
+
167
+ let content: string;
168
+ try {
169
+ content = readFileSync(filePath, 'utf-8');
170
+ } catch {
171
+ return [];
172
+ }
173
+
174
+ const lines = content.split('\n').filter(line => line.trim());
175
+ let entries: MeshLedgerEntry[] = [];
176
+
177
+ for (const line of lines) {
178
+ try {
179
+ const entry = JSON.parse(line) as MeshLedgerEntry;
180
+ if (!entry.id || !entry.kind) continue;
181
+ entries.push(entry);
182
+ } catch {
183
+ // Skip malformed lines
184
+ }
185
+ }
186
+
187
+ // Apply filters
188
+ if (opts?.since) {
189
+ const sinceDate = new Date(opts.since).getTime();
190
+ if (!isNaN(sinceDate)) {
191
+ entries = entries.filter(e => new Date(e.timestamp).getTime() >= sinceDate);
192
+ }
193
+ }
194
+
195
+ if (opts?.kind?.length) {
196
+ const kindSet = new Set(opts.kind);
197
+ entries = entries.filter(e => kindSet.has(e.kind));
198
+ }
199
+
200
+ // Apply tail (return last N entries)
201
+ if (opts?.tail && opts.tail > 0 && entries.length > opts.tail) {
202
+ entries = entries.slice(-opts.tail);
203
+ }
204
+
205
+ return entries;
206
+ }
207
+
208
+ /**
209
+ * Get a summary of mesh activity from the ledger.
210
+ */
211
+ export function getLedgerSummary(meshId: string): MeshLedgerSummary {
212
+ const entries = readLedgerEntries(meshId);
213
+ const now = Date.now();
214
+ const recentFailureCutoff = now - RECENT_FAILURE_WINDOW_MS;
215
+
216
+ const summary: MeshLedgerSummary = {
217
+ meshId,
218
+ totalEntries: entries.length,
219
+ taskDispatched: 0,
220
+ taskCompleted: 0,
221
+ taskFailed: 0,
222
+ taskStalled: 0,
223
+ sessionLaunched: 0,
224
+ checkpointCreated: 0,
225
+ lastActivityAt: null,
226
+ recentFailures: 0,
227
+ };
228
+
229
+ for (const entry of entries) {
230
+ switch (entry.kind) {
231
+ case 'task_dispatched': summary.taskDispatched++; break;
232
+ case 'task_completed': summary.taskCompleted++; break;
233
+ case 'task_failed': {
234
+ summary.taskFailed++;
235
+ if (new Date(entry.timestamp).getTime() >= recentFailureCutoff) {
236
+ summary.recentFailures++;
237
+ }
238
+ break;
239
+ }
240
+ case 'task_stalled': summary.taskStalled++; break;
241
+ case 'session_launched': summary.sessionLaunched++; break;
242
+ case 'checkpoint_created': summary.checkpointCreated++; break;
243
+ }
244
+ }
245
+
246
+ if (entries.length > 0) {
247
+ summary.lastActivityAt = entries[entries.length - 1].timestamp;
248
+ }
249
+
250
+ return summary;
251
+ }
252
+
253
+ // ─── Recovery Context ───────────────────────────
254
+
255
+ export interface SessionRecoveryContext {
256
+ /** The original task message that was dispatched to this session/node */
257
+ lastTaskMessage: string | null;
258
+ /** The node that was running the failed task */
259
+ failedNodeId: string | null;
260
+ /** Session ID of the failed session */
261
+ failedSessionId: string | null;
262
+ /** Provider used for the failed session */
263
+ failedProviderType: string | null;
264
+ /** Number of consecutive failures for this node (within recent window) */
265
+ consecutiveNodeFailures: number;
266
+ /** Number of times this specific task was attempted (matched by truncated message prefix) */
267
+ taskAttemptCount: number;
268
+ /** Whether a retry is recommended based on maxRetries policy */
269
+ retryRecommended: boolean;
270
+ /** Human-readable recovery advice for the coordinator */
271
+ advice: string;
272
+ }
273
+
274
+ /**
275
+ * Build recovery context for a failed session.
276
+ * Looks up the ledger to find the original task, count failures, and advise on retry.
277
+ */
278
+ export function getSessionRecoveryContext(
279
+ meshId: string,
280
+ opts: {
281
+ sessionId?: string;
282
+ nodeId?: string;
283
+ maxRetries?: number;
284
+ },
285
+ ): SessionRecoveryContext {
286
+ const maxRetries = opts.maxRetries ?? 1;
287
+ const entries = readLedgerEntries(meshId);
288
+
289
+ // Find the last task_dispatched for this session or node
290
+ let lastDispatch: MeshLedgerEntry | null = null;
291
+ for (let i = entries.length - 1; i >= 0; i--) {
292
+ const e = entries[i];
293
+ if (e.kind !== 'task_dispatched') continue;
294
+ if (opts.sessionId && e.sessionId === opts.sessionId) { lastDispatch = e; break; }
295
+ if (opts.nodeId && e.nodeId === opts.nodeId) { lastDispatch = e; break; }
296
+ }
297
+
298
+ const lastTaskMessage = typeof lastDispatch?.payload?.message === 'string'
299
+ ? lastDispatch.payload.message
300
+ : null;
301
+
302
+ // Count consecutive recent failures for this node (within 30 min window)
303
+ const now = Date.now();
304
+ const recentWindow = now - RECENT_FAILURE_WINDOW_MS;
305
+ let consecutiveNodeFailures = 0;
306
+ for (let i = entries.length - 1; i >= 0; i--) {
307
+ const e = entries[i];
308
+ if (new Date(e.timestamp).getTime() < recentWindow) break;
309
+ if (opts.nodeId && e.nodeId !== opts.nodeId) continue;
310
+ if (e.kind === 'task_failed') {
311
+ consecutiveNodeFailures++;
312
+ } else if (e.kind === 'task_completed' || e.kind === 'task_dispatched') {
313
+ // A completion or new dispatch breaks the consecutive failure chain
314
+ break;
315
+ }
316
+ }
317
+
318
+ // Count how many times the same task was attempted (match by message prefix)
319
+ let taskAttemptCount = 0;
320
+ if (lastTaskMessage) {
321
+ const prefix = lastTaskMessage.slice(0, 200);
322
+ for (const e of entries) {
323
+ if (e.kind === 'task_dispatched' && typeof e.payload?.message === 'string') {
324
+ if (e.payload.message.startsWith(prefix)) {
325
+ taskAttemptCount++;
326
+ }
327
+ }
328
+ }
329
+ }
330
+
331
+ const retryRecommended = consecutiveNodeFailures <= maxRetries;
332
+
333
+ // Build advice string
334
+ let advice: string;
335
+ if (consecutiveNodeFailures === 0) {
336
+ advice = 'No recent failures detected. This may be a normal stop.';
337
+ } else if (retryRecommended) {
338
+ const remaining = maxRetries - consecutiveNodeFailures + 1;
339
+ advice = `Retry recommended (${consecutiveNodeFailures}/${maxRetries + 1} attempts used, ${remaining} remaining). `
340
+ + (lastTaskMessage
341
+ ? `Re-launch the session and resend the original task.`
342
+ : `Re-launch the session. Original task message not found in ledger.`);
343
+ } else {
344
+ advice = `Max retries exceeded (${consecutiveNodeFailures} consecutive failures). `
345
+ + `Consider: (1) reassigning to a different node, (2) simplifying the task, or (3) escalating to the user.`;
346
+ }
347
+
348
+ return {
349
+ lastTaskMessage,
350
+ failedNodeId: opts.nodeId || null,
351
+ failedSessionId: opts.sessionId || null,
352
+ failedProviderType: null, // filled by caller if available
353
+ consecutiveNodeFailures,
354
+ taskAttemptCount,
355
+ retryRecommended,
356
+ advice,
357
+ };
358
+ }
359
+
360
+ // ─── File Rotation ──────────────────────────────
361
+
362
+ function rotateLedgerFile(meshId: string, currentPath: string): void {
363
+ // Find next rotation index
364
+ let index = 1;
365
+ while (existsSync(getRotatedPath(meshId, index))) {
366
+ index++;
367
+ if (index > 10) break; // Max 10 rotations
368
+ }
369
+
370
+ // If all slots full, overwrite the oldest
371
+ if (index > 10) index = 10;
372
+
373
+ try {
374
+ renameSync(currentPath, getRotatedPath(meshId, index));
375
+ } catch {
376
+ // Rotation failed — the next append will just grow the file
377
+ }
378
+ }
@@ -26,6 +26,8 @@ export interface MeshSyncTransport {
26
26
  }): Promise<{ mesh: RemoteMeshRecord }>;
27
27
  /** DELETE /api/v1/repo-meshes/:id */
28
28
  deleteRemoteMesh(meshId: string): Promise<void>;
29
+ /** POST /api/v1/repo-meshes/:id/ledger/sync */
30
+ syncMeshLedger?(meshId: string, data: { newEntries: any[] }): Promise<{ missingEntries: any[] }>;
29
31
  }
30
32
 
31
33
  export interface RemoteMeshRecord {
@@ -105,5 +107,35 @@ export async function syncMeshes(transport: MeshSyncTransport): Promise<MeshSync
105
107
  }
106
108
  }
107
109
 
110
+ // Sync ledgers for all local meshes if the transport supports it
111
+ if (transport.syncMeshLedger) {
112
+ for (const local of localMeshes) {
113
+ try {
114
+ await syncMeshLedger(local.id, transport);
115
+ } catch (e: any) {
116
+ result.errors.push(`Ledger sync failed for "${local.name}": ${e.message}`);
117
+ }
118
+ }
119
+ }
120
+
108
121
  return result;
109
122
  }
123
+
124
+ /**
125
+ * Sync the task ledger for a specific mesh.
126
+ */
127
+ export async function syncMeshLedger(meshId: string, transport: MeshSyncTransport): Promise<void> {
128
+ if (!transport.syncMeshLedger) return;
129
+ const { readLedgerEntries, appendRemoteLedgerEntries } = await import('./mesh-ledger.js');
130
+
131
+ // Read all local entries (no tail)
132
+ const localEntries = readLedgerEntries(meshId);
133
+
134
+ // Send to cloud and get missing entries back
135
+ const res = await transport.syncMeshLedger(meshId, { newEntries: localEntries });
136
+
137
+ // Append any missing entries from the cloud
138
+ if (res.missingEntries && res.missingEntries.length > 0) {
139
+ appendRemoteLedgerEntries(meshId, res.missingEntries);
140
+ }
141
+ }