@adhdev/daemon-core 0.9.76 → 0.9.77-rc.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-adapters/provider-cli-adapter.d.ts +2 -0
- package/dist/cli-adapters/provider-cli-shared.d.ts +14 -4
- package/dist/index.d.ts +5 -0
- package/dist/index.js +911 -209
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +906 -216
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-events.d.ts +11 -1
- package/dist/mesh/mesh-ledger.d.ts +90 -0
- package/dist/mesh/mesh-sync.d.ts +10 -0
- package/dist/mesh/mesh-work-queue.d.ts +50 -0
- package/dist/repo-mesh-types.d.ts +6 -0
- package/dist/shared-types.d.ts +12 -0
- package/package.json +1 -1
- package/src/cli-adapters/provider-cli-adapter.ts +10 -4
- package/src/cli-adapters/provider-cli-shared.ts +14 -4
- package/src/commands/router.ts +123 -0
- package/src/index.ts +11 -0
- package/src/mesh/coordinator-prompt.ts +27 -12
- package/src/mesh/mesh-events.ts +200 -1
- package/src/mesh/mesh-ledger.ts +378 -0
- package/src/mesh/mesh-sync.ts +32 -0
- package/src/mesh/mesh-work-queue.ts +164 -0
- package/src/repo-mesh-types.ts +7 -0
- package/src/shared-types.ts +12 -0
- package/src/status/builders.ts +13 -0
package/src/mesh/mesh-events.ts
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
|
|
2
2
|
import { getMesh, getMeshByRepo } from '../config/mesh-config.js';
|
|
3
3
|
import { LOG } from '../logging/logger.js';
|
|
4
|
+
import { appendLedgerEntry, getSessionRecoveryContext } from './mesh-ledger.js';
|
|
5
|
+
import type { MeshLedgerKind, SessionRecoveryContext } from './mesh-ledger.js';
|
|
6
|
+
import { claimNextTask, updateSessionTaskStatus, enqueueTask } from './mesh-work-queue.js';
|
|
4
7
|
|
|
5
8
|
// ---------------------------------------------------------------------------
|
|
6
9
|
// MCP coordinator pending-event queue
|
|
@@ -37,6 +40,13 @@ const MESH_COORDINATOR_EVENTS = new Set([
|
|
|
37
40
|
'monitor:long_generating',
|
|
38
41
|
]);
|
|
39
42
|
|
|
43
|
+
const EVENT_TO_LEDGER_KIND: Record<string, MeshLedgerKind> = {
|
|
44
|
+
'agent:generating_completed': 'task_completed',
|
|
45
|
+
'agent:waiting_approval': 'task_approval_needed',
|
|
46
|
+
'agent:stopped': 'task_failed',
|
|
47
|
+
'monitor:long_generating': 'task_stalled',
|
|
48
|
+
};
|
|
49
|
+
|
|
40
50
|
function isMeshCoordinatorEvent(eventName: unknown): eventName is string {
|
|
41
51
|
return typeof eventName === 'string' && MESH_COORDINATOR_EVENTS.has(eventName);
|
|
42
52
|
}
|
|
@@ -50,10 +60,68 @@ function formatCompletionMetadata(event: Record<string, unknown>): string {
|
|
|
50
60
|
return parts.length > 0 ? ` (${parts.join('; ')})` : '';
|
|
51
61
|
}
|
|
52
62
|
|
|
63
|
+
export function tryAssignQueueTask(
|
|
64
|
+
components: { cliManager: any },
|
|
65
|
+
meshId: string,
|
|
66
|
+
nodeId: string,
|
|
67
|
+
sessionId: string,
|
|
68
|
+
providerType: string
|
|
69
|
+
): boolean {
|
|
70
|
+
const task = claimNextTask(meshId, nodeId, sessionId);
|
|
71
|
+
if (!task) return false;
|
|
72
|
+
|
|
73
|
+
LOG.info('MeshQueue', `Node ${nodeId} (${sessionId}) pulled task ${task.id}`);
|
|
74
|
+
|
|
75
|
+
components.cliManager.handleCliCommand('agent_command', {
|
|
76
|
+
targetSessionId: sessionId,
|
|
77
|
+
cliType: providerType,
|
|
78
|
+
action: 'send_chat',
|
|
79
|
+
input: task.message,
|
|
80
|
+
}).catch((e: any) => {
|
|
81
|
+
LOG.error('MeshQueue', `Failed to dispatch task to node ${nodeId}: ${e?.message}`);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
return true;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Triggers a queue check for all nodes in the mesh.
|
|
89
|
+
* Called when a new task is enqueued, in case nodes are already idle.
|
|
90
|
+
*/
|
|
91
|
+
export function triggerMeshQueue(components: { instanceManager: any; cliManager: any }, meshId: string) {
|
|
92
|
+
const mesh = getMesh(meshId);
|
|
93
|
+
if (!mesh) return;
|
|
94
|
+
|
|
95
|
+
// Find all CLI instances that belong to this mesh and are idle
|
|
96
|
+
const cliInstances = components.instanceManager.getByCategory('cli');
|
|
97
|
+
for (const inst of cliInstances) {
|
|
98
|
+
const state = inst.getState();
|
|
99
|
+
const settings = state.settings as Record<string, unknown> || {};
|
|
100
|
+
|
|
101
|
+
const instMeshId = readNonEmptyString(settings.meshNodeFor);
|
|
102
|
+
if (instMeshId !== meshId && !settings.launchedByCoordinator) continue;
|
|
103
|
+
|
|
104
|
+
const nodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
|
|
105
|
+
if (!nodeId) continue;
|
|
106
|
+
|
|
107
|
+
// Is it idle? (online and waiting for input)
|
|
108
|
+
if (state.status !== 'idle' && state.status !== 'stopped' && state.activeChat?.status !== 'waiting_input') continue;
|
|
109
|
+
|
|
110
|
+
const sessionId = state.instanceId;
|
|
111
|
+
const providerType = state.type || readNonEmptyString(settings.providerType);
|
|
112
|
+
|
|
113
|
+
if (providerType) {
|
|
114
|
+
// Try to assign a task to this idle node
|
|
115
|
+
tryAssignQueueTask(components, meshId, nodeId, sessionId, providerType);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
53
120
|
function buildMeshSystemMessage(args: {
|
|
54
121
|
event: string;
|
|
55
122
|
nodeLabel: string;
|
|
56
123
|
metadataEvent: Record<string, unknown>;
|
|
124
|
+
recoveryContext?: SessionRecoveryContext | null;
|
|
57
125
|
}): string {
|
|
58
126
|
const metadata = formatCompletionMetadata(args.metadataEvent);
|
|
59
127
|
if (args.event === 'agent:generating_completed') {
|
|
@@ -63,6 +131,28 @@ function buildMeshSystemMessage(args: {
|
|
|
63
131
|
return `[System] ${args.nodeLabel} is waiting for approval to proceed${metadata}. You may use mesh_read_chat and mesh_approve to handle it.`;
|
|
64
132
|
}
|
|
65
133
|
if (args.event === 'agent:stopped') {
|
|
134
|
+
const rc = args.recoveryContext;
|
|
135
|
+
if (rc && rc.consecutiveNodeFailures > 0) {
|
|
136
|
+
const parts = [
|
|
137
|
+
`[System] ${args.nodeLabel} has stopped unexpectedly${metadata}.`,
|
|
138
|
+
`\n\n**Recovery Context:**`,
|
|
139
|
+
`- Consecutive failures on this node: ${rc.consecutiveNodeFailures}`,
|
|
140
|
+
rc.taskAttemptCount > 0 ? `- This task has been attempted ${rc.taskAttemptCount} time(s)` : '',
|
|
141
|
+
`- Recommendation: ${rc.advice}`,
|
|
142
|
+
];
|
|
143
|
+
if (rc.retryRecommended && rc.lastTaskMessage) {
|
|
144
|
+
parts.push(
|
|
145
|
+
`\n\n**Original task to retry:**`,
|
|
146
|
+
`> ${rc.lastTaskMessage.length > 300 ? rc.lastTaskMessage.slice(0, 300) + '...' : rc.lastTaskMessage}`,
|
|
147
|
+
`\nTo retry: call \`mesh_launch_session\` for this node, then \`mesh_send_task\` with the original task.`,
|
|
148
|
+
);
|
|
149
|
+
} else if (!rc.retryRecommended) {
|
|
150
|
+
parts.push(
|
|
151
|
+
`\nDo NOT retry on this node. Consider reassigning to a different node or asking the user for guidance.`,
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
return parts.filter(Boolean).join('\n');
|
|
155
|
+
}
|
|
66
156
|
return `[System] ${args.nodeLabel} has stopped${metadata}. Use mesh_read_chat once if you need to inspect its last output.`;
|
|
67
157
|
}
|
|
68
158
|
if (args.event === 'monitor:long_generating') {
|
|
@@ -78,6 +168,111 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
78
168
|
event: string;
|
|
79
169
|
metadataEvent: Record<string, unknown>;
|
|
80
170
|
}) {
|
|
171
|
+
// ── Task Queue & Ledger ──
|
|
172
|
+
if (args.event === 'agent:generating_completed') {
|
|
173
|
+
const sessionId = readNonEmptyString(args.metadataEvent.targetSessionId);
|
|
174
|
+
const nodeId = readNonEmptyString(args.metadataEvent.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
175
|
+
const providerType = readNonEmptyString(args.metadataEvent.providerType);
|
|
176
|
+
|
|
177
|
+
if (sessionId) {
|
|
178
|
+
updateSessionTaskStatus(args.meshId, sessionId, 'completed');
|
|
179
|
+
if (nodeId && providerType) {
|
|
180
|
+
// Short delay to allow completion event to propagate before pulling next
|
|
181
|
+
setTimeout(() => {
|
|
182
|
+
tryAssignQueueTask(components, args.meshId, nodeId, sessionId, providerType);
|
|
183
|
+
}, 500);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
} else if (args.event === 'agent:stopped') {
|
|
187
|
+
const sessionId = readNonEmptyString(args.metadataEvent.targetSessionId);
|
|
188
|
+
if (sessionId) {
|
|
189
|
+
updateSessionTaskStatus(args.meshId, sessionId, 'failed');
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
const ledgerKind = EVENT_TO_LEDGER_KIND[args.event];
|
|
194
|
+
if (ledgerKind) {
|
|
195
|
+
try {
|
|
196
|
+
appendLedgerEntry(args.meshId, {
|
|
197
|
+
kind: ledgerKind,
|
|
198
|
+
nodeId: readNonEmptyString(args.metadataEvent.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
|
|
199
|
+
sessionId: readNonEmptyString(args.metadataEvent.targetSessionId) || undefined,
|
|
200
|
+
providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
|
|
201
|
+
payload: {
|
|
202
|
+
event: args.event,
|
|
203
|
+
nodeLabel: args.nodeLabel,
|
|
204
|
+
providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
|
|
205
|
+
},
|
|
206
|
+
});
|
|
207
|
+
} catch (e: any) {
|
|
208
|
+
LOG.warn('MeshLedger', `Failed to record ${ledgerKind}: ${e?.message || e}`);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// ── Recovery Context: enrich agent:stopped with retry intelligence ──
|
|
213
|
+
let recoveryContext: SessionRecoveryContext | null = null;
|
|
214
|
+
if (args.event === 'agent:stopped') {
|
|
215
|
+
try {
|
|
216
|
+
// Resolve maxTaskRetries from mesh policy
|
|
217
|
+
const mesh = getMesh(args.meshId);
|
|
218
|
+
const maxRetries = mesh?.policy?.maxTaskRetries ?? 1;
|
|
219
|
+
|
|
220
|
+
recoveryContext = getSessionRecoveryContext(args.meshId, {
|
|
221
|
+
sessionId: readNonEmptyString(args.metadataEvent.targetSessionId) || undefined,
|
|
222
|
+
nodeId: readNonEmptyString(args.metadataEvent.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
|
|
223
|
+
maxRetries,
|
|
224
|
+
});
|
|
225
|
+
recoveryContext.failedProviderType = readNonEmptyString(args.metadataEvent.providerType) || null;
|
|
226
|
+
|
|
227
|
+
// Record recovery_attempted if retry is recommended
|
|
228
|
+
if (recoveryContext.retryRecommended && recoveryContext.consecutiveNodeFailures > 0) {
|
|
229
|
+
appendLedgerEntry(args.meshId, {
|
|
230
|
+
kind: 'recovery_attempted',
|
|
231
|
+
nodeId: recoveryContext.failedNodeId || undefined,
|
|
232
|
+
sessionId: recoveryContext.failedSessionId || undefined,
|
|
233
|
+
providerType: recoveryContext.failedProviderType || undefined,
|
|
234
|
+
payload: {
|
|
235
|
+
consecutiveFailures: recoveryContext.consecutiveNodeFailures,
|
|
236
|
+
taskAttemptCount: recoveryContext.taskAttemptCount,
|
|
237
|
+
retryRecommended: recoveryContext.retryRecommended,
|
|
238
|
+
advice: recoveryContext.advice,
|
|
239
|
+
},
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
// Auto-Recovery (Phase 5): Automatically re-enqueue the task and re-launch the session
|
|
243
|
+
if (recoveryContext.lastTaskMessage && recoveryContext.failedNodeId && recoveryContext.failedProviderType) {
|
|
244
|
+
const autoNodeId = recoveryContext.failedNodeId;
|
|
245
|
+
try {
|
|
246
|
+
const task = enqueueTask(args.meshId, recoveryContext.lastTaskMessage, {
|
|
247
|
+
targetNodeId: autoNodeId
|
|
248
|
+
});
|
|
249
|
+
LOG.info('MeshRecovery', `Auto-requeued failed task: ${task.id} for node ${autoNodeId}`);
|
|
250
|
+
|
|
251
|
+
const node = mesh?.nodes.find(n => n.id === autoNodeId);
|
|
252
|
+
if (node) {
|
|
253
|
+
components.cliManager.handleCliCommand('launch_cli', {
|
|
254
|
+
cliType: recoveryContext.failedProviderType,
|
|
255
|
+
dir: node.workspace,
|
|
256
|
+
settings: {
|
|
257
|
+
meshNodeFor: args.meshId,
|
|
258
|
+
meshNodeId: node.id,
|
|
259
|
+
spawnedSessionVisibility: mesh?.policy?.spawnedSessionVisibility || 'hidden',
|
|
260
|
+
launchedByCoordinator: true,
|
|
261
|
+
}
|
|
262
|
+
}).catch((e: any) => LOG.error('MeshRecovery', `Failed to auto-relaunch session for ${node.id}: ${e?.message}`));
|
|
263
|
+
}
|
|
264
|
+
} catch (e: any) {
|
|
265
|
+
LOG.warn('MeshRecovery', `Failed to execute auto-recovery: ${e?.message}`);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
LOG.info('MeshRecovery', `Recovery context for ${args.nodeLabel}: ${recoveryContext.advice}`);
|
|
271
|
+
} catch (e: any) {
|
|
272
|
+
LOG.warn('MeshRecovery', `Failed to build recovery context: ${e?.message || e}`);
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
81
276
|
const coordinatorInstances = components.instanceManager.getByCategory('cli').filter((inst) => {
|
|
82
277
|
const instState = inst.getState();
|
|
83
278
|
if (instState.settings?.meshCoordinatorFor !== args.meshId) return false;
|
|
@@ -92,7 +287,10 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
92
287
|
event: args.event,
|
|
93
288
|
meshId: args.meshId,
|
|
94
289
|
nodeLabel: args.nodeLabel,
|
|
95
|
-
metadataEvent:
|
|
290
|
+
metadataEvent: {
|
|
291
|
+
...args.metadataEvent,
|
|
292
|
+
...(recoveryContext ? { recoveryContext } : {}),
|
|
293
|
+
},
|
|
96
294
|
queuedAt: Date.now(),
|
|
97
295
|
});
|
|
98
296
|
LOG.info('MeshEvents', `Queued ${args.event} for MCP coordinator (mesh ${args.meshId})`);
|
|
@@ -104,6 +302,7 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
104
302
|
event: args.event,
|
|
105
303
|
nodeLabel: args.nodeLabel,
|
|
106
304
|
metadataEvent: args.metadataEvent,
|
|
305
|
+
recoveryContext,
|
|
107
306
|
});
|
|
108
307
|
if (!messageText) return { success: false, error: 'unsupported mesh event' };
|
|
109
308
|
|
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Mesh Task Ledger — GasTown-inspired append-only JSONL task history
|
|
3
|
+
*
|
|
4
|
+
* Records all mesh orchestration events (task dispatch, completion, failure,
|
|
5
|
+
* checkpoint, node lifecycle) as an append-only JSONL file per mesh.
|
|
6
|
+
*
|
|
7
|
+
* Inspired by GasTown's "Beads" pattern: every action is a versioned record
|
|
8
|
+
* that persists across agent sessions, enabling recovery, auditing, and
|
|
9
|
+
* continuity when individual sessions fail or context windows are exhausted.
|
|
10
|
+
*
|
|
11
|
+
* Storage: ~/.adhdev/mesh-ledger/<meshId>.jsonl
|
|
12
|
+
* Format: One JSON object per line, newest entries appended at end
|
|
13
|
+
* Safety: mode 0o600, atomic append via appendFileSync
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { existsSync, mkdirSync, readFileSync, appendFileSync, statSync, renameSync } from 'fs';
|
|
17
|
+
import { join } from 'path';
|
|
18
|
+
import { randomUUID } from 'crypto';
|
|
19
|
+
import { getConfigDir } from '../config/config.js';
|
|
20
|
+
import { EventEmitter } from 'events';
|
|
21
|
+
// ─── Types ──────────────────────────────────────
|
|
22
|
+
|
|
23
|
+
export type MeshLedgerKind =
|
|
24
|
+
| 'task_dispatched'
|
|
25
|
+
| 'task_completed'
|
|
26
|
+
| 'task_failed'
|
|
27
|
+
| 'task_stalled'
|
|
28
|
+
| 'task_approval_needed'
|
|
29
|
+
| 'session_launched'
|
|
30
|
+
| 'session_stopped'
|
|
31
|
+
| 'checkpoint_created'
|
|
32
|
+
| 'node_cloned'
|
|
33
|
+
| 'node_removed'
|
|
34
|
+
| 'coordinator_started'
|
|
35
|
+
| 'recovery_attempted'
|
|
36
|
+
;
|
|
37
|
+
|
|
38
|
+
export interface MeshLedgerEntry {
|
|
39
|
+
id: string;
|
|
40
|
+
meshId: string;
|
|
41
|
+
timestamp: string;
|
|
42
|
+
kind: MeshLedgerKind;
|
|
43
|
+
nodeId?: string;
|
|
44
|
+
sessionId?: string;
|
|
45
|
+
providerType?: string;
|
|
46
|
+
payload: Record<string, unknown>;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export interface MeshLedgerSummary {
|
|
50
|
+
meshId: string;
|
|
51
|
+
totalEntries: number;
|
|
52
|
+
taskDispatched: number;
|
|
53
|
+
taskCompleted: number;
|
|
54
|
+
taskFailed: number;
|
|
55
|
+
taskStalled: number;
|
|
56
|
+
sessionLaunched: number;
|
|
57
|
+
checkpointCreated: number;
|
|
58
|
+
lastActivityAt: string | null;
|
|
59
|
+
recentFailures: number; // failures in last 30 minutes
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export interface ReadLedgerOptions {
|
|
63
|
+
tail?: number;
|
|
64
|
+
since?: string;
|
|
65
|
+
kind?: MeshLedgerKind[];
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// ─── Constants ──────────────────────────────────
|
|
69
|
+
|
|
70
|
+
const LEDGER_DIR_NAME = 'mesh-ledger';
|
|
71
|
+
const MAX_FILE_SIZE_BYTES = 10 * 1024 * 1024; // 10 MB
|
|
72
|
+
const RECENT_FAILURE_WINDOW_MS = 30 * 60 * 1000; // 30 minutes
|
|
73
|
+
|
|
74
|
+
// ─── Path Helpers ───────────────────────────────
|
|
75
|
+
|
|
76
|
+
export function getLedgerDir(): string {
|
|
77
|
+
const dir = join(getConfigDir(), LEDGER_DIR_NAME);
|
|
78
|
+
if (!existsSync(dir)) {
|
|
79
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
80
|
+
}
|
|
81
|
+
return dir;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function getLedgerPath(meshId: string): string {
|
|
85
|
+
// Sanitize meshId to prevent path traversal
|
|
86
|
+
const safe = meshId.replace(/[^a-zA-Z0-9_-]/g, '_');
|
|
87
|
+
return join(getLedgerDir(), `${safe}.jsonl`);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function getRotatedPath(meshId: string, index: number): string {
|
|
91
|
+
const safe = meshId.replace(/[^a-zA-Z0-9_-]/g, '_');
|
|
92
|
+
return join(getLedgerDir(), `${safe}.${index}.jsonl`);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// ─── Core API ───────────────────────────────────
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Append a new entry to the mesh ledger.
|
|
99
|
+
* Handles file creation, rotation on size overflow, and atomic writes.
|
|
100
|
+
*/
|
|
101
|
+
export const meshLedgerEvents = new EventEmitter();
|
|
102
|
+
|
|
103
|
+
export function appendLedgerEntry(
|
|
104
|
+
meshId: string,
|
|
105
|
+
partial: Omit<MeshLedgerEntry, 'id' | 'meshId' | 'timestamp'>,
|
|
106
|
+
): MeshLedgerEntry {
|
|
107
|
+
const entry: MeshLedgerEntry = {
|
|
108
|
+
id: randomUUID(),
|
|
109
|
+
meshId,
|
|
110
|
+
timestamp: new Date().toISOString(),
|
|
111
|
+
...partial,
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
const filePath = getLedgerPath(meshId);
|
|
115
|
+
|
|
116
|
+
// Rotate if file exceeds max size
|
|
117
|
+
if (existsSync(filePath)) {
|
|
118
|
+
try {
|
|
119
|
+
const stat = statSync(filePath);
|
|
120
|
+
if (stat.size >= MAX_FILE_SIZE_BYTES) {
|
|
121
|
+
rotateLedgerFile(meshId, filePath);
|
|
122
|
+
}
|
|
123
|
+
} catch {
|
|
124
|
+
// stat failed — proceed with append anyway
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
try {
|
|
129
|
+
const line = JSON.stringify(entry) + '\n';
|
|
130
|
+
appendFileSync(filePath, line, { encoding: 'utf-8', mode: 0o600 });
|
|
131
|
+
meshLedgerEvents.emit('append', meshId, entry);
|
|
132
|
+
return entry;
|
|
133
|
+
} catch (e: any) {
|
|
134
|
+
throw new Error(`Failed to append to ledger for mesh ${meshId}: ${e.message}`);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Append entries received from the cloud to the local ledger.
|
|
140
|
+
* This skips deduplicated entries and just writes new ones.
|
|
141
|
+
*/
|
|
142
|
+
export function appendRemoteLedgerEntries(meshId: string, entries: MeshLedgerEntry[]): void {
|
|
143
|
+
if (entries.length === 0) return;
|
|
144
|
+
const ledgerPath = getLedgerPath(meshId);
|
|
145
|
+
|
|
146
|
+
// Read existing to deduplicate by ID
|
|
147
|
+
const existing = new Set(readLedgerEntries(meshId).map(e => e.id));
|
|
148
|
+
const newEntries = entries.filter(e => !existing.has(e.id));
|
|
149
|
+
|
|
150
|
+
if (newEntries.length === 0) return;
|
|
151
|
+
|
|
152
|
+
try {
|
|
153
|
+
const lines = newEntries.map(e => JSON.stringify(e)).join('\n') + '\n';
|
|
154
|
+
appendFileSync(ledgerPath, lines, { encoding: 'utf-8', mode: 0o600 });
|
|
155
|
+
} catch (e: any) {
|
|
156
|
+
throw new Error(`Failed to append remote ledger entries for mesh ${meshId}: ${e.message}`);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Read ledger entries with optional filtering.
|
|
162
|
+
*/
|
|
163
|
+
export function readLedgerEntries(meshId: string, opts?: ReadLedgerOptions): MeshLedgerEntry[] {
|
|
164
|
+
const filePath = getLedgerPath(meshId);
|
|
165
|
+
if (!existsSync(filePath)) return [];
|
|
166
|
+
|
|
167
|
+
let content: string;
|
|
168
|
+
try {
|
|
169
|
+
content = readFileSync(filePath, 'utf-8');
|
|
170
|
+
} catch {
|
|
171
|
+
return [];
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
const lines = content.split('\n').filter(line => line.trim());
|
|
175
|
+
let entries: MeshLedgerEntry[] = [];
|
|
176
|
+
|
|
177
|
+
for (const line of lines) {
|
|
178
|
+
try {
|
|
179
|
+
const entry = JSON.parse(line) as MeshLedgerEntry;
|
|
180
|
+
if (!entry.id || !entry.kind) continue;
|
|
181
|
+
entries.push(entry);
|
|
182
|
+
} catch {
|
|
183
|
+
// Skip malformed lines
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// Apply filters
|
|
188
|
+
if (opts?.since) {
|
|
189
|
+
const sinceDate = new Date(opts.since).getTime();
|
|
190
|
+
if (!isNaN(sinceDate)) {
|
|
191
|
+
entries = entries.filter(e => new Date(e.timestamp).getTime() >= sinceDate);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
if (opts?.kind?.length) {
|
|
196
|
+
const kindSet = new Set(opts.kind);
|
|
197
|
+
entries = entries.filter(e => kindSet.has(e.kind));
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
// Apply tail (return last N entries)
|
|
201
|
+
if (opts?.tail && opts.tail > 0 && entries.length > opts.tail) {
|
|
202
|
+
entries = entries.slice(-opts.tail);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
return entries;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* Get a summary of mesh activity from the ledger.
|
|
210
|
+
*/
|
|
211
|
+
export function getLedgerSummary(meshId: string): MeshLedgerSummary {
|
|
212
|
+
const entries = readLedgerEntries(meshId);
|
|
213
|
+
const now = Date.now();
|
|
214
|
+
const recentFailureCutoff = now - RECENT_FAILURE_WINDOW_MS;
|
|
215
|
+
|
|
216
|
+
const summary: MeshLedgerSummary = {
|
|
217
|
+
meshId,
|
|
218
|
+
totalEntries: entries.length,
|
|
219
|
+
taskDispatched: 0,
|
|
220
|
+
taskCompleted: 0,
|
|
221
|
+
taskFailed: 0,
|
|
222
|
+
taskStalled: 0,
|
|
223
|
+
sessionLaunched: 0,
|
|
224
|
+
checkpointCreated: 0,
|
|
225
|
+
lastActivityAt: null,
|
|
226
|
+
recentFailures: 0,
|
|
227
|
+
};
|
|
228
|
+
|
|
229
|
+
for (const entry of entries) {
|
|
230
|
+
switch (entry.kind) {
|
|
231
|
+
case 'task_dispatched': summary.taskDispatched++; break;
|
|
232
|
+
case 'task_completed': summary.taskCompleted++; break;
|
|
233
|
+
case 'task_failed': {
|
|
234
|
+
summary.taskFailed++;
|
|
235
|
+
if (new Date(entry.timestamp).getTime() >= recentFailureCutoff) {
|
|
236
|
+
summary.recentFailures++;
|
|
237
|
+
}
|
|
238
|
+
break;
|
|
239
|
+
}
|
|
240
|
+
case 'task_stalled': summary.taskStalled++; break;
|
|
241
|
+
case 'session_launched': summary.sessionLaunched++; break;
|
|
242
|
+
case 'checkpoint_created': summary.checkpointCreated++; break;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
if (entries.length > 0) {
|
|
247
|
+
summary.lastActivityAt = entries[entries.length - 1].timestamp;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
return summary;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// ─── Recovery Context ───────────────────────────
|
|
254
|
+
|
|
255
|
+
export interface SessionRecoveryContext {
|
|
256
|
+
/** The original task message that was dispatched to this session/node */
|
|
257
|
+
lastTaskMessage: string | null;
|
|
258
|
+
/** The node that was running the failed task */
|
|
259
|
+
failedNodeId: string | null;
|
|
260
|
+
/** Session ID of the failed session */
|
|
261
|
+
failedSessionId: string | null;
|
|
262
|
+
/** Provider used for the failed session */
|
|
263
|
+
failedProviderType: string | null;
|
|
264
|
+
/** Number of consecutive failures for this node (within recent window) */
|
|
265
|
+
consecutiveNodeFailures: number;
|
|
266
|
+
/** Number of times this specific task was attempted (matched by truncated message prefix) */
|
|
267
|
+
taskAttemptCount: number;
|
|
268
|
+
/** Whether a retry is recommended based on maxRetries policy */
|
|
269
|
+
retryRecommended: boolean;
|
|
270
|
+
/** Human-readable recovery advice for the coordinator */
|
|
271
|
+
advice: string;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Build recovery context for a failed session.
|
|
276
|
+
* Looks up the ledger to find the original task, count failures, and advise on retry.
|
|
277
|
+
*/
|
|
278
|
+
export function getSessionRecoveryContext(
|
|
279
|
+
meshId: string,
|
|
280
|
+
opts: {
|
|
281
|
+
sessionId?: string;
|
|
282
|
+
nodeId?: string;
|
|
283
|
+
maxRetries?: number;
|
|
284
|
+
},
|
|
285
|
+
): SessionRecoveryContext {
|
|
286
|
+
const maxRetries = opts.maxRetries ?? 1;
|
|
287
|
+
const entries = readLedgerEntries(meshId);
|
|
288
|
+
|
|
289
|
+
// Find the last task_dispatched for this session or node
|
|
290
|
+
let lastDispatch: MeshLedgerEntry | null = null;
|
|
291
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
292
|
+
const e = entries[i];
|
|
293
|
+
if (e.kind !== 'task_dispatched') continue;
|
|
294
|
+
if (opts.sessionId && e.sessionId === opts.sessionId) { lastDispatch = e; break; }
|
|
295
|
+
if (opts.nodeId && e.nodeId === opts.nodeId) { lastDispatch = e; break; }
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
const lastTaskMessage = typeof lastDispatch?.payload?.message === 'string'
|
|
299
|
+
? lastDispatch.payload.message
|
|
300
|
+
: null;
|
|
301
|
+
|
|
302
|
+
// Count consecutive recent failures for this node (within 30 min window)
|
|
303
|
+
const now = Date.now();
|
|
304
|
+
const recentWindow = now - RECENT_FAILURE_WINDOW_MS;
|
|
305
|
+
let consecutiveNodeFailures = 0;
|
|
306
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
307
|
+
const e = entries[i];
|
|
308
|
+
if (new Date(e.timestamp).getTime() < recentWindow) break;
|
|
309
|
+
if (opts.nodeId && e.nodeId !== opts.nodeId) continue;
|
|
310
|
+
if (e.kind === 'task_failed') {
|
|
311
|
+
consecutiveNodeFailures++;
|
|
312
|
+
} else if (e.kind === 'task_completed' || e.kind === 'task_dispatched') {
|
|
313
|
+
// A completion or new dispatch breaks the consecutive failure chain
|
|
314
|
+
break;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
// Count how many times the same task was attempted (match by message prefix)
|
|
319
|
+
let taskAttemptCount = 0;
|
|
320
|
+
if (lastTaskMessage) {
|
|
321
|
+
const prefix = lastTaskMessage.slice(0, 200);
|
|
322
|
+
for (const e of entries) {
|
|
323
|
+
if (e.kind === 'task_dispatched' && typeof e.payload?.message === 'string') {
|
|
324
|
+
if (e.payload.message.startsWith(prefix)) {
|
|
325
|
+
taskAttemptCount++;
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
const retryRecommended = consecutiveNodeFailures <= maxRetries;
|
|
332
|
+
|
|
333
|
+
// Build advice string
|
|
334
|
+
let advice: string;
|
|
335
|
+
if (consecutiveNodeFailures === 0) {
|
|
336
|
+
advice = 'No recent failures detected. This may be a normal stop.';
|
|
337
|
+
} else if (retryRecommended) {
|
|
338
|
+
const remaining = maxRetries - consecutiveNodeFailures + 1;
|
|
339
|
+
advice = `Retry recommended (${consecutiveNodeFailures}/${maxRetries + 1} attempts used, ${remaining} remaining). `
|
|
340
|
+
+ (lastTaskMessage
|
|
341
|
+
? `Re-launch the session and resend the original task.`
|
|
342
|
+
: `Re-launch the session. Original task message not found in ledger.`);
|
|
343
|
+
} else {
|
|
344
|
+
advice = `Max retries exceeded (${consecutiveNodeFailures} consecutive failures). `
|
|
345
|
+
+ `Consider: (1) reassigning to a different node, (2) simplifying the task, or (3) escalating to the user.`;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
return {
|
|
349
|
+
lastTaskMessage,
|
|
350
|
+
failedNodeId: opts.nodeId || null,
|
|
351
|
+
failedSessionId: opts.sessionId || null,
|
|
352
|
+
failedProviderType: null, // filled by caller if available
|
|
353
|
+
consecutiveNodeFailures,
|
|
354
|
+
taskAttemptCount,
|
|
355
|
+
retryRecommended,
|
|
356
|
+
advice,
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// ─── File Rotation ──────────────────────────────
|
|
361
|
+
|
|
362
|
+
function rotateLedgerFile(meshId: string, currentPath: string): void {
|
|
363
|
+
// Find next rotation index
|
|
364
|
+
let index = 1;
|
|
365
|
+
while (existsSync(getRotatedPath(meshId, index))) {
|
|
366
|
+
index++;
|
|
367
|
+
if (index > 10) break; // Max 10 rotations
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
// If all slots full, overwrite the oldest
|
|
371
|
+
if (index > 10) index = 10;
|
|
372
|
+
|
|
373
|
+
try {
|
|
374
|
+
renameSync(currentPath, getRotatedPath(meshId, index));
|
|
375
|
+
} catch {
|
|
376
|
+
// Rotation failed — the next append will just grow the file
|
|
377
|
+
}
|
|
378
|
+
}
|
package/src/mesh/mesh-sync.ts
CHANGED
|
@@ -26,6 +26,8 @@ export interface MeshSyncTransport {
|
|
|
26
26
|
}): Promise<{ mesh: RemoteMeshRecord }>;
|
|
27
27
|
/** DELETE /api/v1/repo-meshes/:id */
|
|
28
28
|
deleteRemoteMesh(meshId: string): Promise<void>;
|
|
29
|
+
/** POST /api/v1/repo-meshes/:id/ledger/sync */
|
|
30
|
+
syncMeshLedger?(meshId: string, data: { newEntries: any[] }): Promise<{ missingEntries: any[] }>;
|
|
29
31
|
}
|
|
30
32
|
|
|
31
33
|
export interface RemoteMeshRecord {
|
|
@@ -105,5 +107,35 @@ export async function syncMeshes(transport: MeshSyncTransport): Promise<MeshSync
|
|
|
105
107
|
}
|
|
106
108
|
}
|
|
107
109
|
|
|
110
|
+
// Sync ledgers for all local meshes if the transport supports it
|
|
111
|
+
if (transport.syncMeshLedger) {
|
|
112
|
+
for (const local of localMeshes) {
|
|
113
|
+
try {
|
|
114
|
+
await syncMeshLedger(local.id, transport);
|
|
115
|
+
} catch (e: any) {
|
|
116
|
+
result.errors.push(`Ledger sync failed for "${local.name}": ${e.message}`);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
108
121
|
return result;
|
|
109
122
|
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Sync the task ledger for a specific mesh.
|
|
126
|
+
*/
|
|
127
|
+
export async function syncMeshLedger(meshId: string, transport: MeshSyncTransport): Promise<void> {
|
|
128
|
+
if (!transport.syncMeshLedger) return;
|
|
129
|
+
const { readLedgerEntries, appendRemoteLedgerEntries } = await import('./mesh-ledger.js');
|
|
130
|
+
|
|
131
|
+
// Read all local entries (no tail)
|
|
132
|
+
const localEntries = readLedgerEntries(meshId);
|
|
133
|
+
|
|
134
|
+
// Send to cloud and get missing entries back
|
|
135
|
+
const res = await transport.syncMeshLedger(meshId, { newEntries: localEntries });
|
|
136
|
+
|
|
137
|
+
// Append any missing entries from the cloud
|
|
138
|
+
if (res.missingEntries && res.missingEntries.length > 0) {
|
|
139
|
+
appendRemoteLedgerEntries(meshId, res.missingEntries);
|
|
140
|
+
}
|
|
141
|
+
}
|