clementine-agent 1.17.0 → 1.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/assistant.d.ts +1 -0
- package/dist/agent/assistant.js +11 -1
- package/dist/agent/orchestrator.d.ts +8 -1
- package/dist/agent/orchestrator.js +49 -2
- package/dist/agent/self-improve-loop.d.ts +2 -0
- package/dist/agent/self-improve-loop.js +15 -0
- package/dist/cli/dashboard.js +95 -1
- package/dist/gateway/cron-scheduler.d.ts +2 -0
- package/dist/gateway/cron-scheduler.js +17 -0
- package/dist/gateway/router.js +7 -1
- package/dist/memory/store.d.ts +22 -0
- package/dist/memory/store.js +81 -0
- package/dist/tools/shared.d.ts +88 -0
- package/package.json +1 -1
|
@@ -267,6 +267,7 @@ export declare class PersonalAssistant {
|
|
|
267
267
|
schema: Record<string, unknown>;
|
|
268
268
|
};
|
|
269
269
|
delegateProfile?: AgentProfile;
|
|
270
|
+
abortSignal?: AbortSignal;
|
|
270
271
|
}): Promise<string>;
|
|
271
272
|
runCronJob(jobName: string, jobPrompt: string, tier?: number, maxTurns?: number, model?: string, workDir?: string, timeoutMs?: number, successCriteria?: string[], agentSlug?: string, opts?: {
|
|
272
273
|
disableAllTools?: boolean;
|
package/dist/agent/assistant.js
CHANGED
|
@@ -4069,10 +4069,19 @@ You have a cost budget per message — not a hard turn limit. Work until the tas
|
|
|
4069
4069
|
}
|
|
4070
4070
|
// ── Plan Step Execution ───────────────────────────────────────────
|
|
4071
4071
|
async runPlanStep(stepId, prompt, opts = {}) {
|
|
4072
|
-
const { tier = 2, maxTurns = 15, model, disableTools = false, outputFormat, delegateProfile } = opts;
|
|
4072
|
+
const { tier = 2, maxTurns = 15, model, disableTools = false, outputFormat, delegateProfile, abortSignal } = opts;
|
|
4073
4073
|
// Don't mutate the global — pass source through the closure instead
|
|
4074
4074
|
// Per-step stall guard so concurrent steps don't cross-contaminate
|
|
4075
4075
|
const stepGuard = new StallGuard();
|
|
4076
|
+
// Per-step AbortController, mirroring the parent signal so the orchestrator
|
|
4077
|
+
// (or gateway, via the session AC) can stop in-flight SDK streams.
|
|
4078
|
+
const stepAc = new AbortController();
|
|
4079
|
+
if (abortSignal) {
|
|
4080
|
+
if (abortSignal.aborted)
|
|
4081
|
+
stepAc.abort(abortSignal.reason);
|
|
4082
|
+
else
|
|
4083
|
+
abortSignal.addEventListener('abort', () => stepAc.abort(abortSignal.reason), { once: true });
|
|
4084
|
+
}
|
|
4076
4085
|
const sdkOptions = await this.buildOptions({
|
|
4077
4086
|
isHeartbeat: false,
|
|
4078
4087
|
cronTier: tier,
|
|
@@ -4085,6 +4094,7 @@ You have a cost budget per message — not a hard turn limit. Work until the tas
|
|
|
4085
4094
|
outputFormat,
|
|
4086
4095
|
stallGuard: stepGuard,
|
|
4087
4096
|
profile: delegateProfile ?? null,
|
|
4097
|
+
abortController: stepAc,
|
|
4088
4098
|
});
|
|
4089
4099
|
const trace = [];
|
|
4090
4100
|
const stream = query({ prompt, options: sdkOptions });
|
|
@@ -39,15 +39,22 @@ export declare class PlanOrchestrator {
|
|
|
39
39
|
private startTime;
|
|
40
40
|
private stateId;
|
|
41
41
|
private agentProfiles;
|
|
42
|
+
private abortSignal?;
|
|
42
43
|
constructor(assistant: PersonalAssistant);
|
|
44
|
+
/** Fire-and-forget autonomy telemetry — must never throw. */
|
|
45
|
+
private logAutonomy;
|
|
43
46
|
private saveState;
|
|
44
47
|
private cleanupState;
|
|
45
48
|
/** Load a previously interrupted plan state (for future resumability). */
|
|
46
49
|
static loadState(stateId: string): PlanState | null;
|
|
47
50
|
/**
|
|
48
51
|
* Main entry: plan → approve → execute → synthesize → return final response.
|
|
52
|
+
*
|
|
53
|
+
* `abortSignal` (typically the gateway session controller) lets the user stop
|
|
54
|
+
* a running plan: the orchestrator checks it between waves and forwards it to
|
|
55
|
+
* every `runPlanStep` call so in-flight SDK streams abort immediately.
|
|
49
56
|
*/
|
|
50
|
-
run(taskDescription: string, onProgress?: (updates: PlanProgressUpdate[]) => Promise<void>, onApproval?: (planSummary: string, steps: PlanStep[]) => Promise<boolean | string>, availableAgents?: AgentProfile[]): Promise<string>;
|
|
57
|
+
run(taskDescription: string, onProgress?: (updates: PlanProgressUpdate[]) => Promise<void>, onApproval?: (planSummary: string, steps: PlanStep[]) => Promise<boolean | string>, availableAgents?: AgentProfile[], abortSignal?: AbortSignal): Promise<string>;
|
|
51
58
|
/**
|
|
52
59
|
* Goal-backward verification pass using Haiku after plan synthesis.
|
|
53
60
|
* Verifies outcomes rather than just rating quality:
|
|
@@ -123,10 +123,22 @@ export class PlanOrchestrator {
|
|
|
123
123
|
startTime = 0;
|
|
124
124
|
stateId;
|
|
125
125
|
agentProfiles = new Map();
|
|
126
|
+
abortSignal;
|
|
126
127
|
constructor(assistant) {
|
|
127
128
|
this.assistant = assistant;
|
|
128
129
|
this.stateId = `plan-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`;
|
|
129
130
|
}
|
|
131
|
+
/** Fire-and-forget autonomy telemetry — must never throw. */
|
|
132
|
+
async logAutonomy(event, details) {
|
|
133
|
+
try {
|
|
134
|
+
const { getStore } = await import('../tools/shared.js');
|
|
135
|
+
const store = await getStore();
|
|
136
|
+
store.logAutonomyEvent({ component: 'orchestrator', event, details });
|
|
137
|
+
}
|
|
138
|
+
catch {
|
|
139
|
+
// Best-effort — telemetry must never break plan execution.
|
|
140
|
+
}
|
|
141
|
+
}
|
|
130
142
|
// ── State persistence ────────────────────────────────────────────────
|
|
131
143
|
saveState(state) {
|
|
132
144
|
try {
|
|
@@ -161,13 +173,18 @@ export class PlanOrchestrator {
|
|
|
161
173
|
}
|
|
162
174
|
/**
|
|
163
175
|
* Main entry: plan → approve → execute → synthesize → return final response.
|
|
176
|
+
*
|
|
177
|
+
* `abortSignal` (typically the gateway session controller) lets the user stop
|
|
178
|
+
* a running plan: the orchestrator checks it between waves and forwards it to
|
|
179
|
+
* every `runPlanStep` call so in-flight SDK streams abort immediately.
|
|
164
180
|
*/
|
|
165
|
-
async run(taskDescription, onProgress, onApproval, availableAgents) {
|
|
181
|
+
async run(taskDescription, onProgress, onApproval, availableAgents, abortSignal) {
|
|
166
182
|
// Reset instance state for reuse safety
|
|
167
183
|
this.stepStatuses.clear();
|
|
168
184
|
this.stepStartTimes.clear();
|
|
169
185
|
this.agentProfiles.clear();
|
|
170
186
|
this.startTime = Date.now();
|
|
187
|
+
this.abortSignal = abortSignal;
|
|
171
188
|
// Index available agents for delegation lookups
|
|
172
189
|
if (availableAgents) {
|
|
173
190
|
for (const agent of availableAgents) {
|
|
@@ -206,6 +223,7 @@ export class PlanOrchestrator {
|
|
|
206
223
|
return this.runSingleStep(taskDescription);
|
|
207
224
|
}
|
|
208
225
|
logger.info({ goal: effectiveTask, stepCount: plan.steps.length, steps: plan.steps.map(s => s.id), revision: revisionCount }, 'Plan generated');
|
|
226
|
+
this.logAutonomy('plan_created', { stepCount: plan.steps.length, revision: revisionCount });
|
|
209
227
|
// 2. Initialize statuses
|
|
210
228
|
this.stepStatuses.clear();
|
|
211
229
|
for (const step of plan.steps) {
|
|
@@ -269,6 +287,15 @@ export class PlanOrchestrator {
|
|
|
269
287
|
};
|
|
270
288
|
this.saveState(state);
|
|
271
289
|
for (const wave of waves) {
|
|
290
|
+
// User-initiated cancellation: bail before starting the next wave so we
|
|
291
|
+
// don't kick off SDK streams the user already asked to stop.
|
|
292
|
+
if (this.abortSignal?.aborted) {
|
|
293
|
+
logger.info({ goal: taskDescription, wavesCompleted: state.wavesCompleted }, 'Plan cancelled by user');
|
|
294
|
+
state.status = 'failed';
|
|
295
|
+
this.saveState(state);
|
|
296
|
+
this.logAutonomy('plan_aborted', { reason: 'user_cancelled', wavesCompleted: state.wavesCompleted });
|
|
297
|
+
return 'Plan cancelled.';
|
|
298
|
+
}
|
|
272
299
|
// Mark running
|
|
273
300
|
for (const step of wave) {
|
|
274
301
|
this.stepStatuses.set(step.id, {
|
|
@@ -291,6 +318,7 @@ export class PlanOrchestrator {
|
|
|
291
318
|
maxTurns: step.maxTurns ?? 15,
|
|
292
319
|
model: step.model,
|
|
293
320
|
delegateProfile,
|
|
321
|
+
abortSignal: this.abortSignal,
|
|
294
322
|
});
|
|
295
323
|
return { stepId: step.id, result };
|
|
296
324
|
}), MAX_CONCURRENT_STEPS);
|
|
@@ -312,6 +340,7 @@ export class PlanOrchestrator {
|
|
|
312
340
|
durationMs: elapsed,
|
|
313
341
|
resultPreview: resultText.slice(0, 100),
|
|
314
342
|
});
|
|
343
|
+
this.logAutonomy('step_completed', { stepId: step.id, description: step.description, durationMs: elapsed, model: step.model });
|
|
315
344
|
}
|
|
316
345
|
else {
|
|
317
346
|
const errMsg = `[FAILED: ${outcome.reason}]`;
|
|
@@ -324,9 +353,20 @@ export class PlanOrchestrator {
|
|
|
324
353
|
resultPreview: errMsg.slice(0, 100),
|
|
325
354
|
});
|
|
326
355
|
logger.error({ stepId: step.id, err: outcome.reason }, 'Plan step failed');
|
|
356
|
+
this.logAutonomy('step_failed', { stepId: step.id, description: step.description, error: String(outcome.reason).slice(0, 500) });
|
|
327
357
|
}
|
|
328
358
|
}
|
|
329
359
|
await safeProgress(this.getAllUpdates());
|
|
360
|
+
// If the user aborted mid-wave, the SDK calls above already threw and
|
|
361
|
+
// `outcome.reason` will reflect the abort. Surface that as a clean
|
|
362
|
+
// cancellation instead of falling through to spot-check / repair / synthesis.
|
|
363
|
+
if (this.abortSignal?.aborted) {
|
|
364
|
+
logger.info({ goal: taskDescription, wavesCompleted: state.wavesCompleted }, 'Plan cancelled by user mid-wave');
|
|
365
|
+
state.status = 'failed';
|
|
366
|
+
this.saveState(state);
|
|
367
|
+
this.logAutonomy('plan_aborted', { reason: 'user_cancelled', wavesCompleted: state.wavesCompleted });
|
|
368
|
+
return 'Plan cancelled.';
|
|
369
|
+
}
|
|
330
370
|
// Inter-wave spot-check with severity levels: critical issues trigger repair
|
|
331
371
|
const spotCheckIssues = this.spotCheckWaveResults(wave, results);
|
|
332
372
|
if (spotCheckIssues.length > 0) {
|
|
@@ -348,6 +388,7 @@ export class PlanOrchestrator {
|
|
|
348
388
|
tier: step.tier ?? 2,
|
|
349
389
|
maxTurns: step.maxTurns ?? 15,
|
|
350
390
|
model: step.model,
|
|
391
|
+
abortSignal: this.abortSignal,
|
|
351
392
|
});
|
|
352
393
|
results.set(step.id, retryResult || '[No output on retry]');
|
|
353
394
|
this.stepStatuses.set(step.id, {
|
|
@@ -367,6 +408,7 @@ export class PlanOrchestrator {
|
|
|
367
408
|
logger.warn({ stepId: issue.stepId }, 'Repair decision: abort plan');
|
|
368
409
|
state.status = 'failed';
|
|
369
410
|
this.saveState(state);
|
|
411
|
+
this.logAutonomy('plan_aborted', { stepId: issue.stepId, reason: issue.issue });
|
|
370
412
|
return `Plan aborted — step "${this.stepStatuses.get(issue.stepId)?.description ?? issue.stepId}" failed critically and could not be repaired.`;
|
|
371
413
|
}
|
|
372
414
|
// 'skip' falls through — annotate and continue
|
|
@@ -434,10 +476,12 @@ export class PlanOrchestrator {
|
|
|
434
476
|
tier: 2,
|
|
435
477
|
maxTurns: 5,
|
|
436
478
|
disableTools: true,
|
|
479
|
+
abortSignal: this.abortSignal,
|
|
437
480
|
});
|
|
438
481
|
}
|
|
439
482
|
catch (err) {
|
|
440
483
|
logger.error({ err }, 'Synthesis step failed');
|
|
484
|
+
this.logAutonomy('synthesis_failed', { error: String(err).slice(0, 500) });
|
|
441
485
|
// Fallback: concatenate results
|
|
442
486
|
finalResult = Array.from(results.entries())
|
|
443
487
|
.map(([id, r]) => `**${this.stepStatuses.get(id)?.description ?? id}:**\n${r}`)
|
|
@@ -453,6 +497,7 @@ export class PlanOrchestrator {
|
|
|
453
497
|
await safeProgress(this.getAllUpdates());
|
|
454
498
|
const totalMs = Date.now() - this.startTime;
|
|
455
499
|
logger.info({ totalMs, steps: plan.steps.length }, 'Plan execution complete');
|
|
500
|
+
this.logAutonomy('plan_completed', { totalMs, steps: plan.steps.length, errors: state.errors.length });
|
|
456
501
|
// Mark state as complete and clean up
|
|
457
502
|
state.status = 'complete';
|
|
458
503
|
this.saveState(state);
|
|
@@ -602,6 +647,7 @@ export class PlanOrchestrator {
|
|
|
602
647
|
maxTurns: 1,
|
|
603
648
|
model: 'haiku',
|
|
604
649
|
disableTools: true,
|
|
650
|
+
abortSignal: this.abortSignal,
|
|
605
651
|
});
|
|
606
652
|
const cleaned = decision.trim().toLowerCase();
|
|
607
653
|
if (cleaned.includes('retry'))
|
|
@@ -628,7 +674,7 @@ export class PlanOrchestrator {
|
|
|
628
674
|
`If a step matches an agent's specialty, add "delegateTo": "agent-slug" to that step. ` +
|
|
629
675
|
`The delegated agent will run the step with their own personality, tools, and expertise.\n`;
|
|
630
676
|
}
|
|
631
|
-
const plannerResult = await this.assistant.runPlanStep('planner', PLANNER_PROMPT + agentContext + task + PLANNER_PROMPT_SUFFIX, { tier: 2, maxTurns: 1, model: 'sonnet', disableTools: true });
|
|
677
|
+
const plannerResult = await this.assistant.runPlanStep('planner', PLANNER_PROMPT + agentContext + task + PLANNER_PROMPT_SUFFIX, { tier: 2, maxTurns: 1, model: 'sonnet', disableTools: true, abortSignal: this.abortSignal });
|
|
632
678
|
// Parse JSON from the planner response
|
|
633
679
|
const parsed = this.parseJsonFromResponse(plannerResult);
|
|
634
680
|
if (!parsed?.steps || !Array.isArray(parsed.steps) || parsed.steps.length === 0) {
|
|
@@ -781,6 +827,7 @@ Work through this task narrating your reasoning:
|
|
|
781
827
|
return this.assistant.runPlanStep('fallback', task, {
|
|
782
828
|
tier: 2,
|
|
783
829
|
maxTurns: 25,
|
|
830
|
+
abortSignal: this.abortSignal,
|
|
784
831
|
});
|
|
785
832
|
}
|
|
786
833
|
getAllUpdates() {
|
|
@@ -112,6 +112,8 @@ export declare class SelfImproveLoop {
|
|
|
112
112
|
/** Coalesce a burst of fs.watch events (multiple triggers landing in
|
|
113
113
|
* quick succession) into a single tick. */
|
|
114
114
|
private scheduleDebouncedTick;
|
|
115
|
+
/** Fire-and-forget autonomy telemetry — must never throw. */
|
|
116
|
+
private logAutonomy;
|
|
115
117
|
/**
|
|
116
118
|
* Process all pending triggers. Public so tests + manual invocations
|
|
117
119
|
* (e.g., a `clementine self-improve tick` CLI command) can call it.
|
|
@@ -313,6 +313,17 @@ export class SelfImproveLoop {
|
|
|
313
313
|
this.tick().catch((err) => logger.error({ err }, 'Self-improve event-driven tick failed'));
|
|
314
314
|
}, WATCH_DEBOUNCE_MS);
|
|
315
315
|
}
|
|
316
|
+
/** Fire-and-forget autonomy telemetry — must never throw. */
|
|
317
|
+
async logAutonomy(event, trigger, details) {
|
|
318
|
+
try {
|
|
319
|
+
const { getStore } = await import('../tools/shared.js');
|
|
320
|
+
const store = await getStore();
|
|
321
|
+
store.logAutonomyEvent({ component: 'self_improve', event, agentSlug: trigger.agentSlug ?? null, details });
|
|
322
|
+
}
|
|
323
|
+
catch {
|
|
324
|
+
// Best-effort — telemetry must never break the self-improve loop.
|
|
325
|
+
}
|
|
326
|
+
}
|
|
316
327
|
/**
|
|
317
328
|
* Process all pending triggers. Public so tests + manual invocations
|
|
318
329
|
* (e.g., a `clementine self-improve tick` CLI command) can call it.
|
|
@@ -346,11 +357,13 @@ export class SelfImproveLoop {
|
|
|
346
357
|
catch { /* ignore */ }
|
|
347
358
|
continue;
|
|
348
359
|
}
|
|
360
|
+
this.logAutonomy('trigger_detected', trigger, { consecutiveErrors: trigger.consecutiveErrors });
|
|
349
361
|
try {
|
|
350
362
|
await this.processOne(trigger, counts);
|
|
351
363
|
}
|
|
352
364
|
catch (err) {
|
|
353
365
|
logger.warn({ err, jobName: trigger.jobName }, 'Failed to process trigger — leaving in place for next tick');
|
|
366
|
+
this.logAutonomy('process_failed', trigger, { error: String(err).slice(0, 500) });
|
|
354
367
|
continue;
|
|
355
368
|
}
|
|
356
369
|
// Successfully handled — remove the trigger
|
|
@@ -419,6 +432,7 @@ export class SelfImproveLoop {
|
|
|
419
432
|
if (recipe.category === 'noop') {
|
|
420
433
|
counts.noop++;
|
|
421
434
|
logger.info({ jobName: trigger.jobName, reason: recipe.description }, 'Self-improve: no-op');
|
|
435
|
+
this.logAutonomy('fix_noop', trigger, { reason: recipe.description });
|
|
422
436
|
return;
|
|
423
437
|
}
|
|
424
438
|
// risky | unknown → write proposal + DM agent
|
|
@@ -435,6 +449,7 @@ export class SelfImproveLoop {
|
|
|
435
449
|
};
|
|
436
450
|
const file = writePendingChange(record, this.pendingDir);
|
|
437
451
|
counts.pending++;
|
|
452
|
+
this.logAutonomy('proposal_written', trigger, { category: recipe.category, proposalId: id });
|
|
438
453
|
await this.notifyAgent(agentSlug, [
|
|
439
454
|
`⚠️ **${trigger.jobName}** has failed ${trigger.consecutiveErrors} times in a row.`,
|
|
440
455
|
'',
|
package/dist/cli/dashboard.js
CHANGED
|
@@ -1297,7 +1297,84 @@ export async function cmdDashboard(opts) {
|
|
|
1297
1297
|
index: false,
|
|
1298
1298
|
}));
|
|
1299
1299
|
// Health check — always responds, no auth, no middleware dependency
|
|
1300
|
-
app.get('/health', (_req, res) => {
|
|
1300
|
+
app.get('/health', async (_req, res) => {
|
|
1301
|
+
const checks = {};
|
|
1302
|
+
let overall = 'healthy';
|
|
1303
|
+
// 1. Daemon process liveness
|
|
1304
|
+
const pid = readPid();
|
|
1305
|
+
const daemonAlive = pid ? isProcessAlive(pid) : false;
|
|
1306
|
+
checks.daemon = daemonAlive ? 'ok' : 'unavailable';
|
|
1307
|
+
if (!daemonAlive)
|
|
1308
|
+
overall = 'unhealthy';
|
|
1309
|
+
// 2. SQLite + memory health
|
|
1310
|
+
try {
|
|
1311
|
+
const { getStore } = await import('../tools/shared.js');
|
|
1312
|
+
const store = await getStore();
|
|
1313
|
+
const health = store.getMemoryHealth();
|
|
1314
|
+
checks.sqlite = 'ok';
|
|
1315
|
+
checks.fts5 = health.lastIntegrityReport?.ftsOk ? 'ok' : 'degraded';
|
|
1316
|
+
checks.writeQueue = (health.writeQueue?.size ?? 0) > 100 ? 'degraded' : 'ok';
|
|
1317
|
+
if (checks.fts5 === 'degraded' || checks.writeQueue === 'degraded') {
|
|
1318
|
+
overall = overall === 'unhealthy' ? 'unhealthy' : 'degraded';
|
|
1319
|
+
}
|
|
1320
|
+
}
|
|
1321
|
+
catch {
|
|
1322
|
+
checks.sqlite = 'unavailable';
|
|
1323
|
+
overall = 'unhealthy';
|
|
1324
|
+
}
|
|
1325
|
+
// 3. Graph store availability
|
|
1326
|
+
try {
|
|
1327
|
+
const { getSharedGraphStore } = await import('../memory/graph-store.js');
|
|
1328
|
+
const graphDbDir = path.join(BASE_DIR, '.graph.db');
|
|
1329
|
+
const gs = await getSharedGraphStore(graphDbDir);
|
|
1330
|
+
checks.graphStore = gs?.isAvailable() ? 'ok' : 'unavailable';
|
|
1331
|
+
if (checks.graphStore === 'unavailable' && overall !== 'unhealthy') {
|
|
1332
|
+
overall = 'degraded';
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
catch {
|
|
1336
|
+
checks.graphStore = 'unavailable';
|
|
1337
|
+
if (overall !== 'unhealthy')
|
|
1338
|
+
overall = 'degraded';
|
|
1339
|
+
}
|
|
1340
|
+
// 4. Recent cron failures (last 24h)
|
|
1341
|
+
try {
|
|
1342
|
+
const runsDir = path.join(BASE_DIR, 'cron', 'runs');
|
|
1343
|
+
let recentFailures = 0;
|
|
1344
|
+
let recentRuns = 0;
|
|
1345
|
+
const dayAgo = Date.now() - 24 * 60 * 60 * 1000;
|
|
1346
|
+
if (existsSync(runsDir)) {
|
|
1347
|
+
for (const file of readdirSync(runsDir).filter(f => f.endsWith('.jsonl'))) {
|
|
1348
|
+
const lines = readFileSync(path.join(runsDir, file), 'utf-8').trim().split('\n').filter(Boolean);
|
|
1349
|
+
for (const line of lines.slice(-50)) {
|
|
1350
|
+
try {
|
|
1351
|
+
const entry = JSON.parse(line);
|
|
1352
|
+
const ts = entry.startedAt ? new Date(entry.startedAt).getTime() : 0;
|
|
1353
|
+
if (ts > dayAgo) {
|
|
1354
|
+
recentRuns++;
|
|
1355
|
+
if (entry.status === 'failed' || entry.status === 'error')
|
|
1356
|
+
recentFailures++;
|
|
1357
|
+
}
|
|
1358
|
+
}
|
|
1359
|
+
catch { /* ignore parse errors */ }
|
|
1360
|
+
}
|
|
1361
|
+
}
|
|
1362
|
+
}
|
|
1363
|
+
checks.cron = recentFailures > 3 ? 'degraded' : recentFailures > 0 ? 'degraded' : 'ok';
|
|
1364
|
+
if (checks.cron === 'degraded' && overall !== 'unhealthy')
|
|
1365
|
+
overall = 'degraded';
|
|
1366
|
+
}
|
|
1367
|
+
catch {
|
|
1368
|
+
checks.cron = 'unavailable';
|
|
1369
|
+
}
|
|
1370
|
+
const statusCode = overall === 'healthy' ? 200 : overall === 'degraded' ? 503 : 503;
|
|
1371
|
+
res.status(statusCode).json({
|
|
1372
|
+
status: overall,
|
|
1373
|
+
checks,
|
|
1374
|
+
daemon: { pid: pid ?? null, alive: daemonAlive },
|
|
1375
|
+
timestamp: new Date().toISOString(),
|
|
1376
|
+
});
|
|
1377
|
+
});
|
|
1301
1378
|
// ── Webhook ingestion (raw-body, HMAC-authed) ─────────────────────
|
|
1302
1379
|
// MUST be registered BEFORE the generic json parser below — otherwise
|
|
1303
1380
|
// that parser consumes the body and HMAC verification fails since we
|
|
@@ -2203,6 +2280,23 @@ export async function cmdDashboard(opts) {
|
|
|
2203
2280
|
res.status(500).json({ error: String(err).slice(0, 200) });
|
|
2204
2281
|
}
|
|
2205
2282
|
});
|
|
2283
|
+
app.get('/api/autonomy', async (req, res) => {
|
|
2284
|
+
try {
|
|
2285
|
+
const { getStore } = await import('../tools/shared.js');
|
|
2286
|
+
const store = await getStore();
|
|
2287
|
+
const logs = store.queryAutonomyLog({
|
|
2288
|
+
component: typeof req.query.component === 'string' ? req.query.component : undefined,
|
|
2289
|
+
event: typeof req.query.event === 'string' ? req.query.event : undefined,
|
|
2290
|
+
agentSlug: typeof req.query.agentSlug === 'string' ? req.query.agentSlug : undefined,
|
|
2291
|
+
limit: typeof req.query.limit === 'string' ? parseInt(req.query.limit, 10) : 100,
|
|
2292
|
+
since: typeof req.query.since === 'string' ? req.query.since : undefined,
|
|
2293
|
+
});
|
|
2294
|
+
res.json({ logs });
|
|
2295
|
+
}
|
|
2296
|
+
catch (err) {
|
|
2297
|
+
res.status(500).json({ error: String(err).slice(0, 200) });
|
|
2298
|
+
}
|
|
2299
|
+
});
|
|
2206
2300
|
app.get('/api/heartbeat/agent/:slug', (req, res) => {
|
|
2207
2301
|
const slug = req.params.slug;
|
|
2208
2302
|
const state = getHeartbeat();
|
|
@@ -93,6 +93,8 @@ export declare class CronScheduler {
|
|
|
93
93
|
private statusChangeListeners;
|
|
94
94
|
private static readonly RUNNING_JOBS_FILE;
|
|
95
95
|
constructor(gateway: Gateway, dispatcher: NotificationDispatcher);
|
|
96
|
+
/** Fire-and-forget autonomy telemetry — must never throw. */
|
|
97
|
+
private logAutonomy;
|
|
96
98
|
/**
|
|
97
99
|
* Atomically persist the current runningJobs set to disk. Uses write-then-
|
|
98
100
|
* rename so a crash mid-write cannot corrupt the file.
|
|
@@ -401,6 +401,17 @@ export class CronScheduler {
|
|
|
401
401
|
// query jobs on connect which happens before start().
|
|
402
402
|
this.loadJobDefinitions();
|
|
403
403
|
}
|
|
404
|
+
/** Fire-and-forget autonomy telemetry — must never throw. */
|
|
405
|
+
async logAutonomy(event, job, details) {
|
|
406
|
+
try {
|
|
407
|
+
const { getStore } = await import('../tools/shared.js');
|
|
408
|
+
const store = await getStore();
|
|
409
|
+
store.logAutonomyEvent({ component: 'cron', event, agentSlug: job.agentSlug ?? null, details });
|
|
410
|
+
}
|
|
411
|
+
catch {
|
|
412
|
+
// Best-effort — telemetry must never break cron execution.
|
|
413
|
+
}
|
|
414
|
+
}
|
|
404
415
|
/**
|
|
405
416
|
* Atomically persist the current runningJobs set to disk. Uses write-then-
|
|
406
417
|
* rename so a crash mid-write cannot corrupt the file.
|
|
@@ -974,6 +985,7 @@ export class CronScheduler {
|
|
|
974
985
|
});
|
|
975
986
|
this.persistRunningJobs(this.runMetadata);
|
|
976
987
|
this.emitStatusChange();
|
|
988
|
+
this.logAutonomy('started', job, { model: job.model, mode: job.mode, tier: job.tier });
|
|
977
989
|
try {
|
|
978
990
|
logger.info(`Running cron job: ${job.name}${job.agentSlug ? ` (agent: ${job.agentSlug})` : ''}`);
|
|
979
991
|
// Set agent profile for scoped cron jobs
|
|
@@ -1084,6 +1096,7 @@ export class CronScheduler {
|
|
|
1084
1096
|
}
|
|
1085
1097
|
}
|
|
1086
1098
|
this._logRun(entry);
|
|
1099
|
+
this.logAutonomy('completed', job, { durationMs: entry.durationMs, deliveryFailed: entry.deliveryFailed, advisorApplied: !!advisorApplied });
|
|
1087
1100
|
// Fire-and-forget: extract procedural skill from successful long-running cron jobs
|
|
1088
1101
|
if (entry.status === 'ok' && entry.durationMs > 30_000 && response && response.length > 500) {
|
|
1089
1102
|
this.gateway.extractCronSkill(job.name, job.prompt, response, entry.durationMs, job.agentSlug)
|
|
@@ -1135,6 +1148,7 @@ export class CronScheduler {
|
|
|
1135
1148
|
}
|
|
1136
1149
|
else {
|
|
1137
1150
|
logger.error({ err, job: job.name }, `Cron job '${job.name}' failed after ${attempt} attempt(s)`);
|
|
1151
|
+
this.logAutonomy('failed', job, { errorType, attempts: attempt, error: String(err).slice(0, 500) });
|
|
1138
1152
|
await this.dispatcher.send(CronScheduler.formatCronError(job.name, err), { agentSlug: job.agentSlug });
|
|
1139
1153
|
}
|
|
1140
1154
|
}
|
|
@@ -1170,6 +1184,7 @@ export class CronScheduler {
|
|
|
1170
1184
|
const consErrors = this.runLog.consecutiveErrors(job.name);
|
|
1171
1185
|
if (consErrors === 5) {
|
|
1172
1186
|
// Circuit breaker just engaged — notify
|
|
1187
|
+
this.logAutonomy('circuit_breaker', job, { consecutiveErrors: consErrors });
|
|
1173
1188
|
this.logAdvisorEvent('circuit-breaker', job.name, `Circuit breaker engaged after ${consErrors} consecutive errors`);
|
|
1174
1189
|
this.dispatcher.send(`⚡ **Circuit breaker engaged** for \`${job.name}\` — ${consErrors} consecutive errors. Will retry in 1 hour.`, { agentSlug: job.agentSlug }).catch(err => logger.debug({ err }, 'Failed to send circuit breaker notification'));
|
|
1175
1190
|
}
|
|
@@ -1189,6 +1204,7 @@ export class CronScheduler {
|
|
|
1189
1204
|
// defined in per-agent CRON.md files (vault/00-System/agents/{slug}/CRON.md)
|
|
1190
1205
|
// rather than only the central one.
|
|
1191
1206
|
if (consErrors >= 3) {
|
|
1207
|
+
this.logAutonomy('self_improve_triggered', job, { consecutiveErrors: consErrors });
|
|
1192
1208
|
try {
|
|
1193
1209
|
const triggerDir = path.join(BASE_DIR, 'self-improve', 'triggers');
|
|
1194
1210
|
mkdirSync(triggerDir, { recursive: true });
|
|
@@ -1218,6 +1234,7 @@ export class CronScheduler {
|
|
|
1218
1234
|
this.scheduledTasks.delete(job.name);
|
|
1219
1235
|
}
|
|
1220
1236
|
logger.error({ job: job.name, consErrors }, `Auto-disabled cron after ${consErrors} consecutive failures`);
|
|
1237
|
+
this.logAutonomy('auto_disabled', job, { consecutiveErrors: consErrors });
|
|
1221
1238
|
this.logAdvisorEvent('auto-disabled', job.name, `Auto-disabled after ${consErrors} consecutive failures`);
|
|
1222
1239
|
this.dispatcher.send(`🛑 **Cron auto-disabled** — \`${job.name}\` failed ${consErrors} times in a row. Fix the job and re-enable it from the dashboard.`, { agentSlug: job.agentSlug }).catch(err => logger.debug({ err }, 'Failed to send auto-disable notification'));
|
|
1223
1240
|
}
|
package/dist/gateway/router.js
CHANGED
|
@@ -1424,9 +1424,15 @@ export class Gateway {
|
|
|
1424
1424
|
}
|
|
1425
1425
|
// Register provenance for the orchestrator session
|
|
1426
1426
|
this.ensureProvenance(sessionKey);
|
|
1427
|
+
// Register a session AbortController so a follow-up message ("stop", or
|
|
1428
|
+
// any new prompt) can interrupt the plan via acquireSessionLock — which
|
|
1429
|
+
// calls abortController.abort('interrupted-by-new-message') when it
|
|
1430
|
+
// sees an in-flight query on this session.
|
|
1431
|
+
const planAc = new AbortController();
|
|
1432
|
+
this.getSession(sessionKey).abortController = planAc;
|
|
1427
1433
|
const { PlanOrchestrator } = await import('../agent/orchestrator.js');
|
|
1428
1434
|
const orchestrator = new PlanOrchestrator(this.assistant);
|
|
1429
|
-
const result = await orchestrator.run(taskDescription, onProgress, onApproval);
|
|
1435
|
+
const result = await orchestrator.run(taskDescription, onProgress, onApproval, undefined, planAc.signal);
|
|
1430
1436
|
scanner.refreshIntegrity();
|
|
1431
1437
|
this.assistant.injectContext(sessionKey, `[Plan: ${taskDescription}]`, result);
|
|
1432
1438
|
return result;
|
package/dist/memory/store.d.ts
CHANGED
|
@@ -1233,6 +1233,28 @@ export declare class MemoryStore {
|
|
|
1233
1233
|
* Reduces salience so they appear lower in search results (but aren't deleted).
|
|
1234
1234
|
*/
|
|
1235
1235
|
markConsolidated(chunkIds: number[]): void;
|
|
1236
|
+
private _stmtLogAutonomy;
|
|
1237
|
+
private _stmtQueryAutonomy;
|
|
1238
|
+
logAutonomyEvent(row: {
|
|
1239
|
+
component: string;
|
|
1240
|
+
event: string;
|
|
1241
|
+
agentSlug?: string | null;
|
|
1242
|
+
details?: Record<string, unknown>;
|
|
1243
|
+
}): void;
|
|
1244
|
+
queryAutonomyLog(opts?: {
|
|
1245
|
+
component?: string;
|
|
1246
|
+
event?: string;
|
|
1247
|
+
agentSlug?: string;
|
|
1248
|
+
limit?: number;
|
|
1249
|
+
since?: string;
|
|
1250
|
+
}): Array<{
|
|
1251
|
+
id: number;
|
|
1252
|
+
component: string;
|
|
1253
|
+
event: string;
|
|
1254
|
+
agentSlug: string | null;
|
|
1255
|
+
details: Record<string, unknown>;
|
|
1256
|
+
createdAt: string;
|
|
1257
|
+
}>;
|
|
1236
1258
|
/**
|
|
1237
1259
|
* Aggregate memory-health snapshot for the dashboard.
|
|
1238
1260
|
*
|
package/dist/memory/store.js
CHANGED
|
@@ -834,6 +834,23 @@ export class MemoryStore {
|
|
|
834
834
|
value TEXT NOT NULL,
|
|
835
835
|
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
836
836
|
);
|
|
837
|
+
`);
|
|
838
|
+
// Autonomy log — structured event stream for the self-improving,
|
|
839
|
+
// proactive, and planning layers. Makes the autonomous subsystem
|
|
840
|
+
// observable without grepping logs.
|
|
841
|
+
this.conn.exec(`
|
|
842
|
+
CREATE TABLE IF NOT EXISTS autonomy_log (
|
|
843
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
844
|
+
component TEXT NOT NULL,
|
|
845
|
+
event TEXT NOT NULL,
|
|
846
|
+
agent_slug TEXT,
|
|
847
|
+
details_json TEXT DEFAULT '{}',
|
|
848
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
849
|
+
);
|
|
850
|
+
CREATE INDEX IF NOT EXISTS idx_autonomy_component ON autonomy_log(component, created_at DESC);
|
|
851
|
+
CREATE INDEX IF NOT EXISTS idx_autonomy_event ON autonomy_log(event, created_at DESC);
|
|
852
|
+
CREATE INDEX IF NOT EXISTS idx_autonomy_agent ON autonomy_log(agent_slug, created_at DESC);
|
|
853
|
+
CREATE INDEX IF NOT EXISTS idx_autonomy_created ON autonomy_log(created_at DESC);
|
|
837
854
|
`);
|
|
838
855
|
}
|
|
839
856
|
// ── Skill usage telemetry ─────────────────────────────────────────
|
|
@@ -4331,6 +4348,70 @@ export class MemoryStore {
|
|
|
4331
4348
|
WHERE id IN (${placeholders})`)
|
|
4332
4349
|
.run(...chunkIds);
|
|
4333
4350
|
}
|
|
4351
|
+
// ── Autonomy log ───────────────────────────────────────────────────
|
|
4352
|
+
_stmtLogAutonomy = null;
|
|
4353
|
+
_stmtQueryAutonomy = null;
|
|
4354
|
+
logAutonomyEvent(row) {
|
|
4355
|
+
try {
|
|
4356
|
+
if (!this._stmtLogAutonomy) {
|
|
4357
|
+
this._stmtLogAutonomy = this.conn.prepare('INSERT INTO autonomy_log (component, event, agent_slug, details_json) VALUES (?, ?, ?, ?)');
|
|
4358
|
+
}
|
|
4359
|
+
this._stmtLogAutonomy.run(row.component, row.event, row.agentSlug ?? null, JSON.stringify(row.details ?? {}));
|
|
4360
|
+
}
|
|
4361
|
+
catch {
|
|
4362
|
+
// Best-effort — autonomy telemetry must never break the system.
|
|
4363
|
+
}
|
|
4364
|
+
}
|
|
4365
|
+
queryAutonomyLog(opts = {}) {
|
|
4366
|
+
const conditions = [];
|
|
4367
|
+
const params = [];
|
|
4368
|
+
if (opts.component) {
|
|
4369
|
+
conditions.push('component = ?');
|
|
4370
|
+
params.push(opts.component);
|
|
4371
|
+
}
|
|
4372
|
+
if (opts.event) {
|
|
4373
|
+
conditions.push('event = ?');
|
|
4374
|
+
params.push(opts.event);
|
|
4375
|
+
}
|
|
4376
|
+
if (opts.agentSlug) {
|
|
4377
|
+
conditions.push('agent_slug = ?');
|
|
4378
|
+
params.push(opts.agentSlug);
|
|
4379
|
+
}
|
|
4380
|
+
if (opts.since) {
|
|
4381
|
+
conditions.push('created_at >= ?');
|
|
4382
|
+
params.push(opts.since);
|
|
4383
|
+
}
|
|
4384
|
+
const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
|
|
4385
|
+
const limit = opts.limit ?? 100;
|
|
4386
|
+
if (!this._stmtQueryAutonomy || conditions.length) {
|
|
4387
|
+
// Ad-hoc query — conditions vary, use generic runner
|
|
4388
|
+
return this.conn
|
|
4389
|
+
.prepare(`SELECT * FROM autonomy_log ${where} ORDER BY created_at DESC LIMIT ?`)
|
|
4390
|
+
.all(...params, limit)
|
|
4391
|
+
.map((row) => {
|
|
4392
|
+
const r = row;
|
|
4393
|
+
return {
|
|
4394
|
+
id: r.id,
|
|
4395
|
+
component: r.component,
|
|
4396
|
+
event: r.event,
|
|
4397
|
+
agentSlug: r.agent_slug,
|
|
4398
|
+
details: JSON.parse(r.details_json || '{}'),
|
|
4399
|
+
createdAt: r.created_at,
|
|
4400
|
+
};
|
|
4401
|
+
});
|
|
4402
|
+
}
|
|
4403
|
+
return this._stmtQueryAutonomy.all(limit).map((row) => {
|
|
4404
|
+
const r = row;
|
|
4405
|
+
return {
|
|
4406
|
+
id: r.id,
|
|
4407
|
+
component: r.component,
|
|
4408
|
+
event: r.event,
|
|
4409
|
+
agentSlug: r.agent_slug,
|
|
4410
|
+
details: JSON.parse(r.details_json || '{}'),
|
|
4411
|
+
createdAt: r.created_at,
|
|
4412
|
+
};
|
|
4413
|
+
});
|
|
4414
|
+
}
|
|
4334
4415
|
/**
|
|
4335
4416
|
* Aggregate memory-health snapshot for the dashboard.
|
|
4336
4417
|
*
|
package/dist/tools/shared.d.ts
CHANGED
|
@@ -459,6 +459,94 @@ export type MemoryStoreType = {
|
|
|
459
459
|
newestUpdated: string | null;
|
|
460
460
|
};
|
|
461
461
|
db: unknown;
|
|
462
|
+
getMemoryHealth(opts?: {
|
|
463
|
+
graphStore?: {
|
|
464
|
+
isAvailable(): boolean;
|
|
465
|
+
nodeCount?(): Promise<number>;
|
|
466
|
+
edgeCount?(): Promise<number>;
|
|
467
|
+
};
|
|
468
|
+
topCitedLimit?: number;
|
|
469
|
+
}): {
|
|
470
|
+
chunks: {
|
|
471
|
+
total: number;
|
|
472
|
+
consolidated: number;
|
|
473
|
+
pinned: number;
|
|
474
|
+
softDeleted: number;
|
|
475
|
+
zombieCount: number;
|
|
476
|
+
};
|
|
477
|
+
chunksByCategory: Array<{
|
|
478
|
+
category: string | null;
|
|
479
|
+
count: number;
|
|
480
|
+
}>;
|
|
481
|
+
tableRowCounts: Record<string, number>;
|
|
482
|
+
topCitedLast30d: Array<{
|
|
483
|
+
chunkId: number;
|
|
484
|
+
sourceFile: string;
|
|
485
|
+
section: string;
|
|
486
|
+
refCount: number;
|
|
487
|
+
}>;
|
|
488
|
+
staleUserModelSlots: Array<{
|
|
489
|
+
slot: string;
|
|
490
|
+
ageDays: number;
|
|
491
|
+
agentSlug: string | null;
|
|
492
|
+
}>;
|
|
493
|
+
staleHighSalienceChunks: Array<{
|
|
494
|
+
chunkId: number;
|
|
495
|
+
sourceFile: string;
|
|
496
|
+
section: string;
|
|
497
|
+
salience: number;
|
|
498
|
+
lastOutcomeScore: number;
|
|
499
|
+
}>;
|
|
500
|
+
chunkCacheStats: {
|
|
501
|
+
hits: number;
|
|
502
|
+
misses: number;
|
|
503
|
+
evictions: number;
|
|
504
|
+
size: number;
|
|
505
|
+
};
|
|
506
|
+
writeQueue: {
|
|
507
|
+
size: number;
|
|
508
|
+
dropped: number;
|
|
509
|
+
} | null;
|
|
510
|
+
lastIntegrityReport: {
|
|
511
|
+
ftsOk: boolean;
|
|
512
|
+
ftsRebuilt: boolean;
|
|
513
|
+
orphanRefsNulled: number;
|
|
514
|
+
missingEmbeddings: number;
|
|
515
|
+
ranAt: string;
|
|
516
|
+
} | null;
|
|
517
|
+
dbSizeBytes: number;
|
|
518
|
+
lastVacuumAt: string | null;
|
|
519
|
+
denseEmbeddings: {
|
|
520
|
+
withDense: number;
|
|
521
|
+
total: number;
|
|
522
|
+
models: Array<{
|
|
523
|
+
model: string;
|
|
524
|
+
count: number;
|
|
525
|
+
}>;
|
|
526
|
+
currentModel: string;
|
|
527
|
+
ready: boolean;
|
|
528
|
+
};
|
|
529
|
+
};
|
|
530
|
+
logAutonomyEvent(row: {
|
|
531
|
+
component: string;
|
|
532
|
+
event: string;
|
|
533
|
+
agentSlug?: string | null;
|
|
534
|
+
details?: Record<string, unknown>;
|
|
535
|
+
}): void;
|
|
536
|
+
queryAutonomyLog(opts?: {
|
|
537
|
+
component?: string;
|
|
538
|
+
event?: string;
|
|
539
|
+
agentSlug?: string;
|
|
540
|
+
limit?: number;
|
|
541
|
+
since?: string;
|
|
542
|
+
}): Array<{
|
|
543
|
+
id: number;
|
|
544
|
+
component: string;
|
|
545
|
+
event: string;
|
|
546
|
+
agentSlug: string | null;
|
|
547
|
+
details: Record<string, unknown>;
|
|
548
|
+
createdAt: string;
|
|
549
|
+
}>;
|
|
462
550
|
};
|
|
463
551
|
export declare function getStore(): Promise<MemoryStoreType>;
|
|
464
552
|
export declare function getStoreSync(): MemoryStoreType | null;
|