clementine-agent 1.1.23 → 1.1.25

package/dist/agent/assistant.js

@@ -22,6 +22,7 @@ import { AgentManager } from './agent-manager.js';
 import { extractLinks } from './link-extractor.js';
 import { StallGuard } from './stall-guard.js';
 import { collectToolCalls, detectContradiction, buildCorrectionPrompt } from './contradiction-validator.js';
+import { recordToolOutcome as recordMcpToolOutcome } from './mcp-circuit-breaker.js';
 import { assembleContext } from '../memory/context-assembler.js';
 import { PromptCache } from './prompt-cache.js';
 import { searchSkills as searchSkillsSync } from './skill-extractor.js';
@@ -813,6 +814,7 @@ export class PersonalAssistant {
                     numTurns: result.num_turns,
                     durationMs: result.duration_ms,
                     agentSlug: agentSlug ?? undefined,
+                    totalCostUsd: 'total_cost_usd' in result ? result.total_cost_usd : undefined,
                 });
             }
             catch (err) {
@@ -2946,6 +2948,15 @@ You have a cost budget per message — not a hard turn limit. Work until the tas
                 if (!contradictionRetried && attempt < PersonalAssistant.RATE_LIMIT_MAX_RETRIES && responseText.trim()) {
                     try {
                         const toolCallRecords = collectToolCalls(collectedSdkMessages);
+                        // Feed every tool outcome to the MCP circuit breaker so flaky
+                        // connectors get tripped + surfaced via the advisor-events
+                        // path that insight-engine already monitors.
+                        for (const r of toolCallRecords) {
+                            try {
+                                recordMcpToolOutcome(r.name, r.resultClass);
+                            }
+                            catch { /* non-fatal */ }
+                        }
                         // Diagnostic — emits once per turn so we can see what the
                         // validator is working with even when it doesn't fire. Without
                         // this we're blind to the "regex missed the phrasing" case.
@@ -3835,6 +3846,17 @@ You have a cost budget per message — not a hard turn limit. Work until the tas
         const cronProfile = agentSlug && agentSlug !== 'clementine'
             ? this.profileManager.get(agentSlug)
             : null;
+        // Per-agent monthly budget gate. Refuse to start the cron run if this
+        // agent has exceeded its cap for the calendar month. The breaker
+        // surfaces via insight-engine so the owner sees it without polling.
+        if (cronProfile && this.memoryStore) {
+            const { checkAgentBudget } = await import('./budget-enforcement.js');
+            const budget = checkAgentBudget(cronProfile, this.memoryStore);
+            if (!budget.allowed) {
+                logger.warn({ jobName, agentSlug, spent: budget.spentCents, limit: budget.limitCents }, 'Cron job skipped — agent over monthly budget');
+                return `[BUDGET_BLOCK] ${budget.message}`;
+            }
+        }
         // Cron jobs deliver via side effects (sent emails, updated records, etc),
         // not chat text — pass mode='cron' so high_effort_low_output guard is
         // disabled. Loop detection and circular-reasoning checks stay active.

package/dist/agent/budget-enforcement.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Per-agent monthly budget enforcement.
+ *
+ * AgentProfile.budgetMonthlyCents is set in agent.md frontmatter (0 or
+ * undefined = unlimited). This module checks the current month's spend
+ * against the cap before letting an autonomous activity (cron, heartbeat,
+ * delegated team task) start.
+ *
+ * Enforcement is intentionally narrow:
+ *   - User-initiated chat is NEVER blocked. The owner needs to be able to
+ *     talk to a paused agent to lift the pause (raise the cap, reset the
+ *     period, etc.).
+ *   - Cron + heartbeat + delegation flows ARE blocked. Those are the
+ *     paths that can run away with cost.
+ *
+ * Surfacing: when the breaker fires, we write a circuit-breaker advisor
+ * event the same shape the MCP and cron breakers use. insight-engine
+ * picks that up and surfaces it in the next signal pull, so the owner
+ * sees "Budget breaker tripped for agent <slug>" in their next insight.
+ */
+import type { AgentProfile } from '../types.js';
+import type { MemoryStore } from '../memory/store.js';
+export interface BudgetCheckResult {
+    allowed: boolean;
+    spentCents: number;
+    limitCents: number;
+    /** Human-readable explanation when blocked. */
+    message?: string;
+}
+/**
+ * Decide whether a profile's autonomous activity may proceed for the
+ * current calendar month. Returns allowed=true if no budget is set,
+ * if the agent is global Clementine (no profile), or if spend < limit.
+ *
+ * Side effect: when the breaker fires for the first time this month for
+ * a given agent, emits an advisor event so insight-engine surfaces it.
+ */
+export declare function checkAgentBudget(profile: AgentProfile | null | undefined, memoryStore: MemoryStore | null | undefined): BudgetCheckResult;
+/** Test seam — clear the "already notified this month" memo. */
+export declare function _resetNotifiedForTesting(): void;
+//# sourceMappingURL=budget-enforcement.d.ts.map

package/dist/agent/budget-enforcement.js

@@ -0,0 +1,98 @@
+/**
+ * Per-agent monthly budget enforcement.
+ *
+ * AgentProfile.budgetMonthlyCents is set in agent.md frontmatter (0 or
+ * undefined = unlimited). This module checks the current month's spend
+ * against the cap before letting an autonomous activity (cron, heartbeat,
+ * delegated team task) start.
+ *
+ * Enforcement is intentionally narrow:
+ *   - User-initiated chat is NEVER blocked. The owner needs to be able to
+ *     talk to a paused agent to lift the pause (raise the cap, reset the
+ *     period, etc.).
+ *   - Cron + heartbeat + delegation flows ARE blocked. Those are the
+ *     paths that can run away with cost.
+ *
+ * Surfacing: when the breaker fires, we write a circuit-breaker advisor
+ * event the same shape the MCP and cron breakers use. insight-engine
+ * picks that up and surfaces it in the next signal pull, so the owner
+ * sees "Budget breaker tripped for agent <slug>" in their next insight.
+ */
+import { appendFileSync, mkdirSync } from 'node:fs';
+import path from 'node:path';
+import pino from 'pino';
+import { BASE_DIR } from '../config.js';
+const logger = pino({ name: 'clementine.budget-enforcement' });
+const ADVISOR_EVENTS_FILE = path.join(BASE_DIR, 'cron', 'advisor-events.jsonl');
+/** Track per-agent "we already notified about this month" so we don't spam. */
+const notifiedThisMonth = new Map();
+function monthKey() {
+    const d = new Date();
+    return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}`;
+}
+/**
+ * Decide whether a profile's autonomous activity may proceed for the
+ * current calendar month. Returns allowed=true if no budget is set,
+ * if the agent is global Clementine (no profile), or if spend < limit.
+ *
+ * Side effect: when the breaker fires for the first time this month for
+ * a given agent, emits an advisor event so insight-engine surfaces it.
+ */
+export function checkAgentBudget(profile, memoryStore) {
+    // No profile (Clementine herself) — global budget is governed elsewhere.
+    if (!profile)
+        return { allowed: true, spentCents: 0, limitCents: 0 };
+    const limit = profile.budgetMonthlyCents ?? 0;
+    // Unlimited.
+    if (!limit || limit <= 0)
+        return { allowed: true, spentCents: 0, limitCents: 0 };
+    if (!memoryStore)
+        return { allowed: true, spentCents: 0, limitCents: limit };
+    let spent = 0;
+    try {
+        spent = memoryStore.getMonthlyCostCents(profile.slug);
+    }
+    catch (err) {
+        logger.debug({ err, slug: profile.slug }, 'Budget query failed — allowing through');
+        return { allowed: true, spentCents: 0, limitCents: limit };
+    }
+    if (spent < limit) {
+        return { allowed: true, spentCents: spent, limitCents: limit };
+    }
+    const usd = (cents) => `$${(cents / 100).toFixed(2)}`;
+    const msg = `Agent "${profile.slug}" has hit its monthly budget (${usd(spent)} of ${usd(limit)}). ` +
+        `Autonomous activity (cron, heartbeat, delegation) is paused for this agent. ` +
+        `Lift by raising budgetMonthlyCents in agent.md or by resetting at month end.`;
+    // Emit the breaker event once per month per agent so insight-engine
+    // surfaces it but we don't spam the owner with the same message every
+    // single tick after the breaker trips.
+    const stamp = `${profile.slug}|${monthKey()}`;
+    if (notifiedThisMonth.get(profile.slug) !== monthKey()) {
+        notifiedThisMonth.set(profile.slug, monthKey());
+        emitAdvisorEvent({
+            type: 'circuit-breaker',
+            jobName: `budget:${profile.slug}`,
+            detail: msg,
+        });
+        logger.warn({ slug: profile.slug, spent, limit }, 'Agent monthly budget tripped');
+    }
+    else {
+        logger.debug({ stamp, spent, limit }, 'Agent budget still tripped (already notified this month)');
+    }
+    return { allowed: false, spentCents: spent, limitCents: limit, message: msg };
+}
+/** Test seam — clear the "already notified this month" memo. */
+export function _resetNotifiedForTesting() {
+    notifiedThisMonth.clear();
+}
+function emitAdvisorEvent(evt) {
+    try {
+        mkdirSync(path.dirname(ADVISOR_EVENTS_FILE), { recursive: true });
+        const line = JSON.stringify({ timestamp: new Date().toISOString(), ...evt }) + '\n';
+        appendFileSync(ADVISOR_EVENTS_FILE, line);
+    }
+    catch (err) {
+        logger.debug({ err }, 'Failed to emit budget advisor event');
+    }
+}
+//# sourceMappingURL=budget-enforcement.js.map

package/dist/agent/crash-forensics.d.ts

@@ -0,0 +1,76 @@
+/**
+ * Crash forensics — capture context when something goes wrong so the next
+ * launch can surface "I crashed at 2:14am because X" instead of leaving
+ * the user wondering why their daemon went quiet.
+ *
+ * Two surfaces:
+ *
+ *   1. installCrashHandlers() wraps process.on('uncaughtException') and
+ *      process.on('unhandledRejection') — when those fire, we write a
+ *      timestamped JSON dump to ~/.clementine/crash-reports/. The existing
+ *      handlers in index.ts keep the daemon alive (deliberate); the dump
+ *      gives us a forensic trail without changing that behavior.
+ *
+ *   2. surfaceUnreadCrashReports(dispatcher) runs at startup, scans for
+ *      report files that haven't been acknowledged, sends a one-line
+ *      summary via the dispatcher, then renames them with a `.ack`
+ *      suffix so they don't re-fire on the next launch.
+ *
+ * The dump shape is intentionally small (under ~10KB) so it survives even
+ * when the underlying problem is "we ran out of memory."
+ */
+export type CrashType = 'uncaughtException' | 'unhandledRejection';
+export interface CrashReport {
+    timestamp: string;
+    type: CrashType;
+    error: string;
+    stack?: string;
+    uptime: number;
+    pid: number;
+    recentLogs: string[];
+}
+/**
+ * Build a crash report payload. Pure function — exported for testing.
+ * Intentionally bounds the size of recentLogs so a runaway log file
+ * doesn't make the dump unwriteable when the system is already wobbly.
+ */
+export declare function buildCrashReport(opts: {
+    type: CrashType;
+    error: unknown;
+    uptime: number;
+    pid: number;
+    baseDir: string;
+}): CrashReport;
+/** Write a single crash report. Best-effort — never throws. */
+export declare function writeCrashReport(opts: {
+    type: CrashType;
+    error: unknown;
+    baseDir: string;
+}): string | null;
+export declare function installCrashHandlers(baseDir: string): void;
+/** Test seam — clear the install flag. */
+export declare function _resetInstalledForTesting(): void;
+/**
+ * Read all unread crash reports (those without a `.ack` sibling),
+ * sorted oldest-first. Returned shape is the parsed payload + the
+ * source filename so the caller can ack it after surfacing.
+ */
+export declare function readUnreadCrashReports(baseDir: string): Array<{
+    report: CrashReport;
+    file: string;
+}>;
+/** Mark a crash report as acknowledged so it doesn't re-surface. */
+export declare function ackCrashReport(file: string): void;
+/**
+ * Format a single crash report as a one-line owner-readable summary.
+ * Intentionally short — the full dump is on disk for deep debugging.
+ */
+export declare function formatCrashSummary(report: CrashReport): string;
+/**
+ * Startup helper: scan for unread reports, send each as a chat
+ * notification via the provided send function, then ack each one.
+ * Send function is the dispatcher's `send` so we don't take a hard
+ * dependency on the dispatcher type from this module.
+ */
+export declare function surfaceUnreadCrashReports(baseDir: string, send: (msg: string) => Promise<void>): Promise<number>;
+//# sourceMappingURL=crash-forensics.d.ts.map

package/dist/agent/crash-forensics.js

@@ -0,0 +1,197 @@
+/**
+ * Crash forensics — capture context when something goes wrong so the next
+ * launch can surface "I crashed at 2:14am because X" instead of leaving
+ * the user wondering why their daemon went quiet.
+ *
+ * Two surfaces:
+ *
+ *   1. installCrashHandlers() wraps process.on('uncaughtException') and
+ *      process.on('unhandledRejection') — when those fire, we write a
+ *      timestamped JSON dump to ~/.clementine/crash-reports/. The existing
+ *      handlers in index.ts keep the daemon alive (deliberate); the dump
+ *      gives us a forensic trail without changing that behavior.
+ *
+ *   2. surfaceUnreadCrashReports(dispatcher) runs at startup, scans for
+ *      report files that haven't been acknowledged, sends a one-line
+ *      summary via the dispatcher, then renames them with a `.ack`
+ *      suffix so they don't re-fire on the next launch.
+ *
+ * The dump shape is intentionally small (under ~10KB) so it survives even
+ * when the underlying problem is "we ran out of memory."
+ */
+import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import pino from 'pino';
+const logger = pino({ name: 'clementine.crash-forensics' });
+/** How many lines of recent log to capture in the dump. */
+const RECENT_LOG_LINES = 30;
+function reportsDir(baseDir) {
+    return path.join(baseDir, 'crash-reports');
+}
+function logFilePath(baseDir) {
+    return path.join(baseDir, 'logs', 'clementine.log');
+}
+function readRecentLogLines(baseDir, n) {
+    try {
+        const p = logFilePath(baseDir);
+        if (!existsSync(p))
+            return [];
+        const all = readFileSync(p, 'utf-8').split('\n').filter(Boolean);
+        return all.slice(-n);
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Build a crash report payload. Pure function — exported for testing.
+ * Intentionally bounds the size of recentLogs so a runaway log file
+ * doesn't make the dump unwriteable when the system is already wobbly.
+ */
+export function buildCrashReport(opts) {
+    const err = opts.error;
+    const errorMsg = err instanceof Error ? err.message : (typeof err === 'string' ? err : JSON.stringify(err));
+    const stack = err instanceof Error ? err.stack : undefined;
+    return {
+        timestamp: new Date().toISOString(),
+        type: opts.type,
+        error: errorMsg.slice(0, 1000),
+        stack: stack?.slice(0, 4000),
+        uptime: opts.uptime,
+        pid: opts.pid,
+        recentLogs: readRecentLogLines(opts.baseDir, RECENT_LOG_LINES),
+    };
+}
+/** Write a single crash report. Best-effort — never throws. */
+export function writeCrashReport(opts) {
+    try {
+        const dir = reportsDir(opts.baseDir);
+        mkdirSync(dir, { recursive: true });
+        const report = buildCrashReport({
+            type: opts.type,
+            error: opts.error,
+            uptime: process.uptime(),
+            pid: process.pid,
+            baseDir: opts.baseDir,
+        });
+        // Keep millisecond precision so back-to-back crashes don't collide
+        // on filename (was a real test failure — two writes within 10ms got
+        // the same name and the second clobbered the first).
+        const safeStamp = report.timestamp.replace(/[:.]/g, '-');
+        const filename = path.join(dir, `${safeStamp}-${opts.type}.json`);
+        writeFileSync(filename, JSON.stringify(report, null, 2), { mode: 0o600 });
+        return filename;
+    }
+    catch (err) {
+        // If we can't even write the dump, log to stderr — the daemon's logger
+        // may itself be the thing that's failed.
+        try {
+            console.error('crash-forensics: failed to write report:', err);
+        }
+        catch { /* nothing to do */ }
+        return null;
+    }
+}
+/**
+ * Wire the global handlers. Idempotent — calling twice is a no-op past
+ * the first install. We DON'T exit the process here: the existing
+ * uncaughtException handler in index.ts keeps the daemon alive on
+ * purpose (segfaults / OOM still kill it; this is for soft errors
+ * where execution can continue).
+ */
+let _installed = false;
+export function installCrashHandlers(baseDir) {
+    if (_installed)
+        return;
+    _installed = true;
+    process.on('uncaughtException', (err) => {
+        const file = writeCrashReport({ type: 'uncaughtException', error: err, baseDir });
+        if (file)
+            logger.warn({ file }, 'Crash report written for uncaughtException');
+    });
+    process.on('unhandledRejection', (err) => {
+        const file = writeCrashReport({ type: 'unhandledRejection', error: err, baseDir });
+        if (file)
+            logger.warn({ file }, 'Crash report written for unhandledRejection');
+    });
+}
+/** Test seam — clear the install flag. */
+export function _resetInstalledForTesting() {
+    _installed = false;
+}
+/**
+ * Read all unread crash reports (those without a `.ack` sibling),
+ * sorted oldest-first. Returned shape is the parsed payload + the
+ * source filename so the caller can ack it after surfacing.
+ */
+export function readUnreadCrashReports(baseDir) {
+    const dir = reportsDir(baseDir);
+    if (!existsSync(dir))
+        return [];
+    const all = readdirSync(dir);
+    const ackedSet = new Set(all.filter(f => f.endsWith('.ack')).map(f => f.replace(/\.ack$/, '')));
+    const unread = all
+        .filter(f => f.endsWith('.json') && !ackedSet.has(f))
+        .sort();
+    const out = [];
+    for (const name of unread) {
+        const filePath = path.join(dir, name);
+        try {
+            const parsed = JSON.parse(readFileSync(filePath, 'utf-8'));
+            out.push({ report: parsed, file: filePath });
+        }
+        catch {
+            // Corrupt dump — ack it anyway so we don't keep tripping on it.
+            ackCrashReport(filePath);
+        }
+    }
+    return out;
+}
+/** Mark a crash report as acknowledged so it doesn't re-surface. */
+export function ackCrashReport(file) {
+    try {
+        renameSync(file, `${file}.ack`);
+    }
+    catch {
+        // Non-fatal — worst case we surface it again next launch.
+    }
+}
+/**
+ * Format a single crash report as a one-line owner-readable summary.
+ * Intentionally short — the full dump is on disk for deep debugging.
+ */
+export function formatCrashSummary(report) {
+    const stamp = report.timestamp.slice(0, 19).replace('T', ' ');
+    const upHours = Math.floor(report.uptime / 3600);
+    const upMin = Math.floor((report.uptime % 3600) / 60);
+    const uptimeStr = upHours > 0 ? `${upHours}h${upMin}m` : `${upMin}m`;
+    const errLine = report.error.split('\n')[0].slice(0, 220);
+    return `${stamp} (after ${uptimeStr} uptime) — ${report.type}: ${errLine}`;
+}
+/**
+ * Startup helper: scan for unread reports, send each as a chat
+ * notification via the provided send function, then ack each one.
+ * Send function is the dispatcher's `send` so we don't take a hard
+ * dependency on the dispatcher type from this module.
+ */
+export async function surfaceUnreadCrashReports(baseDir, send) {
+    const unread = readUnreadCrashReports(baseDir);
+    if (unread.length === 0)
+        return 0;
+    // Group multiple reports into one digest message — one ping per launch
+    // is enough; the file system has the per-event detail.
+    const lines = unread.slice(0, 10).map(u => `• ${formatCrashSummary(u.report)}`);
+    const overflow = unread.length > 10 ? `\n…and ${unread.length - 10} more in ${reportsDir(baseDir)}` : '';
+    const dirHint = `\n\n_Full dumps: ${reportsDir(baseDir)}_`;
+    const msg = `**Recovered from ${unread.length} crash event${unread.length === 1 ? '' : 's'} since last successful run.**\n\n${lines.join('\n')}${overflow}${dirHint}`;
+    try {
+        await send(msg);
+    }
+    catch (err) {
+        logger.warn({ err }, 'Failed to dispatch crash-recovery summary');
+    }
+    for (const u of unread)
+        ackCrashReport(u.file);
+    return unread.length;
+}
+//# sourceMappingURL=crash-forensics.js.map

package/dist/agent/mcp-circuit-breaker.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Per-MCP-server circuit breaker.
+ *
+ * When an MCP server starts returning errors (auth failures, connector
+ * timeouts, "no such tool available") repeatedly, agents keep calling it
+ * — burning tool turns on something that isn't going to work. This module
+ * tracks per-server failure rates with a sliding window and surfaces a
+ * tripped state to the existing insight-engine via advisor-events.jsonl
+ * (same path the cron-side circuit breaker uses).
+ *
+ * Trip rule: K failures of class auth_error/other_error within WINDOW_MS
+ * trips the breaker for COOLDOWN_MS. Argument errors are agent-fault, not
+ * connector-fault, and don't count toward the trip threshold.
+ *
+ * Auto-reset: COOLDOWN_MS after the trip moment, the breaker clears and
+ * the failure window resets. The next failure starts the count fresh.
+ */
+export type ToolResultClass = 'success' | 'arg_error' | 'auth_error' | 'other_error';
+/**
+ * Extract the MCP server name from a fully-qualified tool name. Handles
+ * server names that themselves contain underscores (e.g. `claude_ai_Gmail`)
+ * by treating only the FINAL `__` separator as the server/tool boundary.
+ *
+ *   mcp__clementine-tools__memory_search        → "clementine-tools"
+ *   mcp__claude_ai_Gmail__authenticate          → "claude_ai_Gmail"
+ *   mcp__ElevenLabs__text_to_speech             → "ElevenLabs"
+ *   mcp__plugin_x_y__do_thing                   → "plugin_x_y"
+ *
+ * Returns null for non-MCP tools (Bash, Read, etc.).
+ */
+export declare function extractServerName(toolName: string): string | null;
+/**
+ * Record the outcome of a single tool invocation. Only auth_error /
+ * other_error count toward the failure window — arg_error is the agent's
+ * fault (bad parameters), and success obviously doesn't count.
+ */
+export declare function recordToolOutcome(toolName: string, resultClass: ToolResultClass): void;
+/** True when the named server is currently in the open (failing) state. */
+export declare function isServerTripped(server: string): boolean;
+/** Get all currently-tripped servers — useful for status display + system-prompt injection. */
+export declare function getTrippedServers(): Array<{
+    server: string;
+    trippedAt: string;
+    reason: string;
+    cooldownRemainingMs: number;
+}>;
+/** Manual reset — used by an `mcp circuit reset` admin command (future). */
+export declare function resetServer(server: string): boolean;
+/** Reset every breaker — used by tests. */
+export declare function _resetAll(): void;
+//# sourceMappingURL=mcp-circuit-breaker.d.ts.map

package/dist/agent/mcp-circuit-breaker.js

@@ -0,0 +1,175 @@
+/**
+ * Per-MCP-server circuit breaker.
+ *
+ * When an MCP server starts returning errors (auth failures, connector
+ * timeouts, "no such tool available") repeatedly, agents keep calling it
+ * — burning tool turns on something that isn't going to work. This module
+ * tracks per-server failure rates with a sliding window and surfaces a
+ * tripped state to the existing insight-engine via advisor-events.jsonl
+ * (same path the cron-side circuit breaker uses).
+ *
+ * Trip rule: K failures of class auth_error/other_error within WINDOW_MS
+ * trips the breaker for COOLDOWN_MS. Argument errors are agent-fault, not
+ * connector-fault, and don't count toward the trip threshold.
+ *
+ * Auto-reset: COOLDOWN_MS after the trip moment, the breaker clears and
+ * the failure window resets. The next failure starts the count fresh.
+ */
+import { appendFileSync, existsSync, mkdirSync } from 'node:fs';
+import path from 'node:path';
+import pino from 'pino';
+import { BASE_DIR } from '../config.js';
+const logger = pino({ name: 'clementine.mcp-circuit-breaker' });
+/** Threshold to trip the breaker. */
+const MAX_CONNECTOR_FAILURES = 5;
+/** Sliding window for counting failures. */
+const WINDOW_MS = 5 * 60 * 1000; // 5 minutes
+/** How long the breaker stays open before auto-resetting. */
+const COOLDOWN_MS = 5 * 60 * 1000; // 5 minutes
+const ADVISOR_EVENTS_FILE = path.join(BASE_DIR, 'cron', 'advisor-events.jsonl');
+const state = new Map();
+/**
+ * Extract the MCP server name from a fully-qualified tool name. Handles
+ * server names that themselves contain underscores (e.g. `claude_ai_Gmail`)
+ * by treating only the FINAL `__` separator as the server/tool boundary.
+ *
+ *   mcp__clementine-tools__memory_search        → "clementine-tools"
+ *   mcp__claude_ai_Gmail__authenticate          → "claude_ai_Gmail"
+ *   mcp__ElevenLabs__text_to_speech             → "ElevenLabs"
+ *   mcp__plugin_x_y__do_thing                   → "plugin_x_y"
+ *
+ * Returns null for non-MCP tools (Bash, Read, etc.).
+ */
+export function extractServerName(toolName) {
+    if (!toolName.startsWith('mcp__'))
+        return null;
+    const rest = toolName.slice('mcp__'.length);
+    const lastSep = rest.lastIndexOf('__');
+    if (lastSep <= 0)
+        return null;
+    return rest.slice(0, lastSep);
+}
+function getServerState(server) {
+    let s = state.get(server);
+    if (!s) {
+        s = { failureTimestamps: [] };
+        state.set(server, s);
+    }
+    return s;
+}
+/**
+ * Record the outcome of a single tool invocation. Only auth_error /
+ * other_error count toward the failure window — arg_error is the agent's
+ * fault (bad parameters), and success obviously doesn't count.
+ */
+export function recordToolOutcome(toolName, resultClass) {
+    const server = extractServerName(toolName);
+    if (!server)
+        return; // built-in tool like Bash/Read — not our concern
+    const s = getServerState(server);
+    const now = Date.now();
+    // Auto-reset if the cooldown has expired since the last trip.
+    if (s.trippedAt !== undefined && now - s.trippedAt >= COOLDOWN_MS) {
+        s.trippedAt = undefined;
+        s.trippedReason = undefined;
+        s.failureTimestamps = [];
+        logger.info({ server }, 'MCP circuit breaker auto-reset after cooldown');
+        emitAdvisorEvent({
+            type: 'circuit-breaker',
+            jobName: `mcp:${server}`,
+            detail: 'Connector breaker reset — probing again on next call',
+            reset: true,
+        });
+    }
+    if (resultClass === 'success') {
+        // Successful call inside the window — clear the failure list so a flap
+        // doesn't accumulate forever. Don't auto-reset a tripped breaker on
+        // success though; that needs to wait for the cooldown so we don't
+        // ping-pong on intermittent failures.
+        if (s.trippedAt === undefined) {
+            s.failureTimestamps = [];
+        }
+        return;
+    }
+    if (resultClass === 'arg_error') {
+        // Agent passed bad args — connector itself is fine.
+        return;
+    }
+    // auth_error or other_error — count toward the failure window.
+    s.failureTimestamps.push(now);
+    // Drop old timestamps outside the window.
+    s.failureTimestamps = s.failureTimestamps.filter(t => now - t <= WINDOW_MS);
+    if (s.trippedAt === undefined && s.failureTimestamps.length >= MAX_CONNECTOR_FAILURES) {
+        s.trippedAt = now;
+        s.trippedReason = `${s.failureTimestamps.length} ${resultClass} failure(s) in the last ${Math.round(WINDOW_MS / 60_000)}m`;
+        logger.warn({ server, failures: s.failureTimestamps.length, resultClass }, 'MCP circuit breaker tripped');
+        emitAdvisorEvent({
+            type: 'circuit-breaker',
+            jobName: `mcp:${server}`,
+            detail: `MCP connector "${server}" tripped — ${s.trippedReason}. Prefer alternatives until cooldown expires (~${Math.round(COOLDOWN_MS / 60_000)}m).`,
+        });
+    }
+}
+/** True when the named server is currently in the open (failing) state. */
+export function isServerTripped(server) {
+    const s = state.get(server);
+    if (!s || s.trippedAt === undefined)
+        return false;
+    if (Date.now() - s.trippedAt >= COOLDOWN_MS)
+        return false;
+    return true;
+}
+/** Get all currently-tripped servers — useful for status display + system-prompt injection. */
+export function getTrippedServers() {
+    const now = Date.now();
+    const out = [];
+    for (const [server, s] of state) {
+        if (s.trippedAt === undefined)
+            continue;
+        const remaining = COOLDOWN_MS - (now - s.trippedAt);
+        if (remaining <= 0)
+            continue;
+        out.push({
+            server,
+            trippedAt: new Date(s.trippedAt).toISOString(),
+            reason: s.trippedReason ?? 'unknown',
+            cooldownRemainingMs: remaining,
+        });
+    }
+    return out;
+}
+/** Manual reset — used by an `mcp circuit reset` admin command (future). */
+export function resetServer(server) {
+    const s = state.get(server);
+    if (!s || s.trippedAt === undefined)
+        return false;
+    s.trippedAt = undefined;
+    s.trippedReason = undefined;
+    s.failureTimestamps = [];
+    logger.info({ server }, 'MCP circuit breaker manually reset');
+    emitAdvisorEvent({
+        type: 'circuit-breaker',
+        jobName: `mcp:${server}`,
+        detail: 'Manually reset',
+        reset: true,
+    });
+    return true;
+}
+/** Reset every breaker — used by tests. */
+export function _resetAll() {
+    state.clear();
+}
+function emitAdvisorEvent(evt) {
+    try {
+        mkdirSync(path.dirname(ADVISOR_EVENTS_FILE), { recursive: true });
+        if (!existsSync(path.dirname(ADVISOR_EVENTS_FILE)))
+            return;
+        const line = JSON.stringify({ timestamp: new Date().toISOString(), ...evt }) + '\n';
+        appendFileSync(ADVISOR_EVENTS_FILE, line);
+    }
+    catch (err) {
+        // Non-fatal — observability event, not load-bearing for the breaker logic.
+        logger.debug({ err }, 'Failed to emit advisor event for MCP circuit breaker');
+    }
+}
+//# sourceMappingURL=mcp-circuit-breaker.js.map

package/dist/channels/discord.js

@@ -632,6 +632,30 @@ export async function startDiscord(gateway, heartbeat, cronScheduler, dispatcher
     client.on(Events.Error, (err) => {
         logger.error({ err }, 'Discord client error — will attempt to reconnect');
     });
+    // ── Connection lifecycle observability ─────────────────────────────
+    // discord.js auto-reconnects via the WebSocketManager — these handlers
+    // give us visibility into when shards drop and recover so the daemon
+    // can report "Discord went offline at HH:MM, came back at HH:MM" instead
+    // of leaving the user wondering why nothing was responding.
+    let lastDisconnectAt = null;
+    client.on(Events.ShardDisconnect, (event, shardId) => {
+        lastDisconnectAt = Date.now();
+        logger.warn({ shardId, code: event?.code, reason: event?.reason }, 'Discord shard disconnected');
+    });
+    client.on(Events.ShardReconnecting, (shardId) => {
+        logger.info({ shardId }, 'Discord shard reconnecting...');
+    });
+    client.on(Events.ShardReady, (shardId, unavailableGuilds) => {
+        if (lastDisconnectAt !== null) {
+            const downtimeMs = Date.now() - lastDisconnectAt;
+            const downtimeSec = Math.round(downtimeMs / 1000);
+            logger.info({ shardId, unavailableGuilds: unavailableGuilds?.size, downtimeSec }, 'Discord shard reconnected');
+            lastDisconnectAt = null;
+        }
+        else {
+            logger.info({ shardId, unavailableGuilds: unavailableGuilds?.size }, 'Discord shard ready');
+        }
+    });
     client.once(Events.ClientReady, async (readyClient) => {
         logger.info(`${ASSISTANT_NAME} online as ${readyClient.user.tag}`);
         // Register slash commands (global — takes up to 1hr to propagate, but works in DMs)

package/dist/index.js

@@ -665,6 +665,21 @@ async function asyncMain() {
     const dispatcher = new NotificationDispatcher();
     gateway.setDispatcher(dispatcher);
     gateway.initSkillNotifications();
+    // Crash recovery — surface any forensic dumps written before this start.
+    // Fire-and-forget; if the dispatcher isn't ready yet, the next launch
+    // catches it on retry (the .ack rename only happens after send succeeds).
+    void (async () => {
+        try {
+            const { surfaceUnreadCrashReports } = await import('./agent/crash-forensics.js');
+            const count = await surfaceUnreadCrashReports(config.BASE_DIR, async (msg) => { await dispatcher.send(msg); });
+            if (count > 0) {
+                logger.info({ count }, 'Surfaced crash recovery summary to owner');
+            }
+        }
+        catch (err) {
+            logger.warn({ err }, 'Failed to surface crash recovery summary');
+        }
+    })();
     // Heartbeat + Cron schedulers
     const { HeartbeatScheduler, CronScheduler } = await import('./gateway/heartbeat.js');
     const heartbeat = new HeartbeatScheduler(gateway, dispatcher);
@@ -1106,6 +1121,12 @@ function main() {
     process.on('unhandledRejection', (err) => {
         logger.error({ err }, 'Unhandled promise rejection — daemon staying alive');
     });
+    // Crash forensics — write a JSON dump alongside the existing log line
+    // so the next launch can surface "I crashed because X" via chat.
+    // Fire-and-forget: failure to load shouldn't block daemon startup.
+    import('./agent/crash-forensics.js')
+        .then(({ installCrashHandlers }) => installCrashHandlers(config.BASE_DIR))
+        .catch((err) => logger.warn({ err }, 'Failed to install crash forensics handlers — continuing without them'));
     // First-run auto-setup
     const envFile = path.join(config.BASE_DIR, '.env');
     if (!existsSync(envFile)) {

package/dist/memory/store.d.ts

@@ -593,7 +593,10 @@ export declare class MemoryStore {
     }>;
     /**
      * Log token usage from an SDK query result.
-     * Iterates modelUsage record and inserts one row per model.
+     * Iterates modelUsage record and inserts one row per model. Cost is
+     * apportioned across models proportionally to total tokens (input +
+     * output) so per-agent monthly aggregations stay accurate when a turn
+     * uses more than one model.
      */
     logUsage(entry: {
         sessionKey: string;
@@ -607,7 +610,15 @@ export declare class MemoryStore {
         numTurns: number;
         durationMs: number;
         agentSlug?: string;
+        /** Total cost in USD for the whole turn (from SDK result.total_cost_usd). */
+        totalCostUsd?: number;
     }): void;
+    /**
+     * Get the current month's spend in cents for an agent (or for global
+     * Clementine if agentSlug is null/undefined). "Month" = first day of
+     * the current calendar month in UTC.
+     */
+    getMonthlyCostCents(agentSlug: string | null | undefined): number;
     /**
      * Get aggregated usage summary, optionally filtered by time.
      */

package/dist/memory/store.js

@@ -405,6 +405,12 @@ export class MemoryStore {
             this.conn.exec(`CREATE INDEX IF NOT EXISTS idx_usage_agent ON usage_log(agent_slug)`);
         }
         catch { /* column already exists */ }
+        // Migration: add cost_cents for budget enforcement (per-agent monthly caps).
+        // Stored as INTEGER cents to avoid float precision drift across aggregations.
+        try {
+            this.conn.exec(`ALTER TABLE usage_log ADD COLUMN cost_cents INTEGER DEFAULT 0`);
+        }
+        catch { /* column already exists */ }
         // ── SDR Operational Tables ───────────────────────────────────────
         // Leads — structured prospect records for SDR workflows
         this.conn.exec(`
@@ -2545,15 +2551,51 @@ export class MemoryStore {
     // ── Usage Tracking ────────────────────────────────────────────────
     /**
      * Log token usage from an SDK query result.
-     * Iterates modelUsage record and inserts one row per model.
+     * Iterates modelUsage record and inserts one row per model. Cost is
+     * apportioned across models proportionally to total tokens (input +
+     * output) so per-agent monthly aggregations stay accurate when a turn
+     * uses more than one model.
      */
     logUsage(entry) {
         if (!this._stmtInsertUsage) {
-            this._stmtInsertUsage = this.conn.prepare(`INSERT INTO usage_log (session_key, source, model, input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens, num_turns, duration_ms, agent_slug)
-         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
-        }
+            this._stmtInsertUsage = this.conn.prepare(`INSERT INTO usage_log (session_key, source, model, input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens, num_turns, duration_ms, agent_slug, cost_cents)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
+        }
+        // Apportion the total cost across models by token share.
+        const totalCostCents = entry.totalCostUsd != null
+            ? Math.max(0, Math.round(entry.totalCostUsd * 100))
+            : 0;
+        const totalTokens = Object.values(entry.modelUsage).reduce((sum, u) => sum + (u.inputTokens ?? 0) + (u.outputTokens ?? 0), 0);
         for (const [model, usage] of Object.entries(entry.modelUsage)) {
-            this._stmtInsertUsage.run(entry.sessionKey, entry.source, model, usage.inputTokens ?? 0, usage.outputTokens ?? 0, usage.cacheReadInputTokens ?? 0, usage.cacheCreationInputTokens ?? 0, entry.numTurns ?? 0, entry.durationMs ?? 0, entry.agentSlug ?? null);
+            const modelTokens = (usage.inputTokens ?? 0) + (usage.outputTokens ?? 0);
+            const shareCents = totalCostCents > 0 && totalTokens > 0
+                ? Math.round(totalCostCents * (modelTokens / totalTokens))
+                : 0;
+            this._stmtInsertUsage.run(entry.sessionKey, entry.source, model, usage.inputTokens ?? 0, usage.outputTokens ?? 0, usage.cacheReadInputTokens ?? 0, usage.cacheCreationInputTokens ?? 0, entry.numTurns ?? 0, entry.durationMs ?? 0, entry.agentSlug ?? null, shareCents);
+        }
+    }
+    /**
+     * Get the current month's spend in cents for an agent (or for global
+     * Clementine if agentSlug is null/undefined). "Month" = first day of
+     * the current calendar month in UTC.
+     */
+    getMonthlyCostCents(agentSlug) {
+        const startOfMonth = new Date();
+        startOfMonth.setUTCDate(1);
+        startOfMonth.setUTCHours(0, 0, 0, 0);
+        const sinceIso = startOfMonth.toISOString();
+        const where = agentSlug
+            ? 'WHERE agent_slug = ? AND created_at >= ?'
+            : 'WHERE agent_slug IS NULL AND created_at >= ?';
+        const params = agentSlug ? [agentSlug, sinceIso] : [sinceIso];
+        try {
+            const row = this.conn
+                .prepare(`SELECT COALESCE(SUM(cost_cents), 0) as total FROM usage_log ${where}`)
+                .get(...params);
+            return row?.total ?? 0;
+        }
+        catch {
+            return 0;
         }
     }
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clementine-agent",
-  "version": "1.1.23",
+  "version": "1.1.25",
   "description": "Clementine — Personal AI Assistant (TypeScript)",
   "type": "module",
   "main": "dist/index.js",