clementine-agent 1.0.13 → 1.0.15

package/dist/gateway/failure-monitor.js

@@ -0,0 +1,416 @@
+/**
+ * Clementine TypeScript — Cron failure monitor.
+ *
+ * Surfaces cron jobs that have been failing repeatedly so they don't sit
+ * silently broken (which is what happened to ross-the-sdr:reply-detection —
+ * the existing circuit breaker fired ONCE at consErrors=5 and then went
+ * quiet for days).
+ *
+ * Threshold: a job is "broken" if either
+ *   - it has >= 3 error/retried entries in the last 48h, OR
+ *   - the circuit breaker engaged for it within the last 48h.
+ *
+ * Per-job 24h cooldown prevents re-spamming the owner with the same news.
+ *
+ * Read-only with respect to the cron run logs and advisor events; mutates
+ * only its own state file (cron/failure-monitor.json).
+ */
+import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync, } from 'node:fs';
+import path from 'node:path';
+import pino from 'pino';
+import { BASE_DIR } from '../config.js';
+const logger = pino({ name: 'clementine.failure-monitor' });
+const RUNS_DIR = path.join(BASE_DIR, 'cron', 'runs');
+const ADVISOR_EVENTS_FILE = path.join(BASE_DIR, 'cron', 'advisor-events.jsonl');
+const STATE_FILE = path.join(BASE_DIR, 'cron', 'failure-monitor.json');
+const SELF_IMPROVE_STATE_FILE = path.join(BASE_DIR, 'self-improve', 'state.json');
+const SELF_IMPROVE_LOG_FILE = path.join(BASE_DIR, 'self-improve', 'experiment-log.jsonl');
+/** A job is broken if it crosses any of these thresholds in the lookback window. */
+const ERRORS_IN_WINDOW = 3;
+const WINDOW_HOURS = 48;
+/**
+ * Independent of the window — a job whose last N runs are all failures is
+ * broken even if they're spread over days (daily cron jobs can't accumulate
+ * 3 failures in 48h, but 2 consecutive BLOCKED days is still broken).
+ */
+const CONSECUTIVE_FAILURES = 2;
+/** Don't re-DM the owner about the same broken job within this window. */
+const NOTIFY_COOLDOWN_HOURS = 24;
+function loadState() {
+    try {
+        if (!existsSync(STATE_FILE))
+            return { notified: {} };
+        const raw = JSON.parse(readFileSync(STATE_FILE, 'utf-8'));
+        return { notified: raw.notified ?? {} };
+    }
+    catch {
+        return { notified: {} };
+    }
+}
+function saveState(state) {
+    try {
+        mkdirSync(path.dirname(STATE_FILE), { recursive: true });
+        writeFileSync(STATE_FILE, JSON.stringify(state, null, 2));
+    }
+    catch (err) {
+        logger.warn({ err }, 'Failed to persist failure-monitor state');
+    }
+}
+function readRunLog(filePath) {
+    try {
+        return readFileSync(filePath, 'utf-8')
+            .trim()
+            .split('\n')
+            .filter(Boolean)
+            .map(line => {
+            try {
+                return JSON.parse(line);
+            }
+            catch {
+                return null;
+            }
+        })
+            .filter((e) => e !== null);
+    }
+    catch {
+        return [];
+    }
+}
+function isFailure(entry) {
+    return entry.status === 'error' || entry.status === 'retried' || isSemanticFailure(entry);
+}
+/**
+ * "Semantic failure" — a run the scheduler called `ok` but whose agent output
+ * self-reports the task didn't actually complete. We only flag on explicit
+ * block/failure markers in the preview; the duration-vs-output heuristic was
+ * tested against the live corpus and produced too many false positives on
+ * legitimately quiet jobs (healthchecks, inbox probes that return empty
+ * when there's nothing to report).
+ *
+ * Markers are drawn from observed failure modes in Ross's cron jobs
+ * (kernel-vs-local Bash, "BLOCKED (no local bash access)") plus generic
+ * agent self-reports.
+ */
+function isSemanticFailure(entry) {
+    if (entry.status !== 'ok')
+        return false;
+    const preview = (entry.outputPreview ?? '').trim();
+    if (!preview)
+        return false;
+    const previewLower = preview.toLowerCase();
+    // Match on word boundaries so "BLOCKED" matches "Result: BLOCKED" but
+    // "blockedBy" in a stray JSON fragment doesn't.
+    const markerRegexes = [
+        /\b(blocked|task_blocked|task_incomplete)\b/,
+        /\b(failed|could not|unable to|no local bash|permission denied)\b/,
+        /__nothing__/,
+    ];
+    for (const re of markerRegexes) {
+        if (re.test(previewLower))
+            return true;
+    }
+    return false;
+}
+/**
+ * Pull the most recent circuit-breaker engagement for a job, looking at the
+ * entire advisor log (not just the 48h window). A stuck breaker counts as a
+ * broken job even if it last fired weeks ago, because while engaged the job
+ * stops running entirely and produces no new failure entries.
+ *
+ * Returns the engagement timestamp (if currently engaged with no subsequent
+ * recovery) and the most recent advisor opinion string, if any.
+ */
+function lastCircuitBreakerEvent(jobName) {
+    if (!existsSync(ADVISOR_EVENTS_FILE))
+        return { engagedAt: null, lastOpinion: null };
+    let engagedAt = null;
+    let lastOpinion = null;
+    try {
+        const lines = readFileSync(ADVISOR_EVENTS_FILE, 'utf-8').trim().split('\n');
+        for (const line of lines) {
+            try {
+                const evt = JSON.parse(line);
+                if (evt.jobName !== jobName)
+                    continue;
+                // Capture the most recent opinion regardless of type
+                lastOpinion = `${evt.type}: ${evt.detail}`;
+                if (evt.type === 'circuit-breaker')
+                    engagedAt = evt.timestamp;
+                if (evt.type === 'circuit-recovery' || evt.type === 'auto-disabled')
+                    engagedAt = null;
+            }
+            catch { /* skip malformed */ }
+        }
+    }
+    catch { /* non-fatal */ }
+    return { engagedAt, lastOpinion };
+}
+/**
+ * Compute the current set of broken jobs by scanning all run logs.
+ * Pure function (state-free) — used both by the monitor sweep and the dashboard endpoint.
+ */
+export function computeBrokenJobs(now = Date.now()) {
+    if (!existsSync(RUNS_DIR))
+        return [];
+    const sinceMs = now - WINDOW_HOURS * 60 * 60 * 1000;
+    const broken = [];
+    let files = [];
+    try {
+        files = readdirSync(RUNS_DIR).filter(f => f.endsWith('.jsonl'));
+    }
+    catch {
+        return [];
+    }
+    const dormantCutoffMs = now - 7 * 24 * 60 * 60 * 1000;
+    for (const file of files) {
+        const entries = readRunLog(path.join(RUNS_DIR, file));
+        if (entries.length === 0)
+            continue;
+        const jobName = entries[0].jobName;
+        // Skip dormant jobs — if the last run is >7 days old the job is
+        // probably removed or renamed and its historical failures aren't
+        // actionable. Circuit breaker still counts because an engaged breaker
+        // is itself "the job stopped running".
+        const lastEntry = entries[entries.length - 1];
+        const lastRunMs = Date.parse(lastEntry.startedAt);
+        // Always consult the breaker state — a stuck breaker is the primary
+        // signal for "job has been silently broken for days".
+        const cb = lastCircuitBreakerEvent(jobName);
+        if (!cb.engagedAt && Number.isFinite(lastRunMs) && lastRunMs < dormantCutoffMs) {
+            continue;
+        }
+        const inWindow = entries.filter(e => {
+            const ts = Date.parse(e.startedAt);
+            return Number.isFinite(ts) && ts >= sinceMs;
+        });
+        const failures = inWindow.filter(isFailure);
+        // Consecutive-failure signal: scan from most recent entry backward.
+        // Stops at the first non-failure (ignoring 'skipped' which is neither
+        // signal). Catches daily jobs that fail every run without accumulating
+        // 3 in a 48h window.
+        let consecutiveFailures = 0;
+        for (let i = entries.length - 1; i >= 0; i--) {
+            const e = entries[i];
+            if (e.status === 'skipped')
+                continue;
+            if (isFailure(e))
+                consecutiveFailures++;
+            else
+                break;
+        }
+        const meetsThreshold = failures.length >= ERRORS_IN_WINDOW
+            || consecutiveFailures >= CONSECUTIVE_FAILURES
+            || !!cb.engagedAt;
+        if (!meetsThreshold)
+            continue;
+        // Gather up to 3 distinct error messages, newest first. Prefer in-window
+        // errors; if the breaker is engaged and there are no recent runs, fall
+        // back to the most recent errors anywhere in the log.
+        const errSource = failures.length > 0
+            ? failures
+            : entries.filter(isFailure);
+        const distinctErrors = [];
+        const seen = new Set();
+        for (let i = errSource.length - 1; i >= 0 && distinctErrors.length < 3; i--) {
+            const err = (errSource[i].error ?? '').trim();
+            if (!err)
+                continue;
+            const key = err.slice(0, 120);
+            if (seen.has(key))
+                continue;
+            seen.add(key);
+            distinctErrors.push(err.slice(0, 400));
+        }
+        const lastFailureEntry = failures[failures.length - 1] ?? errSource[errSource.length - 1] ?? null;
+        const agentSlug = jobName.includes(':') ? jobName.split(':')[0] : undefined;
+        broken.push({
+            jobName,
+            agentSlug,
+            errorCount48h: failures.length,
+            totalRuns48h: inWindow.length,
+            lastErrorAt: lastFailureEntry?.startedAt ?? null,
+            lastErrors: distinctErrors,
+            circuitBreakerEngagedAt: cb.engagedAt,
+            lastAdvisorOpinion: cb.lastOpinion,
+        });
+    }
+    // Also check the self-improve loop — it has its own log (not cron/runs/).
+    const siBroken = detectSelfImproveBreakage(now);
+    if (siBroken)
+        broken.push(siBroken);
+    // Most recently failing first
+    broken.sort((a, b) => {
+        const aT = a.lastErrorAt ? Date.parse(a.lastErrorAt) : 0;
+        const bT = b.lastErrorAt ? Date.parse(b.lastErrorAt) : 0;
+        return bT - aT;
+    });
+    return broken;
+}
+/**
+ * The self-improve loop writes to its own experiment-log.jsonl, not cron/runs/.
+ * Its breakage pattern is: state.lastRunAt keeps getting updated nightly but
+ * no new experiments are being appended (they're all failing pre-iteration),
+ * OR the most recent experiments are all errors, OR state.infraError is set.
+ *
+ * Returns a synthetic BrokenJob for the self-improve pseudo-job, or null if
+ * healthy / no data.
+ */
+function detectSelfImproveBreakage(now) {
+    if (!existsSync(SELF_IMPROVE_STATE_FILE))
+        return null;
+    let state = {};
+    try {
+        state = JSON.parse(readFileSync(SELF_IMPROVE_STATE_FILE, 'utf-8'));
+    }
+    catch {
+        return null;
+    }
+    const experiments = [];
+    if (existsSync(SELF_IMPROVE_LOG_FILE)) {
+        try {
+            const lines = readFileSync(SELF_IMPROVE_LOG_FILE, 'utf-8').trim().split('\n').filter(Boolean);
+            for (const line of lines.slice(-10)) {
+                try {
+                    experiments.push(JSON.parse(line));
+                }
+                catch { /* skip */ }
+            }
+        }
+        catch { /* non-fatal */ }
+    }
+    const lastRunMs = state.lastRunAt ? Date.parse(state.lastRunAt) : 0;
+    const lookback48h = now - 48 * 60 * 60 * 1000;
+    const staleLookback = now - 7 * 24 * 60 * 60 * 1000; // 7 days
+    const recentExperiments = experiments.filter(e => {
+        const ts = e.startedAt ? Date.parse(e.startedAt) : 0;
+        return Number.isFinite(ts) && ts >= staleLookback;
+    });
+    const recentErrors = recentExperiments.filter(e => e.approvalStatus === 'denied' && (e.reason?.startsWith('Error') ?? false));
+    // Three break modes:
+    //  a. state.infraError is set (loop detected unfixable infra issue)
+    //  b. all 3+ most recent experiments within lookback are errors
+    //  c. loop ran recently but no new experiments appeared (silent early-exit)
+    const hasInfraError = !!state.infraError;
+    const allRecentErrored = recentExperiments.length >= 3
+        && recentExperiments.every(e => e.approvalStatus === 'denied');
+    const silentEarlyExit = lastRunMs > lookback48h
+        && recentExperiments.length === 0;
+    if (!hasInfraError && !allRecentErrored && !silentEarlyExit)
+        return null;
+    const lastErrors = [];
+    for (let i = experiments.length - 1; i >= 0 && lastErrors.length < 3; i--) {
+        const err = (experiments[i].error ?? '').trim();
+        if (!err)
+            continue;
+        lastErrors.push(err.slice(0, 400));
+    }
+    // If we don't have an explicit infraError but the last recorded error
+    // looks schema-related, surface it — this captures the state where all
+    // iterations died with the same API 400 but state.infraError never got
+    // persisted (happens when MAX_INFRA_ERRORS isn't crossed within a run).
+    const lastLoggedError = experiments.length > 0 ? (experiments[experiments.length - 1].error ?? '') : '';
+    const inferredInfraSchema = /input_schema|tools\.\d+\.custom/i.test(lastLoggedError);
+    let opinion;
+    if (hasInfraError) {
+        opinion = `infra: ${state.infraError.category} — ${state.infraError.diagnostic.slice(0, 200)}`;
+    }
+    else if (silentEarlyExit && inferredInfraSchema) {
+        opinion = 'loop ran but produced no experiments — last logged error was an MCP tool schema validation (API 400). Check external MCP servers (claude_desktop_config.json, Claude Code settings) for a recently-updated package exposing a malformed input_schema.';
+    }
+    else if (silentEarlyExit) {
+        opinion = 'loop ran but produced no experiments — likely crashing before iteration (check metrics gathering or hypothesis generation)';
+    }
+    else {
+        opinion = `${recentErrors.length}/${recentExperiments.length} recent iterations errored`;
+    }
+    return {
+        jobName: 'self-improve',
+        agentSlug: undefined,
+        errorCount48h: recentErrors.length,
+        totalRuns48h: recentExperiments.length,
+        lastErrorAt: experiments[experiments.length - 1]?.startedAt ?? state.lastRunAt ?? null,
+        lastErrors,
+        circuitBreakerEngagedAt: hasInfraError ? state.lastRunAt ?? null : null,
+        lastAdvisorOpinion: opinion,
+    };
+}
+/** Format a broken-job report for the owner DM. */
+function formatReport(jobs) {
+    const lines = [];
+    lines.push(`🚨 **${jobs.length} cron job${jobs.length === 1 ? '' : 's'} repeatedly failing** (last ${WINDOW_HOURS}h)`);
+    lines.push('');
+    for (const j of jobs) {
+        const breaker = j.circuitBreakerEngagedAt ? ' · circuit breaker engaged' : '';
+        lines.push(`• \`${j.jobName}\` — ${j.errorCount48h}/${j.totalRuns48h} runs failed${breaker}`);
+        if (j.lastErrors.length > 0) {
+            const preview = j.lastErrors[0].split('\n')[0].slice(0, 140);
+            lines.push(`  Last error: ${preview}`);
+        }
+        if (j.lastAdvisorOpinion) {
+            lines.push(`  Advisor: ${j.lastAdvisorOpinion.slice(0, 140)}`);
+        }
+    }
+    lines.push('');
+    lines.push('Open the dashboard → Broken Jobs panel for the full picture.');
+    return lines.join('\n');
+}
+/**
+ * Run a sweep: identify currently-broken jobs, pick the ones we haven't
+ * notified about recently, and dispatch one consolidated DM.
+ *
+ * Returns the jobs that triggered a fresh notification (mostly for tests/logs).
+ */
+export async function runFailureSweep(send, now = Date.now()) {
+    const broken = computeBrokenJobs(now);
+    if (broken.length === 0) {
+        // Clear cooldowns for jobs that recovered so future failures notify promptly.
+        const state = loadState();
+        let mutated = false;
+        for (const name of Object.keys(state.notified)) {
+            if (!broken.find(b => b.jobName === name)) {
+                delete state.notified[name];
+                mutated = true;
+            }
+        }
+        if (mutated)
+            saveState(state);
+        return [];
+    }
+    const state = loadState();
+    const cooldownMs = NOTIFY_COOLDOWN_HOURS * 60 * 60 * 1000;
+    const fresh = [];
+    for (const job of broken) {
+        const prev = state.notified[job.jobName];
+        if (prev && now - Date.parse(prev.lastNotifiedAt) < cooldownMs)
+            continue;
+        fresh.push(job);
+    }
+    if (fresh.length === 0)
+        return [];
+    try {
+        await send(formatReport(fresh));
+        const stamp = new Date(now).toISOString();
+        for (const job of fresh) {
+            state.notified[job.jobName] = { lastNotifiedAt: stamp, lastErrorCount: job.errorCount48h };
+        }
+        saveState(state);
+        appendAuditLog('notified', fresh.map(j => j.jobName));
+        logger.info({ count: fresh.length, jobs: fresh.map(j => j.jobName) }, 'Failure monitor: notified owner');
+    }
+    catch (err) {
+        logger.warn({ err }, 'Failure monitor: notification dispatch failed');
+    }
+    return fresh;
+}
+function appendAuditLog(action, jobNames) {
+    try {
+        const auditPath = path.join(BASE_DIR, 'cron', 'failure-monitor.log');
+        appendFileSync(auditPath, JSON.stringify({
+            action,
+            jobs: jobNames,
+            timestamp: new Date().toISOString(),
+        }) + '\n');
+    }
+    catch { /* non-fatal */ }
+}
+//# sourceMappingURL=failure-monitor.js.map

package/dist/gateway/fix-verification.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Clementine TypeScript — Cron fix verification.
+ *
+ * When a CRON.md (global or per-agent) is edited, we record a "pending
+ * verification" for any job whose definition changed AND that is currently
+ * in a failing state. After that job's next non-skipped run, we DM the
+ * owner with the verdict — succeeded or still failing — so a self-reported
+ * "fix" can't go untested again.
+ */
+import type { CronJobDefinition, CronRunEntry } from '../types.js';
+interface PendingVerification {
+    jobName: string;
+    recordedAt: string;
+    preFailureCount: number;
+    preLastError: string | null;
+}
+/**
+ * Compare an old and new jobs list and record verifications for any job that:
+ *   - exists in both lists (new jobs aren't "fixes" of existing problems)
+ *   - has its definition hash changed
+ *   - is currently in a failing state per failure-monitor
+ *
+ * Disabled jobs and removed jobs are tracked too: if a previously failing
+ * job gets disabled or removed in the edit, we surface that as a "removed
+ * pending verification" rather than waiting for a run that will never come.
+ */
+export declare function recordEditsForFailingJobs(oldJobs: CronJobDefinition[], newJobs: CronJobDefinition[]): void;
+/**
+ * After a cron run completes, check whether we were waiting on a fix
+ * verification for this job. If so, send the owner a verdict and clear it.
+ *
+ * Skipped runs (circuit breaker, pre-check exit, etc.) don't carry signal
+ * and shouldn't count as a verdict either way.
+ */
+export declare function checkAndDeliverVerification(entry: CronRunEntry, send: (text: string) => Promise<unknown>): Promise<void>;
+/** Read-only accessor for dashboards or debugging. */
+export declare function listPendingVerifications(): PendingVerification[];
+export {};
+//# sourceMappingURL=fix-verification.d.ts.map

package/dist/gateway/fix-verification.js

@@ -0,0 +1,144 @@
+/**
+ * Clementine TypeScript — Cron fix verification.
+ *
+ * When a CRON.md (global or per-agent) is edited, we record a "pending
+ * verification" for any job whose definition changed AND that is currently
+ * in a failing state. After that job's next non-skipped run, we DM the
+ * owner with the verdict — succeeded or still failing — so a self-reported
+ * "fix" can't go untested again.
+ */
+import { existsSync, mkdirSync, readFileSync, writeFileSync, } from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import pino from 'pino';
+import { BASE_DIR } from '../config.js';
+import { computeBrokenJobs } from './failure-monitor.js';
+const logger = pino({ name: 'clementine.fix-verification' });
+const STATE_FILE = path.join(BASE_DIR, 'cron', 'fix-verifications.json');
+function loadState() {
+    try {
+        if (!existsSync(STATE_FILE))
+            return { pending: {} };
+        const raw = JSON.parse(readFileSync(STATE_FILE, 'utf-8'));
+        return { pending: raw.pending ?? {} };
+    }
+    catch {
+        return { pending: {} };
+    }
+}
+function saveState(state) {
+    try {
+        mkdirSync(path.dirname(STATE_FILE), { recursive: true });
+        writeFileSync(STATE_FILE, JSON.stringify(state, null, 2));
+    }
+    catch (err) {
+        logger.warn({ err }, 'Failed to persist fix-verification state');
+    }
+}
+/**
+ * Hash the job fields a fix could touch. Schedule + prompt + tier + mode +
+ * model + maxTurns + maxHours + workDir + preCheck + successCriteria are the
+ * only fields a "fix" would realistically change. We deliberately ignore
+ * `enabled` because disabling isn't a fix.
+ */
+function jobHash(j) {
+    const data = JSON.stringify({
+        schedule: j.schedule,
+        prompt: j.prompt,
+        tier: j.tier,
+        maxTurns: j.maxTurns,
+        model: j.model,
+        workDir: j.workDir,
+        mode: j.mode,
+        maxHours: j.maxHours,
+        preCheck: j.preCheck,
+        successCriteria: j.successCriteria,
+    });
+    return crypto.createHash('sha1').update(data).digest('hex').slice(0, 12);
+}
+/**
+ * Compare an old and new jobs list and record verifications for any job that:
+ *   - exists in both lists (new jobs aren't "fixes" of existing problems)
+ *   - has its definition hash changed
+ *   - is currently in a failing state per failure-monitor
+ *
+ * Disabled jobs and removed jobs are tracked too: if a previously failing
+ * job gets disabled or removed in the edit, we surface that as a "removed
+ * pending verification" rather than waiting for a run that will never come.
+ */
+export function recordEditsForFailingJobs(oldJobs, newJobs) {
+    const oldByName = new Map(oldJobs.map(j => [j.name, j]));
+    const newByName = new Map(newJobs.map(j => [j.name, j]));
+    const broken = computeBrokenJobs();
+    const brokenByName = new Map(broken.map(b => [b.jobName, b]));
+    const state = loadState();
+    const stamp = new Date().toISOString();
+    let mutated = false;
+    for (const [name, oj] of oldByName) {
+        const b = brokenByName.get(name);
+        if (!b)
+            continue; // not currently broken — nothing to verify
+        const nj = newByName.get(name);
+        if (!nj) {
+            // Job removed entirely. Treat as resolved by removal.
+            delete state.pending[name];
+            mutated = true;
+            logger.info({ job: name }, 'Failing job removed from CRON.md — verification cleared');
+            continue;
+        }
+        if (!nj.enabled) {
+            // Job disabled. Don't wait for a run; clear and note.
+            delete state.pending[name];
+            mutated = true;
+            logger.info({ job: name }, 'Failing job disabled in CRON.md — verification cleared');
+            continue;
+        }
+        if (jobHash(oj) === jobHash(nj))
+            continue; // no relevant changes
+        state.pending[name] = {
+            jobName: name,
+            recordedAt: stamp,
+            preFailureCount: b.errorCount48h,
+            preLastError: b.lastErrors[0] ?? null,
+        };
+        mutated = true;
+        logger.info({ job: name, preFailureCount: b.errorCount48h }, 'Recorded pending fix verification');
+    }
+    if (mutated)
+        saveState(state);
+}
+/**
+ * After a cron run completes, check whether we were waiting on a fix
+ * verification for this job. If so, send the owner a verdict and clear it.
+ *
+ * Skipped runs (circuit breaker, pre-check exit, etc.) don't carry signal
+ * and shouldn't count as a verdict either way.
+ */
+export async function checkAndDeliverVerification(entry, send) {
+    if (entry.status === 'skipped')
+        return;
+    const state = loadState();
+    const pending = state.pending[entry.jobName];
+    if (!pending)
+        return;
+    delete state.pending[entry.jobName];
+    saveState(state);
+    const ok = entry.status === 'ok';
+    const verdict = ok ? '✅ succeeded' : '⚠️ still failing';
+    const ageMin = Math.max(1, Math.round((Date.now() - Date.parse(pending.recordedAt)) / 60000));
+    const detail = ok
+        ? ''
+        : `\nError: ${(entry.error ?? 'unknown').split('\n')[0].slice(0, 200)}`;
+    const msg = `**[Fix verification]** \`${entry.jobName}\` ${verdict} on its first run after edit (${ageMin}m later).${detail}`;
+    try {
+        await send(msg);
+    }
+    catch (err) {
+        logger.warn({ err, job: entry.jobName }, 'Failed to send fix verification DM');
+    }
+}
+/** Read-only accessor for dashboards or debugging. */
+export function listPendingVerifications() {
+    return Object.values(loadState().pending);
+}
+//# sourceMappingURL=fix-verification.js.map

package/dist/gateway/heartbeat-scheduler.js

@@ -103,6 +103,13 @@ export class HeartbeatScheduler {
         catch (err) {
             logger.warn({ err }, 'Session eviction failed');
         }
+        // Cron failure sweep — surface jobs that have been silently failing.
+        // Runs every tick; per-job 24h cooldown lives inside the monitor.
+        import('./failure-monitor.js').then(({ runFailureSweep }) => {
+            runFailureSweep((text) => this.dispatcher.send(text, {})).catch(err => {
+                logger.warn({ err }, 'Failure sweep failed');
+            });
+        }).catch(err => logger.warn({ err }, 'Failure sweep import failed'));
         const now = new Date();
         const hour = now.getHours();
         // ── Nightly tasks: run regardless of active hours ─────────────────
@@ -174,6 +181,25 @@ export class HeartbeatScheduler {
                 logger.warn({ err }, 'Per-agent self-improvement scheduling error');
             }
         }
+        // Daily stale-skill archival: run once per day at 3 AM. Skills never
+        // retrieved in 90+ days (both frontmatter useCount and skill_usage empty)
+        // get moved to .archive/ so they stop competing in trigger matching.
+        if (hour === 3 && this.lastState.lastSkillDecayDate !== todayISO()) {
+            this.lastState.lastSkillDecayDate = todayISO();
+            this.saveState();
+            import('../agent/skill-extractor.js').then(({ archiveStaleSkills }) => {
+                try {
+                    const store = this.gateway.getMemoryStore();
+                    const archived = archiveStaleSkills(90, store ? (name) => store.skillRetrievalCount(name) : undefined);
+                    if (archived.length > 0) {
+                        logger.info({ count: archived.length, names: archived.slice(0, 5) }, 'Archived stale skills');
+                    }
+                }
+                catch (err) {
+                    logger.warn({ err }, 'Stale skill archival failed');
+                }
+            }).catch(err => logger.warn({ err }, 'Stale skill archival import failed'));
+        }
         // Evening memory consolidation: once per day between 7-9 PM
         if (hour >= 19 && hour < 21 && this.lastConsolidationDate !== todayISO()) {
             this.lastConsolidationDate = todayISO();
@@ -607,10 +633,41 @@ export class HeartbeatScheduler {
         const prompt = buildInsightPrompt(signals);
         if (!prompt)
             return;
-        // Run lightweight LLM call via gateway
-        const response = await this.gateway.handleCronJob('insight-check', prompt, 1, // tier 1
-        1, // max 1 turn (just rating + message)
-        'haiku');
+        // Run lightweight LLM call via gateway. Log success AND failure to the
+        // cron run log so the failure monitor can see hourly breakage.
+        // maxTurns bumped 1 → 3 because the agent needs to fan out ~4 parallel
+        // tool calls (activity_history, outlook_inbox, goal_list, task_list)
+        // before composing its rating — at 1 turn it always crashes with
+        // "Reached maximum number of turns".
+        const icStartedAt = new Date();
+        let response = null;
+        try {
+            response = await this.gateway.handleCronJob('insight-check', prompt, 1, // tier 1
+            3, // max 3 turns (parallel tool fan-out + synthesis)
+            'haiku');
+            this.runLog.append({
+                jobName: 'insight-check',
+                startedAt: icStartedAt.toISOString(),
+                finishedAt: new Date().toISOString(),
+                status: 'ok',
+                durationMs: Date.now() - icStartedAt.getTime(),
+                attempt: 1,
+                outputPreview: (response ?? '').slice(0, 200),
+            });
+        }
+        catch (err) {
+            this.runLog.append({
+                jobName: 'insight-check',
+                startedAt: icStartedAt.toISOString(),
+                finishedAt: new Date().toISOString(),
+                status: 'error',
+                durationMs: Date.now() - icStartedAt.getTime(),
+                attempt: 1,
+                error: String(err).slice(0, 400),
+                errorType: 'transient',
+            });
+            throw err;
+        }
         if (!response)
             return;
         const insight = parseInsightResponse(response);