clementine-agent 1.0.21 → 1.0.22

package/dist/agent/self-improve.js

@@ -611,63 +611,108 @@ export class SelfImproveLoop {
     async hypothesize(metrics, history) {
         // Read targeted triggers (written by cron scheduler when jobs fail repeatedly)
         let targetedTriggers = '';
+        const triggerBullets = [];
+        // Source 1: explicit triggers written by the cron scheduler at 3+
+        // consecutive errors (legacy path — we still honor and drain).
         const triggersDir = path.join(SELF_IMPROVE_DIR, 'triggers');
         if (existsSync(triggersDir)) {
             const triggerFiles = readdirSync(triggersDir).filter(f => f.endsWith('.json'));
-            if (triggerFiles.length > 0) {
-                const triggers = triggerFiles.slice(0, 3).map(f => {
-                    try {
-                        const t = JSON.parse(readFileSync(path.join(triggersDir, f), 'utf-8'));
-                        // Clean up trigger after reading
-                        unlinkSync(path.join(triggersDir, f));
-                        return t;
-                    }
-                    catch {
-                        return null;
-                    }
-                }).filter(Boolean);
-                if (triggers.length > 0) {
-                    targetedTriggers = `\n\n## PRIORITY: Failing Jobs Needing Attention\n` +
-                        `These jobs have been failing repeatedly and need prompt/config fixes:\n` +
-                        triggers.map((t) => `- **${t.jobName}**: ${t.consecutiveErrors} consecutive errors. Recent: ${(t.recentErrors ?? []).join('; ')}`).join('\n') +
-                        `\n\nFocus your improvement hypothesis on fixing these jobs first.\n`;
+            const triggers = triggerFiles.slice(0, 3).map(f => {
+                try {
+                    const t = JSON.parse(readFileSync(path.join(triggersDir, f), 'utf-8'));
+                    unlinkSync(path.join(triggersDir, f));
+                    return t;
+                }
+                catch {
+                    return null;
                 }
+            }).filter(Boolean);
+            for (const t of triggers) {
+                triggerBullets.push(`- **${t.jobName}**: ${t.consecutiveErrors} consecutive errors. Recent: ${(t.recentErrors ?? []).join('; ')}`);
             }
         }
+        // Source 2: broken-jobs from the failure monitor. These are jobs the
+        // user hasn't applied a fix for yet — real, current gaps the hypothesizer
+        // should target. Complements the diversity constraint: even if the area
+        // has been over-targeted historically, a specific broken job is a fresh
+        // concrete signal.
+        try {
+            const { computeBrokenJobs } = await import('../gateway/failure-monitor.js');
+            const broken = computeBrokenJobs();
+            for (const b of broken.slice(0, 3)) {
+                const diagHint = b.diagnosis
+                    ? ` Diagnosis: ${b.diagnosis.rootCause.slice(0, 120)}`
+                    : '';
+                triggerBullets.push(`- **${b.jobName}**: ${b.errorCount48h}/${b.totalRuns48h} failed in 48h${b.circuitBreakerEngagedAt ? ' (breaker engaged)' : ''}.${diagHint}`);
+            }
+        }
+        catch { /* failure-monitor module optional */ }
+        if (triggerBullets.length > 0) {
+            targetedTriggers = `\n\n## PRIORITY: Failing Jobs Needing Attention\n` +
+                `These jobs have been failing recently and need prompt/config fixes:\n` +
+                triggerBullets.join('\n') +
+                `\n\nFocus your improvement hypothesis on fixing these jobs first.\n`;
+        }
         // Format experiment history for the prompt
         const historyText = history.slice(-20).map(e => `#${e.iteration} | ${e.area} | "${e.hypothesis.slice(0, 60)}" | ${(e.score * 10).toFixed(1)}/10 ${e.accepted ? '✅' : '❌'}`).join('\n') || '(no prior experiments)';
-        // Enforce diversity: count recent proposals per area:target AND per area
+        // Enforce diversity: count recent proposals per area:target AND per area.
+        // A pair is only "over-targeted" if its MOST RECENT attempt was within
+        // the last 30 days — otherwise it's fair game to retry with fresh data.
+        // Stops the saturation state where after ~60 experiments the loop has
+        // blocked every area:target pair permanently and produces no new
+        // hypotheses (the Apr 11-19 plateau).
+        const DIVERSITY_WINDOW_MS = 30 * 24 * 60 * 60 * 1000;
+        const diversityCutoff = Date.now() - DIVERSITY_WINDOW_MS;
         const recentTargets = new Map();
         const recentAreas = new Map();
-        for (const e of history.slice(-10)) {
+        for (const e of history.slice(-50)) {
             const key = `${e.area}:${e.target}`;
-            recentTargets.set(key, (recentTargets.get(key) ?? 0) + 1);
-            recentAreas.set(e.area, (recentAreas.get(e.area) ?? 0) + 1);
+            const ts = Date.parse(e.startedAt);
+            const tsMs = Number.isFinite(ts) ? ts : 0;
+            const cur = recentTargets.get(key);
+            recentTargets.set(key, {
+                count: (cur?.count ?? 0) + 1,
+                newestMs: Math.max(cur?.newestMs ?? 0, tsMs),
+            });
+            const curA = recentAreas.get(e.area);
+            recentAreas.set(e.area, {
+                count: (curA?.count ?? 0) + 1,
+                newestMs: Math.max(curA?.newestMs ?? 0, tsMs),
+            });
         }
         for (const p of this.getPendingChanges()) {
             const key = `${p.area}:${p.target}`;
-            recentTargets.set(key, (recentTargets.get(key) ?? 0) + 1);
-            recentAreas.set(p.area, (recentAreas.get(p.area) ?? 0) + 1);
+            const now = Date.now();
+            const cur = recentTargets.get(key);
+            recentTargets.set(key, {
+                count: (cur?.count ?? 0) + 1,
+                newestMs: Math.max(cur?.newestMs ?? 0, now),
+            });
+            const curA = recentAreas.get(p.area);
+            recentAreas.set(p.area, {
+                count: (curA?.count ?? 0) + 1,
+                newestMs: Math.max(curA?.newestMs ?? 0, now),
+            });
         }
-        // Block area:target pairs with >= 2 recent proposals
+        // Block only when both (a) count is high enough AND (b) the last attempt
+        // was within the diversity window.
         const overTargeted = [...recentTargets.entries()]
-            .filter(([, count]) => count >= 2)
+            .filter(([, v]) => v.count >= 2 && v.newestMs > diversityCutoff)
             .map(([key]) => key);
-        // Block entire areas with >= 3 recent proposals
         const overTargetedAreas = [...recentAreas.entries()]
-            .filter(([, count]) => count >= 3)
+            .filter(([, v]) => v.count >= 3 && v.newestMs > diversityCutoff)
             .map(([area]) => area);
         // Build area coverage stats to nudge the LLM toward unexplored areas
         const allAreas = this.config.areas;
         const areaCoverage = allAreas.map(area => {
-            const count = recentAreas.get(area) ?? 0;
+            const count = recentAreas.get(area)?.count ?? 0;
             return `- ${area}: ${count} recent proposals`;
         }).join('\n');
         const diversityConstraint = `\n\n## AREA COVERAGE (target under-explored areas)\n${areaCoverage}\n` +
             (overTargeted.length > 0 || overTargetedAreas.length > 0
                 ? `\n## DIVERSITY CONSTRAINT\n` +
                     (overTargetedAreas.length > 0
-                        ? `These AREAS have been over-targeted and MUST NOT be chosen:\n${overTargetedAreas.map(a => `- ${a} (${recentAreas.get(a)} proposals)`).join('\n')}\n`
+                        ? `These AREAS have been over-targeted and MUST NOT be chosen:\n${overTargetedAreas.map(a => `- ${a} (${recentAreas.get(a)?.count ?? 0} proposals)`).join('\n')}\n`
                         : '') +
                     (overTargeted.length > 0
                         ? `These specific targets MUST NOT be re-targeted:\n${overTargeted.map(t => `- ${t}`).join('\n')}\n`

package/dist/gateway/claim-tracker.d.ts

@@ -13,6 +13,7 @@
  * Extraction is regex-only to keep cost at $0 per DM. For nuanced
  * claims the dashboard's manual verify/fail path covers the gap.
  */
+import type { Gateway } from './router.js';
 export type ClaimType = 'scheduled' | 'fixed' | 'will_do' | 'sent' | 'added' | 'unknown';
 export type VerifyStrategy = 'cron_run_check' | 'config_inspect' | 'manual';
 export type ClaimStatus = 'pending' | 'verified' | 'failed' | 'expired' | 'dismissed';
@@ -35,6 +36,13 @@ export interface Claim {
  * Caller supplies sessionKey for traceability. Never throws.
  */
 export declare function extractClaims(text: string, sessionKey?: string | null, agentSlug?: string | null): Omit<Claim, 'status' | 'extractedAt' | 'verifiedAt' | 'verdict'>[];
+/**
+ * Drain the LLM-fallback queue: pick up to N enqueued DMs, ask Haiku
+ * to extract claims via the same shape the regex patterns use, persist
+ * any found. Best-effort — errors just leave the queue unchanged for
+ * the next sweep.
+ */
+export declare function drainLLMFallback(gateway: Gateway, maxPerSweep?: number): Promise<number>;
 export declare function recordClaims(claims: Omit<Claim, 'status' | 'extractedAt' | 'verifiedAt' | 'verdict'>[]): Promise<void>;
 export declare function listClaims(opts?: {
     status?: ClaimStatus;

package/dist/gateway/claim-tracker.js

@@ -13,7 +13,7 @@
  * Extraction is regex-only to keep cost at $0 per DM. For nuanced
  * claims the dashboard's manual verify/fail path covers the gap.
  */
-import { randomBytes } from 'node:crypto';
+import { createHash, randomBytes } from 'node:crypto';
 import path from 'node:path';
 import pino from 'pino';
 import { BASE_DIR, MEMORY_DB_PATH, VAULT_DIR } from '../config.js';
@@ -115,6 +115,42 @@ const PATTERNS = [
         verifyStrategy: 'manual',
     },
 ];
+/**
+ * In-memory queue of DMs that regex-extraction missed but that look like
+ * they might contain claims (long enough, user-facing session). The
+ * heartbeat sweep drains this queue and runs the LLM fallback.
+ *
+ * Bounded to prevent memory growth — oldest entries are evicted.
+ */
+const MAX_PENDING_LLM = 20;
+const pendingLLMExtraction = [];
+function enqueueForLLM(text, sessionKey, agentSlug) {
+    // De-dup by text hash within the queue — don't re-enqueue the same DM.
+    const hash = sha1(text);
+    if (pendingLLMExtraction.some(e => sha1(e.text) === hash))
+        return;
+    pendingLLMExtraction.push({ text, sessionKey, agentSlug, queuedAt: Date.now() });
+    while (pendingLLMExtraction.length > MAX_PENDING_LLM)
+        pendingLLMExtraction.shift();
+}
+function sha1(s) {
+    return createHash('sha1').update(s).digest('hex');
+}
+/** Should a non-matching DM be considered for LLM fallback? */
+function isLLMFallbackCandidate(text, sessionKey) {
+    if (!sessionKey)
+        return false;
+    if (text.length < 100)
+        return false;
+    // Owner-facing DMs only. Skip heartbeat check-ins (they have their own
+    // gate) and skip cron notification messages that are the system talking
+    // about itself.
+    if (!sessionKey.startsWith('discord:') && !sessionKey.startsWith('slack:') && !sessionKey.startsWith('telegram:'))
+        return false;
+    if (text.startsWith('**[') && text.includes('check-in]'))
+        return false;
+    return true;
+}
 /**
  * Extract claims from a message. Returns empty array if nothing matched.
  * Caller supplies sessionKey for traceability. Never throws.
@@ -157,8 +193,116 @@ export function extractClaims(text, sessionKey, agentSlug) {
             agentSlug: agentSlug ?? null,
         });
     }
+    // Regex missed this DM but it looks like it could contain a claim the
+    // regex patterns can't catch ("Got that done", "Sent it, you should see
+    // it in a minute"). Queue for LLM fallback on the next heartbeat.
+    if (out.length === 0 && isLLMFallbackCandidate(text, sessionKey ?? null)) {
+        enqueueForLLM(text, sessionKey ?? null, agentSlug ?? null);
+    }
     return out;
 }
+/**
+ * Drain the LLM-fallback queue: pick up to N enqueued DMs, ask Haiku
+ * to extract claims via the same shape the regex patterns use, persist
+ * any found. Best-effort — errors just leave the queue unchanged for
+ * the next sweep.
+ */
+export async function drainLLMFallback(gateway, maxPerSweep = 3) {
+    let drained = 0;
+    const batch = pendingLLMExtraction.splice(0, Math.min(maxPerSweep, pendingLLMExtraction.length));
+    for (const item of batch) {
+        try {
+            const claims = await llmExtractClaims(item.text, gateway);
+            if (claims.length === 0)
+                continue;
+            const toRecord = claims.map(c => ({
+                id: randomBytes(6).toString('hex'),
+                sessionKey: item.sessionKey,
+                messageSnippet: item.text.slice(0, 400),
+                claimType: c.claimType,
+                subject: c.subject,
+                dueAt: c.dueAt,
+                verifyStrategy: c.verifyStrategy,
+                agentSlug: item.agentSlug,
+            }));
+            await recordClaims(toRecord);
+            drained += claims.length;
+        }
+        catch (err) {
+            logger.debug({ err }, 'LLM fallback extraction failed for one DM');
+        }
+    }
+    return drained;
+}
+async function llmExtractClaims(text, gateway) {
+    const prompt = [
+        'You are analyzing a chat message Clementine (an AI assistant) just sent to her owner. Did Clementine make any commitments, promises, or claims about something she did or will do?',
+        '',
+        'Only extract claims where there\'s a clear, concrete action. Do NOT extract:',
+        '- Status updates ("inbox has 5 messages")',
+        '- Questions ("Should I proceed?")',
+        '- Suggestions ("You might want to check X")',
+        '- Routine check-ins or greetings',
+        '',
+        'DO extract:',
+        '- "I scheduled X" / "I added Y to your tasks" / "I fixed Z"',
+        '- "I\'ll send X at Ypm" / "Will run Y tomorrow"',
+        '- "Sent email to X" / "Posted to #channel"',
+        '',
+        '## Message:',
+        text.slice(0, 1500),
+        '',
+        'Output a JSON object only (no fences):',
+        '{',
+        '  "claims": [',
+        '    {',
+        '      "claimType": "scheduled|fixed|will_do|sent|added",',
+        '      "subject": "short description of what (the noun phrase)",',
+        '      "dueAt": "ISO timestamp if a specific time was mentioned, else null"',
+        '    }',
+        '  ]',
+        '}',
+        'Empty array if no real commitments.',
+    ].join('\n');
+    let raw;
+    try {
+        raw = await gateway.handleCronJob('llm-claim-extract', prompt, 1, // tier 1
+        3, // tight maxTurns
+        'haiku');
+    }
+    catch {
+        return [];
+    }
+    try {
+        const m = raw.match(/\{[\s\S]*\}/);
+        if (!m)
+            return [];
+        const parsed = JSON.parse(m[0]);
+        const claims = parsed.claims ?? [];
+        const out = [];
+        const validTypes = ['scheduled', 'fixed', 'will_do', 'sent', 'added'];
+        for (const c of claims) {
+            if (typeof c.subject !== 'string' || !c.subject.trim())
+                continue;
+            const type = typeof c.claimType === 'string' && validTypes.includes(c.claimType)
+                ? c.claimType
+                : 'unknown';
+            if (type === 'unknown')
+                continue;
+            const dueAt = typeof c.dueAt === 'string' && /^\d{4}-\d{2}-\d{2}T/.test(c.dueAt) ? c.dueAt : null;
+            out.push({
+                claimType: type,
+                subject: c.subject.trim().slice(0, 200),
+                dueAt,
+                verifyStrategy: type === 'scheduled' ? 'cron_run_check' : 'manual',
+            });
+        }
+        return out;
+    }
+    catch {
+        return [];
+    }
+}
 // ── Persistence ──────────────────────────────────────────────────────
 async function getStore() {
     const { MemoryStore } = await import('../memory/store.js');

package/dist/gateway/failure-monitor.js

@@ -76,8 +76,48 @@ function readRunLog(filePath) {
         return [];
     }
 }
-function isFailure(entry) {
-    return entry.status === 'error' || entry.status === 'retried' || isSemanticFailure(entry);
+function isFailure(entry, gradeCache) {
+    if (entry.status === 'error' || entry.status === 'retried')
+        return true;
+    if (isSemanticFailure(entry))
+        return true;
+    // Outcome grader verdict, if we have one for this (job, time) tuple.
+    // Key format: `${jobName}|${startedAt}`. A `false` grade means the LLM
+    // judged the apparent-ok run as semantically failed.
+    if (gradeCache) {
+        const key = `${entry.jobName}|${entry.startedAt}`;
+        const passed = gradeCache.get(key);
+        if (passed === false)
+            return true;
+    }
+    return false;
+}
+/**
+ * Pre-load outcome grades for recent runs of all jobs so the synchronous
+ * isFailure check can consult them without hitting SQLite per call.
+ * Returns a map keyed by `${jobName}|${startedAt}` with the `passed` verdict.
+ */
+function loadGradeCache() {
+    const cache = new Map();
+    try {
+        const { MEMORY_DB_PATH } = require('../config.js');
+        if (!existsSync(MEMORY_DB_PATH))
+            return cache;
+        // eslint-disable-next-line @typescript-eslint/no-require-imports
+        const Database = require('better-sqlite3');
+        const db = new Database(MEMORY_DB_PATH, { readonly: true });
+        try {
+            const rows = db.prepare(`SELECT job_name, started_at, passed FROM graded_runs
+         WHERE graded_at >= datetime('now', '-14 days')`).all();
+            for (const r of rows) {
+                cache.set(`${r.job_name}|${r.started_at}`, r.passed === 1);
+            }
+        }
+        catch { /* graded_runs may not exist on older DBs */ }
+        db.close();
+    }
+    catch { /* non-fatal */ }
+    return cache;
 }
 /**
  * "Semantic failure" — a run the scheduler called `ok` but whose agent output
@@ -162,6 +202,7 @@ export function computeBrokenJobs(now = Date.now()) {
         return [];
     }
     const dormantCutoffMs = now - 7 * 24 * 60 * 60 * 1000;
+    const gradeCache = loadGradeCache();
     for (const file of files) {
         const entries = readRunLog(path.join(RUNS_DIR, file));
         if (entries.length === 0)
@@ -196,7 +237,7 @@ export function computeBrokenJobs(now = Date.now()) {
             const ts = Date.parse(e.startedAt);
             return Number.isFinite(ts) && ts >= sinceMs;
         });
-        const failures = inWindow.filter(isFailure);
+        const failures = inWindow.filter(e => isFailure(e, gradeCache));
         // Consecutive-failure signal: scan from most recent entry backward.
         // Stops at the first non-failure (ignoring 'skipped' which is neither
         // signal). Catches daily jobs that fail every run without accumulating
@@ -206,7 +247,7 @@ export function computeBrokenJobs(now = Date.now()) {
             const e = entries[i];
             if (e.status === 'skipped')
                 continue;
-            if (isFailure(e))
+            if (isFailure(e, gradeCache))
                 consecutiveFailures++;
             else
                 break;
@@ -221,7 +262,7 @@ export function computeBrokenJobs(now = Date.now()) {
         // back to the most recent errors anywhere in the log.
         const errSource = failures.length > 0
             ? failures
-            : entries.filter(isFailure);
+            : entries.filter(e => isFailure(e, gradeCache));
         const distinctErrors = [];
         const seen = new Set();
         for (let i = errSource.length - 1; i >= 0 && distinctErrors.length < 3; i--) {
@@ -413,6 +454,18 @@ function formatReport(jobs) {
  * Returns the jobs that triggered a fresh notification (mostly for tests/logs).
  */
 export async function runFailureSweep(send, gateway, now = Date.now()) {
+    // Opportunistically grade suspicious ok runs BEFORE computing broken
+    // jobs, so fresh grades feed into this same sweep's detection.
+    // Scoped to a handful of recent suspicious entries per job to keep cost
+    // bounded (~$0.01 per grade; cached forever).
+    if (gateway) {
+        try {
+            await gradeSuspiciousRecentRuns(gateway, now);
+        }
+        catch (err) {
+            logger.warn({ err }, 'Suspicious-run grading pre-pass failed (non-fatal)');
+        }
+    }
     const broken = computeBrokenJobs(now);
     if (broken.length === 0) {
         // Clear cooldowns AND diagnostic cache entries for jobs that recovered.
@@ -499,4 +552,54 @@ function appendAuditLog(action, jobNames) {
     }
     catch { /* non-fatal */ }
 }
+/**
+ * Scan each job's recent runs for suspicious apparent-ok entries and grade
+ * them. Each job contributes at most 2 LLM calls per sweep. Results are
+ * cached per (jobName, startedAt), so a suspicious run grades exactly once.
+ */
+async function gradeSuspiciousRecentRuns(gateway, now) {
+    const { isSuspicious, gradeRun, getGrade } = await import('./outcome-grader.js');
+    if (!existsSync(RUNS_DIR))
+        return;
+    // Only look at the last 48h so we're not burning grades on ancient entries.
+    const sinceMs = now - 48 * 60 * 60 * 1000;
+    let files = [];
+    try {
+        files = readdirSync(RUNS_DIR).filter(f => f.endsWith('.jsonl'));
+    }
+    catch {
+        return;
+    }
+    for (const file of files) {
+        const entries = readRunLog(path.join(RUNS_DIR, file));
+        const recent = entries
+            .filter(e => {
+            const ts = Date.parse(e.startedAt);
+            return Number.isFinite(ts) && ts >= sinceMs;
+        })
+            .filter(isSuspicious);
+        // Budget: at most 2 per job per sweep. Take the newest.
+        for (const entry of recent.slice(-2)) {
+            const cached = await getGrade(entry.jobName, entry.startedAt);
+            if (cached)
+                continue;
+            // Attempt to read the job prompt for richer grading context.
+            const jobPrompt = await loadJobPrompt(entry.jobName);
+            await gradeRun(entry, gateway, jobPrompt ?? undefined);
+        }
+    }
+}
+/** Load the current CRON.md prompt for a job. Returns null if not found. */
+async function loadJobPrompt(jobName) {
+    try {
+        const { parseCronJobs, parseAgentCronJobs } = await import('./cron-scheduler.js');
+        const { AGENTS_DIR } = await import('../config.js');
+        const allJobs = [...parseCronJobs(), ...parseAgentCronJobs(AGENTS_DIR)];
+        const job = allJobs.find(j => j.name === jobName);
+        return job?.prompt ?? null;
+    }
+    catch {
+        return null;
+    }
+}
 //# sourceMappingURL=failure-monitor.js.map

package/dist/gateway/heartbeat-scheduler.js

@@ -114,12 +114,25 @@ export class HeartbeatScheduler {
         }).catch(err => logger.warn({ err }, 'Failure sweep import failed'));
         // Claim verification sweep — auto-verify pending claims whose due
         // times have passed (e.g. "I scheduled X for 8am" → check at 9am).
-        import('./claim-tracker.js').then(({ verifyDueClaims }) => {
-            verifyDueClaims().then(({ verified, failed, expired }) => {
+        import('./claim-tracker.js').then(async ({ verifyDueClaims, drainLLMFallback }) => {
+            try {
+                const { verified, failed, expired } = await verifyDueClaims();
                 if (verified + failed + expired > 0) {
                     logger.info({ verified, failed, expired }, 'Claim verification sweep complete');
                 }
-            }).catch(err => logger.warn({ err }, 'Claim verification sweep failed'));
+            }
+            catch (err) {
+                logger.warn({ err }, 'Claim verification sweep failed');
+            }
+            // LLM fallback for regex-missed DMs — bounded batch per sweep
+            try {
+                const drained = await drainLLMFallback(this.gateway, 3);
+                if (drained > 0)
+                    logger.info({ count: drained }, 'LLM claim fallback extracted');
+            }
+            catch (err) {
+                logger.debug({ err }, 'LLM claim fallback failed (non-fatal)');
+            }
         }).catch(err => logger.warn({ err }, 'Claim tracker import failed'));
         const now = new Date();
         const hour = now.getHours();

package/dist/gateway/outcome-grader.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Clementine TypeScript — Outcome grader.
+ *
+ * Second-pass check for cron runs the scheduler called `ok` but that
+ * might actually be semantic failures. Covers the gap between the
+ * marker-based semantic detection in failure-monitor (which caught
+ * BLOCKED / FAILED / etc. in output) and the empty-output-too-short
+ * case (which false-positives on legitimate quiet healthchecks).
+ *
+ * Strategy: only invoke the LLM when the run is SUSPICIOUS (empty
+ * preview with non-trivial duration, or ambiguous content). Cost:
+ * bounded to ~$0.01 per suspicious run, cached forever per
+ * (job_name, started_at) tuple.
+ */
+import type { CronRunEntry } from '../types.js';
+import type { Gateway } from './router.js';
+export interface Grade {
+    jobName: string;
+    startedAt: string;
+    passed: boolean;
+    score: number;
+    reasoning: string;
+    gradedAt: string;
+}
+/**
+ * Decide whether a run warrants LLM grading. Heuristic — designed to
+ * fire on the exact pattern that slipped through today: apparent-ok
+ * runs with empty output + duration suggesting real work happened.
+ */
+export declare function isSuspicious(entry: CronRunEntry): boolean;
+export declare function getGrade(jobName: string, startedAt: string): Promise<Grade | null>;
+export declare function recordGrade(grade: Grade): Promise<void>;
+/**
+ * Grade a single cron run. Returns cached grade if we've already graded
+ * this (job, startedAt) tuple. Returns null if grading fails — caller
+ * should fall back to existing signals.
+ */
+export declare function gradeRun(entry: CronRunEntry, gateway: Gateway, jobPrompt?: string): Promise<Grade | null>;
+/** Look up recent grades for a job — used by the dashboard. */
+export declare function recentGrades(jobName: string, limit?: number): Promise<Grade[]>;
+//# sourceMappingURL=outcome-grader.d.ts.map

package/dist/gateway/outcome-grader.js

@@ -0,0 +1,173 @@
+/**
+ * Clementine TypeScript — Outcome grader.
+ *
+ * Second-pass check for cron runs the scheduler called `ok` but that
+ * might actually be semantic failures. Covers the gap between the
+ * marker-based semantic detection in failure-monitor (which caught
+ * BLOCKED / FAILED / etc. in output) and the empty-output-too-short
+ * case (which false-positives on legitimate quiet healthchecks).
+ *
+ * Strategy: only invoke the LLM when the run is SUSPICIOUS (empty
+ * preview with non-trivial duration, or ambiguous content). Cost:
+ * bounded to ~$0.01 per suspicious run, cached forever per
+ * (job_name, started_at) tuple.
+ */
+import pino from 'pino';
+import { MEMORY_DB_PATH, VAULT_DIR } from '../config.js';
+const logger = pino({ name: 'clementine.outcome-grader' });
+async function getStore() {
+    const { MemoryStore } = await import('../memory/store.js');
+    const store = new MemoryStore(MEMORY_DB_PATH, VAULT_DIR);
+    store.initialize();
+    return store;
+}
+/**
+ * Decide whether a run warrants LLM grading. Heuristic — designed to
+ * fire on the exact pattern that slipped through today: apparent-ok
+ * runs with empty output + duration suggesting real work happened.
+ */
+export function isSuspicious(entry) {
+    if (entry.status !== 'ok')
+        return false;
+    const preview = (entry.outputPreview ?? '').trim();
+    // Case 1: empty or near-empty preview with meaningful duration.
+    // 20s threshold catches today's empty-market-leader-followup pattern
+    // (23s + $0.57 cost, returned nothing). Legitimate quiet healthchecks
+    // can run 15-33s too — we'll grade them once, the LLM correctly judges
+    // "nothing to report" as passed, and the cached result means no re-grade.
+    if (preview.length < 20 && entry.durationMs > 20_000)
+        return true;
+    // Case 2: reasonable preview but contains soft-negative language that
+    // marker-based detection might miss. Kept tight so we don't spend on
+    // normal variance.
+    const lower = preview.toLowerCase();
+    if (/\b(partial|skipped\s+\d+|could\s+not\s+complete|insufficient|timeout(?!ed)|not\s+enough|attempting|retrying)\b/.test(lower)) {
+        return true;
+    }
+    return false;
+}
+export async function getGrade(jobName, startedAt) {
+    try {
+        const store = await getStore();
+        const db = store.conn;
+        const row = db.prepare(`SELECT job_name AS jobName, started_at AS startedAt, passed, score, reasoning, graded_at AS gradedAt
+       FROM graded_runs WHERE job_name = ? AND started_at = ?`).get(jobName, startedAt);
+        store.close();
+        if (!row)
+            return null;
+        return { ...row, passed: row.passed === 1 };
+    }
+    catch {
+        return null;
+    }
+}
+export async function recordGrade(grade) {
+    try {
+        const store = await getStore();
+        const db = store.conn;
+        db.prepare(`INSERT OR REPLACE INTO graded_runs (job_name, started_at, passed, score, reasoning)
+       VALUES (?, ?, ?, ?, ?)`).run(grade.jobName, grade.startedAt, grade.passed ? 1 : 0, grade.score, grade.reasoning);
+        store.close();
+    }
+    catch (err) {
+        logger.warn({ err, jobName: grade.jobName }, 'Failed to record grade');
+    }
+}
+function buildPrompt(entry, jobPrompt) {
+    return [
+        'You are judging whether a cron job execution actually accomplished its intent.',
+        '',
+        `## Job: ${entry.jobName}`,
+        `## Duration: ${Math.round(entry.durationMs / 1000)}s`,
+        '',
+        '## Job instructions (the prompt the agent was given):',
+        jobPrompt ? jobPrompt.slice(0, 2000) : '(instructions unavailable)',
+        '',
+        '## What the agent produced (output preview, may be truncated):',
+        (entry.outputPreview ?? '(empty)').slice(0, 1500),
+        '',
+        '## Your job',
+        'Decide: did the agent actually accomplish the task, or did it superficially succeed while failing semantically?',
+        'Examples of semantic success that looks like failure: a healthcheck that returns nothing because everything is healthy; a reply-detection sweep that returns nothing because no replies came in.',
+        'Examples of semantic failure that looks like success: the agent hits a blocker, logs status=ok, returns empty; the agent fails auth and returns a generic "cannot proceed"; the agent reports "attempting X" but never actually does X.',
+        '',
+        'Output ONLY a JSON object, no fences:',
+        '{',
+        '  "passed": true|false,',
+        '  "score": 0-5 (5 = clearly accomplished, 0 = clearly failed),',
+        '  "reasoning": "one sentence explaining your judgment"',
+        '}',
+    ].join('\n');
+}
+function parseGrade(raw) {
+    try {
+        const m = raw.match(/\{[\s\S]*\}/);
+        if (!m)
+            return null;
+        const p = JSON.parse(m[0]);
+        if (typeof p.passed !== 'boolean')
+            return null;
+        const score = typeof p.score === 'number' ? Math.max(0, Math.min(5, Math.round(p.score))) : (p.passed ? 4 : 1);
+        return {
+            passed: p.passed,
+            score,
+            reasoning: typeof p.reasoning === 'string' ? p.reasoning.slice(0, 300) : '(no reasoning)',
+        };
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Grade a single cron run. Returns cached grade if we've already graded
+ * this (job, startedAt) tuple. Returns null if grading fails — caller
+ * should fall back to existing signals.
+ */
+export async function gradeRun(entry, gateway, jobPrompt) {
+    // Cache lookup
+    const cached = await getGrade(entry.jobName, entry.startedAt);
+    if (cached)
+        return cached;
+    if (!isSuspicious(entry))
+        return null;
+    const prompt = buildPrompt(entry, jobPrompt ?? null);
+    let raw;
+    try {
+        raw = await gateway.handleCronJob(`grade:${entry.jobName}`, prompt, 1, // tier 1
+        3, // maxTurns — tight
+        'haiku');
+    }
+    catch (err) {
+        logger.warn({ err, jobName: entry.jobName }, 'Outcome grader LLM call failed');
+        return null;
+    }
+    const parsed = parseGrade(raw);
+    if (!parsed) {
+        logger.warn({ jobName: entry.jobName, rawHead: raw.slice(0, 200) }, 'Outcome grader returned unparseable response');
+        return null;
+    }
+    const grade = {
+        jobName: entry.jobName,
+        startedAt: entry.startedAt,
+        ...parsed,
+        gradedAt: new Date().toISOString(),
+    };
+    await recordGrade(grade);
+    logger.info({ jobName: grade.jobName, passed: grade.passed, score: grade.score }, 'Graded run');
+    return grade;
+}
+/** Look up recent grades for a job — used by the dashboard. */
+export async function recentGrades(jobName, limit = 10) {
+    try {
+        const store = await getStore();
+        const db = store.conn;
+        const rows = db.prepare(`SELECT job_name AS jobName, started_at AS startedAt, passed, score, reasoning, graded_at AS gradedAt
+       FROM graded_runs WHERE job_name = ? ORDER BY started_at DESC LIMIT ?`).all(jobName, limit);
+        store.close();
+        return rows.map(r => ({ ...r, passed: r.passed === 1 }));
+    }
+    catch {
+        return [];
+    }
+}
+//# sourceMappingURL=outcome-grader.js.map

package/dist/memory/store.js

@@ -436,6 +436,17 @@ export class MemoryStore {
       CREATE INDEX IF NOT EXISTS idx_claims_status ON claims(status, extracted_at DESC);
       CREATE INDEX IF NOT EXISTS idx_claims_due ON claims(due_at) WHERE status = 'pending';
       CREATE INDEX IF NOT EXISTS idx_claims_extracted ON claims(extracted_at DESC);
+
+      CREATE TABLE IF NOT EXISTS graded_runs (
+        job_name TEXT NOT NULL,
+        started_at TEXT NOT NULL,
+        passed INTEGER NOT NULL,
+        score INTEGER NOT NULL,
+        reasoning TEXT,
+        graded_at TEXT NOT NULL DEFAULT (datetime('now')),
+        PRIMARY KEY (job_name, started_at)
+      );
+      CREATE INDEX IF NOT EXISTS idx_graded_runs_job ON graded_runs(job_name, started_at DESC);
     `);
     }
     // ── Skill usage telemetry ─────────────────────────────────────────

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clementine-agent",
-  "version": "1.0.21",
+  "version": "1.0.22",
   "description": "Clementine — Personal AI Assistant (TypeScript)",
   "type": "module",
   "main": "dist/index.js",