clementine-agent 1.18.8 → 1.18.9

package/dist/agent/assistant.d.ts

@@ -251,8 +251,10 @@ export declare class PersonalAssistant {
     /** Fire-and-forget: extract a reusable skill from a successful execution. */
     private extractSkillFromExecution;
     private preRotationFlush;
-    private lastExtractionTime;
-    private worthExtracting;
+    private lastExtractionTimes;
+    private memoryExtractionKey;
+    private assessMemoryExtraction;
+    private logMemoryExtractionSkip;
     private spawnMemoryExtraction;
     private static readonly MEMORY_TOOL_NAMES;
     private extractMemory;

package/dist/agent/assistant.js

@@ -2872,12 +2872,16 @@ You have a cost budget per message — not a hard turn limit. Work until the tas
                 logger.warn({ err, sessionKey: key }, 'Transcript save failed');
             }
         }
-        // Fire background memory extraction (non-blocking)
-        if (text.length >= AUTO_MEMORY_MIN_LENGTH &&
-            responseText &&
-            !responseText.startsWith('Error:') &&
-            this.worthExtracting(text, responseText)) {
-            this.spawnMemoryExtraction(text, responseText, key, profile).catch(err => logger.debug({ err }, 'Memory extraction failed'));
+        // Fire background memory extraction (non-blocking). Skips are logged with
+        // structured reasons so memory gaps are diagnosable without reading chats.
+        if (responseText && !responseText.startsWith('Error:')) {
+            const extractionDecision = this.assessMemoryExtraction(text, responseText, key, profile);
+            if (extractionDecision.ok) {
+                this.spawnMemoryExtraction(text, responseText, key, profile).catch(err => logger.debug({ err }, 'Memory extraction failed'));
+            }
+            else {
+                this.logMemoryExtractionSkip(extractionDecision.reason, text, responseText, key, profile);
+            }
         }
         // Score outcome-driven salience: for the chunks we retrieved this turn,
         // check which actually showed up in the response and adjust their
@@ -4188,14 +4192,15 @@ You have a cost budget per message — not a hard turn limit. Work until the tas
         catch { /* non-fatal */ }
     }
     // ── Auto-Memory Extraction ────────────────────────────────────────
-    lastExtractionTime = 0;
-    worthExtracting(prompt, response) {
-        if (response.length < 100)
-            return false;
-        // Skip very short acknowledgment responses
-        if (response.length < 100)
-            return false;
-        // Only skip pure greetings with no substance at all
+    lastExtractionTimes = new Map();
+    memoryExtractionKey(sessionKey, profile) {
+        return `${profile?.slug ?? 'global'}:${sessionKey ?? 'no-session'}`;
+    }
+    assessMemoryExtraction(prompt, response, sessionKey, profile) {
+        if (!this.memoryStore)
+            return { ok: false, reason: 'no_memory_store' };
+        // Only skip pure greetings with no substance at all. Check this before
+        // length so "hey" is diagnosable as intent, not generic shortness.
         const pureGreetings = [
             'hello', 'hi', 'hey', 'thanks', 'thank you',
             'ok', 'okay', 'sure', 'got it', 'sounds good',
@@ -4203,20 +4208,54 @@ You have a cost budget per message — not a hard turn limit. Work until the tas
         ];
         const lower = prompt.toLowerCase().trim();
         if (pureGreetings.some((g) => lower === g || lower === g + '!' || lower === g + '.')) {
-            return false;
+            return { ok: false, reason: 'pure_greeting' };
+        }
+        if (prompt.length < AUTO_MEMORY_MIN_LENGTH || response.length < 100) {
+            return { ok: false, reason: 'too_short' };
         }
-        // Rate limit: max 1 extraction per 45 seconds per session
+        // Rate limit per session/agent. The old process-wide throttle meant one
+        // active chat could suppress memory extraction for unrelated agents.
         const now = Date.now();
-        if (now - this.lastExtractionTime < 45_000)
-            return false;
-        this.lastExtractionTime = now;
-        return true;
+        const key = this.memoryExtractionKey(sessionKey, profile);
+        const last = this.lastExtractionTimes.get(key) ?? 0;
+        if (now - last < 45_000)
+            return { ok: false, reason: 'rate_limited' };
+        this.lastExtractionTimes.set(key, now);
+        return { ok: true };
+    }
+    logMemoryExtractionSkip(reason, userMessage, assistantResponse, sessionKey, profile) {
+        logger.debug({
+            reason,
+            sessionKey,
+            agentSlug: profile?.slug,
+            promptChars: userMessage.length,
+            responseChars: assistantResponse.length,
+        }, 'Auto-memory extraction skipped');
+        if (!this.memoryStore)
+            return;
+        try {
+            this.memoryStore.logExtraction({
+                sessionKey: sessionKey ?? 'unknown',
+                userMessage: userMessage.slice(0, 500),
+                toolName: 'auto_memory_skip',
+                toolInput: JSON.stringify({
+                    reason,
+                    promptChars: userMessage.length,
+                    responseChars: assistantResponse.length,
+                }),
+                extractedAt: new Date().toISOString(),
+                status: `skipped:${reason}`,
+                agentSlug: profile?.slug,
+            });
+        }
+        catch { /* telemetry only */ }
     }
     async spawnMemoryExtraction(userMessage, assistantResponse, sessionKey, profile) {
         // Guard: skip memory extraction if the user message looks like injection
         const memScan = scanner.scan(userMessage);
         if (memScan.verdict === 'block') {
             logger.info('Skipping memory extraction — message was flagged as injection');
+            this.logMemoryExtractionSkip('injection_blocked', userMessage, assistantResponse, sessionKey, profile);
             return;
         }
         let currentMemory = '';

package/dist/agent/self-improve.d.ts

@@ -10,6 +10,37 @@
  */
 import type { SelfImproveConfig, SelfImproveExperiment, SelfImproveState } from '../types.js';
 import type { PersonalAssistant } from './assistant.js';
+export declare const USER_MODEL_SLOT_KEYS: readonly ["user_facts", "goals", "relationships", "agent_persona"];
+export type UserModelSlotKey = typeof USER_MODEL_SLOT_KEYS[number];
+export interface SelfImproveEvidenceSnapshot {
+    feedbackCreatedAt?: string[];
+    reflectionCreatedAt?: string[];
+    cronErrorStartedAt?: string[];
+    cronReflectionAt?: string[];
+    triggerUpdatedAt?: string[];
+    triggerCount?: number;
+    userModelNeedsSeed?: boolean;
+}
+export declare function isPlateauExperiment(entry: Partial<SelfImproveExperiment>): boolean;
+export declare function latestPlateauTime(history: Array<Partial<SelfImproveExperiment>>): number;
+export declare function newestSelfImproveEvidenceTime(evidence: SelfImproveEvidenceSnapshot): number;
+export declare function shouldSkipSelfImproveForPlateau(history: Array<Partial<SelfImproveExperiment>>, evidence: SelfImproveEvidenceSnapshot): {
+    skip: boolean;
+    reason?: string;
+};
+export declare function shouldAppendPlateauMarker(history: Array<Partial<SelfImproveExperiment>>, nowMs?: number, windowMs?: number): boolean;
+export declare function reconcileSelfImproveStateSnapshot(state: SelfImproveState, actualPending: number, opts?: {
+    nowMs?: number;
+    maxDurationMs?: number;
+    graceMs?: number;
+}): {
+    state: SelfImproveState;
+    changed: boolean;
+    diagnostics: string[];
+};
+export declare function normalizeUserModelSlots(slots: unknown): Partial<Record<UserModelSlotKey, string>>;
+export declare function sanitizeUserModelFrontmatter(frontmatterYaml: string): string;
+export declare function buildUserModelMarkdown(frontmatterYaml: string, slots: Partial<Record<UserModelSlotKey, string>>, updatedAt?: string): string;
 export declare class SelfImproveLoop {
     private config;
     private assistant;
@@ -18,6 +49,9 @@ export declare class SelfImproveLoop {
     /** Run a focused self-improvement cycle for a specific agent. */
     runForAgent(agentSlug: string, onProposal?: (experiment: SelfImproveExperiment) => Promise<void>): Promise<SelfImproveState>;
     private gatherMetrics;
+    private evidenceSnapshot;
+    private pendingTriggerEvidence;
+    private userModelNeedsSeed;
     private hypothesize;
     private readCurrentState;
     private evaluate;

package/dist/agent/self-improve.js

@@ -9,13 +9,14 @@
  * All proposed changes require Discord approval before being applied.
  */
 import { randomBytes } from 'node:crypto';
-import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, unlinkSync, writeFileSync, } from 'node:fs';
+import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync, unlinkSync, writeFileSync, } from 'node:fs';
 import matter from 'gray-matter';
 import { load as yamlLoad } from 'js-yaml';
 import path from 'node:path';
 import pino from 'pino';
 import { BASE_DIR, SELF_IMPROVE_DIR, SOUL_FILE, AGENTS_FILE, CRON_FILE, WORKFLOWS_DIR, VAULT_DIR, MEMORY_DB_PATH, AGENTS_DIR, CRON_REFLECTIONS_DIR, GOALS_DIR, } from '../config.js';
 import { listAllGoals } from '../tools/shared.js';
+import { MemoryStore } from '../memory/store.js';
 const logger = pino({ name: 'clementine.self-improve' });
 // ── Defaults ─────────────────────────────────────────────────────────
 const DEFAULT_CONFIG = {
@@ -157,6 +158,129 @@ function classifyRisk(area) {
         default: return 'high';
     }
 }
+export const USER_MODEL_SLOT_KEYS = ['user_facts', 'goals', 'relationships', 'agent_persona'];
+export function isPlateauExperiment(entry) {
+    return entry.reason?.startsWith('Plateau') === true
+        || entry.hypothesis?.startsWith('No new hypothesis') === true;
+}
+function entryTimeMs(entry) {
+    const ts = Date.parse(entry.finishedAt ?? entry.startedAt ?? '');
+    return Number.isFinite(ts) ? ts : 0;
+}
+export function latestPlateauTime(history) {
+    let latest = 0;
+    for (const entry of history) {
+        if (!isPlateauExperiment(entry))
+            continue;
+        latest = Math.max(latest, entryTimeMs(entry));
+    }
+    return latest;
+}
+function latestIsoTime(values = []) {
+    let latest = 0;
+    for (const value of values) {
+        const ts = Date.parse(value ?? '');
+        if (Number.isFinite(ts))
+            latest = Math.max(latest, ts);
+    }
+    return latest;
+}
+export function newestSelfImproveEvidenceTime(evidence) {
+    if (evidence.userModelNeedsSeed)
+        return Date.now();
+    return Math.max(latestIsoTime(evidence.feedbackCreatedAt), latestIsoTime(evidence.reflectionCreatedAt), latestIsoTime(evidence.cronErrorStartedAt), latestIsoTime(evidence.cronReflectionAt), latestIsoTime(evidence.triggerUpdatedAt));
+}
+export function shouldSkipSelfImproveForPlateau(history, evidence) {
+    const plateauAt = latestPlateauTime(history);
+    if (plateauAt === 0)
+        return { skip: false };
+    const newestEvidenceAt = newestSelfImproveEvidenceTime(evidence);
+    if (newestEvidenceAt > plateauAt)
+        return { skip: false };
+    return {
+        skip: true,
+        reason: `No new self-improvement evidence since latest plateau at ${new Date(plateauAt).toISOString()}`,
+    };
+}
+export function shouldAppendPlateauMarker(history, nowMs = Date.now(), windowMs = 24 * 60 * 60 * 1000) {
+    const plateauAt = latestPlateauTime(history);
+    return plateauAt === 0 || nowMs - plateauAt > windowMs;
+}
+export function reconcileSelfImproveStateSnapshot(state, actualPending, opts = {}) {
+    const next = { ...state };
+    const diagnostics = [];
+    let changed = false;
+    if (next.pendingApprovals !== actualPending) {
+        next.pendingApprovals = actualPending;
+        diagnostics.push(`Pending approvals reconciled to ${actualPending}`);
+        changed = true;
+    }
+    if (next.status === 'running' && next.lastRunAt) {
+        const started = Date.parse(next.lastRunAt);
+        const now = opts.nowMs ?? Date.now();
+        const staleAfter = (opts.maxDurationMs ?? DEFAULT_CONFIG.maxDurationMs) + (opts.graceMs ?? 5 * 60 * 1000);
+        if (Number.isFinite(started) && now - started > staleAfter) {
+            next.status = 'failed';
+            next.currentIteration = 0;
+            next.lastDiagnostic = `Stale self-improve run cleared after ${Math.round((now - started) / 60_000)} minutes`;
+            diagnostics.push(next.lastDiagnostic);
+            changed = true;
+        }
+    }
+    return { state: next, changed, diagnostics };
+}
+export function normalizeUserModelSlots(slots) {
+    if (!slots || typeof slots !== 'object')
+        return {};
+    const input = slots;
+    const out = {};
+    for (const key of USER_MODEL_SLOT_KEYS) {
+        const value = input[key];
+        if (typeof value === 'string' && value.trim()) {
+            out[key] = value.trim();
+        }
+        else if (Array.isArray(value)) {
+            const lines = value
+                .map(v => typeof v === 'string' ? v.trim() : '')
+                .filter(Boolean);
+            if (lines.length > 0)
+                out[key] = lines.map(v => `- ${v.replace(/^[-*]\s*/, '')}`).join('\n');
+        }
+    }
+    return out;
+}
+export function sanitizeUserModelFrontmatter(frontmatterYaml) {
+    const lines = frontmatterYaml
+        .trim()
+        .split(/\r?\n/)
+        .map(line => line.replace(/\s+$/, ''));
+    if (/^```(?:ya?ml)?\s*$/i.test(lines[0]?.trim() ?? '')) {
+        lines.shift();
+    }
+    if (/^```\s*$/.test(lines[lines.length - 1]?.trim() ?? '')) {
+        lines.pop();
+    }
+    while (lines[0]?.trim() === '---') {
+        lines.shift();
+    }
+    while (lines[lines.length - 1]?.trim() === '---') {
+        lines.pop();
+    }
+    const sanitized = lines
+        .filter(line => !/^\s*last_updated\s*:/i.test(line))
+        .join('\n')
+        .trim();
+    return sanitized || 'confidence_scores: {}';
+}
+export function buildUserModelMarkdown(frontmatterYaml, slots, updatedAt = new Date().toISOString()) {
+    const body = USER_MODEL_SLOT_KEYS
+        .filter(key => slots[key])
+        .map(key => `## ${key}\n\n${slots[key]}`)
+        .join('\n\n');
+    return `---\n${sanitizeUserModelFrontmatter(frontmatterYaml)}\nlast_updated: "${updatedAt}"\n---\n\n# User Model\n\n` +
+        `This file is auto-generated by the self-improvement loop. The DB-backed user_model_blocks table is the prompt source of truth.\n\n` +
+        `${body}\n`;
+}
 // ── SelfImproveLoop ──────────────────────────────────────────────────
 export class SelfImproveLoop {
     config;
@@ -204,6 +328,7 @@ export class SelfImproveLoop {
         state.status = 'running';
         state.lastRunAt = new Date().toISOString();
         state.currentIteration = 0;
+        delete state.lastDiagnostic;
         this.saveState(state);
         const loopStart = Date.now();
         const history = this.loadExperimentLog();
@@ -223,6 +348,15 @@ export class SelfImproveLoop {
                 cronSuccessRate: metrics.cronSuccessRate,
                 avgResponseQuality: 0, // Updated as we evaluate
             };
+            const plateauSkip = shouldSkipSelfImproveForPlateau(history, this.evidenceSnapshot(metrics));
+            if (plateauSkip.skip) {
+                logger.info({ reason: plateauSkip.reason }, 'Self-improve skipped after plateau — no new evidence');
+                state.status = 'completed';
+                state.currentIteration = 0;
+                state.lastDiagnostic = plateauSkip.reason;
+                this.saveState(state);
+                return state;
+            }
             // Synthesize feedback patterns and update user model before experiment loop
             await this.synthesizeFeedbackPatterns();
             await this.updateUserModel();
@@ -238,23 +372,31 @@ export class SelfImproveLoop {
                     // Record the plateau in the experiment log so it's not silently
                     // invisible. Helps the dashboard and failure monitor distinguish
                     // "exhausted diverse hypotheses" from "crashed mid-run".
-                    const plateauExperiment = {
-                        id: randomBytes(4).toString('hex'),
-                        iteration: i,
-                        startedAt: new Date(loopStart).toISOString(),
-                        finishedAt: new Date().toISOString(),
-                        durationMs: Date.now() - loopStart,
-                        area: 'soul',
-                        target: 'n/a',
-                        hypothesis: 'No new hypothesis — diversity constraint exhausted',
-                        proposedChange: '',
-                        baselineScore: 0,
-                        score: 0,
-                        accepted: false,
-                        approvalStatus: 'denied',
-                        reason: 'Plateau: no novel improvement area remaining',
-                    };
-                    this.appendExperimentLog(plateauExperiment);
+                    if (shouldAppendPlateauMarker(history)) {
+                        const plateauExperiment = {
+                            id: randomBytes(4).toString('hex'),
+                            iteration: i,
+                            startedAt: new Date(loopStart).toISOString(),
+                            finishedAt: new Date().toISOString(),
+                            durationMs: Date.now() - loopStart,
+                            area: 'soul',
+                            target: 'n/a',
+                            hypothesis: 'No new hypothesis — diversity constraint exhausted',
+                            proposedChange: '',
+                            baselineScore: 0,
+                            score: 0,
+                            accepted: false,
+                            approvalStatus: 'denied',
+                            reason: 'Plateau: no novel improvement area remaining',
+                        };
+                        this.appendExperimentLog(plateauExperiment);
+                        history.push(plateauExperiment);
+                        state.totalExperiments++;
+                    }
+                    else {
+                        logger.info('Recent plateau marker already exists — not appending duplicate');
+                    }
+                    state.lastDiagnostic = 'Plateau: no novel improvement area remaining';
                     break;
                 }
                 state.currentIteration = i;
@@ -644,6 +786,58 @@ export class SelfImproveLoop {
             advisorInsights,
         };
     }
+    evidenceSnapshot(metrics) {
+        const triggerEvidence = this.pendingTriggerEvidence();
+        return {
+            feedbackCreatedAt: metrics.negativeFeedback.map(f => f.createdAt).filter((v) => !!v),
+            reflectionCreatedAt: metrics.cronReflections.map(r => r.timestamp).filter(Boolean),
+            cronErrorStartedAt: metrics.cronErrors.map(e => e.startedAt).filter((v) => !!v),
+            cronReflectionAt: metrics.cronReflections.map(r => r.timestamp).filter(Boolean),
+            triggerUpdatedAt: triggerEvidence.updatedAt,
+            triggerCount: triggerEvidence.count,
+            userModelNeedsSeed: this.userModelNeedsSeed(metrics),
+        };
+    }
+    pendingTriggerEvidence() {
+        const triggersDir = path.join(SELF_IMPROVE_DIR, 'triggers');
+        try {
+            if (!existsSync(triggersDir))
+                return { count: 0, updatedAt: [] };
+            const files = readdirSync(triggersDir).filter(f => f.endsWith('.json'));
+            return {
+                count: files.length,
+                updatedAt: files.map(f => {
+                    try {
+                        return statSync(path.join(triggersDir, f)).mtime.toISOString();
+                    }
+                    catch {
+                        return '';
+                    }
+                }).filter(Boolean),
+            };
+        }
+        catch {
+            return { count: 0, updatedAt: [] };
+        }
+    }
+    userModelNeedsSeed(metrics) {
+        try {
+            const store = new MemoryStore(MEMORY_DB_PATH, VAULT_DIR);
+            store.initialize();
+            try {
+                const signalCount = store.getRecentReflections(30).length + store.getRecentFeedback(30).length;
+                if (signalCount < 5 && metrics.negativeFeedback.length + metrics.cronReflections.length < 5)
+                    return false;
+                return store.getAllUserModelBlocks().filter(b => b.content.trim()).length === 0;
+            }
+            finally {
+                store.close();
+            }
+        }
+        catch {
+            return false;
+        }
+    }
     // ── Steps 2-3: Diagnose + Hypothesize ────────────────────────────
     async hypothesize(metrics, history) {
         // Read targeted triggers (written by cron scheduler when jobs fail repeatedly)
@@ -948,11 +1142,17 @@ export class SelfImproveLoop {
                 : rawParsed ? [rawParsed] : [];
         if (opportunities.length === 0)
             return null;
-        // Pick the first opportunity that isn't over-targeted
-        const selected = opportunities.find((o) => {
+        // Pick the first opportunity that isn't over-targeted. Down-rank SOUL.md
+        // after plateau/denied loops unless feedback synthesis produced fresh
+        // SOUL candidates; most operational signals should become prompt
+        // overrides, advisor rules, agent changes, or cron fixes.
+        const isViableOpportunity = (o) => {
             const key = `${o.area}:${o.target}`;
             return !overTargeted.includes(key) && !overTargetedAreas.includes(o.area);
-        }) ?? opportunities[0];
+        };
+        const hasSoulCandidates = !!soulCandidatesText && !/No candidates/i.test(soulCandidatesText);
+        const recentSoulPlateau = history.slice(-10).some(e => e.area === 'soul' && isPlateauExperiment(e));
+        const selected = opportunities.find((o) => isViableOpportunity(o) && (o.area !== 'soul' || hasSoulCandidates || !recentSoulPlateau)) ?? opportunities.find(isViableOpportunity) ?? opportunities[0];
         // ── Step 2: Proposal — load only the target file, generate specific change ──
         const currentContent = await this.readCurrentState(selected.area, selected.target);
         const proposalPrompt = `You identified this as the highest-impact improvement:\n` +
@@ -1373,12 +1573,12 @@ export class SelfImproveLoop {
     /** Update the structured user model from interaction data. */
     async updateUserModel() {
         try {
-            const { MemoryStore } = await import('../memory/store.js');
             const store = new MemoryStore(MEMORY_DB_PATH, VAULT_DIR);
             store.initialize();
             const reflections = store.getRecentReflections(30);
             const feedback = store.getRecentFeedback(30);
             const patterns = store.getBehavioralPatterns(1);
+            const existingDbModel = store.renderUserModel();
             store.close();
             if (reflections.length + feedback.length < 5) {
                 logger.info('Not enough interaction data for user model update');
@@ -1397,36 +1597,74 @@ export class SelfImproveLoop {
             const feedbackSummary = feedback.slice(0, 15).map(f => `- [${f.rating}] ${f.channel}: ${f.comment || f.messageSnippet || '(no detail)'}`.slice(0, 120)).join('\n');
             const patternSummary = patterns.map(p => `- "${p.correction}" [${p.category}] x${p.count}`).join('\n');
             const prompt = `You are updating a structured user model based on interaction data. The model tracks the owner's expertise, priorities, communication preferences, and behavioral patterns.\n\n` +
-                `## Current Model\n${existingModel || '(empty — first synthesis)'}\n\n` +
+                `## Current Markdown Model\n${existingModel || '(empty — first synthesis)'}\n\n` +
+                `## Current DB Slots\n${existingDbModel || '(empty — no user_model_blocks populated yet)'}\n\n` +
                 `## Recent Session Reflections (${reflections.length})\n${reflectionSummary || '(none)'}\n\n` +
                 `## Recent Feedback (${feedback.length})\n${feedbackSummary || '(none)'}\n\n` +
                 `## Recurring Behavioral Patterns\n${patternSummary || '(none)'}\n\n` +
                 `## Instructions\n` +
-                `Output a YAML frontmatter block for USER_MODEL.md. Include these sections:\n` +
-                `- expertise: map of domain → level (beginner/intermediate/expert) based on how they interact\n` +
-                `- priorities: list of current focus areas with priority level\n` +
-                `- communication: style, verbosity, decision_making, time_sensitivity\n` +
-                `- patterns: morning/afternoon/evening behavioral patterns\n` +
-                `- confidence_scores: how confident each section is (0-1)\n\n` +
-                `Preserve existing data that's still accurate. Only update fields where new evidence supports a change.\n` +
-                `Output ONLY the YAML frontmatter block (--- delimited), no other text.`;
+                `Return a JSON object with:\n` +
+                `- frontmatterYaml: YAML body for USER_MODEL.md, without --- delimiters. Include expertise, priorities, communication, patterns, confidence_scores.\n` +
+                `- slots: object with exactly these keys: user_facts, goals, relationships, agent_persona.\n\n` +
+                `Slot guidance:\n` +
+                `- user_facts: durable facts about the owner, expertise, working style, identifiers that should always be top-of-mind.\n` +
+                `- goals: active intents/priorities only; omit stale or weakly inferred goals.\n` +
+                `- relationships: people, companies, agents, projects, and how they relate to the owner.\n` +
+                `- agent_persona: durable guidance for Clementine's behavior with this owner.\n\n` +
+                `Keep each slot concise (prefer under 1200 chars). Preserve existing data that's still accurate. Only update fields where evidence supports a change. Use an empty string for a slot with no evidence.\n` +
+                `Output ONLY valid JSON, no markdown.`;
             const result = await this.assistant.runPlanStep('si-user-model', prompt, {
                 tier: 1,
                 maxTurns: 1,
                 disableTools: true,
+                outputFormat: {
+                    type: 'json_schema',
+                    schema: {
+                        type: 'object',
+                        properties: {
+                            frontmatterYaml: { type: 'string' },
+                            slots: {
+                                type: 'object',
+                                properties: {
+                                    user_facts: { type: 'string' },
+                                    goals: { type: 'string' },
+                                    relationships: { type: 'string' },
+                                    agent_persona: { type: 'string' },
+                                },
+                                required: ['user_facts', 'goals', 'relationships', 'agent_persona'],
+                            },
+                        },
+                        required: ['frontmatterYaml', 'slots'],
+                    },
+                },
             });
-            // Extract YAML frontmatter from response
-            const yamlMatch = result.match(/---\s*\n([\s\S]*?)\n---/);
-            if (!yamlMatch) {
-                logger.warn('User model synthesis returned no YAML block');
+            const parsed = this.parseJsonResponse(result);
+            const slots = normalizeUserModelSlots(parsed?.slots);
+            const frontmatterYaml = typeof parsed?.frontmatterYaml === 'string' && parsed.frontmatterYaml.trim()
+                ? parsed.frontmatterYaml.trim()
+                : 'confidence_scores: {}';
+            if (Object.keys(slots).length === 0) {
+                logger.warn('User model synthesis returned no populated DB slots');
                 return;
             }
             const modelDir = path.join(VAULT_DIR, '00-System');
             if (!existsSync(modelDir))
                 mkdirSync(modelDir, { recursive: true });
-            const content = `---\n${yamlMatch[1].trim()}\nlast_updated: "${new Date().toISOString()}"\n---\n\n# User Model\n\nThis file is auto-generated by the self-improvement loop. It captures a structured understanding of the owner based on interaction patterns, feedback, and behavioral corrections.\n`;
+            const writeStore = new MemoryStore(MEMORY_DB_PATH, VAULT_DIR);
+            writeStore.initialize();
+            try {
+                for (const key of USER_MODEL_SLOT_KEYS) {
+                    const content = slots[key];
+                    if (content)
+                        writeStore.setUserModelBlock({ slot: key, content });
+                }
+            }
+            finally {
+                writeStore.close();
+            }
+            const content = buildUserModelMarkdown(frontmatterYaml, slots);
             writeFileSync(modelFile, content);
-            logger.info('User model updated: USER_MODEL.md');
+            logger.info({ slots: Object.keys(slots) }, 'User model updated: DB slots + USER_MODEL.md');
         }
         catch (err) {
             logger.error({ err }, 'User model update failed');
@@ -1481,12 +1719,14 @@ export class SelfImproveLoop {
     reconcileState() {
         const state = this.loadState();
         const actualPending = this.getPendingChanges().length;
-        if (state.pendingApprovals !== actualPending) {
-            logger.warn({ stored: state.pendingApprovals, actual: actualPending }, 'Pending approvals counter drift — reconciling');
-            state.pendingApprovals = actualPending;
-            this.saveState(state);
+        const reconciled = reconcileSelfImproveStateSnapshot(state, actualPending, {
+            maxDurationMs: this.config.maxDurationMs,
+        });
+        if (reconciled.changed) {
+            logger.warn({ diagnostics: reconciled.diagnostics, storedPending: state.pendingApprovals, actualPending }, 'Self-improve state drift — reconciling');
+            this.saveState(reconciled.state);
         }
-        return state;
+        return reconciled.state;
     }
     /** Expire pending proposals older than APPROVAL_TTL_MS. */
     expireStaleProposals() {

package/dist/gateway/router.js

@@ -1684,6 +1684,7 @@ export class Gateway {
                         `Last run: ${state.lastRunAt || 'never'}\n` +
                         `Total experiments: ${state.totalExperiments}\n` +
                         `Pending approvals: ${state.pendingApprovals}\n` +
+                        (state.lastDiagnostic ? `Diagnostic: ${state.lastDiagnostic}\n` : '') +
                         `Baseline — Feedback: ${(m.feedbackPositiveRatio * 100).toFixed(0)}% positive, ` +
                         `Cron: ${(m.cronSuccessRate * 100).toFixed(0)}% success, ` +
                         `Quality: ${m.avgResponseQuality.toFixed(2)}`;

package/dist/memory/store.d.ts

@@ -1282,12 +1282,23 @@ export declare class MemoryStore {
             count: number;
         }>;
         tableRowCounts: Record<string, number>;
+        recentActivity: {
+            recallTracesLast7d: number;
+            recallTracesLast30d: number;
+            extractionSkipsLast30d: number;
+        };
         topCitedLast30d: Array<{
             chunkId: number;
             sourceFile: string;
             section: string;
             refCount: number;
         }>;
+        userModelSlots: {
+            total: number;
+            populated: number;
+            global: number;
+            agentScoped: number;
+        };
         staleUserModelSlots: Array<{
             slot: string;
             ageDays: number;
@@ -1300,6 +1311,7 @@ export declare class MemoryStore {
             salience: number;
             lastOutcomeScore: number;
         }>;
+        selfImprovePlateausLast7d: number;
         chunkCacheStats: ReturnType<HotCache<number, unknown>['stats']>;
         writeQueue: {
             size: number;