clementine-agent 1.0.32 → 1.0.34

package/dist/memory/context-assembler.js

@@ -33,14 +33,15 @@ export async function assembleContext(options) {
             priority: 0,
             maxChars: 500,
             minRemainingBudget: 0,
-            resolve: () => {
+            resolve: (budget) => {
                 if (!fs.existsSync(idPath))
                     return '';
                 try {
                     const content = fs.readFileSync(idPath, 'utf-8').trim();
                     if (!content)
                         return '';
-                    return `## Identity\n\n${content}`;
+                    const block = `## Identity\n\n${content}`;
+                    return block.length > budget ? block.slice(0, budget) : block;
                 }
                 catch {
                     return '';
@@ -56,14 +57,15 @@ export async function assembleContext(options) {
             priority: 1,
             maxChars: isAutonomous ? 1000 : 2000,
             minRemainingBudget: 0,
-            resolve: () => {
+            resolve: (budget) => {
                 if (!fs.existsSync(wmPath))
                     return '';
                 try {
                     const content = fs.readFileSync(wmPath, 'utf-8').trim();
                     if (!content)
                         return '';
-                    return `## Working Memory (scratchpad)\n\n${content}`;
+                    const block = `## Working Memory (scratchpad)\n\n${content}`;
+                    return block.length > budget ? block.slice(0, budget) : block;
                 }
                 catch {
                     return '';
@@ -79,10 +81,15 @@ export async function assembleContext(options) {
             priority: 2,
             maxChars: isAutonomous ? 1000 : 2000,
             minRemainingBudget: 500,
-            resolve: () => skillCtx,
+            resolve: (budget) => skillCtx.length > budget ? skillCtx.slice(0, budget) : skillCtx,
         });
     }
     // Slot 3: Memory search results (core recall)
+    // formatResultsForPrompt respects the effective budget and breaks on
+    // entry boundaries (not mid-string), so we don't need the outer
+    // slice-truncation to kick in here. Previously this slot was double-
+    // truncated: formatter used its own 8000 cap, then the outer loop cut
+    // further by Math.min(maxChars, remaining), chopping entries in half.
     if (options.memoryResults && options.memoryResults.length > 0) {
         const results = options.memoryResults;
         slots.push({
@@ -90,10 +97,7 @@ export async function assembleContext(options) {
             priority: 3,
             maxChars: isAutonomous ? 2000 : 8000,
             minRemainingBudget: 200,
-            resolve: () => {
-                // formatResultsForPrompt already handles truncation within its own budget
-                return formatResultsForPrompt(results, isAutonomous ? 2000 : 8000);
-            },
+            resolve: (budget) => formatResultsForPrompt(results, budget),
         });
     }
     // Slot 4: Graph relationships (supplementary)
@@ -104,7 +108,7 @@ export async function assembleContext(options) {
             priority: 4,
             maxChars: 2000,
             minRemainingBudget: 500,
-            resolve: () => graphCtx,
+            resolve: (budget) => graphCtx.length > budget ? graphCtx.slice(0, budget) : graphCtx,
         });
     }
     // Sort by priority (lower number = higher priority)
@@ -121,18 +125,27 @@ export async function assembleContext(options) {
             continue;
         }
         try {
-            let content = await slot.resolve();
+            // The slot's effective budget is the smaller of its own maxChars and
+            // what's actually remaining across all slots. Passed into resolve so
+            // the slot produces right-sized content up front, not a mid-entry
+            // truncation after the fact.
+            const effectiveBudget = Math.min(slot.maxChars, remaining);
+            const content = await slot.resolve(effectiveBudget);
             if (!content) {
                 skipped.push(slot.name);
                 continue;
             }
-            // Truncate to the smaller of slot max and remaining budget
-            const limit = Math.min(slot.maxChars, remaining);
-            if (content.length > limit) {
-                content = content.slice(0, limit) + '\n...(truncated)';
+            // Safety net: if resolve() ignored the budget and returned too much,
+            // clip at a line boundary rather than a character boundary so we don't
+            // leave a malformed half-block in the prompt.
+            let finalContent = content;
+            if (content.length > effectiveBudget) {
+                const trimmed = content.slice(0, effectiveBudget);
+                const lastNewline = trimmed.lastIndexOf('\n');
+                finalContent = (lastNewline > 0 ? trimmed.slice(0, lastNewline) : trimmed) + '\n...(truncated)';
             }
-            parts.push(content);
-            remaining -= content.length;
+            parts.push(finalContent);
+            remaining -= finalContent.length;
             included.push(slot.name);
         }
         catch {

package/dist/memory/embeddings.d.ts

@@ -35,4 +35,11 @@ export declare function deserializeEmbedding(buf: Buffer): Float32Array;
  * Check if the embedding system is ready (vocabulary loaded with sufficient words).
  */
 export declare function isReady(): boolean;
+/**
+ * Stable hash of the current vocabulary's word→dimension mapping. When this
+ * changes, previously-stored embedding vectors become silently incorrect
+ * because dimension N now represents a different word. Callers (MemoryStore
+ * backfill) use this hash to detect staleness and invalidate stored vectors.
+ */
+export declare function getVocabHash(): string;
 //# sourceMappingURL=embeddings.d.ts.map

package/dist/memory/embeddings.js

@@ -9,6 +9,7 @@
  * Query-time: embed the query, compute cosine similarity against stored vectors.
  */
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { createHash } from 'node:crypto';
 import path from 'node:path';
 import pino from 'pino';
 import { BASE_DIR } from '../config.js';
@@ -163,6 +164,19 @@ export function isReady() {
     loadVocab();
     return vocabWords.length >= 50; // need at least 50 vocab words
 }
+/**
+ * Stable hash of the current vocabulary's word→dimension mapping. When this
+ * changes, previously-stored embedding vectors become silently incorrect
+ * because dimension N now represents a different word. Callers (MemoryStore
+ * backfill) use this hash to detect staleness and invalidate stored vectors.
+ */
+export function getVocabHash() {
+    loadVocab();
+    if (vocabWords.length === 0)
+        return '';
+    // Order-sensitive: dimension assignment depends on insertion order.
+    return createHash('sha1').update(vocabWords.join('|')).digest('hex').slice(0, 16);
+}
 const STOP_WORDS = new Set([
     'the', 'be', 'to', 'of', 'and', 'in', 'that', 'have', 'it', 'for',
     'not', 'on', 'with', 'he', 'as', 'you', 'do', 'at', 'this', 'but',

package/dist/memory/graph-store.d.ts

@@ -61,6 +61,22 @@ export declare class GraphStore {
     syncFromVault(vaultDir: string, agentsDir: string): Promise<GraphSyncStats>;
     extractAndStoreRelationships(triplets: RelationshipTriplet[]): Promise<void>;
     enrichWithGraphContext(entityIds: string[], _maxHops?: number): Promise<string>;
+    /**
+     * Drop Note nodes whose slug isn't in the caller-provided set of valid IDs.
+     * Wikilinks into deleted vault files leave dangling Note nodes with
+     * MENTIONS edges pointing at them — this cleans those up.
+     *
+     * Deliberately NOT auto-scheduled: blast radius is significant, and the
+     * caller (dashboard action, MCP tool, manual script) should supply the
+     * authoritative valid-IDs set. Runs DETACH DELETE so incoming edges go
+     * with the node.
+     *
+     * Returns counts of what was removed.
+     */
+    invalidateOrphanedNotes(validIds: Set<string>): Promise<{
+        scanned: number;
+        deleted: number;
+    }>;
 }
 export declare function getSharedGraphStore(persistenceDir: string): Promise<GraphStore | null>;
 //# sourceMappingURL=graph-store.d.ts.map

package/dist/memory/graph-store.js

@@ -580,6 +580,53 @@ export class GraphStore {
             return '';
         return '\n## Relationship Context\n' + lines.join('\n');
     }
+    /**
+     * Drop Note nodes whose slug isn't in the caller-provided set of valid IDs.
+     * Wikilinks into deleted vault files leave dangling Note nodes with
+     * MENTIONS edges pointing at them — this cleans those up.
+     *
+     * Deliberately NOT auto-scheduled: blast radius is significant, and the
+     * caller (dashboard action, MCP tool, manual script) should supply the
+     * authoritative valid-IDs set. Runs DETACH DELETE so incoming edges go
+     * with the node.
+     *
+     * Returns counts of what was removed.
+     */
+    async invalidateOrphanedNotes(validIds) {
+        if (!this.available)
+            return { scanned: 0, deleted: 0 };
+        if (validIds.size === 0) {
+            // Defense: refuse to run with an empty set — would delete every Note.
+            logger.warn('invalidateOrphanedNotes called with empty validIds — refusing to run');
+            return { scanned: 0, deleted: 0 };
+        }
+        let scanned = 0;
+        let deleted = 0;
+        try {
+            const res = await this.graph.query('MATCH (n:Note) RETURN n.id AS id');
+            const rows = (res.data ?? []);
+            scanned = rows.length;
+            for (const row of rows) {
+                const id = row.id;
+                if (!id || validIds.has(id))
+                    continue;
+                try {
+                    await this.graph.query('MATCH (n:Note {id: $id}) DETACH DELETE n', { params: { id } });
+                    deleted++;
+                }
+                catch (err) {
+                    logger.debug({ err, id }, 'Orphan Note deletion failed');
+                }
+            }
+        }
+        catch (err) {
+            logger.warn({ err }, 'invalidateOrphanedNotes query failed');
+        }
+        if (deleted > 0) {
+            logger.info({ scanned, deleted, validIdsSize: validIds.size }, 'Invalidated orphan Note nodes');
+        }
+        return { scanned, deleted };
+    }
 }
 // ── Shared Client Helper ───────────────────────────────────────────────
 /**

package/dist/memory/store.d.ts

@@ -174,10 +174,15 @@ export declare class MemoryStore {
         salienceThreshold?: number;
         accessLogRetentionDays?: number;
         transcriptRetentionDays?: number;
+        behavioralRetentionDays?: number;
     }): {
         episodicPruned: number;
         accessLogPruned: number;
         transcriptsPruned: number;
+        skillUsagePruned: number;
+        feedbackPruned: number;
+        reflectionsPruned: number;
+        usageLogPruned: number;
     };
     /**
      * Get chunks within a date range, ordered chronologically.
@@ -533,6 +538,7 @@ export declare class MemoryStore {
     buildEmbeddings(): {
         vocabSize: number;
         backfilled: number;
+        invalidated: number;
     };
     /**
      * Delete all chunks, wikilinks, file hash, and access log for a given file.

package/dist/memory/store.js

@@ -10,9 +10,11 @@
  * (single-user, one MCP subprocess handles all writes).
  */
 import { createHash } from 'node:crypto';
-import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync } from 'node:fs';
+import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs';
 import path from 'node:path';
 import Database from 'better-sqlite3';
+import { BASE_DIR } from '../config.js';
+import { temporalDecay } from './search.js';
 import * as embeddingsModule from './embeddings.js';
 import { chunkFile } from './chunker.js';
 import { mmrRerank } from './mmr.js';
@@ -184,6 +186,24 @@ export class MemoryStore {
         catch {
             // Index already exists
         }
+        // Hot-path indices: every chat turn sorts/filters chunks by updated_at
+        // (recency) and by (agent_slug, updated_at) for agent-scoped recent
+        // context. Without these the queries do full table scans.
+        try {
+            this.conn.exec('CREATE INDEX idx_chunks_updated_at ON chunks(updated_at DESC)');
+        }
+        catch { /* already exists */ }
+        try {
+            this.conn.exec('CREATE INDEX idx_chunks_agent_updated ON chunks(agent_slug, updated_at DESC)');
+        }
+        catch { /* already exists */ }
+        // Embedding filter — searchByEmbedding's base predicate is
+        // `embedding IS NOT NULL`; a partial index turns that into an
+        // index-only scan for the candidate set.
+        try {
+            this.conn.exec('CREATE INDEX idx_chunks_has_embedding ON chunks(id) WHERE embedding IS NOT NULL');
+        }
+        catch { /* already exists */ }
         // Access log table for salience tracking
         this.conn.exec(`
       CREATE TABLE IF NOT EXISTS access_log (
@@ -581,32 +601,35 @@ export class MemoryStore {
                 stats.filesDeleted++;
             }
         }
-        // Process changed/new files
-        for (const filePath of filesToUpdate) {
+        // Process changed/new files inside a single transaction so a 1000-file
+        // sync produces one WAL commit instead of 1000+. Prepared statements are
+        // hoisted out of the loop — better-sqlite3 caches by SQL text anyway, but
+        // the explicit handle avoids re-parsing and makes the intent clear.
+        const insertStmt = this.conn.prepare(`INSERT INTO chunks
+       (source_file, section, content, chunk_type, frontmatter_json, content_hash, category, topic)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
+        const upsertHashStmt = this.conn.prepare(`INSERT OR REPLACE INTO file_hashes (rel_path, content_hash, last_synced)
+       VALUES (?, ?, datetime('now'))`);
+        const processFile = (filePath) => {
             const rel = path.relative(this.vaultDir, filePath);
             const chunks = chunkFile(filePath, this.vaultDir);
             if (chunks.length === 0)
-                continue;
-            // Delete old chunks for this file
+                return;
             this.deleteFileChunks(rel);
-            // Insert new chunks
-            const insertStmt = this.conn.prepare(`INSERT INTO chunks
-         (source_file, section, content, chunk_type, frontmatter_json, content_hash, category, topic)
-         VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
             for (const chunk of chunks) {
                 insertStmt.run(chunk.sourceFile, chunk.section, chunk.content, chunk.chunkType, chunk.frontmatterJson, chunk.contentHash, chunk.category ?? null, chunk.topic ?? null);
             }
-            // Parse and index wikilinks
             this.indexWikilinks(rel, filePath);
-            // Update file hash
             const bytes = readFileSync(filePath);
             const fileHash = createHash('sha256').update(bytes).digest('hex').slice(0, 16);
-            this.conn
-                .prepare(`INSERT OR REPLACE INTO file_hashes (rel_path, content_hash, last_synced)
-           VALUES (?, ?, datetime('now'))`)
-                .run(rel, fileHash);
+            upsertHashStmt.run(rel, fileHash);
             stats.filesUpdated++;
-        }
+        };
+        const processAll = this.conn.transaction((files) => {
+            for (const f of files)
+                processFile(f);
+        });
+        processAll(filesToUpdate);
         // Count total chunks
         const countRow = this.conn
             .prepare('SELECT COUNT(*) as cnt FROM chunks')
@@ -711,20 +734,30 @@ export class MemoryStore {
      * Get the most recently updated chunks.
      */
     getRecentChunks(limit = 5, agentSlug, filters, strict = false) {
-        const mapRow = (row) => ({
-            sourceFile: row.source_file,
-            section: row.section,
-            content: row.content,
-            score: 0,
-            chunkType: row.chunk_type,
-            matchType: 'recency',
-            lastUpdated: row.updated_at ?? '',
-            chunkId: row.id,
-            salience: row.salience ?? 0,
-            agentSlug: row.agent_slug ?? null,
-            category: row.category,
-            topic: row.topic,
-        });
+        const now = Date.now();
+        const mapRow = (row) => {
+            // Score recency by exponential decay (half-life 30 days). Previously
+            // every recent row got score=0, which meant MMR's min-max normalization
+            // ranked them at the floor — a two-day-old chunk and a six-month-old
+            // chunk were indistinguishable. Decay lets recent results actually
+            // compete with FTS and vector matches during rerank.
+            const daysOld = row.updated_at ? (now - Date.parse(row.updated_at)) / 86_400_000 : 0;
+            const decayed = temporalDecay(daysOld);
+            return {
+                sourceFile: row.source_file,
+                section: row.section,
+                content: row.content,
+                score: decayed,
+                chunkType: row.chunk_type,
+                matchType: 'recency',
+                lastUpdated: row.updated_at ?? '',
+                chunkId: row.id,
+                salience: row.salience ?? 0,
+                agentSlug: row.agent_slug ?? null,
+                category: row.category,
+                topic: row.topic,
+            };
+        };
         // Build optional WHERE clauses for category/topic
         let filterSql = '';
         const filterParams = [];
@@ -838,17 +871,20 @@ export class MemoryStore {
      * Scans chunks that have stored embeddings and returns top matches.
      */
     searchByEmbedding(queryVec, limit, agentSlug, strict = false) {
-        const rows = this.conn
-            .prepare(`SELECT id, source_file, section, content, chunk_type, embedding, salience, agent_slug, updated_at, category, topic
-         FROM chunks
-         WHERE embedding IS NOT NULL`)
-            .all();
+        // Push agent-isolation into SQL so we don't deserialize embeddings for
+        // rows we'd immediately reject. Soft isolation (non-strict) still loads
+        // all embeddings because the boost is applied post-scoring, but at
+        // least strict mode no longer scans foreign-agent chunks.
+        let sql = 'SELECT id, source_file, section, content, chunk_type, embedding, salience, agent_slug, updated_at, category, topic FROM chunks WHERE embedding IS NOT NULL';
+        const params = [];
+        if (strict && agentSlug) {
+            sql += ' AND (agent_slug IS NULL OR agent_slug = ?)';
+            params.push(agentSlug);
+        }
+        const rows = this.conn.prepare(sql).all(...params);
         const scored = [];
         for (const row of rows) {
             try {
-                // Hard isolation: skip chunks from other agents (allow own + global)
-                if (strict && agentSlug && row.agent_slug !== null && row.agent_slug !== agentSlug)
-                    continue;
                 const vec = embeddingsModule.deserializeEmbedding(row.embedding);
                 const sim = embeddingsModule.cosineSimilarity(queryVec, vec);
                 if (sim < 0.15)
@@ -1148,6 +1184,10 @@ export class MemoryStore {
         const threshold = opts.salienceThreshold ?? 0.01;
         const accessRetention = opts.accessLogRetentionDays ?? 60;
         const transcriptRetention = opts.transcriptRetentionDays ?? 90;
+        // Behavioral telemetry kept longer than transcripts so the feedback loop
+        // (getFeedbackStats, getBehavioralPatterns, getSkillsToSuppress) has a
+        // wide enough window to aggregate meaningful signal.
+        const behavioralRetention = opts.behavioralRetentionDays ?? 180;
         // Prune stale episodic chunks (not vault-sourced content)
         const episodicResult = this.conn
             .prepare(`DELETE FROM chunks
@@ -1167,10 +1207,30 @@ export class MemoryStore {
             .prepare(`DELETE FROM transcripts
          WHERE created_at < datetime('now', ?)`)
             .run(`-${transcriptRetention} days`);
+        // Behavioral telemetry pruning — these tables were previously unbounded.
+        // Each is append-only, so a rolling window is safe; aggregate stats
+        // consume the window directly rather than historical totals.
+        const skillUsageResult = this.conn
+            .prepare(`DELETE FROM skill_usage WHERE retrieved_at < datetime('now', ?)`)
+            .run(`-${behavioralRetention} days`);
+        const feedbackResult = this.conn
+            .prepare(`DELETE FROM feedback WHERE created_at < datetime('now', ?)`)
+            .run(`-${behavioralRetention} days`);
+        const reflectionsResult = this.conn
+            .prepare(`DELETE FROM session_reflections WHERE created_at < datetime('now', ?)`)
+            .run(`-${behavioralRetention} days`);
+        // Usage log is denser (per-exchange) — keep a shorter window.
+        const usageResult = this.conn
+            .prepare(`DELETE FROM usage_log WHERE created_at < datetime('now', ?)`)
+            .run(`-${Math.min(behavioralRetention, 90)} days`);
         return {
             episodicPruned: episodicResult.changes,
             accessLogPruned: accessResult.changes,
             transcriptsPruned: transcriptResult.changes,
+            skillUsagePruned: skillUsageResult.changes,
+            feedbackPruned: feedbackResult.changes,
+            reflectionsPruned: reflectionsResult.changes,
+            usageLogPruned: usageResult.changes,
         };
     }
     // ── Timeline Query ─────────────────────────────────────────────
@@ -2045,25 +2105,55 @@ export class MemoryStore {
             .prepare('SELECT id, content FROM chunks')
             .all();
         if (rows.length === 0)
-            return { vocabSize: 0, backfilled: 0 };
+            return { vocabSize: 0, backfilled: 0, invalidated: 0 };
+        // Capture prior vocab hash BEFORE rebuild. If buildVocab produces a
+        // different word→dimension mapping, previously-stored embedding vectors
+        // become silently wrong (dimension N now represents a different word).
+        const hashFile = path.join(BASE_DIR, '.embedding-vocab.hash');
+        let priorHash = '';
+        try {
+            if (existsSync(hashFile))
+                priorHash = readFileSync(hashFile, 'utf-8').trim();
+        }
+        catch { /* first run */ }
         // Build vocabulary from entire corpus (including consolidated summaries)
         embeddingsModule.buildVocab(rows.map((r) => r.content));
         if (!embeddingsModule.isReady())
-            return { vocabSize: 0, backfilled: 0 };
+            return { vocabSize: 0, backfilled: 0, invalidated: 0 };
+        // If the vocab shifted, invalidate every stored vector so they re-embed
+        // against the new word→dim mapping. Without this, old vectors silently
+        // mismatch query vectors and cosine similarity returns nonsense.
+        const newHash = embeddingsModule.getVocabHash();
+        let invalidated = 0;
+        if (priorHash && priorHash !== newHash) {
+            const res = this.conn.prepare('UPDATE chunks SET embedding = NULL WHERE embedding IS NOT NULL').run();
+            invalidated = res.changes;
+            // Count is returned in the result object — callers (maintenance cycle)
+            // log it there. No local logger in this file to avoid the import.
+        }
+        try {
+            writeFileSync(hashFile, newHash);
+        }
+        catch { /* non-fatal */ }
         // Backfill embeddings for all chunks that don't have one
         const missing = this.conn
             .prepare('SELECT id, content FROM chunks WHERE embedding IS NULL')
             .all();
         const updateStmt = this.conn.prepare('UPDATE chunks SET embedding = ? WHERE id = ?');
         let backfilled = 0;
-        for (const row of missing) {
-            const vec = embeddingsModule.embed(row.content);
-            if (vec) {
-                updateStmt.run(embeddingsModule.serializeEmbedding(vec), row.id);
-                backfilled++;
+        // Wrap backfill in a transaction — potentially thousands of UPDATEs
+        // per vocab shift, and a single WAL commit is dramatically faster.
+        const backfillAll = this.conn.transaction((items) => {
+            for (const row of items) {
+                const vec = embeddingsModule.embed(row.content);
+                if (vec) {
+                    updateStmt.run(embeddingsModule.serializeEmbedding(vec), row.id);
+                    backfilled++;
+                }
             }
-        }
-        return { vocabSize: rows.length, backfilled };
+        });
+        backfillAll(missing);
+        return { vocabSize: rows.length, backfilled, invalidated };
     }
     // ── Helpers ───────────────────────────────────────────────────────
     /**

package/dist/tools/shared.d.ts

@@ -86,10 +86,15 @@ export type MemoryStoreType = {
         salienceThreshold?: number;
         accessLogRetentionDays?: number;
         transcriptRetentionDays?: number;
+        behavioralRetentionDays?: number;
     }): {
         episodicPruned: number;
         accessLogPruned: number;
         transcriptsPruned: number;
+        skillUsagePruned: number;
+        feedbackPruned: number;
+        reflectionsPruned: number;
+        usageLogPruned: number;
     };
     checkDuplicate(content: string, sourceFile?: string): {
         isDuplicate: boolean;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clementine-agent",
-  "version": "1.0.32",
+  "version": "1.0.34",
   "description": "Clementine — Personal AI Assistant (TypeScript)",
   "type": "module",
   "main": "dist/index.js",