clementine-agent 1.0.0

package/dist/memory/mmr.js

@@ -0,0 +1,75 @@
+/**
+ * Clementine TypeScript — Maximal Marginal Relevance re-ranking.
+ *
+ * Ensures search results are both relevant AND diverse by penalizing
+ * results too similar to already-selected ones.
+ * Uses Jaccard similarity on tokenized content (no embedding provider needed).
+ */
+/** Tokenize text into lowercase word set for Jaccard similarity. */
+export function tokenize(text) {
+    return new Set(text.toLowerCase()
+        .replace(/[^\w\s]/g, ' ')
+        .split(/\s+/)
+        .filter(w => w.length > 2));
+}
+/** Jaccard similarity between two token sets: |A∩B| / |A∪B| */
+export function jaccard(a, b) {
+    if (a.size === 0 && b.size === 0)
+        return 1;
+    let intersection = 0;
+    for (const token of a) {
+        if (b.has(token))
+            intersection++;
+    }
+    const union = a.size + b.size - intersection;
+    return union === 0 ? 0 : intersection / union;
+}
+/**
+ * Re-rank results using Maximal Marginal Relevance.
+ *
+ * @param results - Scored search results (higher score = more relevant)
+ * @param lambda - Balance: 0 = max diversity, 1 = max relevance (default 0.7)
+ * @param limit - Max results to return
+ */
+export function mmrRerank(results, lambda = 0.7, limit) {
+    if (results.length === 0)
+        return [];
+    const maxResults = limit ?? results.length;
+    if (results.length <= 1)
+        return results.slice(0, maxResults);
+    // Normalize scores to [0, 1]
+    const scores = results.map(r => r.score);
+    const minScore = Math.min(...scores);
+    const maxScore = Math.max(...scores);
+    const range = maxScore - minScore;
+    const normScores = scores.map(s => range === 0 ? 1 : (s - minScore) / range);
+    // Cache tokenized content
+    const tokens = results.map(r => tokenize(r.content));
+    const selected = [];
+    const remaining = new Set(results.map((_, i) => i));
+    for (let step = 0; step < maxResults && remaining.size > 0; step++) {
+        let bestIdx = -1;
+        let bestMmr = -Infinity;
+        for (const i of remaining) {
+            const relevance = normScores[i];
+            // Max similarity to any already-selected result
+            let maxSim = 0;
+            for (const j of selected) {
+                const sim = jaccard(tokens[i], tokens[j]);
+                if (sim > maxSim)
+                    maxSim = sim;
+            }
+            const mmr = lambda * relevance - (1 - lambda) * maxSim;
+            if (mmr > bestMmr) {
+                bestMmr = mmr;
+                bestIdx = i;
+            }
+        }
+        if (bestIdx === -1)
+            break;
+        selected.push(bestIdx);
+        remaining.delete(bestIdx);
+    }
+    return selected.map(i => results[i]);
+}
+//# sourceMappingURL=mmr.js.map

package/dist/memory/search.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Clementine TypeScript — Search result helpers.
+ *
+ * Utility functions for temporal decay, deduplication, and formatting
+ * of search results for system prompt injection.
+ */
+import type { SearchResult } from '../types.js';
+export { mmrRerank } from './mmr.js';
+/**
+ * Exponential decay multiplier based on age.
+ *
+ * score = exp(-0.693 * days / halfLife)
+ * At halfLife days, score = 0.5. At 0 days, score = 1.0.
+ */
+export declare function temporalDecay(daysOld: number, halfLife?: number): number;
+/**
+ * Deduplicate results by (sourceFile, section), keeping the highest-scored.
+ */
+export declare function deduplicateResults(results: SearchResult[]): SearchResult[];
+/**
+ * Format search results as a context block for the system prompt.
+ *
+ * Truncates to stay within maxChars.
+ */
+export declare function formatResultsForPrompt(results: SearchResult[], maxChars?: number): string;
+//# sourceMappingURL=search.d.ts.map

package/dist/memory/search.js

@@ -0,0 +1,67 @@
+/**
+ * Clementine TypeScript — Search result helpers.
+ *
+ * Utility functions for temporal decay, deduplication, and formatting
+ * of search results for system prompt injection.
+ */
+export { mmrRerank } from './mmr.js';
+/**
+ * Exponential decay multiplier based on age.
+ *
+ * score = exp(-0.693 * days / halfLife)
+ * At halfLife days, score = 0.5. At 0 days, score = 1.0.
+ */
+export function temporalDecay(daysOld, halfLife = 30) {
+    if (daysOld <= 0)
+        return 1.0;
+    return Math.exp(-0.693 * daysOld / halfLife);
+}
+/**
+ * Deduplicate results by (sourceFile, section), keeping the highest-scored.
+ */
+export function deduplicateResults(results) {
+    const seen = new Map();
+    for (const r of results) {
+        const key = `${r.sourceFile}\0${r.section}`;
+        const existing = seen.get(key);
+        if (!existing || r.score > existing.score) {
+            seen.set(key, r);
+        }
+    }
+    return Array.from(seen.values());
+}
+/**
+ * Format search results as a context block for the system prompt.
+ *
+ * Truncates to stay within maxChars.
+ */
+export function formatResultsForPrompt(results, maxChars = 8000) {
+    if (results.length === 0)
+        return '';
+    const parts = [];
+    let total = 0;
+    for (const r of results) {
+        // Add relative time annotation so the agent can reference recency naturally
+        let timeHint = '';
+        if (r.lastUpdated) {
+            const daysAgo = Math.floor((Date.now() - new Date(r.lastUpdated).getTime()) / 86_400_000);
+            if (daysAgo === 0)
+                timeHint = ' (today)';
+            else if (daysAgo === 1)
+                timeHint = ' (yesterday)';
+            else if (daysAgo < 7)
+                timeHint = ` (${daysAgo} days ago)`;
+            else if (daysAgo < 30)
+                timeHint = ` (${Math.floor(daysAgo / 7)} weeks ago)`;
+            else if (daysAgo < 365)
+                timeHint = ` (${Math.floor(daysAgo / 30)} months ago)`;
+        }
+        const entry = `### ${r.sourceFile} > ${r.section}${timeHint}\n${r.content}\n`;
+        if (total + entry.length > maxChars)
+            break;
+        parts.push(entry);
+        total += entry.length;
+    }
+    return parts.join('\n');
+}
+//# sourceMappingURL=search.js.map

package/dist/memory/store.d.ts

@@ -0,0 +1,530 @@
+/**
+ * Clementine TypeScript — SQLite FTS5 memory store.
+ *
+ * Mirrors the Obsidian vault as a search-optimized index. The vault remains
+ * the source of truth; this is a read-optimized cache.
+ *
+ * FTS5 = full-text search built into SQLite. Zero cost. Zero latency.
+ *
+ * Concurrency: WAL mode allows concurrent readers. Writes are serialized
+ * (single-user, one MCP subprocess handles all writes).
+ */
+import type { Feedback, MemoryExtraction, SearchResult, SessionSummary, SyncStats, TranscriptTurn, WikilinkConnection } from '../types.js';
+export declare class MemoryStore {
+    private dbPath;
+    private vaultDir;
+    private db;
+    private _stmtChunkCount;
+    private _stmtInsertTranscript;
+    private _stmtInsertUsage;
+    constructor(dbPath: string, vaultDir: string);
+    /**
+     * Create the database and schema if needed.
+     */
+    initialize(): void;
+    /**
+     * Close the database connection.
+     */
+    close(): void;
+    /** Lazily-initializing accessor for the database connection. */
+    private get conn();
+    /** Return the total number of indexed chunks. */
+    getChunkCount(): number;
+    /**
+     * Scan the entire vault, hash-compare, and re-index changed files.
+     */
+    fullSync(): SyncStats;
+    /**
+     * Re-index a single file after a write operation.
+     */
+    updateFile(relPath: string, agentSlug?: string): void;
+    /**
+     * Full-text search using FTS5 with BM25 ranking.
+     */
+    searchFts(query: string, limit?: number, filters?: {
+        category?: string;
+        topic?: string;
+    }, isolateAgentSlug?: string): SearchResult[];
+    /**
+     * Get the most recently updated chunks.
+     */
+    getRecentChunks(limit?: number, agentSlug?: string, filters?: {
+        category?: string;
+        topic?: string;
+    }, strict?: boolean): SearchResult[];
+    /**
+     * Combined FTS5 relevance + recency search for context injection.
+     *
+     * Layer 3 of the memory architecture:
+     * 1. FTS5 search -> top N relevant
+     * 2. Recency fetch -> N most recent chunks
+     * 3. Deduplicate by (source_file, section)
+     * 4. Apply salience boost to FTS results
+     */
+    searchContext(query: string, limitOrOpts?: number | {
+        limit?: number;
+        recencyLimit?: number;
+        agentSlug?: string;
+        category?: string;
+        topic?: string;
+        strict?: boolean;
+    }, recencyLimitArg?: number): SearchResult[];
+    /**
+     * Search chunks by embedding cosine similarity.
+     * Scans chunks that have stored embeddings and returns top matches.
+     */
+    private searchByEmbedding;
+    /**
+     * Promote a memory chunk to global visibility (agent_slug = NULL).
+     * Used by agents to deliberately share an insight across the agent ecosystem.
+     * Does NOT copy the chunk — it promotes the existing chunk in-place.
+     *
+     * @param chunkId - ID of the chunk to promote
+     * @param promotedBy - slug of the agent promoting it (for audit log)
+     * @returns description of what was promoted, or error message
+     */
+    promoteToGlobal(chunkId: number, promotedBy?: string): string;
+    /**
+     * Get all notes connected to/from the given note via wikilinks.
+     */
+    getConnections(noteName: string): WikilinkConnection[];
+    /**
+     * Save a conversation turn to the transcripts table.
+     */
+    saveTurn(sessionKey: string, role: string, content: string, model?: string): void;
+    /**
+     * Get all turns for a given session, ordered chronologically.
+     */
+    getSessionTranscript(sessionKey: string): TranscriptTurn[];
+    /**
+     * Get recent transcript activity across all sessions since a given timestamp.
+     * Returns a compact summary of what happened (sessions, message counts, snippets).
+     */
+    getRecentActivity(sinceIso: string, maxEntries?: number): Array<{
+        sessionKey: string;
+        role: string;
+        content: string;
+        createdAt: string;
+    }>;
+    /**
+     * Search transcripts by keyword. Returns matching turns with context.
+     */
+    searchTranscripts(query: string, limit?: number, sessionKey?: string): TranscriptTurn[];
+    /**
+     * Save a session summary for cross-session context.
+     */
+    saveSessionSummary(sessionKey: string, summary: string, exchangeCount?: number): void;
+    /**
+     * Get the most recent session summaries.
+     */
+    getRecentSummaries(limit?: number): SessionSummary[];
+    /**
+     * Record that chunks were accessed (retrieved/displayed).
+     */
+    recordAccess(chunkIds: number[], accessType?: string): void;
+    /**
+     * Recompute salience score for a chunk based on access patterns.
+     *
+     * salience = frequency_bonus + recency_bonus
+     * frequency_bonus = log(access_count + 1) * 0.15
+     * recency_bonus = decay(days_since_last_access, half_life=7) * 0.3
+     */
+    private recomputeSalience;
+    /**
+     * Apply temporal decay to all chunk salience scores.
+     *
+     * Call daily (or on startup). Reduces salience for chunks that
+     * haven't been accessed recently, so stale memories naturally
+     * sink below active ones.
+     *
+     * decay = exp(-0.693 * daysSinceLastAccess / halfLife)
+     */
+    decaySalience(halfLifeDays?: number): number;
+    /**
+     * Prune stale data to keep the database bounded.
+     *
+     * - Deletes episodic chunks with salience < threshold and age > maxDays
+     * - Trims access_log entries older than retentionDays
+     * - Trims transcripts older than retentionDays
+     *
+     * Returns counts of deleted items.
+     */
+    pruneStaleData(opts?: {
+        maxAgeDays?: number;
+        salienceThreshold?: number;
+        accessLogRetentionDays?: number;
+        transcriptRetentionDays?: number;
+    }): {
+        episodicPruned: number;
+        accessLogPruned: number;
+        transcriptsPruned: number;
+    };
+    /**
+     * Get chunks within a date range, ordered chronologically.
+     * Useful for "what happened last week" type queries.
+     */
+    getTimeline(startDate: string, endDate: string, limit?: number): SearchResult[];
+    /**
+     * Index a session summary as an episodic memory chunk.
+     *
+     * These chunks have sector='episodic' and a synthetic source_file
+     * so they can be found by search but distinguished from vault content.
+     */
+    indexEpisodicChunk(sessionKey: string, summaryText: string): void;
+    /**
+     * Check if content is a duplicate of something already stored.
+     * Returns match info or null if content is unique.
+     *
+     * Strategy:
+     * 1. Exact match via content_hash (fast)
+     * 2. Near-duplicate via FTS5 BM25 + word overlap (conservative)
+     */
+    checkDuplicate(content: string, sourceFile?: string): {
+        isDuplicate: boolean;
+        matchType: 'exact' | 'near' | null;
+        matchId?: number;
+    };
+    /**
+     * Bump a chunk's salience and update its timestamp when a duplicate is detected.
+     * Instead of discarding duplicate mentions, this reinforces the existing chunk
+     * so frequently-mentioned facts surface higher in search results.
+     */
+    bumpChunkSalience(chunkId: number, boost?: number): void;
+    /**
+     * Log a memory extraction event for transparency tracking.
+     */
+    logExtraction(extraction: Omit<MemoryExtraction, 'id'>): void;
+    /**
+     * Get recent memory extractions, optionally filtered by status.
+     */
+    getRecentExtractions(limit?: number, status?: string): MemoryExtraction[];
+    /**
+     * Mark an extraction as corrected with a replacement fact.
+     * Also removes the wrong content from the search index so it stops surfacing.
+     */
+    correctExtraction(id: number, correction: string): void;
+    /**
+     * Dismiss an extraction (mark as invalid).
+     * Also removes the content from the search index.
+     */
+    dismissExtraction(id: number): void;
+    /**
+     * Get recent corrections to use as negative examples in auto-memory extraction.
+     * Returns corrections from the last 30 days so the extraction prompt knows
+     * what facts have been corrected and shouldn't be re-extracted.
+     */
+    getRecentCorrections(limit?: number): Array<{
+        toolInput: string;
+        correction: string;
+    }>;
+    /**
+     * Log feedback about response quality.
+     */
+    logFeedback(feedback: {
+        sessionKey?: string;
+        channel: string;
+        messageSnippet?: string;
+        responseSnippet?: string;
+        rating: 'positive' | 'negative' | 'mixed';
+        comment?: string;
+    }): void;
+    /**
+     * Get recent feedback entries.
+     */
+    getRecentFeedback(limit?: number): Feedback[];
+    /**
+     * Get aggregate feedback statistics.
+     */
+    getFeedbackStats(): {
+        positive: number;
+        negative: number;
+        mixed: number;
+        total: number;
+    };
+    saveSessionReflection(reflection: {
+        sessionKey: string;
+        exchangeCount: number;
+        frictionSignals: string[];
+        qualityScore: number;
+        behavioralCorrections: Array<{
+            correction: string;
+            category: string;
+            strength: string;
+        }>;
+        preferencesLearned: Array<{
+            preference: string;
+            confidence: string;
+        }>;
+        agentSlug?: string;
+    }): void;
+    getRecentReflections(limit?: number, agentSlug?: string): Array<{
+        sessionKey: string;
+        exchangeCount: number;
+        frictionSignals: string[];
+        qualityScore: number;
+        behavioralCorrections: Array<{
+            correction: string;
+            category: string;
+            strength: string;
+        }>;
+        preferencesLearned: Array<{
+            preference: string;
+            confidence: string;
+        }>;
+        agentSlug: string | null;
+        createdAt: string;
+    }>;
+    /** Get recurring behavioral corrections (appeared in 2+ sessions). */
+    getBehavioralPatterns(minOccurrences?: number): Array<{
+        correction: string;
+        category: string;
+        count: number;
+        lastSeen: string;
+    }>;
+    /**
+     * Log token usage from an SDK query result.
+     * Iterates modelUsage record and inserts one row per model.
+     */
+    logUsage(entry: {
+        sessionKey: string;
+        source: string;
+        modelUsage: Record<string, {
+            inputTokens: number;
+            outputTokens: number;
+            cacheReadInputTokens: number;
+            cacheCreationInputTokens: number;
+        }>;
+        numTurns: number;
+        durationMs: number;
+        agentSlug?: string;
+    }): void;
+    /**
+     * Get aggregated usage summary, optionally filtered by time.
+     */
+    getUsageSummary(sinceIso?: string): {
+        totalInput: number;
+        totalOutput: number;
+        totalCacheRead: number;
+        totalCacheCreation: number;
+        totalTokens: number;
+        byModel: Array<{
+            model: string;
+            input: number;
+            output: number;
+            cacheRead: number;
+        }>;
+        bySource: Array<{
+            source: string;
+            input: number;
+            output: number;
+        }>;
+        byDay: Array<{
+            day: string;
+            input: number;
+            output: number;
+        }>;
+    };
+    /**
+     * Get per-agent usage stats for observability dashboard.
+     */
+    getAgentStats(agentSlug: string, sinceIso?: string): {
+        totalInput: number;
+        totalOutput: number;
+        totalTokens: number;
+        numQueries: number;
+        avgTurns: number;
+        avgDurationMs: number;
+        bySource: Array<{
+            source: string;
+            count: number;
+            tokens: number;
+        }>;
+        byDay: Array<{
+            day: string;
+            tokens: number;
+            count: number;
+        }>;
+    };
+    /**
+     * Compare all agents by usage. Returns a leaderboard.
+     */
+    getAgentComparison(sinceIso?: string): Array<{
+        agentSlug: string;
+        totalTokens: number;
+        numQueries: number;
+        avgTurns: number;
+    }>;
+    /**
+     * Get usage summary for a specific session.
+     */
+    getSessionUsage(sessionKey: string): {
+        totalInput: number;
+        totalOutput: number;
+        totalTokens: number;
+        numQueries: number;
+    };
+    /**
+     * Get chunks that are candidates for consolidation:
+     * - Older than `minAgeDays`
+     * - Not already consolidated
+     * - Grouped by source file prefix (topic area)
+     *
+     * Returns groups with 3+ chunks that can be synthesized into summaries.
+     */
+    getConsolidationCandidates(minAgeDays?: number): Array<{
+        topic: string;
+        chunkIds: number[];
+        contents: string[];
+        totalChars: number;
+    }>;
+    /**
+     * Mark chunks as consolidated after they've been synthesized into a summary.
+     * Reduces salience so they appear lower in search results (but aren't deleted).
+     */
+    markConsolidated(chunkIds: number[]): void;
+    /**
+     * Get consolidation stats for monitoring.
+     */
+    getConsolidationStats(): {
+        totalChunks: number;
+        consolidated: number;
+        unconsolidated: number;
+    };
+    /**
+     * Insert a summary chunk created by the consolidation engine.
+     * Gets higher initial salience than regular chunks.
+     */
+    insertSummaryChunk(sourceFile: string, section: string, content: string): void;
+    upsertLead(lead: {
+        agentSlug: string;
+        email: string;
+        name: string;
+        company?: string;
+        title?: string;
+        status?: string;
+        source?: string;
+        sfId?: string;
+        metadata?: Record<string, unknown>;
+    }): {
+        id: number;
+        created: boolean;
+    };
+    searchLeads(filters: {
+        agentSlug?: string;
+        status?: string;
+        company?: string;
+        query?: string;
+        limit?: number;
+    }): Array<Record<string, unknown>>;
+    getLeadByEmail(email: string): Record<string, unknown> | undefined;
+    getLeadById(id: number): Record<string, unknown> | undefined;
+    enrollSequence(enrollment: {
+        leadId: number;
+        sequenceName: string;
+        nextStepDueAt?: string;
+    }): number;
+    advanceSequence(id: number, updates: {
+        currentStep?: number;
+        status?: string;
+        nextStepDueAt?: string | null;
+    }): void;
+    getDueSequences(agentSlug?: string): Array<Record<string, unknown>>;
+    getSequencesByLead(leadId: number): Array<Record<string, unknown>>;
+    logActivity(activity: {
+        leadId?: number;
+        agentSlug: string;
+        type: string;
+        subject?: string;
+        detail?: string;
+        templateUsed?: string;
+    }): number;
+    getActivities(filters: {
+        leadId?: number;
+        agentSlug?: string;
+        type?: string;
+        limit?: number;
+        sinceIso?: string;
+    }): Array<Record<string, unknown>>;
+    addSuppression(email: string, reason: string, addedBy?: string): void;
+    isSuppressed(email: string): boolean;
+    getSuppressionList(limit?: number): Array<Record<string, unknown>>;
+    logSend(entry: {
+        agentSlug: string;
+        recipient: string;
+        subject?: string;
+        templateUsed?: string;
+        policyRef?: string;
+    }): void;
+    getDailySendCount(agentSlug: string): number;
+    getSendLog(filters: {
+        agentSlug?: string;
+        limit?: number;
+        sinceIso?: string;
+    }): Array<Record<string, unknown>>;
+    addApproval(entry: {
+        agentSlug: string;
+        actionType: string;
+        summary: string;
+        detail?: Record<string, unknown>;
+    }): number;
+    resolveApproval(id: number, status: 'approved' | 'rejected', resolvedBy?: string): void;
+    getPendingApprovals(agentSlug?: string): Array<Record<string, unknown>>;
+    getApprovalById(id: number): Record<string, unknown> | undefined;
+    getAgentKpis(agentSlug: string, sinceIso?: string): Record<string, number>;
+    /** Get current month's token spend for an agent (in cents, estimated from token counts). */
+    getAgentMonthlySpend(agentSlug: string): number;
+    /** Check if an agent has exceeded its monthly budget. */
+    isOverBudget(agentSlug: string, budgetCents: number): boolean;
+    /** Snapshot a config file before writing changes. */
+    snapshotConfig(agentSlug: string, fileName: string, content: string, changedBy?: string): void;
+    /** Get revision history for an agent's config files. */
+    getConfigRevisions(agentSlug: string, fileName?: string, limit?: number): Array<Record<string, unknown>>;
+    /** Get a specific config revision's content. */
+    getConfigRevisionContent(id: number): string | null;
+    logSfSync(record: {
+        localTable: string;
+        localId: number;
+        sfObjectType: string;
+        sfId: string;
+        syncDirection: string;
+        syncStatus?: string;
+        errorMessage?: string;
+    }): number;
+    getSfSyncHistory(opts?: {
+        limit?: number;
+        sfObjectType?: string;
+        syncStatus?: string;
+    }): Array<Record<string, unknown>>;
+    getLeadBySfId(sfId: string): Record<string, unknown> | undefined;
+    getUnsyncedLeads(agentSlug?: string): Array<Record<string, unknown>>;
+    getLeadsModifiedSince(since: string, agentSlug?: string): Array<Record<string, unknown>>;
+    /**
+     * Build the TF-IDF vocabulary from all chunk contents, then backfill
+     * embeddings for any chunks that don't have one yet.
+     * Safe to call repeatedly — skips chunks that already have embeddings.
+     */
+    buildEmbeddings(): {
+        vocabSize: number;
+        backfilled: number;
+    };
+    /**
+     * Delete all chunks, wikilinks, file hash, and access log for a given file.
+     */
+    private deleteFileChunks;
+    /**
+     * Sanitize a query for FTS5 syntax.
+     *
+     * Quotes each word and joins with OR to match any word (not all).
+     * This works better for natural language queries.
+     */
+    static sanitizeFtsQuery(query: string): string;
+    /**
+     * Parse and index [[wikilinks]] from a file.
+     */
+    private indexWikilinks;
+    /**
+     * Recursively walk a directory for .md files.
+     */
+    private walkMdFiles;
+}
+//# sourceMappingURL=store.d.ts.map