clementine-agent 1.0.0

package/dist/memory/chunker.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Clementine TypeScript — Vault file chunker for memory indexing.
+ *
+ * Parses Markdown files into chunks by ## headers, extracts frontmatter,
+ * and splits oversized sections at paragraph boundaries.
+ */
+import type { Chunk } from '../types.js';
+/**
+ * Parse a Markdown file into chunks by ## headers.
+ *
+ * @param filePath - Absolute path to the Markdown file.
+ * @param vaultDir - Absolute path to the vault root.
+ * @returns List of Chunk objects. Empty if file should be skipped.
+ */
+export declare function chunkFile(filePath: string, vaultDir: string): Chunk[];
+/**
+ * Split Markdown body by ## headers.
+ *
+ * Content before the first ## header is labeled "preamble".
+ *
+ * @returns Array of [sectionName, sectionContent] tuples.
+ */
+export declare function splitByHeaders(body: string): [string, string][];
+/**
+ * Split text at paragraph boundaries (double newlines) to stay under maxChars.
+ */
+export declare function splitAtParagraphs(text: string, maxChars: number): string[];
+//# sourceMappingURL=chunker.d.ts.map

package/dist/memory/chunker.js

@@ -0,0 +1,226 @@
+/**
+ * Clementine TypeScript — Vault file chunker for memory indexing.
+ *
+ * Parses Markdown files into chunks by ## headers, extracts frontmatter,
+ * and splits oversized sections at paragraph boundaries.
+ */
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import path from 'node:path';
+import matter from 'gray-matter';
+/** Directories to skip when scanning the vault. */
+const SKIP_DIRS = new Set(['06-Templates', '.obsidian']);
+/** Maximum chunk size before splitting at paragraph boundaries. */
+const MAX_CHUNK_CHARS = 3000;
+/** Directory-to-category mapping for vault structure. */
+const DIR_CATEGORY_MAP = {
+    '00-System': 'advice',
+    '01-Daily-Notes': 'events',
+    '02-People': 'facts',
+    '03-Projects': 'discoveries',
+    '04-Topics': 'facts',
+    '05-Tasks': 'advice',
+    '07-Inbox': 'events',
+};
+/** Content keyword patterns for category detection (used as fallback). */
+const CATEGORY_KEYWORDS = [
+    [/\b(prefer|always use|never use|i like|i don'?t like|i hate)\b/i, 'preferences'],
+    [/\b(learned|discovered|TIL|turns out|insight|breakthrough)\b/i, 'discoveries'],
+    [/\b(reminder|tip|rule of thumb|always|never|best practice)\b/i, 'advice'],
+];
+/**
+ * Detect category and topic for a chunk based on vault path, frontmatter, and content.
+ */
+function detectCategoryAndTopic(relPath, frontmatter, content) {
+    // Category detection (cascade)
+    let category = null;
+    // 1. Explicit frontmatter category
+    if (frontmatter.category) {
+        const fm = String(frontmatter.category).toLowerCase();
+        if (['facts', 'events', 'discoveries', 'preferences', 'advice'].includes(fm)) {
+            category = fm;
+        }
+    }
+    // 2. Directory-based
+    if (!category) {
+        const topDir = relPath.split('/')[0];
+        category = DIR_CATEGORY_MAP[topDir] ?? null;
+    }
+    // 3. Content keyword heuristics (only if nothing else matched)
+    if (!category) {
+        for (const [pattern, cat] of CATEGORY_KEYWORDS) {
+            if (pattern.test(content)) {
+                category = cat;
+                break;
+            }
+        }
+    }
+    // Topic detection (cascade)
+    let topic = null;
+    // 1. Explicit frontmatter topic or first tag
+    if (frontmatter.topic) {
+        topic = String(frontmatter.topic);
+    }
+    else if (Array.isArray(frontmatter.tags) && frontmatter.tags.length > 0) {
+        topic = String(frontmatter.tags[0]);
+    }
+    // 2. Second path segment (subdirectory name)
+    if (!topic) {
+        const parts = relPath.split('/');
+        if (parts.length >= 3) {
+            topic = parts[1];
+        }
+    }
+    return { category, topic };
+}
+/**
+ * Compute a truncated SHA-256 content hash (first 16 hex chars).
+ */
+function contentHash(text) {
+    return createHash('sha256').update(text).digest('hex').slice(0, 16);
+}
+/**
+ * Parse a Markdown file into chunks by ## headers.
+ *
+ * @param filePath - Absolute path to the Markdown file.
+ * @param vaultDir - Absolute path to the vault root.
+ * @returns List of Chunk objects. Empty if file should be skipped.
+ */
+export function chunkFile(filePath, vaultDir) {
+    const relPath = path.relative(vaultDir, filePath);
+    // Skip templates and .obsidian
+    for (const skip of SKIP_DIRS) {
+        if (relPath.startsWith(skip)) {
+            return [];
+        }
+    }
+    let raw;
+    try {
+        raw = readFileSync(filePath, 'utf-8');
+    }
+    catch {
+        return [];
+    }
+    let parsed;
+    try {
+        parsed = matter(raw);
+    }
+    catch {
+        return [];
+    }
+    const fmJson = parsed.data && Object.keys(parsed.data).length > 0
+        ? JSON.stringify(parsed.data)
+        : '';
+    const chunks = [];
+    const { category, topic } = detectCategoryAndTopic(relPath, parsed.data ?? {}, parsed.content);
+    // Add frontmatter as its own chunk if present
+    if (parsed.data && Object.keys(parsed.data).length > 0) {
+        const fmText = Object.entries(parsed.data)
+            .map(([k, v]) => `${k}: ${v}`)
+            .join('\n');
+        chunks.push({
+            sourceFile: relPath,
+            section: 'frontmatter',
+            content: fmText,
+            chunkType: 'frontmatter',
+            frontmatterJson: fmJson,
+            contentHash: contentHash(fmText),
+            category,
+            topic,
+        });
+    }
+    // Split body by ## headers
+    const sections = splitByHeaders(parsed.content);
+    for (const [sectionName, sectionContent] of sections) {
+        const content = sectionContent.trim();
+        if (!content)
+            continue;
+        const chunkType = sectionName === 'preamble' ? 'preamble' : 'heading';
+        // Split oversized sections at paragraph boundaries
+        if (content.length > MAX_CHUNK_CHARS) {
+            const subChunks = splitAtParagraphs(content, MAX_CHUNK_CHARS);
+            for (let i = 0; i < subChunks.length; i++) {
+                const label = subChunks.length > 1 ? `${sectionName} (part ${i + 1})` : sectionName;
+                chunks.push({
+                    sourceFile: relPath,
+                    section: label,
+                    chunkType,
+                    content: subChunks[i],
+                    frontmatterJson: fmJson,
+                    contentHash: contentHash(subChunks[i]),
+                    category,
+                    topic,
+                });
+            }
+        }
+        else {
+            chunks.push({
+                sourceFile: relPath,
+                section: sectionName,
+                chunkType,
+                content,
+                frontmatterJson: fmJson,
+                contentHash: contentHash(content),
+                category,
+                topic,
+            });
+        }
+    }
+    return chunks;
+}
+/**
+ * Split Markdown body by ## headers.
+ *
+ * Content before the first ## header is labeled "preamble".
+ *
+ * @returns Array of [sectionName, sectionContent] tuples.
+ */
+export function splitByHeaders(body) {
+    const sections = [];
+    let currentName = 'preamble';
+    let currentLines = [];
+    for (const line of body.split('\n')) {
+        if (line.startsWith('## ')) {
+            // Save previous section
+            if (currentLines.length > 0) {
+                sections.push([currentName, currentLines.join('\n')]);
+            }
+            currentName = line.slice(3).trim();
+            currentLines = [];
+        }
+        else {
+            currentLines.push(line);
+        }
+    }
+    // Don't forget the last section
+    if (currentLines.length > 0) {
+        sections.push([currentName, currentLines.join('\n')]);
+    }
+    return sections;
+}
+/**
+ * Split text at paragraph boundaries (double newlines) to stay under maxChars.
+ */
+export function splitAtParagraphs(text, maxChars) {
+    const paragraphs = text.split('\n\n');
+    const chunks = [];
+    let current = [];
+    let currentLen = 0;
+    for (const para of paragraphs) {
+        const paraLen = para.length + 2; // +2 for the \n\n separator
+        if (currentLen + paraLen > maxChars && current.length > 0) {
+            chunks.push(current.join('\n\n'));
+            current = [para];
+            currentLen = para.length;
+        }
+        else {
+            current.push(para);
+            currentLen += paraLen;
+        }
+    }
+    if (current.length > 0) {
+        chunks.push(current.join('\n\n'));
+    }
+    return chunks;
+}
+//# sourceMappingURL=chunker.js.map

package/dist/memory/consolidation.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Clementine TypeScript — Memory Consolidation Engine.
+ *
+ * Three strategies that run during the evening consolidation window:
+ * 1. Fact Dedup — merge chunks with high Jaccard similarity (pure SQL + local compute)
+ * 2. Topic Summarization — LLM (Haiku) summarizes groups of 5+ chunks into a single summary
+ * 3. Principle Extraction — distill repeated behavioral corrections into permanent rules
+ *
+ * Source chunks are marked consolidated (never deleted) — they remain searchable at lower salience.
+ */
+export interface ConsolidationResult {
+    deduped: number;
+    summarized: number;
+    principlesExtracted: number;
+    errors: string[];
+}
+interface MemoryStoreHandle {
+    getConsolidationCandidates(minAgeDays: number): Array<{
+        topic: string;
+        chunkIds: number[];
+        contents: string[];
+        totalChars: number;
+    }>;
+    markConsolidated(chunkIds: number[]): void;
+    getConsolidationStats(): {
+        totalChunks: number;
+        consolidated: number;
+        unconsolidated: number;
+    };
+    insertSummaryChunk(sourceFile: string, section: string, content: string): void;
+    getBehavioralPatterns(minOccurrences: number): Array<{
+        correction: string;
+        count: number;
+        category: string;
+        lastSeen: string;
+    }>;
+}
+/**
+ * Run all consolidation strategies in sequence.
+ * Budget-aware: stops if token/cost estimate exceeds limits.
+ */
+export declare function runConsolidation(store: MemoryStoreHandle, llmCall?: (prompt: string) => Promise<string>): Promise<ConsolidationResult>;
+export {};
+//# sourceMappingURL=consolidation.d.ts.map

package/dist/memory/consolidation.js

@@ -0,0 +1,171 @@
+/**
+ * Clementine TypeScript — Memory Consolidation Engine.
+ *
+ * Three strategies that run during the evening consolidation window:
+ * 1. Fact Dedup — merge chunks with high Jaccard similarity (pure SQL + local compute)
+ * 2. Topic Summarization — LLM (Haiku) summarizes groups of 5+ chunks into a single summary
+ * 3. Principle Extraction — distill repeated behavioral corrections into permanent rules
+ *
+ * Source chunks are marked consolidated (never deleted) — they remain searchable at lower salience.
+ */
+import pino from 'pino';
+import { tokenize, jaccard } from './mmr.js';
+const logger = pino({ name: 'clementine.consolidation' });
+/**
+ * Run all consolidation strategies in sequence.
+ * Budget-aware: stops if token/cost estimate exceeds limits.
+ */
+export async function runConsolidation(store, llmCall) {
+    const result = {
+        deduped: 0,
+        summarized: 0,
+        principlesExtracted: 0,
+        errors: [],
+    };
+    // Strategy 1: Fact Deduplication (no LLM needed)
+    try {
+        result.deduped = deduplicateChunks(store);
+        logger.info({ deduped: result.deduped }, 'Fact deduplication complete');
+    }
+    catch (err) {
+        const msg = `Dedup failed: ${err}`;
+        result.errors.push(msg);
+        logger.warn({ err }, msg);
+    }
+    // Strategy 2: Topic Summarization (requires LLM)
+    if (llmCall) {
+        try {
+            result.summarized = await summarizeTopics(store, llmCall);
+            logger.info({ summarized: result.summarized }, 'Topic summarization complete');
+        }
+        catch (err) {
+            const msg = `Summarization failed: ${err}`;
+            result.errors.push(msg);
+            logger.warn({ err }, msg);
+        }
+    }
+    // Strategy 3: Principle Extraction (requires LLM)
+    if (llmCall) {
+        try {
+            result.principlesExtracted = await extractPrinciples(store, llmCall);
+            logger.info({ principlesExtracted: result.principlesExtracted }, 'Principle extraction complete');
+        }
+        catch (err) {
+            const msg = `Principle extraction failed: ${err}`;
+            result.errors.push(msg);
+            logger.warn({ err }, msg);
+        }
+    }
+    const stats = store.getConsolidationStats();
+    logger.info({
+        ...result,
+        totalChunks: stats.totalChunks,
+        consolidated: stats.consolidated,
+        unconsolidated: stats.unconsolidated,
+    }, 'Consolidation cycle complete');
+    return result;
+}
+/**
+ * Strategy 1: Merge chunks with >0.7 Jaccard similarity within the same topic.
+ * Pure local computation — no LLM call needed.
+ * Marks the lower-salience duplicate as consolidated.
+ */
+function deduplicateChunks(store) {
+    const candidates = store.getConsolidationCandidates(7); // chunks older than 7 days
+    let deduped = 0;
+    for (const group of candidates) {
+        if (group.chunkIds.length < 2)
+            continue;
+        // Tokenize all chunks in this topic group
+        const tokenSets = group.contents.map(c => tokenize(c));
+        const toConsolidate = [];
+        // Compare each pair — mark the shorter one as consolidated
+        for (let i = 0; i < tokenSets.length; i++) {
+            if (toConsolidate.includes(group.chunkIds[i]))
+                continue;
+            for (let j = i + 1; j < tokenSets.length; j++) {
+                if (toConsolidate.includes(group.chunkIds[j]))
+                    continue;
+                const sim = jaccard(tokenSets[i], tokenSets[j]);
+                if (sim > 0.7) {
+                    // Mark the shorter content as duplicate
+                    const shorter = group.contents[i].length <= group.contents[j].length
+                        ? group.chunkIds[i]
+                        : group.chunkIds[j];
+                    toConsolidate.push(shorter);
+                }
+            }
+        }
+        if (toConsolidate.length > 0) {
+            store.markConsolidated(toConsolidate);
+            deduped += toConsolidate.length;
+        }
+        // Cap per cycle to avoid long-running operations
+        if (deduped >= 50)
+            break;
+    }
+    return deduped;
+}
+/**
+ * Strategy 2: For topic groups with 5+ unconsolidated chunks,
+ * generate a summary via Haiku and insert as a new summary chunk.
+ * Mark source chunks as consolidated.
+ */
+async function summarizeTopics(store, llmCall) {
+    const candidates = store.getConsolidationCandidates(14); // chunks older than 14 days
+    let summarized = 0;
+    // Process top 3 topic groups per cycle
+    for (const group of candidates.slice(0, 3)) {
+        if (group.chunkIds.length < 5)
+            continue;
+        // Build prompt for summarization
+        const contentSample = group.contents.slice(0, 10).join('\n---\n').slice(0, 3000);
+        const prompt = `Summarize the following ${group.chunkIds.length} memory chunks about "${group.topic}" ` +
+            `into 3-5 concise sentences that capture the essential facts. ` +
+            `Preserve specific names, dates, decisions, and identifiers. ` +
+            `Drop redundant or trivial information.\n\n` +
+            `Chunks:\n${contentSample}\n\n` +
+            `Output only the summary text, nothing else.`;
+        try {
+            const summary = await llmCall(prompt);
+            if (summary && summary.length > 50) {
+                // Insert summary as a new chunk
+                store.insertSummaryChunk(group.topic, `Consolidated Summary (${group.chunkIds.length} chunks)`, summary.slice(0, 3000));
+                // Mark source chunks as consolidated
+                store.markConsolidated(group.chunkIds);
+                summarized += group.chunkIds.length;
+            }
+        }
+        catch (err) {
+            logger.debug({ err, topic: group.topic }, 'Failed to summarize topic group');
+        }
+    }
+    return summarized;
+}
+/**
+ * Strategy 3: Behavioral corrections that appear 3+ times in session reflections
+ * are distilled into permanent rules. Returns the number of new principles created.
+ */
+async function extractPrinciples(store, llmCall) {
+    const patterns = store.getBehavioralPatterns(3); // corrections appearing 3+ times
+    if (patterns.length === 0)
+        return 0;
+    const prompt = `The following behavioral corrections have been received repeatedly from the user.\n\n` +
+        `${patterns.map(p => `- "${p.correction}" (${p.category}, ${p.count} times)`).join('\n')}\n\n` +
+        `Distill these into concise, actionable rules (1 sentence each) that should be ` +
+        `permanently applied. Merge similar corrections into a single rule.\n\n` +
+        `Output only the rules as a numbered list, nothing else.`;
+    try {
+        const rules = await llmCall(prompt);
+        if (rules && rules.length > 20) {
+            // Store as a consolidated principle chunk
+            store.insertSummaryChunk('00-System/MEMORY', 'Behavioral Principles (auto-consolidated)', rules.slice(0, 2000));
+            return patterns.length;
+        }
+    }
+    catch (err) {
+        logger.debug({ err }, 'Failed to extract behavioral principles');
+    }
+    return 0;
+}
+//# sourceMappingURL=consolidation.js.map

package/dist/memory/context-assembler.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Clementine TypeScript — Token-Budgeted Context Assembler.
+ *
+ * Assembles context for system prompt injection within a strict character budget.
+ * Fills slots by priority so the most important context always fits, with
+ * graceful degradation when budget is tight.
+ *
+ * Inspired by Phantom's MemoryContextBuilder pattern.
+ */
+import type { SearchResult } from '../types.js';
+export interface AssembledContext {
+    /** Combined context string ready for prompt injection. */
+    text: string;
+    /** Total characters used. */
+    charsUsed: number;
+    /** Which slots were included. */
+    slotsIncluded: string[];
+    /** Which slots were skipped (due to budget or empty). */
+    slotsSkipped: string[];
+}
+export interface AssemblerOptions {
+    /** Total character budget for all context. Default: 12000. */
+    totalBudget?: number;
+    /** Working memory file path (null to skip). */
+    workingMemoryPath?: string | null;
+    /** Pre-searched memory results. */
+    memoryResults?: SearchResult[];
+    /** Pre-resolved skill context string. */
+    skillContext?: string;
+    /** Pre-resolved graph context string. */
+    graphContext?: string;
+    /** Whether this is an autonomous run (truncates aggressively). */
+    isAutonomous?: boolean;
+    /** Identity file path (null to skip). */
+    identityPath?: string | null;
+}
+/**
+ * Assemble context from multiple sources within a character budget.
+ *
+ * Priority order:
+ *  1. Working memory (persistent scratchpad — always relevant)
+ *  2. Procedural skills (high-value, rarely large)
+ *  3. Memory search results (core recall)
+ *  4. Graph relationships (supplementary enrichment)
+ *
+ * Each slot truncates independently to its own maxChars limit.
+ * If total budget is exhausted, lower-priority slots are skipped entirely.
+ */
+export declare function assembleContext(options: AssemblerOptions): Promise<AssembledContext>;
+//# sourceMappingURL=context-assembler.d.ts.map

package/dist/memory/context-assembler.js

@@ -0,0 +1,149 @@
+/**
+ * Clementine TypeScript — Token-Budgeted Context Assembler.
+ *
+ * Assembles context for system prompt injection within a strict character budget.
+ * Fills slots by priority so the most important context always fits, with
+ * graceful degradation when budget is tight.
+ *
+ * Inspired by Phantom's MemoryContextBuilder pattern.
+ */
+import fs from 'node:fs';
+import { formatResultsForPrompt } from './search.js';
+/**
+ * Assemble context from multiple sources within a character budget.
+ *
+ * Priority order:
+ *  1. Working memory (persistent scratchpad — always relevant)
+ *  2. Procedural skills (high-value, rarely large)
+ *  3. Memory search results (core recall)
+ *  4. Graph relationships (supplementary enrichment)
+ *
+ * Each slot truncates independently to its own maxChars limit.
+ * If total budget is exhausted, lower-priority slots are skipped entirely.
+ */
+export async function assembleContext(options) {
+    const totalBudget = options.totalBudget ?? 12_000;
+    const isAutonomous = options.isAutonomous ?? false;
+    const slots = [];
+    // Slot 0: Identity seed (always loaded, tiny footprint)
+    if (options.identityPath) {
+        const idPath = options.identityPath;
+        slots.push({
+            name: 'identity',
+            priority: 0,
+            maxChars: 500,
+            minRemainingBudget: 0,
+            resolve: () => {
+                if (!fs.existsSync(idPath))
+                    return '';
+                try {
+                    const content = fs.readFileSync(idPath, 'utf-8').trim();
+                    if (!content)
+                        return '';
+                    return `## Identity\n\n${content}`;
+                }
+                catch {
+                    return '';
+                }
+            },
+        });
+    }
+    // Slot 1: Working memory (highest priority — always fits)
+    if (options.workingMemoryPath) {
+        const wmPath = options.workingMemoryPath;
+        slots.push({
+            name: 'working-memory',
+            priority: 1,
+            maxChars: isAutonomous ? 1000 : 2000,
+            minRemainingBudget: 0,
+            resolve: () => {
+                if (!fs.existsSync(wmPath))
+                    return '';
+                try {
+                    const content = fs.readFileSync(wmPath, 'utf-8').trim();
+                    if (!content)
+                        return '';
+                    return `## Working Memory (scratchpad)\n\n${content}`;
+                }
+                catch {
+                    return '';
+                }
+            },
+        });
+    }
+    // Slot 2: Procedural skills (high value, small footprint)
+    if (options.skillContext) {
+        const skillCtx = options.skillContext;
+        slots.push({
+            name: 'skills',
+            priority: 2,
+            maxChars: isAutonomous ? 1000 : 2000,
+            minRemainingBudget: 500,
+            resolve: () => skillCtx,
+        });
+    }
+    // Slot 3: Memory search results (core recall)
+    if (options.memoryResults && options.memoryResults.length > 0) {
+        const results = options.memoryResults;
+        slots.push({
+            name: 'memory-recall',
+            priority: 3,
+            maxChars: isAutonomous ? 2000 : 8000,
+            minRemainingBudget: 200,
+            resolve: () => {
+                // formatResultsForPrompt already handles truncation within its own budget
+                return formatResultsForPrompt(results, isAutonomous ? 2000 : 8000);
+            },
+        });
+    }
+    // Slot 4: Graph relationships (supplementary)
+    if (options.graphContext) {
+        const graphCtx = options.graphContext;
+        slots.push({
+            name: 'graph',
+            priority: 4,
+            maxChars: 2000,
+            minRemainingBudget: 500,
+            resolve: () => graphCtx,
+        });
+    }
+    // Sort by priority (lower number = higher priority)
+    slots.sort((a, b) => a.priority - b.priority);
+    // Fill slots within budget
+    let remaining = totalBudget;
+    const parts = [];
+    const included = [];
+    const skipped = [];
+    for (const slot of slots) {
+        // Check if we have enough remaining budget
+        if (remaining < slot.minRemainingBudget) {
+            skipped.push(slot.name);
+            continue;
+        }
+        try {
+            let content = await slot.resolve();
+            if (!content) {
+                skipped.push(slot.name);
+                continue;
+            }
+            // Truncate to the smaller of slot max and remaining budget
+            const limit = Math.min(slot.maxChars, remaining);
+            if (content.length > limit) {
+                content = content.slice(0, limit) + '\n...(truncated)';
+            }
+            parts.push(content);
+            remaining -= content.length;
+            included.push(slot.name);
+        }
+        catch {
+            skipped.push(slot.name);
+        }
+    }
+    return {
+        text: parts.join('\n\n'),
+        charsUsed: totalBudget - remaining,
+        slotsIncluded: included,
+        slotsSkipped: skipped,
+    };
+}
+//# sourceMappingURL=context-assembler.js.map