clementine-agent 1.0.0

package/dist/memory/embeddings.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Clementine TypeScript — Embedding Provider.
+ *
+ * Provides vector embeddings for memory chunks to enable semantic search.
+ * Uses a lightweight local approach (TF-IDF vectors stored as Float32Array blobs)
+ * that runs without external API calls or heavy WASM dependencies.
+ *
+ * The embedding column (BLOB) in the chunks table stores serialized Float32Arrays.
+ * Query-time: embed the query, compute cosine similarity against stored vectors.
+ */
+/**
+ * Build the IDF vocabulary from a corpus of text chunks.
+ * Should be called periodically (e.g., during evening consolidation) with all chunk contents.
+ */
+export declare function buildVocab(documents: string[]): void;
+/**
+ * Compute a TF-IDF embedding vector for a text string.
+ * Returns a Float32Array of length EMBEDDING_DIM, or null if vocab isn't ready.
+ */
+export declare function embed(text: string): Float32Array | null;
+/**
+ * Compute cosine similarity between two embedding vectors.
+ * Returns a value in [-1, 1] where 1 = identical, 0 = orthogonal.
+ */
+export declare function cosineSimilarity(a: Float32Array, b: Float32Array): number;
+/**
+ * Serialize a Float32Array to a Buffer for SQLite BLOB storage.
+ */
+export declare function serializeEmbedding(vec: Float32Array): Buffer;
+/**
+ * Deserialize a Buffer (from SQLite BLOB) back to Float32Array.
+ */
+export declare function deserializeEmbedding(buf: Buffer): Float32Array;
+/**
+ * Check if the embedding system is ready (vocabulary loaded with sufficient words).
+ */
+export declare function isReady(): boolean;
+//# sourceMappingURL=embeddings.d.ts.map

package/dist/memory/embeddings.js

@@ -0,0 +1,180 @@
+/**
+ * Clementine TypeScript — Embedding Provider.
+ *
+ * Provides vector embeddings for memory chunks to enable semantic search.
+ * Uses a lightweight local approach (TF-IDF vectors stored as Float32Array blobs)
+ * that runs without external API calls or heavy WASM dependencies.
+ *
+ * The embedding column (BLOB) in the chunks table stores serialized Float32Arrays.
+ * Query-time: embed the query, compute cosine similarity against stored vectors.
+ */
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import pino from 'pino';
+import { BASE_DIR } from '../config.js';
+const logger = pino({ name: 'clementine.embeddings' });
+/** Dimension of the TF-IDF embedding vectors. */
+const EMBEDDING_DIM = 512;
+/** IDF vocabulary — built from corpus, cached to disk. */
+let idfVocab = null;
+let vocabWords = [];
+const VOCAB_PATH = path.join(BASE_DIR, '.embedding-vocab.json');
+/**
+ * Load or initialize the IDF vocabulary.
+ * The vocabulary maps the top-N words to their IDF weights.
+ */
+function loadVocab() {
+    if (idfVocab)
+        return;
+    if (existsSync(VOCAB_PATH)) {
+        try {
+            const data = JSON.parse(readFileSync(VOCAB_PATH, 'utf-8'));
+            idfVocab = new Map(Object.entries(data));
+            vocabWords = [...idfVocab.keys()];
+            return;
+        }
+        catch {
+            logger.debug('Failed to load embedding vocab — will rebuild');
+        }
+    }
+    // Start with empty vocab — will be built on first buildVocab() call
+    idfVocab = new Map();
+    vocabWords = [];
+}
+/**
+ * Build the IDF vocabulary from a corpus of text chunks.
+ * Should be called periodically (e.g., during evening consolidation) with all chunk contents.
+ */
+export function buildVocab(documents) {
+    if (documents.length === 0)
+        return;
+    // Compute document frequency for each word
+    const df = new Map();
+    const N = documents.length;
+    for (const doc of documents) {
+        const words = new Set(tokenize(doc));
+        for (const word of words) {
+            df.set(word, (df.get(word) ?? 0) + 1);
+        }
+    }
+    // Select top EMBEDDING_DIM words by document frequency (must appear in at least 2 docs)
+    const sorted = [...df.entries()]
+        .filter(([, count]) => count >= 2 && count < N * 0.95) // skip too-rare and too-common
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, EMBEDDING_DIM);
+    idfVocab = new Map();
+    vocabWords = [];
+    for (const [word, count] of sorted) {
+        const idf = Math.log(N / (1 + count));
+        idfVocab.set(word, idf);
+        vocabWords.push(word);
+    }
+    // Persist to disk
+    try {
+        mkdirSync(path.dirname(VOCAB_PATH), { recursive: true });
+        writeFileSync(VOCAB_PATH, JSON.stringify(Object.fromEntries(idfVocab)));
+        logger.info({ vocabSize: vocabWords.length, corpusSize: N }, 'Embedding vocabulary built');
+    }
+    catch (err) {
+        logger.debug({ err }, 'Failed to persist embedding vocabulary');
+    }
+}
+/**
+ * Tokenize text into lowercase words, filtering short words and stop words.
+ */
+function tokenize(text) {
+    return text
+        .toLowerCase()
+        .replace(/[^\w\s]/g, ' ')
+        .split(/\s+/)
+        .filter(w => w.length > 2 && !STOP_WORDS.has(w));
+}
+/**
+ * Compute a TF-IDF embedding vector for a text string.
+ * Returns a Float32Array of length EMBEDDING_DIM, or null if vocab isn't ready.
+ */
+export function embed(text) {
+    loadVocab();
+    if (!idfVocab || vocabWords.length === 0)
+        return null;
+    const words = tokenize(text);
+    if (words.length === 0)
+        return null;
+    // Compute term frequency
+    const tf = new Map();
+    for (const word of words) {
+        tf.set(word, (tf.get(word) ?? 0) + 1);
+    }
+    // Build TF-IDF vector
+    const vec = new Float32Array(vocabWords.length);
+    for (let i = 0; i < vocabWords.length; i++) {
+        const word = vocabWords[i];
+        const termFreq = tf.get(word) ?? 0;
+        const idf = idfVocab.get(word) ?? 0;
+        vec[i] = termFreq * idf;
+    }
+    // L2 normalize
+    let norm = 0;
+    for (let i = 0; i < vec.length; i++) {
+        norm += vec[i] * vec[i];
+    }
+    norm = Math.sqrt(norm);
+    if (norm > 0) {
+        for (let i = 0; i < vec.length; i++) {
+            vec[i] /= norm;
+        }
+    }
+    return vec;
+}
+/**
+ * Compute cosine similarity between two embedding vectors.
+ * Returns a value in [-1, 1] where 1 = identical, 0 = orthogonal.
+ */
+export function cosineSimilarity(a, b) {
+    if (a.length !== b.length)
+        return 0;
+    let dot = 0;
+    for (let i = 0; i < a.length; i++) {
+        dot += a[i] * b[i];
+    }
+    return dot; // Already L2-normalized, so dot product = cosine similarity
+}
+/**
+ * Serialize a Float32Array to a Buffer for SQLite BLOB storage.
+ */
+export function serializeEmbedding(vec) {
+    return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
+}
+/**
+ * Deserialize a Buffer (from SQLite BLOB) back to Float32Array.
+ */
+export function deserializeEmbedding(buf) {
+    const ab = new ArrayBuffer(buf.length);
+    const view = new Uint8Array(ab);
+    for (let i = 0; i < buf.length; i++) {
+        view[i] = buf[i];
+    }
+    return new Float32Array(ab);
+}
+/**
+ * Check if the embedding system is ready (vocabulary loaded with sufficient words).
+ */
+export function isReady() {
+    loadVocab();
+    return vocabWords.length >= 50; // need at least 50 vocab words
+}
+const STOP_WORDS = new Set([
+    'the', 'be', 'to', 'of', 'and', 'in', 'that', 'have', 'it', 'for',
+    'not', 'on', 'with', 'he', 'as', 'you', 'do', 'at', 'this', 'but',
+    'his', 'by', 'from', 'they', 'we', 'say', 'her', 'she', 'or', 'an',
+    'will', 'my', 'one', 'all', 'would', 'there', 'their', 'what', 'so',
+    'up', 'out', 'if', 'about', 'who', 'get', 'which', 'go', 'me', 'when',
+    'make', 'can', 'like', 'time', 'no', 'just', 'him', 'know', 'take',
+    'people', 'into', 'year', 'your', 'good', 'some', 'could', 'them',
+    'see', 'other', 'than', 'then', 'now', 'look', 'only', 'come', 'its',
+    'over', 'think', 'also', 'back', 'after', 'use', 'two', 'how', 'our',
+    'work', 'first', 'well', 'way', 'even', 'new', 'want', 'because',
+    'any', 'these', 'give', 'day', 'most', 'was', 'are', 'has', 'had',
+    'been', 'were', 'did', 'does', 'done', 'being',
+]);
+//# sourceMappingURL=embeddings.js.map

package/dist/memory/graph-store.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Clementine TypeScript — FalkorDBLite graph memory layer.
+ *
+ * Adds entity graph, typed relationships, and multi-hop traversal on top
+ * of the existing SQLite FTS5 memory store. The vault remains the source
+ * of truth; the graph is a derived index that can be rebuilt at any time.
+ *
+ * Architecture:
+ *   - The daemon calls `initialize()` which starts an embedded FalkorDB
+ *     server and writes its Unix socket path to SOCKET_FILE.
+ *   - MCP tools, dashboard, and assistant.ts call `connectToRunning()`
+ *     which reads the socket file and connects as a client (no new server).
+ *   - If no running instance is found, all graph features degrade gracefully.
+ *
+ * Graceful degradation: if FalkorDBLite fails to initialize, `isAvailable()`
+ * returns false and all graph features are silently skipped.
+ */
+import type { EntityNode, EntityRef, GraphSyncStats, PathResult, RelationshipTriplet, TraversalResult } from '../types.js';
+export declare class GraphStore {
+    private db;
+    private client;
+    private graph;
+    private available;
+    private persistenceDir;
+    private ownsServer;
+    constructor(persistenceDir: string);
+    /** Get the socket file path for this instance's data dir. */
+    private get socketFilePath();
+    /**
+     * Start an embedded FalkorDB server. Only the daemon should call this.
+     * Writes the socket path to a file so other processes can connect.
+     */
+    initialize(): Promise<void>;
+    /**
+     * Connect to an already-running FalkorDB instance via its socket file.
+     * Does NOT start a new server. Returns false if no running instance.
+     */
+    connectToRunning(): Promise<boolean>;
+    isAvailable(): boolean;
+    close(): Promise<void>;
+    upsertEntity(label: string, id: string, props: Record<string, any>): Promise<void>;
+    getEntity(label: string, id: string): Promise<EntityNode | null>;
+    createRelationship(from: EntityRef, to: EntityRef, type: string, props?: Record<string, any>, temporal?: {
+        validFrom?: string;
+        validTo?: string;
+    }): Promise<void>;
+    /**
+     * Mark a relationship as no longer active by setting its valid_to timestamp.
+     */
+    invalidateRelationship(fromId: string, toId: string, relType: string, asOf?: string): Promise<boolean>;
+    getRelationships(entityId: string, direction?: 'in' | 'out' | 'both', relType?: string, asOf?: string): Promise<Array<{
+        from: string;
+        to: string;
+        type: string;
+        properties: Record<string, any>;
+    }>>;
+    traverse(startId: string, maxDepth?: number, relTypes?: string[], asOf?: string): Promise<TraversalResult[]>;
+    shortestPath(fromId: string, toId: string): Promise<PathResult | null>;
+    findConnected(entityId: string, targetLabel: string, maxHops?: number): Promise<EntityNode[]>;
+    query(cypher: string, params?: Record<string, any>): Promise<any[]>;
+    syncFromVault(vaultDir: string, agentsDir: string): Promise<GraphSyncStats>;
+    extractAndStoreRelationships(triplets: RelationshipTriplet[]): Promise<void>;
+    enrichWithGraphContext(entityIds: string[], _maxHops?: number): Promise<string>;
+}
+export declare function getSharedGraphStore(persistenceDir: string): Promise<GraphStore | null>;
+//# sourceMappingURL=graph-store.d.ts.map