@aperdomoll90/ledger-ai 1.0.1 → 1.1.0

package/dist/lib/notes.js

@@ -1,26 +1,29 @@
+import { randomUUID } from 'crypto';
 import { fatal, ExitCode } from './errors.js';
 import { contentHash } from './hash.js';
+const DELIVERY_BY_TYPE = {
+    'user-preference': 'persona',
+    'feedback': 'persona',
+    'architecture-decision': 'project',
+    'project-status': 'project',
+    'error': 'project',
+    'event': 'project',
+    'reference': 'knowledge',
+    'general': 'knowledge',
+};
+export function inferDelivery(noteType) {
+    return DELIVERY_BY_TYPE[noteType] ?? 'knowledge';
+}
 // --- Queries ---
-export async function fetchCachedNotes(supabase) {
-    const { data: cachedNotes, error: cacheError } = await supabase
-        .from('notes')
-        .select('id, content, metadata, created_at, updated_at')
-        .eq('metadata->>local_cache', 'true');
-    if (cacheError) {
-        fatal(`Error querying Ledger: ${cacheError.message}`, ExitCode.SUPABASE_ERROR);
-    }
-    const { data: ruleNotes, error: ruleError } = await supabase
+export async function fetchPersonaNotes(supabase) {
+    const { data, error } = await supabase
         .from('notes')
         .select('id, content, metadata, created_at, updated_at')
-        .in('metadata->>type', ['feedback', 'user-preference']);
-    if (ruleError) {
-        fatal(`Error querying rule notes: ${ruleError.message}`, ExitCode.SUPABASE_ERROR);
-    }
-    const allNotes = new Map();
-    for (const note of [...(cachedNotes || []), ...(ruleNotes || [])]) {
-        allNotes.set(note.id, note);
+        .eq('metadata->>delivery', 'persona');
+    if (error) {
+        fatal(`Error querying persona notes: ${error.message}`, ExitCode.SUPABASE_ERROR);
     }
-    return Array.from(allNotes.values());
+    return (data || []);
 }
 export async function findNoteByFile(supabase, filename) {
     const { data: byFile } = await supabase
@@ -75,7 +78,7 @@ export async function searchNotes(supabase, openai, query, threshold = 0.3, maxR
     return (data || []);
 }
 export async function fetchNoteHashes(supabase) {
-    const notes = await fetchCachedNotes(supabase);
+    const notes = await fetchPersonaNotes(supabase);
     return notes
         .filter(n => n.metadata.local_file)
         .map(n => ({
@@ -85,6 +88,36 @@ export async function fetchNoteHashes(supabase) {
         content: n.content,
     }));
 }
+// --- Chunking ---
+export function chunkText(text, maxChars, overlap) {
+    if (text.length <= maxChars)
+        return [text];
+    const paragraphs = text.split(/\n\n+/);
+    const chunks = [];
+    let current = '';
+    for (const paragraph of paragraphs) {
+        if (current.length + paragraph.length + 2 > maxChars && current.length > 0) {
+            chunks.push(current.trim());
+            const overlapText = current.slice(-overlap);
+            current = overlapText + '\n\n' + paragraph;
+        }
+        else {
+            current = current ? current + '\n\n' + paragraph : paragraph;
+        }
+    }
+    if (current.trim()) {
+        chunks.push(current.trim());
+    }
+    return chunks.flatMap((chunk) => {
+        if (chunk.length <= maxChars)
+            return [chunk];
+        const forced = [];
+        for (let i = 0; i < chunk.length; i += maxChars - overlap) {
+            forced.push(chunk.slice(i, i + maxChars));
+        }
+        return forced;
+    });
+}
 export async function updateNoteHash(supabase, noteId, hash) {
     const { data: note, error: fetchError } = await supabase
         .from('notes')
@@ -103,3 +136,382 @@ export async function updateNoteHash(supabase, noteId, hash) {
         fatal(`Error updating note hash: ${error.message}`, ExitCode.SUPABASE_ERROR);
     }
 }
+// --- Constants ---
+const MAX_CHARS_PER_CHUNK = 25_000;
+const CHUNK_OVERLAP = 2_000;
+// --- Shared Helpers ---
+export async function getEmbedding(openai, text) {
+    const response = await openai.embeddings.create({
+        model: 'text-embedding-3-small',
+        input: text,
+    });
+    return response.data[0].embedding;
+}
+function formatNotePreview(id, meta, content, maxLen = 300) {
+    const uKey = meta.upsert_key;
+    const noteType = meta.type;
+    const project = meta.project;
+    const desc = meta.description;
+    const label = uKey || `id-${id}`;
+    const preview = content.slice(0, maxLen).replace(/\n/g, '\n  ');
+    const truncated = content.length > maxLen ? '...' : '';
+    const descLine = desc ? `\nDescription: ${desc}` : '';
+    return `"${label}" (id: ${id}) | type: ${noteType || '-'} | project: ${project || '-'}${descLine}\n  ${preview}${truncated}`;
+}
+// --- Shared Operations (called by both MCP and CLI) ---
+export async function opSearchNotes(clients, query, threshold, limit, type, project) {
+    const embedding = await getEmbedding(clients.openai, query);
+    const fetchLimit = (type || project) ? limit * 3 : limit;
+    const { data, error } = await clients.supabase.rpc('match_notes', {
+        q_emb: JSON.stringify(embedding),
+        threshold,
+        max_results: fetchLimit,
+    });
+    if (error) {
+        return { status: 'error', message: `Error: ${error.message}` };
+    }
+    let results = data;
+    if (type)
+        results = results.filter(n => n.metadata.type === type);
+    if (project)
+        results = results.filter(n => n.metadata.project === project);
+    results = results.slice(0, limit);
+    // Fallback: retry at 0.3 when requested threshold returns empty
+    if ((!results || results.length === 0) && threshold > 0.3) {
+        const { data: fallbackData, error: fallbackError } = await clients.supabase.rpc('match_notes', {
+            q_emb: JSON.stringify(embedding),
+            threshold: 0.3,
+            max_results: fetchLimit,
+        });
+        if (!fallbackError && fallbackData && fallbackData.length > 0) {
+            let fallbackResults = fallbackData;
+            if (type)
+                fallbackResults = fallbackResults.filter(n => n.metadata.type === type);
+            if (project)
+                fallbackResults = fallbackResults.filter(n => n.metadata.project === project);
+            fallbackResults = fallbackResults.slice(0, limit);
+            if (fallbackResults.length > 0) {
+                results = fallbackResults;
+                const output = await reassembleResults(clients.supabase, results);
+                return { status: 'ok', message: '⚠ No strong matches found. Showing low-confidence results (similarity 0.3–0.5):\n\n' + output };
+            }
+        }
+    }
+    if (!results || results.length === 0) {
+        return { status: 'ok', message: 'No matching notes found.' };
+    }
+    const output = await reassembleResults(clients.supabase, results);
+    return { status: 'ok', message: output };
+}
+async function reassembleResults(supabase, results) {
+    const seenGroups = new Set();
+    const output = [];
+    for (const note of results) {
+        const meta = note.metadata;
+        const groupId = meta.chunk_group;
+        if (groupId) {
+            if (seenGroups.has(groupId))
+                continue;
+            seenGroups.add(groupId);
+            const { data: siblings, error: sibError } = await supabase
+                .from('notes')
+                .select('id, content, metadata, created_at')
+                .eq('metadata->>chunk_group', groupId)
+                .order('metadata->>chunk_index', { ascending: true });
+            if (sibError || !siblings || siblings.length === 0) {
+                output.push(`[${note.id}] (similarity: ${note.similarity.toFixed(3)}) [chunked, sibling fetch failed]\n${note.content}\nMetadata: ${JSON.stringify(note.metadata)}`);
+            }
+            else {
+                const reassembled = siblings.map((s) => s.content).join('\n\n');
+                output.push(`[${siblings[0].id}] (similarity: ${note.similarity.toFixed(3)}) [${siblings.length} chunks reassembled]\n${reassembled}\nMetadata: ${JSON.stringify({ ...meta, chunks: siblings.length })}`);
+            }
+        }
+        else {
+            output.push(`[${note.id}] (similarity: ${note.similarity.toFixed(3)})\n${note.content}\nMetadata: ${JSON.stringify(note.metadata)}`);
+        }
+    }
+    return output.join('\n\n---\n\n');
+}
+export async function opListNotes(clients, limit, type, project) {
+    let query = clients.supabase
+        .from('notes')
+        .select('id, content, metadata, created_at')
+        .order('created_at', { ascending: false })
+        .limit(limit);
+    if (type)
+        query = query.eq('metadata->>type', type);
+    if (project)
+        query = query.eq('metadata->>project', project);
+    const { data, error } = await query;
+    if (error) {
+        return { status: 'error', message: `Error: ${error.message}` };
+    }
+    if (!data || data.length === 0) {
+        return { status: 'ok', message: 'No notes found.' };
+    }
+    const formatted = data.map((note) => {
+        const meta = note.metadata;
+        const chunkInfo = meta.chunk_group ? ` [chunk ${meta.chunk_index + 1}/${meta.total_chunks}]` : '';
+        return `[${note.id}]${chunkInfo} ${note.created_at}\n${note.content.slice(0, 200)}${note.content.length > 200 ? '...' : ''}\nMetadata: ${JSON.stringify(note.metadata)}`;
+    }).join('\n\n---\n\n');
+    return { status: 'ok', message: formatted };
+}
+export async function opAddNote(clients, content, type, agent, metadata, force) {
+    const fullMetadata = { ...metadata, type, agent, content_hash: contentHash(content) };
+    const upsertKey = metadata.upsert_key;
+    // Duplicate guard: if no upsert_key and not forced, check for similar notes
+    if (!upsertKey && !force) {
+        try {
+            const embedding = await getEmbedding(clients.openai, content.slice(0, 2000));
+            const { data: similar } = await clients.supabase.rpc('match_notes', {
+                q_emb: JSON.stringify(embedding),
+                threshold: 0.6,
+                max_results: 5,
+            });
+            if (similar && similar.length > 0) {
+                const suggestions = similar.map((n) => {
+                    const meta = n.metadata;
+                    const key = meta.upsert_key;
+                    const proj = meta.project;
+                    const nType = meta.type;
+                    const preview = n.content.slice(0, 200).replace(/\n/g, ' ');
+                    return `  [${n.id}] similarity: ${n.similarity.toFixed(3)} | key: "${key || 'none'}" | project: ${proj || '-'} | type: ${nType || '-'}\n    ${preview}...`;
+                }).join('\n\n');
+                return {
+                    status: 'confirm',
+                    message: `SIMILAR NOTES FOUND — ask the user before proceeding:\n\n${suggestions}\n\nOptions:\n  1. Update an existing note: re-call add_note with that note's upsert_key\n  2. Create new note anyway: re-call add_note with force: true\n\nAsk the user: "Should I update [note description] or create a new note?"`,
+                };
+            }
+        }
+        catch {
+            // If similarity check fails, proceed with creation
+        }
+    }
+    // Upsert: if upsert_key provided, update existing note
+    if (upsertKey) {
+        const { data: existing } = await clients.supabase
+            .from('notes')
+            .select('id, metadata, created_at')
+            .eq('metadata->>upsert_key', upsertKey)
+            .limit(1)
+            .single();
+        if (existing) {
+            return upsertExistingNote(clients, existing, content, fullMetadata);
+        }
+    }
+    // Create new note
+    return createNewNote(clients, content, fullMetadata);
+}
+async function upsertExistingNote(clients, existing, content, fullMetadata) {
+    const oldGroup = existing.metadata.chunk_group;
+    const oldCreatedAt = existing.created_at;
+    const newChunks = chunkText(content, MAX_CHARS_PER_CHUNK, CHUNK_OVERLAP);
+    const upsertKey = fullMetadata.upsert_key;
+    // Single-chunk → single-chunk: real SQL UPDATE (preserves ID + created_at)
+    if (!oldGroup && newChunks.length === 1) {
+        const embedding = await getEmbedding(clients.openai, content);
+        const { data, error } = await clients.supabase
+            .from('notes')
+            .update({ content, metadata: fullMetadata, embedding })
+            .eq('id', existing.id)
+            .select('id, created_at')
+            .single();
+        if (error)
+            return { status: 'error', message: `Error: ${error.message}` };
+        const preview = content.length > 100 ? content.slice(0, 100) + '...' : content;
+        return { status: 'ok', message: `Updated "${upsertKey}" (id: ${data.id}, ${content.length} chars)\nPreview: ${preview}` };
+    }
+    // Chunk count changed — delete old, preserve ID for first insert
+    const preserveId = existing.id;
+    if (oldGroup) {
+        await clients.supabase.from('notes').delete().eq('metadata->>chunk_group', oldGroup);
+    }
+    else {
+        await clients.supabase.from('notes').delete().eq('id', existing.id);
+    }
+    if (newChunks.length === 1) {
+        const embedding = await getEmbedding(clients.openai, content);
+        const { data, error } = await clients.supabase
+            .from('notes')
+            .insert({ id: preserveId, content, metadata: fullMetadata, embedding, created_at: oldCreatedAt })
+            .select('id, created_at')
+            .single();
+        if (error)
+            return { status: 'error', message: `Error: ${error.message}` };
+        const preview = content.length > 100 ? content.slice(0, 100) + '...' : content;
+        return { status: 'ok', message: `Updated "${upsertKey}" (id: ${data.id}, ${content.length} chars)\nPreview: ${preview}` };
+    }
+    // Multiple new chunks
+    const newGroupId = randomUUID();
+    const ids = [];
+    for (let i = 0; i < newChunks.length; i++) {
+        const chunkMeta = { ...fullMetadata, chunk_group: newGroupId, chunk_index: i, total_chunks: newChunks.length };
+        const embedding = await getEmbedding(clients.openai, newChunks[i]);
+        const insertData = { content: newChunks[i], metadata: chunkMeta, embedding, created_at: oldCreatedAt };
+        if (i === 0)
+            insertData.id = preserveId;
+        const { data, error } = await clients.supabase.from('notes').insert(insertData).select('id').single();
+        if (error)
+            return { status: 'error', message: `Error saving chunk ${i + 1}/${newChunks.length}: ${error.message}` };
+        ids.push(data.id);
+    }
+    return { status: 'ok', message: `Updated "${upsertKey}" as ${newChunks.length} chunks (ids: ${ids.join(', ')}, ${content.length} chars total)` };
+}
+async function createNewNote(clients, content, fullMetadata) {
+    const chunks = chunkText(content, MAX_CHARS_PER_CHUNK, CHUNK_OVERLAP);
+    const upsertKey = fullMetadata.upsert_key;
+    if (chunks.length === 1) {
+        const embedding = await getEmbedding(clients.openai, content);
+        const { data, error } = await clients.supabase
+            .from('notes')
+            .insert({ content, metadata: fullMetadata, embedding })
+            .select('id, created_at')
+            .single();
+        if (error)
+            return { status: 'error', message: `Error: ${error.message}` };
+        const label = upsertKey || `id ${data.id}`;
+        const preview = content.length > 100 ? content.slice(0, 100) + '...' : content;
+        return { status: 'ok', message: `Saved "${label}" (id: ${data.id}, type: ${fullMetadata.type}, ${content.length} chars)\nPreview: ${preview}` };
+    }
+    const groupId = randomUUID();
+    const ids = [];
+    for (let i = 0; i < chunks.length; i++) {
+        const chunkMeta = { ...fullMetadata, chunk_group: groupId, chunk_index: i, total_chunks: chunks.length };
+        const embedding = await getEmbedding(clients.openai, chunks[i]);
+        const { data, error } = await clients.supabase.from('notes').insert({ content: chunks[i], metadata: chunkMeta, embedding }).select('id').single();
+        if (error)
+            return { status: 'error', message: `Error saving chunk ${i + 1}/${chunks.length}: ${error.message}` };
+        ids.push(data.id);
+    }
+    return { status: 'ok', message: `Saved "${upsertKey || 'chunked'}" as ${chunks.length} chunks (ids: ${ids.join(', ')}, ${content.length} chars total)` };
+}
+export async function opUpdateNote(clients, id, content, metadata, confirmed = false) {
+    const { data: existing, error: fetchError } = await clients.supabase
+        .from('notes')
+        .select('id, content, metadata, created_at')
+        .eq('id', id)
+        .single();
+    if (fetchError || !existing) {
+        return { status: 'error', message: `Error: note ${id} not found.` };
+    }
+    // Confirmation gate
+    if (!confirmed) {
+        const currentPreview = formatNotePreview(existing.id, existing.metadata, existing.content);
+        const newPreview = content.slice(0, 300).replace(/\n/g, '\n  ');
+        return {
+            status: 'confirm',
+            message: `CONFIRM UPDATE — Is this the correct note to apply this change to?\n\n${currentPreview}\n\nNew content:\n  ${newPreview}${content.length > 300 ? '...' : ''}\n\nTo proceed, call update_note again with confirmed: true.`,
+        };
+    }
+    const existingMeta = existing.metadata;
+    const groupId = existingMeta.chunk_group;
+    const oldCreatedAt = existing.created_at;
+    const baseMeta = metadata ?? existingMeta;
+    const chunks = chunkText(content, MAX_CHARS_PER_CHUNK, CHUNK_OVERLAP);
+    // Single-chunk → single-chunk: real SQL UPDATE
+    if (!groupId && chunks.length === 1) {
+        const embedding = await getEmbedding(clients.openai, content);
+        const updatedMeta = { ...baseMeta, content_hash: contentHash(content) };
+        const { data, error } = await clients.supabase
+            .from('notes')
+            .update({ content, metadata: updatedMeta, embedding })
+            .eq('id', id)
+            .select('id, created_at')
+            .single();
+        if (error)
+            return { status: 'error', message: `Error: ${error.message}` };
+        const uKey = baseMeta.upsert_key;
+        const label = uKey || `id ${data.id}`;
+        const preview = content.length > 100 ? content.slice(0, 100) + '...' : content;
+        return { status: 'ok', message: `Updated "${label}" (id: ${data.id}, ${content.length} chars)\nPreview: ${preview}` };
+    }
+    // Chunk count changed — delete old, re-insert with preserved ID
+    const preserveId = id;
+    if (groupId) {
+        await clients.supabase.from('notes').delete().eq('metadata->>chunk_group', groupId);
+    }
+    else {
+        await clients.supabase.from('notes').delete().eq('id', id);
+    }
+    if (chunks.length === 1) {
+        const embedding = await getEmbedding(clients.openai, content);
+        const { chunk_group, chunk_index, total_chunks, ...cleanMeta } = baseMeta;
+        const updatedMeta = { ...cleanMeta, content_hash: contentHash(content) };
+        const { data, error } = await clients.supabase
+            .from('notes')
+            .insert({ id: preserveId, content, metadata: updatedMeta, embedding, created_at: oldCreatedAt })
+            .select('id, created_at')
+            .single();
+        if (error)
+            return { status: 'error', message: `Error: ${error.message}` };
+        const uKey = baseMeta.upsert_key;
+        const label = uKey || `id ${data.id}`;
+        const preview = content.length > 100 ? content.slice(0, 100) + '...' : content;
+        return { status: 'ok', message: `Updated "${label}" (id: ${data.id}, ${content.length} chars)\nPreview: ${preview}` };
+    }
+    const newGroupId = randomUUID();
+    const ids = [];
+    for (let i = 0; i < chunks.length; i++) {
+        const { chunk_group, chunk_index, total_chunks, ...cleanMeta } = baseMeta;
+        const chunkMeta = { ...cleanMeta, content_hash: contentHash(content), chunk_group: newGroupId, chunk_index: i, total_chunks: chunks.length };
+        const embedding = await getEmbedding(clients.openai, chunks[i]);
+        const insertData = { content: chunks[i], metadata: chunkMeta, embedding, created_at: oldCreatedAt };
+        if (i === 0)
+            insertData.id = preserveId;
+        const { data, error } = await clients.supabase.from('notes').insert(insertData).select('id').single();
+        if (error)
+            return { status: 'error', message: `Error updating chunk ${i + 1}/${chunks.length}: ${error.message}` };
+        ids.push(data.id);
+    }
+    return { status: 'ok', message: `Note updated as ${chunks.length} chunks (ids: ${ids.join(', ')}, group: ${newGroupId})` };
+}
+export async function opUpdateMetadata(clients, id, metadata) {
+    const { data: existing, error: fetchError } = await clients.supabase
+        .from('notes')
+        .select('id, metadata')
+        .eq('id', id)
+        .single();
+    if (fetchError || !existing) {
+        return { status: 'error', message: `Error: note ${id} not found.` };
+    }
+    const merged = { ...existing.metadata, ...metadata };
+    const { error } = await clients.supabase
+        .from('notes')
+        .update({ metadata: merged })
+        .eq('id', id);
+    if (error)
+        return { status: 'error', message: `Error: ${error.message}` };
+    const uKey = merged.upsert_key;
+    return { status: 'ok', message: `Updated metadata for "${uKey || `id ${id}`}"` };
+}
+export async function opDeleteNote(clients, id, confirmed = false) {
+    const { data: existing } = await clients.supabase
+        .from('notes')
+        .select('id, content, metadata')
+        .eq('id', id)
+        .single();
+    if (!existing) {
+        return { status: 'error', message: `Error: note ${id} not found.` };
+    }
+    const meta = existing.metadata;
+    const groupId = meta.chunk_group;
+    // Confirmation gate
+    if (!confirmed) {
+        const chunkInfo = groupId ? ` (chunked, all chunks will be deleted)` : '';
+        const preview = formatNotePreview(existing.id, meta, existing.content);
+        return {
+            status: 'confirm',
+            message: `CONFIRM DELETE — This note will be permanently removed${chunkInfo}:\n\n${preview}\n\nTo proceed, call delete_note again with confirmed: true.`,
+        };
+    }
+    if (groupId) {
+        const { error } = await clients.supabase.from('notes').delete().eq('metadata->>chunk_group', groupId);
+        if (error)
+            return { status: 'error', message: `Error: ${error.message}` };
+        return { status: 'ok', message: `Deleted all chunks in group ${groupId}.` };
+    }
+    const { error } = await clients.supabase.from('notes').delete().eq('id', id);
+    if (error)
+        return { status: 'error', message: `Error: ${error.message}` };
+    return { status: 'ok', message: `Note ${id} deleted.` };
+}