@openanonymity/nanomem 0.1.0 → 0.1.1

package/src/engine/executors.js

@@ -9,7 +9,7 @@
  * Each factory takes a storage backend and returns an object mapping
  * tool names to async functions: { tool_name: async (args) => resultString }
  */
-/** @import { ExtractionExecutorHooks, StorageBackend } from '../types.js' */
+/** @import { ChatCompletionResponse, ExtractionExecutorHooks, LLMClient, StorageBackend, ToolDefinition } from '../types.js' */
 import {
     compactBullets,
     inferTopicFromPath,
@@ -17,6 +17,176 @@ import {
     parseBullets,
     renderCompactedDocument,
 } from '../bullets/index.js';
+import { trimRecentConversation } from './recentConversation.js';
+
+const MAX_AUGMENT_QUERY_FILES = 8;
+const MAX_AUGMENT_FILE_CHARS = 1800;
+const MAX_AUGMENT_TOTAL_CHARS = 12000;
+const MAX_AUGMENT_RECENT_CONTEXT_CHARS = 3000;
+const AUGMENT_CRAFTER_MAX_ATTEMPTS = 3;
+const AUGMENT_CRAFTER_RETRY_BASE_DELAY_MS = 350;
+
+function normalizeLookupPath(value, { stripExtension = false } = {}) {
+    let normalized = String(value || '')
+        .trim()
+        .replace(/\\/g, '/')
+        .replace(/^\.\//, '')
+        .replace(/^\/+/, '')
+        .replace(/\/+/g, '/');
+
+    if (stripExtension) {
+        normalized = normalized.replace(/\.md$/i, '');
+    }
+
+    if (typeof normalized.normalize === 'function') {
+        normalized = normalized.normalize('NFKD').replace(/[\u0300-\u036f]/g, '');
+    }
+
+    return normalizeFactText(normalized.replace(/[\/_]/g, ' '));
+}
+
+function pathMatchesQuery(path, query) {
+    const rawPath = String(path || '');
+    const rawQuery = String(query || '').trim().toLowerCase();
+    if (!rawPath || !rawQuery) return false;
+    if (rawPath.toLowerCase().includes(rawQuery)) return true;
+
+    const normalizedQuery = normalizeFactText(rawQuery);
+    if (!normalizedQuery) return false;
+
+    return normalizeLookupPath(rawPath).includes(normalizedQuery)
+        || normalizeLookupPath(rawPath, { stripExtension: true }).includes(normalizedQuery);
+}
+
+const AUGMENT_QUERY_EXECUTOR_SYSTEM_PROMPT = `You craft delegation prompts for a frontier model.
+
+Your job is to turn a user's request plus selected memory into a minimized, self-contained prompt with explicit [[user_data]] tagging.
+
+Return JSON only with this exact shape:
+{"reviewPrompt":"string"}
+
+Core rules:
+- The frontier model has zero prior context. Include everything it actually needs in one pass.
+- Include only the minimum user-specific data required to answer well.
+- If memory is not actually needed, keep the prompt generic.
+- Keep the user's current request in normal prose.
+- Every additional fact sourced from memory files or recent conversation that you include must be wrapped in [[user_data]]...[[/user_data]].
+- Do not wrap generic instructions, output-format guidance, or your own reasoning in tags.
+- Strip personal identifiers unless they are strictly necessary.
+- No real names unless the task genuinely requires the specific name.
+- No specific location unless the task depends on location.
+- Put everything into one final minimized prompt in reviewPrompt.
+- Do not include markdown fences or any text outside the JSON object.
+
+Privacy and minimization:
+- Every included fact should pass this test: "Does the frontier model need this specific fact to answer well?" If no, leave it out.
+- If a memory fact only repeats or confirms what the current query already makes obvious, leave it out.
+- Generalize when possible. Prefer "their partner is vegetarian" or just "vegetarian-friendly options" over a partner's real name.
+- Open-ended everyday questions usually need less context than planning or personalized analysis questions.
+- Do not assume household members are part of the request unless the user's question or the retrieved memory makes that clearly necessary.
+
+Common over-sharing patterns to avoid:
+- Do not include background facts that merely restate the topic, interest, or domain already obvious from the user's current query.
+- Do not include descriptive biography when the answer only needs concrete constraints, preferences, specs, or requirements.
+- Only include memory when it changes the answer: constraints, tradeoffs, personalization, or disambiguation.
+- Prefer concise, answer-shaping facts over broad user background.
+
+The user will review the exact prompt before it is sent. Keep it useful, minimal, and explicit.`;
+
+function clipText(value, limit) {
+    const text = typeof value === 'string' ? value.trim() : '';
+    if (!text) return '';
+    if (text.length <= limit) return text;
+    return `${text.slice(0, limit)}\n...(truncated)`;
+}
+
+function renderFiles(files) {
+    const normalizedFiles = Array.isArray(files) ? files : [];
+    let usedChars = 0;
+
+    return normalizedFiles.map((file, index) => {
+        const path = typeof file?.path === 'string' ? file.path : `memory-${index + 1}.md`;
+        let content = typeof file?.content === 'string' ? file.content.trim() : '';
+        if (!content) content = '(empty)';
+
+        const remaining = MAX_AUGMENT_TOTAL_CHARS - usedChars;
+        if (remaining <= 0) {
+            content = '(omitted for length)';
+        } else {
+            content = clipText(content, Math.min(MAX_AUGMENT_FILE_CHARS, remaining));
+            usedChars += content.length;
+        }
+
+        return `## ${path}\n${content}`;
+    }).join('\n\n');
+}
+
+function buildCrafterInput({ userQuery, files, conversationText }) {
+    const sections = [
+        `User query:\n${userQuery.trim()}`,
+        `Retrieved memory files:\n${renderFiles(files)}`
+    ];
+
+    const clippedConversation = trimRecentContext(conversationText);
+    if (clippedConversation) {
+        sections.push(`Recent conversation:\n${clippedConversation}`);
+    }
+
+    sections.push(`Produce the JSON now. Remember:
+- reviewPrompt should be the exact final prompt that will be shown to the user
+- keep the current user request in normal prose
+- any extra facts injected from memory or recent conversation must stay wrapped in [[user_data]] tags
+- if a memory fact only restates the domain already obvious from the query, omit it
+- omit names, relationship labels, and locations unless the prompt really needs them`);
+
+    return sections.join('\n\n');
+}
+
+function extractResponseText(response) {
+    if (!response) return '';
+    if (typeof response.content === 'string') return response.content;
+    return '';
+}
+
+function parseCrafterJson(rawText) {
+    const text = typeof rawText === 'string' ? rawText.trim() : '';
+    if (!text) {
+        throw new Error('augment_query prompt crafter returned an empty response.');
+    }
+
+    const codeFenceMatch = text.match(/```(?:json)?\s*([\s\S]*?)```/i);
+    const candidate = codeFenceMatch?.[1]?.trim() || text;
+    const start = candidate.indexOf('{');
+    const end = candidate.lastIndexOf('}');
+    const jsonText = (start !== -1 && end !== -1 && end >= start)
+        ? candidate.slice(start, end + 1)
+        : candidate;
+
+    let parsed;
+    try {
+        parsed = JSON.parse(jsonText);
+    } catch (error) {
+        throw new Error(`augment_query prompt crafter returned invalid JSON: ${error instanceof Error ? error.message : String(error)}`);
+    }
+
+    return {
+        reviewPrompt: typeof parsed?.reviewPrompt === 'string' ? parsed.reviewPrompt.trim() : ''
+    };
+}
+
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function getCrafterRetryDelay(attemptIndex) {
+    const exponential = AUGMENT_CRAFTER_RETRY_BASE_DELAY_MS * Math.pow(2, attemptIndex);
+    const jitter = Math.random() * AUGMENT_CRAFTER_RETRY_BASE_DELAY_MS;
+    return exponential + jitter;
+}
+
+function normalizeQueryText(text) {
+    return String(text || '').trim().replace(/\s+/g, ' ');
+}
 
 /**
  * Build tool executors for the retrieval (read) flow.
@@ -33,9 +203,10 @@ export function createRetrievalExecutors(backend) {
             const contentPaths = results.map(r => r.path);
 
             const allFiles = await backend.exportAll();
-            const queryLower = query.toLowerCase();
             const pathMatches = allFiles
-                .filter(f => !f.path.endsWith('_tree.md') && f.path.toLowerCase().includes(queryLower))
+                .filter((file) => typeof file?.path === 'string' && typeof file?.content === 'string')
+                .filter((file) => !file.path.endsWith('_tree.md'))
+                .filter((file) => pathMatchesQuery(file.path, query))
                 .map(f => f.path);
 
             const seen = new Set();
@@ -47,13 +218,178 @@ export function createRetrievalExecutors(backend) {
             return JSON.stringify({ paths: paths.slice(0, 5), count: Math.min(paths.length, 5) });
         },
         read_file: async ({ path }) => {
-            const content = await backend.read(path);
+            const resolvedPath = typeof backend.resolvePath === 'function'
+                ? await backend.resolvePath(path)
+                : null;
+            const content = await backend.read(resolvedPath || path);
             if (content === null) return JSON.stringify({ error: `File not found: ${path}` });
             return content.length > 1500 ? content.slice(0, 1500) + '...(truncated)' : content;
         }
     };
 }
 
+/**
+ * Build the executed augment_query tool for the retrieval flow.
+ *
+ * The outer memory-agent loop chooses relevant files. This executor then runs a
+ * dedicated prompt-crafter pass that turns those raw inputs into the final
+ * tagged prompt, keeping prompt-crafting fully inside nanomem.
+ *
+ * @param {object} options
+ * @param {StorageBackend} options.backend
+ * @param {LLMClient} options.llmClient
+ * @param {string} options.model
+ * @param {string} options.query
+ * @param {string} [options.conversationText]
+ * @param {(event: { stage: 'loading', message: string, attempt?: number }) => void} [options.onProgress]
+ */
+export function createAugmentQueryExecutor({ backend, llmClient, model, query, conversationText, onProgress }) {
+    return async ({ user_query, memory_files }) => {
+        const selectedPaths = Array.isArray(memory_files)
+            ? [...new Set(memory_files.filter((path) => typeof path === 'string' && path.trim()))].slice(0, MAX_AUGMENT_QUERY_FILES)
+            : [];
+        const originalQuery = normalizeQueryText(query);
+        const providedQuery = normalizeQueryText(user_query);
+        const effectiveQuery = (typeof user_query === 'string' && providedQuery && providedQuery === originalQuery)
+            ? user_query.trim()
+            : query;
+
+        if (!effectiveQuery || !effectiveQuery.trim()) {
+            return JSON.stringify({ error: 'augment_query requires the original user_query.' });
+        }
+
+        if (selectedPaths.length === 0) {
+            return JSON.stringify({
+                noRelevantMemory: true,
+                files: []
+            });
+        }
+
+        const files = [];
+        for (const path of selectedPaths) {
+            const resolvedPath = typeof backend.resolvePath === 'function'
+                ? await backend.resolvePath(path)
+                : null;
+            const canonicalPath = resolvedPath || path;
+            const raw = await backend.read(canonicalPath);
+            if (!raw) continue;
+            files.push({ path: canonicalPath, content: raw });
+        }
+
+        if (files.length === 0) {
+            return JSON.stringify({ error: 'augment_query could not load any selected memory files.' });
+        }
+
+        let reviewPrompt = '';
+        let crafterError = '';
+        const messages = /** @type {import('../types.js').LLMMessage[]} */ ([
+            { role: 'system', content: AUGMENT_QUERY_EXECUTOR_SYSTEM_PROMPT },
+            {
+                role: 'user',
+                content: buildCrafterInput({
+                    userQuery: effectiveQuery,
+                    files,
+                    conversationText
+                })
+            }
+        ]);
+
+        for (let attempt = 1; attempt <= AUGMENT_CRAFTER_MAX_ATTEMPTS; attempt += 1) {
+            let response;
+            try {
+                onProgress?.({
+                    stage: 'loading',
+                    message: attempt === 1
+                        ? 'Crafting minimized prompt...'
+                        : `Retrying prompt crafting (${attempt}/${AUGMENT_CRAFTER_MAX_ATTEMPTS})...`,
+                    attempt
+                });
+                if (typeof llmClient.streamChatCompletion === 'function') {
+                    let emittedReasoningPhase = false;
+                    let emittedOutputPhase = false;
+                    response = /** @type {ChatCompletionResponse} */ (await llmClient.streamChatCompletion({
+                        model,
+                        messages,
+                        temperature: 0,
+                        onDelta: (chunk) => {
+                            if (!chunk || emittedOutputPhase) return;
+                            emittedOutputPhase = true;
+                            onProgress?.({
+                                stage: 'loading',
+                                message: 'Finalizing prompt...',
+                                attempt
+                            });
+                        },
+                        onReasoning: (chunk) => {
+                            if (!chunk || emittedReasoningPhase) return;
+                            emittedReasoningPhase = true;
+                            onProgress?.({
+                                stage: 'loading',
+                                message: 'Minimizing personal context...',
+                                attempt
+                            });
+                        }
+                    }));
+                } else {
+                    response = /** @type {ChatCompletionResponse} */ (await llmClient.createChatCompletion({
+                        model,
+                        messages,
+                        temperature: 0
+                    }));
+                }
+            } catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                return JSON.stringify({ error: `augment_query prompt crafting failed: ${message}` });
+            }
+
+            try {
+                const parsed = parseCrafterJson(extractResponseText(response));
+                reviewPrompt = parsed.reviewPrompt;
+                if (!reviewPrompt) {
+                    throw new Error('augment_query did not produce a reviewPrompt.');
+                }
+                crafterError = '';
+                break;
+            } catch (error) {
+                crafterError = error instanceof Error ? error.message : String(error);
+                if (attempt >= AUGMENT_CRAFTER_MAX_ATTEMPTS) {
+                    break;
+                }
+                const delay = getCrafterRetryDelay(attempt - 1);
+                onProgress?.({
+                    stage: 'loading',
+                    message: `Prompt crafter retry ${attempt + 1}/${AUGMENT_CRAFTER_MAX_ATTEMPTS} after: ${crafterError}`,
+                    attempt: attempt + 1
+                });
+                console.warn(`[nanomem/augment_query] prompt crafter attempt ${attempt}/${AUGMENT_CRAFTER_MAX_ATTEMPTS} failed: ${crafterError}. Retrying in ${Math.round(delay)}ms.`);
+                await sleep(delay);
+            }
+        }
+
+        if (crafterError) {
+            return JSON.stringify({ error: `${crafterError} (after ${AUGMENT_CRAFTER_MAX_ATTEMPTS} attempts)` });
+        }
+
+        if (!/\[\[user_data\]\]/.test(reviewPrompt)) {
+            return JSON.stringify({
+                noRelevantMemory: true,
+                files: []
+            });
+        }
+
+        const apiPrompt = stripUserDataTags(reviewPrompt);
+
+        return JSON.stringify({
+            reviewPrompt,
+            apiPrompt,
+            files: files.map((file) => ({
+                path: file.path,
+                content: clipText(file.content, MAX_AUGMENT_FILE_CHARS)
+            }))
+        });
+    };
+}
+
 /**
  * Build tool executors for the extraction (write) flow.
  * @param {StorageBackend} backend
@@ -117,6 +453,70 @@ export function createExtractionExecutors(backend, hooks = {}) {
     };
 }
 
+/**
+ * Build tool executors for the deletion flow.
+ * @param {StorageBackend} backend
+ * @param {{ refreshIndex?: Function, onWrite?: Function }} [hooks]
+ */
+export function createDeletionExecutors(backend, hooks = {}) {
+    const { refreshIndex, onWrite } = hooks;
+
+    return {
+        list_directory: async ({ dir_path }) => {
+            const { files, dirs } = await backend.ls(dir_path || '');
+            return JSON.stringify({ files, dirs });
+        },
+        retrieve_file: async ({ query }) => {
+            const results = await backend.search(query);
+            const contentPaths = results.map(r => r.path);
+
+            const allFiles = await backend.exportAll();
+            const queryLower = query.toLowerCase();
+            const pathMatches = allFiles
+                .filter(f => !f.path.endsWith('_tree.md') && f.path.toLowerCase().includes(queryLower))
+                .map(f => f.path);
+
+            const seen = new Set();
+            const paths = [];
+            for (const p of [...pathMatches, ...contentPaths]) {
+                if (!seen.has(p)) { seen.add(p); paths.push(p); }
+            }
+
+            return JSON.stringify({ paths: paths.slice(0, 5), count: Math.min(paths.length, 5) });
+        },
+        read_file: async ({ path }) => {
+            const content = await backend.read(path);
+            if (content === null) return JSON.stringify({ error: `File not found: ${path}` });
+            return content;
+        },
+        delete_bullet: async ({ path, bullet_text }) => {
+            const before = await backend.read(path);
+            if (!before) return JSON.stringify({ error: `File not found: ${path}` });
+            // Strip pipe-delimited metadata if present — removeArchivedItem matches
+            // against bullet.text (fact text only), not the full line with metadata.
+            const factText = bullet_text.includes('|')
+                ? bullet_text.split('|')[0].trim()
+                : bullet_text.trim();
+            const after = removeArchivedItem(before, factText, path);
+            if (after === null) {
+                return JSON.stringify({ error: `No exact match found for the given bullet text in: ${path}` });
+            }
+            // If no bullets remain, delete the file entirely instead of leaving empty headers.
+            const remaining = parseBullets(after);
+            if (remaining.length === 0) {
+                await backend.delete(path);
+                if (refreshIndex) await refreshIndex(path);
+                onWrite?.(path, before, null);
+                return JSON.stringify({ success: true, path, action: 'file_deleted', removed: factText });
+            }
+            await backend.write(path, after);
+            if (refreshIndex) await refreshIndex(path);
+            onWrite?.(path, before, after);
+            return JSON.stringify({ success: true, path, action: 'deleted', removed: factText });
+        },
+    };
+}
+
 function removeArchivedItem(content, itemText, path) {
     const raw = String(content || '');
     const target = normalizeFactText(itemText);
@@ -150,3 +550,15 @@ function removeArchivedItem(content, itemText, path) {
     if (!removed) return null;
     return filtered.join('\n').trim();
 }
+
+function trimRecentContext(conversationText) {
+    return trimRecentConversation(conversationText, {
+        maxChars: MAX_AUGMENT_RECENT_CONTEXT_CHARS
+    });
+}
+
+function stripUserDataTags(text) {
+    return String(text ?? '')
+        .replace(/\[\[user_data\]\]/g, '')
+        .replace(/\[\[\/user_data\]\]/g, '');
+}

package/src/engine/ingester.js

@@ -19,68 +19,86 @@ import {
 
 const MAX_CONVERSATION_CHARS = 128000;
 
-/** @type {ToolDefinition[]} */
-const EXTRACTION_TOOLS = [
-    {
-        type: 'function',
-        function: {
-            name: 'read_file',
-            description: 'Read an existing memory file before writing.',
-            parameters: {
-                type: 'object',
-                properties: {
-                    path: { type: 'string', description: 'File path (e.g. personal/about.md)' }
-                },
-                required: ['path']
-            }
+/** @type {ToolDefinition} */
+const T_READ_FILE = {
+    type: 'function',
+    function: {
+        name: 'read_file',
+        description: 'Read an existing memory file before writing.',
+        parameters: {
+            type: 'object',
+            properties: {
+                path: { type: 'string', description: 'File path (e.g. personal/about.md)' }
+            },
+            required: ['path']
         }
-    },
-    {
-        type: 'function',
-        function: {
-            name: 'create_new_file',
-            description: 'Create a new memory file for a topic not covered by any existing file.',
-            parameters: {
-                type: 'object',
-                properties: {
-                    path: { type: 'string', description: 'File path (e.g. projects/recipe-app.md)' },
-                    content: { type: 'string', description: 'Bullet-point content to write' }
-                },
-                required: ['path', 'content']
-            }
+    }
+};
+
+/** @type {ToolDefinition} */
+const T_CREATE_NEW_FILE = {
+    type: 'function',
+    function: {
+        name: 'create_new_file',
+        description: 'Create a new memory file for a topic not covered by any existing file.',
+        parameters: {
+            type: 'object',
+            properties: {
+                path: { type: 'string', description: 'File path (e.g. projects/recipe-app.md)' },
+                content: { type: 'string', description: 'Bullet-point content to write' }
+            },
+            required: ['path', 'content']
         }
-    },
-    {
-        type: 'function',
-        function: {
-            name: 'append_memory',
-            description: 'Append new bullet points to an existing memory file.',
-            parameters: {
-                type: 'object',
-                properties: {
-                    path: { type: 'string', description: 'File path to append to' },
-                    content: { type: 'string', description: 'Bullet-point content to append' }
-                },
-                required: ['path', 'content']
-            }
+    }
+};
+
+/** @type {ToolDefinition} */
+const T_APPEND_MEMORY = {
+    type: 'function',
+    function: {
+        name: 'append_memory',
+        description: 'Append new bullet points to an existing memory file.',
+        parameters: {
+            type: 'object',
+            properties: {
+                path: { type: 'string', description: 'File path to append to' },
+                content: { type: 'string', description: 'Bullet-point content to append' }
+            },
+            required: ['path', 'content']
         }
-    },
-    {
-        type: 'function',
-        function: {
-            name: 'update_memory',
-            description: 'Overwrite an existing memory file. Use when existing content is stale or contradicted.',
-            parameters: {
-                type: 'object',
-                properties: {
-                    path: { type: 'string', description: 'File path to update' },
-                    content: { type: 'string', description: 'Complete new content for the file' }
-                },
-                required: ['path', 'content']
-            }
+    }
+};
+
+/** @type {ToolDefinition} */
+const T_UPDATE_MEMORY = {
+    type: 'function',
+    function: {
+        name: 'update_memory',
+        description: 'Overwrite an existing memory file. Use when existing content is stale or contradicted.',
+        parameters: {
+            type: 'object',
+            properties: {
+                path: { type: 'string', description: 'File path to update' },
+                content: { type: 'string', description: 'Complete new content for the file' }
+            },
+            required: ['path', 'content']
         }
     }
-];
+};
+
+/**
+ * Tool sets per ingestion mode.
+ * `add`    — can only write new content (no update_memory).
+ * `update` — can only edit existing files (no create/append).
+ * Others   — full access.
+ * @type {Record<string, ToolDefinition[]>}
+ */
+const TOOLS_BY_MODE = {
+    add:          [T_READ_FILE, T_CREATE_NEW_FILE, T_APPEND_MEMORY],
+    update:       [T_READ_FILE, T_UPDATE_MEMORY],
+};
+
+const EXTRACTION_TOOLS = [T_READ_FILE, T_CREATE_NEW_FILE, T_APPEND_MEMORY, T_UPDATE_MEMORY];
 
 class MemoryIngester {
     constructor({ backend, bulletIndex, llmClient, model, onToolCall }) {
@@ -128,12 +146,14 @@ class MemoryIngester {
             ? `Document content:\n\`\`\`\n${conversationText}\n\`\`\``
             : `Conversation:\n\`\`\`\n${conversationText}\n\`\`\``;
 
+        const tools = TOOLS_BY_MODE[mode] || EXTRACTION_TOOLS;
+
         let toolCallLog;
         try {
             const result = await runAgenticToolLoop({
                 llmClient: this._llmClient,
                 model: this._model,
-                tools: EXTRACTION_TOOLS,
+                tools,
                 toolExecutors,
                 messages: [
                     { role: 'system', content: systemPrompt },
@@ -142,8 +162,8 @@ class MemoryIngester {
                 maxIterations: 12,
                 maxOutputTokens: 4000,
                 temperature: 0,
-                onToolCall: (name, args, result) => {
-                    onToolCall?.(name, args, result);
+                onToolCall: (name, args, result, meta) => {
+                    onToolCall?.(name, args, result, meta);
                 }
             });
             toolCallLog = result.toolCallLog;
@@ -187,7 +207,9 @@ class MemoryIngester {
 
         const defaultTopic = inferTopicFromPath(path);
         const normalized = incomingBullets.map((bullet) => {
-            const b = ensureBulletMetadata({ ...bullet, updatedAt: null }, { defaultTopic, updatedAt });
+            // Preserve existing updatedAt so update_memory doesn't re-stamp unchanged bullets.
+            // Bullets without a date still fall back to updatedAt (today).
+            const b = ensureBulletMetadata(bullet, { defaultTopic, updatedAt });
             if (isDocument && b.source === 'user_statement') b.source = 'document';
             return b;
         });