@openanonymity/nanomem 0.1.0

package/src/engine/compactor.js

@@ -0,0 +1,247 @@
+/**
+ * MemoryCompactor — Periodic dedup + archive of stale facts.
+ *
+ * Two-phase compaction:
+ *   1. Deterministic — dedup, tier assignment, expiry (no LLM, cheap)
+ *   2. Semantic review — LLM pass on Working bullets only, to catch stale
+ *      plans/goals that deterministic logic can't detect. Uses cross-file
+ *      oneLiner summaries as context so inter-file resolutions are visible.
+ *
+ * Unstructured/legacy files use a full LLM rewrite (existing behaviour).
+ *
+ * Usage:
+ * - compactAll(): Force-compact all memory files immediately.
+ * - maybeCompact(): Only runs if >=6 hours have passed since last run.
+ */
+/** @import { LLMClient, StorageBackend } from '../types.js' */
+import {
+    compactBullets,
+    inferTopicFromPath,
+    parseBullets,
+    todayIsoDate,
+    renderCompactedDocument
+} from '../bullets/index.js';
+
+
+const MAX_FILE_CHARS = 8000;
+
+// ─── Prompts ─────────────────────────────────────────────────────
+
+const COMPACTION_PROMPT = `You are compacting a markdown memory file into a stable memory format.
+
+Input is one memory file. Rewrite it into:
+
+# Memory: <Topic>
+
+## Working
+### <Topic>
+- fact | topic=<topic> | tier=working | status=active | source=user_statement|assistant_summary|inference|system | confidence=high|medium|low | updated_at=YYYY-MM-DD | review_at=YYYY-MM-DD(optional) | expires_at=YYYY-MM-DD(optional)
+
+## Long-Term
+### <Topic>
+- fact | topic=<topic> | tier=long_term | status=active | source=user_statement|assistant_summary|inference|system | confidence=high|medium|low | updated_at=YYYY-MM-DD | expires_at=YYYY-MM-DD(optional)
+
+## History
+### <Topic>
+- fact | topic=<topic> | tier=history | status=superseded|expired|uncertain | source=user_statement|assistant_summary|inference|system | confidence=high|medium|low | updated_at=YYYY-MM-DD | expires_at=YYYY-MM-DD(optional)
+
+Rules:
+- Write facts in a timeless, archival format: use absolute dates (YYYY-MM-DD) rather than relative terms like "recently", "currently", "just", or "last week". A fact must be interpretable correctly even years after it was written.
+- Keep only concrete reusable facts.
+- Merge semantic duplicates and keep the most recent/best phrasing.
+- Resolve contradictions: newer user statements beat older ones; user statements beat inferences; higher confidence beats lower.
+- Put stable facts in Long-Term: identity/background, durable preferences, recurring constraints, persistent health facts, long-running roles, durable relationships.
+- Put temporary or in-progress context in Working: active plans, current tasks, temporary situations, near-term goals.
+- Expired facts (expires_at in the past) go to History with status=expired.
+- Working facts should include review_at or expires_at when possible.
+- Keep Working concise. Move stale/low-priority facts to History.
+- Preserve meaning; do not invent facts.
+- Output markdown only (no fences, no explanations).
+
+Today: {TODAY}
+Path: {PATH}
+
+File content:
+\`\`\`
+{CONTENT}
+\`\`\``;
+
+/**
+ * Targeted semantic review of Working bullets only.
+ * Output is minimal (KEEP/SUPERSEDED per bullet) to keep cost low.
+ */
+const SEMANTIC_REVIEW_PROMPT = `Today is {TODAY}. Review these short-term (Working) memory bullets and identify which are stale, completed, or superseded.
+
+{FILE_SUMMARIES_SECTION}{LONG_TERM_SECTION}Working bullets to review:
+{NUMBERED_BULLETS}
+
+For each numbered bullet, output exactly one line in the format:
+N: KEEP
+or
+N: SUPERSEDED — brief reason
+
+Output only these lines, one per bullet, nothing else.`;
+
+// ─── Compactor ───────────────────────────────────────────────────
+
+class MemoryCompactor {
+    constructor({ backend, bulletIndex, llmClient, model, onProgress }) {
+        this._backend = backend;
+        this._bulletIndex = bulletIndex;
+        this._llmClient = llmClient;
+        this._model = model;
+        this._onProgress = onProgress || null;
+        this._running = false;
+        this._fileSummaries = [];
+    }
+
+    async compactAll() {
+        if (this._running) return;
+        this._running = true;
+        try {
+            await this._backend.init();
+            const allFiles = await this._backend.exportAll();
+            const realFiles = allFiles.filter((file) => !file.path.endsWith('_tree.md'));
+
+            // Collect one-liner summaries for cross-file context in semantic review.
+            this._fileSummaries = realFiles
+                .filter(f => f.oneLiner)
+                .map(f => ({ path: f.path, oneLiner: f.oneLiner }));
+
+            let filesChanged = 0;
+            const total = realFiles.length;
+
+            for (let i = 0; i < total; i++) {
+                const file = realFiles[i];
+                this._onProgress?.({ stage: 'file', file: file.path, current: i + 1, total });
+
+                const compacted = await this._compactFile(file.path, file.content || '');
+                if (!compacted) continue;
+                if (compacted.trim() === String(file.content || '').trim()) continue;
+                await this._backend.write(file.path, compacted);
+                await this._bulletIndex.refreshPath(file.path);
+                filesChanged++;
+            }
+
+            return { filesChanged, filesTotal: total };
+
+        } finally {
+            this._running = false;
+            this._fileSummaries = [];
+        }
+    }
+
+    async _compactFile(path, content) {
+        const raw = String(content || '').trim();
+        if (!raw) return null;
+
+        const parsed = parseBullets(raw);
+
+        // Unstructured/legacy files: full LLM rewrite (unchanged behaviour).
+        if (parsed.length === 0) {
+            return this._llmRewrite(path, raw);
+        }
+
+        // Phase 1: deterministic dedup, tier assignment, expiry.
+        const defaultTopic = inferTopicFromPath(path);
+        const det = compactBullets(parsed, { defaultTopic });
+        const deduplicated = parsed.length - (det.working.length + det.longTerm.length + det.history.length);
+        const expired = det.history.filter(b => b.status === 'expired').length;
+
+        // Phase 2: semantic review of Working bullets.
+        // Only fires when Working bullets exist — skipped for stable long-term-only files.
+        let working = det.working;
+        let superseded = 0;
+        if (working.length > 0) {
+            this._onProgress?.({ stage: 'semantic', file: path });
+            working = await this._semanticReviewWorking(working, det.longTerm, path);
+            superseded = working.filter(b => b.status === 'superseded').length;
+        }
+
+        // Re-run deterministic compaction so newly-superseded bullets flow to History.
+        const allBullets = [...working, ...det.longTerm, ...det.history];
+        const final = compactBullets(allBullets, { defaultTopic });
+
+        this._onProgress?.({ stage: 'file_done', file: path, deduplicated, superseded, expired });
+
+        return renderCompactedDocument(
+            final.working, final.longTerm, final.history,
+            { titleTopic: defaultTopic }
+        );
+    }
+
+    /**
+     * Ask the LLM which Working bullets are now stale/completed/superseded.
+     * Returns the same array with superseded bullets marked status=superseded.
+     */
+    async _semanticReviewWorking(working, longTerm, path) {
+        const numberedBullets = working
+            .map((b, i) => `${i + 1}: ${b.text}`)
+            .join('\n');
+
+        const longTermSection = longTerm.length > 0
+            ? `Long-term facts in this file (for resolution context):\n${longTerm.map(b => `- ${b.text}`).join('\n')}\n\n`
+            : '';
+
+        // Cross-file context: exclude this file's own summary.
+        const otherFiles = this._fileSummaries.filter(f => f.path !== path);
+        const fileSummariesSection = otherFiles.length > 0
+            ? `Other memory files:\n${otherFiles.map(f => `- ${f.path}: ${f.oneLiner}`).join('\n')}\n\n`
+            : '';
+
+        const prompt = SEMANTIC_REVIEW_PROMPT
+            .replace('{TODAY}', todayIsoDate())
+            .replace('{FILE_SUMMARIES_SECTION}', fileSummariesSection)
+            .replace('{LONG_TERM_SECTION}', longTermSection)
+            .replace('{NUMBERED_BULLETS}', numberedBullets);
+
+        let responseText = '';
+        try {
+            const response = await this._llmClient.createChatCompletion({
+                model: this._model,
+                messages: [{ role: 'user', content: prompt }],
+                max_tokens: 6000,
+                temperature: 0,
+            });
+            responseText = response.content || '';
+        } catch {
+            // On failure, leave Working bullets unchanged rather than silently corrupting data.
+            return working;
+        }
+
+        // Parse "N: KEEP" / "N: SUPERSEDED — reason" lines.
+        const decisions = new Map();
+        for (const line of responseText.split('\n')) {
+            const match = line.match(/^(\d+)\s*:\s*(KEEP|SUPERSEDED)/i);
+            if (match) {
+                decisions.set(parseInt(match[1], 10), match[2].toUpperCase());
+            }
+        }
+
+        return working.map((b, i) => {
+            if (decisions.get(i + 1) === 'SUPERSEDED') {
+                return { ...b, status: 'superseded', tier: 'history', section: 'history' };
+            }
+            return b;
+        });
+    }
+
+    async _llmRewrite(path, raw) {
+        const prompt = COMPACTION_PROMPT
+            .replace('{TODAY}', todayIsoDate())
+            .replace('{PATH}', path)
+            .replace('{CONTENT}', raw.length > MAX_FILE_CHARS ? raw.slice(0, MAX_FILE_CHARS) + '\n...(truncated)' : raw);
+
+        const response = await this._llmClient.createChatCompletion({
+            model: this._model,
+            messages: [{ role: 'user', content: prompt }],
+            max_tokens: 1800,
+            temperature: 0,
+        });
+
+        const text = (response.content || '').trim();
+        return text || null;
+    }
+}
+
+export { MemoryCompactor };

package/src/engine/executors.js

@@ -0,0 +1,152 @@
+/**
+ * Tool executor factories — the bridge between the agentic tool loop and storage.
+ *
+ * Architecture:
+ *   retrieval.js / extractor.js  — define tool schemas (what the LLM sees)
+ *   executors.js (this file)     — implement those tools (what runs when called)
+ *   toolLoop.js                  — generic engine that connects the two
+ *
+ * Each factory takes a storage backend and returns an object mapping
+ * tool names to async functions: { tool_name: async (args) => resultString }
+ */
+/** @import { ExtractionExecutorHooks, StorageBackend } from '../types.js' */
+import {
+    compactBullets,
+    inferTopicFromPath,
+    normalizeFactText,
+    parseBullets,
+    renderCompactedDocument,
+} from '../bullets/index.js';
+
+/**
+ * Build tool executors for the retrieval (read) flow.
+ * @param {StorageBackend} backend
+ */
+export function createRetrievalExecutors(backend) {
+    return {
+        list_directory: async ({ dir_path }) => {
+            const { files, dirs } = await backend.ls(dir_path || '');
+            return JSON.stringify({ files, dirs });
+        },
+        retrieve_file: async ({ query }) => {
+            const results = await backend.search(query);
+            const contentPaths = results.map(r => r.path);
+
+            const allFiles = await backend.exportAll();
+            const queryLower = query.toLowerCase();
+            const pathMatches = allFiles
+                .filter(f => !f.path.endsWith('_tree.md') && f.path.toLowerCase().includes(queryLower))
+                .map(f => f.path);
+
+            const seen = new Set();
+            const paths = [];
+            for (const p of [...pathMatches, ...contentPaths]) {
+                if (!seen.has(p)) { seen.add(p); paths.push(p); }
+            }
+
+            return JSON.stringify({ paths: paths.slice(0, 5), count: Math.min(paths.length, 5) });
+        },
+        read_file: async ({ path }) => {
+            const content = await backend.read(path);
+            if (content === null) return JSON.stringify({ error: `File not found: ${path}` });
+            return content.length > 1500 ? content.slice(0, 1500) + '...(truncated)' : content;
+        }
+    };
+}
+
+/**
+ * Build tool executors for the extraction (write) flow.
+ * @param {StorageBackend} backend
+ * @param {ExtractionExecutorHooks} [hooks]
+ */
+export function createExtractionExecutors(backend, hooks = {}) {
+    const { normalizeContent, mergeWithExisting, refreshIndex, onWrite } = hooks;
+
+    return {
+        read_file: async ({ path }) => {
+            const content = await backend.read(path);
+            if (content === null) return JSON.stringify({ error: `File not found: ${path}` });
+            return content.length > 2000 ? content.slice(0, 2000) + '...(truncated)' : content;
+        },
+        create_new_file: async ({ path, content }) => {
+            const exists = await backend.exists(path);
+            if (exists) return JSON.stringify({ error: `File already exists: ${path}. Use append_memory or update_memory instead.` });
+            const normalized = normalizeContent ? normalizeContent(content, path) : content;
+            await backend.write(path, normalized);
+            if (refreshIndex) await refreshIndex(path);
+            onWrite?.(path, '', normalized);
+            return JSON.stringify({ success: true, path });
+        },
+        append_memory: async ({ path, content }) => {
+            const existing = await backend.read(path);
+            const newContent = mergeWithExisting
+                ? mergeWithExisting(existing, content, path)
+                : (existing ? existing + '\n\n' + content : content);
+            await backend.write(path, newContent);
+            if (refreshIndex) await refreshIndex(path);
+            onWrite?.(path, existing ?? '', newContent);
+            return JSON.stringify({ success: true, path, action: 'appended' });
+        },
+        update_memory: async ({ path, content }) => {
+            const before = await backend.read(path);
+            const normalized = normalizeContent ? normalizeContent(content, path) : content;
+            await backend.write(path, normalized);
+            if (refreshIndex) await refreshIndex(path);
+            onWrite?.(path, before ?? '', normalized);
+            return JSON.stringify({ success: true, path, action: 'updated' });
+        },
+        archive_memory: async ({ path, item_text }) => {
+            const existing = await backend.read(path);
+            if (!existing) return JSON.stringify({ error: `File not found: ${path}` });
+            const newContent = removeArchivedItem(existing, item_text, path);
+            if (newContent === null) {
+                return JSON.stringify({ error: `Could not find an exact memory item match in: ${path}` });
+            }
+            await backend.write(path, newContent);
+            if (refreshIndex) await refreshIndex(path);
+            return JSON.stringify({ success: true, path, action: 'archived', removed: item_text });
+        },
+        delete_memory: async ({ path }) => {
+            if (path.endsWith('_tree.md')) {
+                return JSON.stringify({ error: 'Cannot delete index files' });
+            }
+            await backend.delete(path);
+            if (refreshIndex) await refreshIndex(path);
+            return JSON.stringify({ success: true, path, action: 'deleted' });
+        }
+    };
+}
+
+function removeArchivedItem(content, itemText, path) {
+    const raw = String(content || '');
+    const target = normalizeFactText(itemText);
+    if (!target) return null;
+
+    const parsed = parseBullets(raw);
+    if (parsed.length > 0) {
+        const remaining = parsed.filter((bullet) => normalizeFactText(bullet.text) !== target);
+        if (remaining.length === parsed.length) return null;
+        const compacted = compactBullets(remaining, { defaultTopic: inferTopicFromPath(path), maxActivePerTopic: 1000 });
+        return renderCompactedDocument(
+            compacted.working, compacted.longTerm, compacted.history,
+            { titleTopic: inferTopicFromPath(path) }
+        );
+    }
+
+    const lines = raw.split('\n');
+    let removed = false;
+    const filtered = lines.filter((line) => {
+        const trimmed = line.trim();
+        const normalized = trimmed.startsWith('- ')
+            ? normalizeFactText(trimmed.slice(2))
+            : normalizeFactText(trimmed);
+        if (!removed && normalized === target) {
+            removed = true;
+            return false;
+        }
+        return true;
+    });
+
+    if (!removed) return null;
+    return filtered.join('\n').trim();
+}

package/src/engine/ingester.js

@@ -0,0 +1,229 @@
+/**
+ * MemoryIngester — Write path for agentic memory.
+ *
+ * Takes a conversation (array of messages) and uses tool-calling via the
+ * agentic loop to decide whether to create/append/update memory files.
+ */
+/** @import { IngestOptions, IngestResult, LLMClient, Message, StorageBackend, ToolDefinition } from '../types.js' */
+import { runAgenticToolLoop } from './toolLoop.js';
+import { createExtractionExecutors } from './executors.js';
+import { resolvePromptSet } from '../prompt_sets/index.js';
+import {
+    compactBullets,
+    ensureBulletMetadata,
+    inferTopicFromPath,
+    parseBullets,
+    renderCompactedDocument,
+    todayIsoDate
+} from '../bullets/index.js';
+
+const MAX_CONVERSATION_CHARS = 128000;
+
+/** @type {ToolDefinition[]} */
+const EXTRACTION_TOOLS = [
+    {
+        type: 'function',
+        function: {
+            name: 'read_file',
+            description: 'Read an existing memory file before writing.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    path: { type: 'string', description: 'File path (e.g. personal/about.md)' }
+                },
+                required: ['path']
+            }
+        }
+    },
+    {
+        type: 'function',
+        function: {
+            name: 'create_new_file',
+            description: 'Create a new memory file for a topic not covered by any existing file.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    path: { type: 'string', description: 'File path (e.g. projects/recipe-app.md)' },
+                    content: { type: 'string', description: 'Bullet-point content to write' }
+                },
+                required: ['path', 'content']
+            }
+        }
+    },
+    {
+        type: 'function',
+        function: {
+            name: 'append_memory',
+            description: 'Append new bullet points to an existing memory file.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    path: { type: 'string', description: 'File path to append to' },
+                    content: { type: 'string', description: 'Bullet-point content to append' }
+                },
+                required: ['path', 'content']
+            }
+        }
+    },
+    {
+        type: 'function',
+        function: {
+            name: 'update_memory',
+            description: 'Overwrite an existing memory file. Use when existing content is stale or contradicted.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    path: { type: 'string', description: 'File path to update' },
+                    content: { type: 'string', description: 'Complete new content for the file' }
+                },
+                required: ['path', 'content']
+            }
+        }
+    }
+];
+
+class MemoryIngester {
+    constructor({ backend, bulletIndex, llmClient, model, onToolCall }) {
+        this._backend = backend;
+        this._bulletIndex = bulletIndex;
+        this._llmClient = llmClient;
+        this._model = model;
+        this._onToolCall = onToolCall || null;
+    }
+
+    /**
+     * Ingest memory from a conversation.
+     *
+     * @param {Message[]} messages
+     * @param {IngestOptions} [options]
+     * @returns {Promise<IngestResult>}
+     */
+    async ingest(messages, options = {}) {
+        const updatedAt = options.updatedAt || todayIsoDate();
+        const onToolCall = this._onToolCall;
+        if (!messages || messages.length === 0) return { status: 'skipped', writeCalls: 0 };
+
+        // Support both `mode` and legacy `extractionMode`
+        const mode = options.mode || options.extractionMode || 'conversation';
+        const isDocument = mode === 'document';
+        const conversationText = isDocument
+            ? this._buildDocumentText(messages)
+            : this._buildConversationText(messages);
+        if (!conversationText) return { status: 'skipped', writeCalls: 0 };
+
+        await this._backend.init();
+        const index = await this._backend.getTree() || '';
+
+        const { ingestionPrompt } = resolvePromptSet(mode);
+        const systemPrompt = ingestionPrompt.replace('{INDEX}', index);
+        const writes = [];
+        const toolExecutors = createExtractionExecutors(this._backend, {
+            normalizeContent: (content, path) => this._normalizeGeneratedContent(content, path, updatedAt, isDocument),
+            mergeWithExisting: (existing, incoming, path) => this._mergeWithExisting(existing, incoming, path, updatedAt, isDocument),
+            refreshIndex: (path) => this._bulletIndex.refreshPath(path),
+            onWrite: (path, before, after) => writes.push({ path, before, after }),
+        });
+
+        const userMessage = isDocument
+            ? `Document content:\n\`\`\`\n${conversationText}\n\`\`\``
+            : `Conversation:\n\`\`\`\n${conversationText}\n\`\`\``;
+
+        let toolCallLog;
+        try {
+            const result = await runAgenticToolLoop({
+                llmClient: this._llmClient,
+                model: this._model,
+                tools: EXTRACTION_TOOLS,
+                toolExecutors,
+                messages: [
+                    { role: 'system', content: systemPrompt },
+                    { role: 'user', content: userMessage }
+                ],
+                maxIterations: 12,
+                maxOutputTokens: 4000,
+                temperature: 0,
+                onToolCall: (name, args, result) => {
+                    onToolCall?.(name, args, result);
+                }
+            });
+            toolCallLog = result.toolCallLog;
+        } catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            return { status: 'error', writeCalls: 0, error: message };
+        }
+
+        const writeTools = ['create_new_file', 'append_memory', 'update_memory', 'archive_memory', 'delete_memory'];
+        const writeCalls = toolCallLog.filter(e => writeTools.includes(e.name));
+
+        return { status: 'processed', writeCalls: writeCalls.length, writes };
+    }
+
+    _buildConversationText(messages) {
+        let text = '';
+        for (const msg of messages) {
+            const role = msg.role === 'user' ? 'User' : 'Assistant';
+            const content = msg.content || '';
+            text += `${role}: ${content}\n\n`;
+            if (text.length > MAX_CONVERSATION_CHARS) break;
+        }
+        return text.trim();
+    }
+
+    _buildDocumentText(messages) {
+        // For documents, concatenate content blocks without role labels.
+        // Multiple messages are treated as sections of the same document.
+        return messages
+            .map(m => (m.content || '').trim())
+            .filter(Boolean)
+            .join('\n\n')
+            .slice(0, MAX_CONVERSATION_CHARS);
+    }
+
+    _normalizeGeneratedContent(content, path, updatedAt, isDocument = false) {
+        const incomingBullets = parseBullets(content);
+        if (incomingBullets.length === 0) {
+            return content;
+        }
+
+        const defaultTopic = inferTopicFromPath(path);
+        const normalized = incomingBullets.map((bullet) => {
+            const b = ensureBulletMetadata({ ...bullet, updatedAt: null }, { defaultTopic, updatedAt });
+            if (isDocument && b.source === 'user_statement') b.source = 'document';
+            return b;
+        });
+        const compacted = compactBullets(normalized, { defaultTopic, maxActivePerTopic: 1000 });
+        return renderCompactedDocument(compacted.working, compacted.longTerm, compacted.history, { titleTopic: defaultTopic });
+    }
+
+    _mergeWithExisting(existing, incoming, path, updatedAt, isDocument = false) {
+        const existingText = String(existing || '');
+        const incomingText = String(incoming || '');
+        const defaultTopic = inferTopicFromPath(path);
+
+        const existingBullets = parseBullets(existingText)
+            .map((bullet) => ensureBulletMetadata(bullet, { defaultTopic }));
+        const incomingBullets = parseBullets(incomingText)
+            .map((bullet) => {
+                const b = ensureBulletMetadata({ ...bullet, updatedAt: null }, { defaultTopic, updatedAt });
+                if (isDocument && b.source === 'user_statement') b.source = 'document';
+                return b;
+            });
+
+        if (incomingBullets.length === 0) {
+            return existingText
+                ? `${existingText}\n\n${incomingText}`
+                : incomingText;
+        }
+
+        if (existingBullets.length === 0) {
+            const compacted = compactBullets(incomingBullets, { defaultTopic, maxActivePerTopic: 1000 });
+            return renderCompactedDocument(compacted.working, compacted.longTerm, compacted.history, { titleTopic: defaultTopic });
+        }
+
+        const merged = [...existingBullets, ...incomingBullets];
+        const compacted = compactBullets(merged, { defaultTopic, maxActivePerTopic: 1000 });
+        return renderCompactedDocument(compacted.working, compacted.longTerm, compacted.history, { titleTopic: defaultTopic });
+    }
+}
+
+export { MemoryIngester };