@openanonymity/nanomem 0.1.0

package/src/imports/chatgpt.js

@@ -0,0 +1,160 @@
+/**
+ * ChatGPT export parser.
+ *
+ * Handles conversations.json from ChatGPT's "Export data" feature.
+ * Format: array of conversation objects, each with a tree-structured `mapping`.
+ *
+ * Based on the OA Fastchat chatgptImporter.
+ */
+/** @import { ChatGptSession } from '../types.js' */
+import { safeDateIso } from '../bullets/normalize.js';
+
+const SKIP_CONTENT_TYPES = new Set([
+    'user_editable_context',
+    'reasoning_recap',
+    'thoughts',
+]);
+
+/**
+ * Detect whether parsed JSON is a ChatGPT export.
+ * ChatGPT exports are arrays of objects with `mapping` and `current_node`.
+ * @param {unknown} parsed
+ * @returns {boolean}
+ */
+export function isChatGptExport(parsed) {
+    if (!Array.isArray(parsed)) return false;
+    if (parsed.length === 0) return false;
+    const first = parsed[0];
+    return first && typeof first === 'object' && ('mapping' in first);
+}
+
+/**
+ * Parse a ChatGPT export into normalized sessions.
+ * @param {unknown[]} conversations — the parsed JSON array
+ * @returns {ChatGptSession[]}
+ */
+export function parseChatGptExport(conversations) {
+    if (!Array.isArray(conversations)) {
+        throw new Error('ChatGPT export should be an array of conversations.');
+    }
+
+    return conversations
+        .map(normalizeChatGptConversation)
+        .filter(session => session.messages.length > 0);
+}
+
+function normalizeChatGptConversation(conversation) {
+    const rawMessages = getConversationPathMessages(conversation);
+    const messages = [];
+
+    for (const message of rawMessages) {
+        const authorRole = message?.author?.role;
+
+        // Skip tool and system messages
+        if (authorRole === 'tool' || authorRole === 'system' || authorRole === 'developer') continue;
+        if (message?.metadata?.is_visually_hidden_from_conversation) continue;
+
+        const contentType = message?.content?.content_type || '';
+        if (SKIP_CONTENT_TYPES.has(contentType)) continue;
+
+        /** @type {'assistant' | 'user'} */
+        const role = authorRole === 'assistant' ? 'assistant' : 'user';
+        const text = extractText(message?.content);
+
+        if (!text.trim()) continue;
+
+        messages.push({ role, content: text });
+    }
+
+    const title = (conversation?.title || '').trim() || null;
+    const updatedAt = conversation?.update_time
+        ? safeDateIso(String(conversation.update_time * 1000))
+        : null;
+    return { title, messages, updatedAt };
+}
+
+function getConversationPathMessages(conversation) {
+    const mapping = conversation?.mapping || {};
+    const currentNode = conversation?.current_node;
+
+    if (!currentNode || !mapping[currentNode]) {
+        // Fallback: all messages sorted by time
+        return Object.values(mapping)
+            .map(node => node?.message)
+            .filter(Boolean)
+            .sort((a, b) => (a.create_time || 0) - (b.create_time || 0));
+    }
+
+    // Walk from current_node to root, then reverse
+    const ordered = [];
+    const visited = new Set();
+    let nodeId = currentNode;
+    while (nodeId && mapping[nodeId] && !visited.has(nodeId)) {
+        visited.add(nodeId);
+        const node = mapping[nodeId];
+        if (node?.message) {
+            ordered.push(node.message);
+        }
+        nodeId = node.parent;
+    }
+    return ordered.reverse();
+}
+
+function isInternalToolCodeBlock(content) {
+    if (!content || typeof content !== 'object') return false;
+    const contentType = content.content_type || '';
+    if (contentType !== 'code') return false;
+    if (typeof content.text !== 'string') return false;
+
+    const lang = (content.language || '').trim().toLowerCase();
+    if (lang && lang !== 'unknown') return false;
+
+    const raw = content.text.trim();
+
+    // Function-call style: search("..."), web.search("..."), etc.
+    if (/^[a-z_][a-z0-9_.]*\s*\(/i.test(raw)) return true;
+
+    // JSON tool payloads
+    if (raw.startsWith('{') && raw.endsWith('}')) {
+        try {
+            const parsed = JSON.parse(raw);
+            if (Array.isArray(parsed?.search_query) || Array.isArray(parsed?.image_query)) return true;
+            if (Array.isArray(parsed?.open)) return true;
+            if (typeof parsed?.response_length === 'string' && !parsed.text && !parsed.content) return true;
+        } catch { /* not JSON, keep it */ }
+    }
+
+    return false;
+}
+
+function extractText(content) {
+    if (!content || typeof content !== 'object') return '';
+
+    const contentType = content.content_type || '';
+    const parts = [];
+
+    if (contentType === 'code' && typeof content.text === 'string') {
+        if (isInternalToolCodeBlock(content)) return '';
+        const lang = content.language ? content.language.trim() : '';
+        parts.push(lang ? `\`\`\`${lang}\n${content.text}\n\`\`\`` : `\`\`\`\n${content.text}\n\`\`\``);
+    } else {
+        if (Array.isArray(content.parts)) {
+            for (const part of content.parts) {
+                if (typeof part === 'string') {
+                    parts.push(part);
+                } else if (part && typeof part === 'object') {
+                    if (typeof part.text === 'string') parts.push(part.text);
+                    else if (typeof part.content === 'string') parts.push(part.content);
+                }
+            }
+        }
+        if (typeof content.text === 'string' && !parts.length) {
+            parts.push(content.text);
+        }
+        if (typeof content.result === 'string') {
+            parts.push(content.result);
+        }
+    }
+
+    return parts.join('\n').trim();
+}

package/src/imports/index.js

@@ -0,0 +1,14 @@
+export {
+    extractSessionsFromOAFastchatExport,
+    extractConversationFromOAFastchatExport,
+    listOAFastchatSessions,
+} from './oaFastchat.js';
+
+export {
+    isChatGptExport,
+    parseChatGptExport,
+} from './chatgpt.js';
+
+export {
+    parseMarkdownFiles,
+} from './markdown.js';

package/src/imports/markdown.js

@@ -0,0 +1,104 @@
+/**
+ * Plain markdown file importer.
+ *
+ * Converts markdown files into sessions that can be fed to ingest().
+ * Each file becomes a single-message conversation with the file content
+ * as a user message, so the LLM extracts structured facts from it.
+ *
+ * Usage (library):
+ *   import { parseMarkdownFiles } from '@openanonymity/nanomem/imports'
+ *   const sessions = parseMarkdownFiles([{ path: 'notes/health.md', content: '...' }])
+ *   for (const session of sessions) await mem.ingest(session.messages)
+ *
+ * Usage (CLI):
+ *   memory import notes.md --format markdown
+ *   memory import notes-dir/ --format markdown
+ */
+/** @import { Message, ChatGptSession } from '../types.js' */
+
+/**
+ * Parse one or more markdown documents into sessions for ingestion.
+ *
+ * Accepts either a single markdown string or an array of { path, content } records
+ * (e.g. from reading a directory of .md files).
+ *
+ * @param {string | { path: string; content: string }[]} input
+ * @returns {ChatGptSession[]}
+ */
+export function parseMarkdownFiles(input) {
+    if (typeof input === 'string') {
+        return parseMarkdownString(input);
+    }
+
+    if (!Array.isArray(input) || input.length === 0) {
+        throw new Error('Expected a markdown string or an array of { path, content } records.');
+    }
+
+    return input
+        .filter(f => f.content && f.content.trim())
+        .map(f => ({
+            title: titleFromPath(f.path),
+            messages: /** @type {Message[]} */ ([{ role: 'user', content: f.content.trim() }]),
+            updatedAt: null,
+        }));
+}
+
+/**
+ * Split a single markdown string into sections by top-level headings.
+ * If the document has multiple `# ` headings, each becomes a separate session.
+ * Otherwise the whole string is a single session.
+ */
+function parseMarkdownString(input) {
+    const trimmed = input.trim();
+    if (!trimmed) return [];
+
+    const sections = splitByTopHeadings(trimmed);
+
+    if (sections.length <= 1) {
+        return [/** @type {ChatGptSession} */ ({
+            title: sections[0]?.heading ?? null,
+            messages: /** @type {Message[]} */ ([{ role: 'user', content: trimmed }]),
+            updatedAt: null,
+        })];
+    }
+
+    return sections
+        .filter(s => s.content.trim())
+        .map(s => /** @type {ChatGptSession} */ ({
+            title: s.heading,
+            messages: /** @type {Message[]} */ ([{ role: 'user', content: s.content.trim() }]),
+            updatedAt: null,
+        }));
+}
+
+function splitByTopHeadings(text) {
+    const lines = text.split('\n');
+    /** @type {{ heading: string | null, content: string }[]} */
+    const sections = [];
+    /** @type {{ heading: string | null, lines: string[] }} */
+    let current = { heading: null, lines: [] };
+
+    for (const line of lines) {
+        const match = line.match(/^#\s+(.*)/);
+        if (match) {
+            if (current.lines.length > 0 || current.heading) {
+                sections.push({ heading: current.heading, content: current.lines.join('\n') });
+            }
+            current = { heading: match[1].trim() || null, lines: [] };
+        } else {
+            current.lines.push(line);
+        }
+    }
+
+    if (current.lines.length > 0 || current.heading) {
+        sections.push({ heading: current.heading, content: current.lines.join('\n') });
+    }
+
+    return sections;
+}
+
+function titleFromPath(filePath) {
+    if (!filePath) return null;
+    const name = filePath.split('/').pop() || '';
+    return name.replace(/\.md$/i, '').replace(/[_-]/g, ' ').trim() || null;
+}

package/src/imports/oaFastchat.js

@@ -0,0 +1,124 @@
+/** @import { Message, SessionSummary, SessionWithConversation } from '../types.js' */
+
+function toArray(value) {
+    return Array.isArray(value) ? value : [];
+}
+
+function normalizeContent(content) {
+    if (typeof content === 'string') return content;
+    if (content == null) return '';
+    return String(content);
+}
+
+function matchesSession(session, { sessionId, sessionTitle } = /** @type {{ sessionId?: string, sessionTitle?: string }} */ ({})) {
+    if (sessionId) return session?.id === sessionId;
+    if (sessionTitle) return session?.title === sessionTitle;
+    return true;
+}
+
+function toSessionSummary(session) {
+    return {
+        id: session.id,
+        title: session.title || '',
+        createdAt: session.createdAt || null,
+        updatedAt: session.updatedAt || null,
+        messageCount: session.messageCountAtGeneration || null,
+        model: session.model || null
+    };
+}
+
+function toConversationMessages(messages, sessionOrder) {
+    return messages
+        .sort((a, b) => {
+            const sessionDiff = (sessionOrder.get(a?.sessionId) || 0) - (sessionOrder.get(b?.sessionId) || 0);
+            if (sessionDiff !== 0) return sessionDiff;
+            const timeDiff = (a?.timestamp || 0) - (b?.timestamp || 0);
+            if (timeDiff !== 0) return timeDiff;
+            return String(a?.id || '').localeCompare(String(b?.id || ''));
+        })
+        .map((message) => /** @type {Message} */ ({
+            role: message?.role === 'assistant' ? 'assistant' : 'user',
+            content: normalizeContent(message?.content)
+        }))
+        .filter((message) => message.content.trim());
+}
+
+/**
+ * @param {any} exportJson
+ * @returns {SessionSummary[]}
+ */
+export function listOAFastchatSessions(exportJson) {
+    const sessions = toArray(exportJson?.data?.chats?.sessions);
+    return sessions.map((session) => toSessionSummary(session));
+}
+
+/**
+ * @param {any} exportJson
+ * @param {{ sessionId?: string, sessionTitle?: string }} [options]
+ * @returns {SessionWithConversation[]}
+ */
+export function extractSessionsFromOAFastchatExport(exportJson, options = {}) {
+    const sessions = toArray(exportJson?.data?.chats?.sessions);
+    const messages = toArray(exportJson?.data?.chats?.messages);
+
+    if (sessions.length === 0) {
+        throw new Error('OAFastChat export does not contain any chat sessions.');
+    }
+
+    const selectedSessions = sessions.filter((session) => matchesSession(session, options));
+    if (selectedSessions.length === 0) {
+        const hint = options.sessionId
+            ? `sessionId=${options.sessionId}`
+            : `sessionTitle=${options.sessionTitle}`;
+        throw new Error(`Could not find a chat session matching ${hint}.`);
+    }
+
+    const sortedSessions = [...selectedSessions].sort((a, b) => {
+        const createdDiff = (a?.createdAt || 0) - (b?.createdAt || 0);
+        if (createdDiff !== 0) return createdDiff;
+        return String(a?.id || '').localeCompare(String(b?.id || ''));
+    });
+    const sessionOrder = new Map(sortedSessions.map((session, index) => [session.id, index]));
+    const sessionsWithConversation = sortedSessions.map((session) => {
+        const sessionMessages = messages.filter((message) => message?.sessionId === session.id);
+        const conversation = toConversationMessages(sessionMessages, sessionOrder);
+        return {
+            session: toSessionSummary(session),
+            conversation
+        };
+    }).filter((entry) => entry.conversation.length > 0);
+
+    if (sessionsWithConversation.length === 0) {
+        throw new Error('The selected chat session set does not contain any messages.');
+    }
+
+    return sessionsWithConversation;
+}
+
+/**
+ * @param {any} exportJson
+ * @param {{ sessionId?: string, sessionTitle?: string }} [options]
+ * @returns {{ session: SessionSummary | null, sessions: SessionSummary[], conversation: Message[] }}
+ */
+export function extractConversationFromOAFastchatExport(exportJson, options = {}) {
+    const sessionsWithConversation = extractSessionsFromOAFastchatExport(exportJson, options);
+    const sessionOrder = new Map(sessionsWithConversation.map((entry, index) => [entry.session.id, index]));
+    const conversation = toConversationMessages(
+        sessionsWithConversation.flatMap((entry) =>
+            entry.conversation.map((message, index) => ({
+                id: `${entry.session.id}:${index}`,
+                sessionId: entry.session.id,
+                timestamp: index,
+                role: message.role,
+                content: message.content
+            }))
+        ),
+        sessionOrder
+    );
+
+    return {
+        session: sessionsWithConversation.length === 1 ? sessionsWithConversation[0].session : null,
+        sessions: sessionsWithConversation.map((entry) => entry.session),
+        conversation
+    };
+}

package/src/index.js

@@ -0,0 +1,199 @@
+/**
+ * @openanonymity/nanomem — LLM-driven personal memory.
+ *
+ * createMemoryBank(config) is the main entry point.
+ *
+ * Returned object has three named groups:
+ *
+ *   Engine   (LLM-driven):  init, retrieve, ingest, compact
+ *   Backends (storage ops): mem.storage.{ read, write, delete, exists,
+ *                                           search, ls, getTree,
+ *                                           rebuildTree, exportAll }
+ *   Utilities  (portability): mem.serialize(), mem.toZip()
+ */
+/** @import { LLMClient, MemoryBank, MemoryBankConfig, MemoryBankLLMConfig, Message, IngestOptions, RetrievalResult, StorageBackend } from './types.js' */
+
+import { createOpenAIClient } from './llm/openai.js';
+import { createAnthropicClient } from './llm/anthropic.js';
+import { MemoryBulletIndex } from './bullets/bulletIndex.js';
+import { MemoryRetriever } from './engine/retriever.js';
+import { MemoryIngester } from './engine/ingester.js';
+import { MemoryCompactor } from './engine/compactor.js';
+import { InMemoryStorage } from './backends/ram.js';
+import { serialize, toZip } from './utils/portability.js';
+
+/**
+ * Create a memory instance.
+ *
+ * @param {MemoryBankConfig} [config]
+ * @returns {MemoryBank}
+ */
+export function createMemoryBank(config = {}) {
+    const llmClient = config.llmClient || _createLlmClient(config.llm);
+    const model = config.model || config.llm?.model || 'gpt-4o';
+    const backend = _createBackend(config.storage, config.storagePath);
+    const bulletIndex = new MemoryBulletIndex(backend);
+
+    const retrieval = new MemoryRetriever({
+        backend, bulletIndex, llmClient, model,
+        onProgress: config.onProgress,
+        onModelText: config.onModelText,
+    });
+    const ingester = new MemoryIngester({
+        backend, bulletIndex, llmClient, model,
+        onToolCall: config.onToolCall,
+    });
+    const compactor = new MemoryCompactor({ backend, bulletIndex, llmClient, model, onProgress: config.onCompactProgress });
+
+    async function write(path, content) {
+        await backend.write(path, content);
+        await bulletIndex.refreshPath(path);
+    }
+
+    async function remove(path) {
+        await backend.delete(path);
+        await bulletIndex.refreshPath(path);
+    }
+
+    async function rebuildTree() {
+        await backend.rebuildTree();
+        await bulletIndex.rebuild();
+    }
+
+    // ─── Public API ──────────────────────────────────────────────
+    return {
+        /** Initialize the storage backend (creates seed files if empty). */
+        init: () => backend.init(),
+
+        // ─── High-level (LLM-driven) ──────────────────────────────
+
+        /**
+         * Retrieve relevant memory context for a query.
+         * @param {string} query
+         * @param {string} [conversationText] current session text for reference resolution
+         * @returns {Promise<RetrievalResult | null>}
+         */
+        retrieve: (query, conversationText) => retrieval.retrieveForQuery(query, conversationText),
+
+        /**
+         * Ingest facts from a conversation into memory.
+         * @param {Message[]} messages
+         * @param {IngestOptions} [options]
+         */
+        ingest: (messages, options) => ingester.ingest(messages, options),
+
+        /** Compact all memory files (dedup, archive stale facts). */
+        compact: () => compactor.compactAll(),
+
+        // ─── Low-level (direct storage ops) ──────────────────────
+
+        storage: {
+            read:         (path)          => backend.read(path),
+            write:        (path, content) => write(path, content),
+            delete:       (path)          => remove(path),
+            exists:       (path)          => backend.exists(path),
+            search:       (query)         => backend.search(query),
+            ls:           (dirPath)       => backend.ls(dirPath),
+            getTree:     ()              => backend.getTree(),
+            rebuildTree: ()              => rebuildTree(),
+            exportAll:    ()              => backend.exportAll(),
+            clear:        ()              => backend.clear(),
+        },
+
+        // ─── Utilities (portability) ──────────────────────────────
+
+        /** Serialize entire memory state to a single portable string. */
+        serialize: async () => serialize(await backend.exportAll()),
+
+        /** Serialize entire memory state to a ZIP archive (Uint8Array). */
+        toZip: async () => toZip(await backend.exportAll()),
+
+        // ─── Internals (for advanced use / testing) ──────────────
+        _backend: backend,
+        _bulletIndex: bulletIndex,
+    };
+}
+
+// ─── Internal Helpers ────────────────────────────────────────────
+
+function _createLlmClient(llmConfig = /** @type {MemoryBankLLMConfig} */ ({ apiKey: '' })) {
+    const { apiKey, baseUrl, headers, provider } = llmConfig;
+    if (!apiKey) {
+        throw new Error('createMemoryBank: config.llm.apiKey is required (or provide config.llmClient)');
+    }
+
+    const detectedProvider = provider || _detectProvider(baseUrl);
+
+    if (detectedProvider === 'anthropic') {
+        return createAnthropicClient({ apiKey, baseUrl, headers });
+    }
+
+    return createOpenAIClient({ apiKey, baseUrl, headers });
+}
+
+function _detectProvider(baseUrl) {
+    if (!baseUrl) return 'openai';
+    const lower = baseUrl.toLowerCase();
+    if (lower.includes('anthropic.com')) return 'anthropic';
+    return 'openai';
+}
+
+function _createBackend(storage, storagePath) {
+    // Custom backend object
+    if (storage && typeof storage === 'object' && typeof storage.read === 'function') {
+        return storage;
+    }
+
+    const storageType = typeof storage === 'string' ? storage : 'ram';
+
+    switch (storageType) {
+        case 'indexeddb':
+            return _asyncBackend(() => import('./backends/indexeddb.js').then(m => new m.IndexedDBStorage()));
+        case 'filesystem':
+            return _asyncBackend(() => import('./backends/filesystem.js').then(m => new m.FileSystemStorage(storagePath || 'nanomem')));
+        case 'ram':
+        default:
+            return new InMemoryStorage();
+    }
+}
+
+function _asyncBackend(loader) {
+    let _backend = null;
+    let _loading = null;
+
+    async function resolve() {
+        if (_backend) return _backend;
+        if (!_loading) _loading = loader().then(b => { _backend = b; return b; });
+        return _loading;
+    }
+
+    const methods = ['init', 'read', 'write', 'delete', 'exists', 'ls', 'search', 'getTree', 'rebuildTree', 'exportAll', 'clear'];
+    const proxy = {};
+    for (const method of methods) {
+        proxy[method] = async (...args) => {
+            const backend = await resolve();
+            return backend[method](...args);
+        };
+    }
+    return /** @type {StorageBackend} */ (proxy);
+}
+
+// ─── Re-exports ──────────────────────────────────────────────────
+
+/** Re-export all shared type definitions for consumers. */
+export * from './types.js';
+export { createOpenAIClient } from './llm/openai.js';
+export { createAnthropicClient } from './llm/anthropic.js';
+export { InMemoryStorage } from './backends/ram.js';
+export { BaseStorage } from './backends/BaseStorage.js';
+export { MemoryBulletIndex } from './bullets/bulletIndex.js';
+export { MemoryRetriever } from './engine/retriever.js';
+export { MemoryIngester } from './engine/ingester.js';
+export { MemoryCompactor } from './engine/compactor.js';
+export { createRetrievalExecutors, createExtractionExecutors } from './engine/executors.js';
+export { serialize, deserialize, toZip } from './utils/portability.js';
+export {
+    extractSessionsFromOAFastchatExport,
+    extractConversationFromOAFastchatExport,
+    listOAFastchatSessions
+} from './imports/oaFastchat.js';