@totalreclaw/totalreclaw 3.3.1-rc.2 → 3.3.1-rc.21

package/dist/import-adapters/claude-adapter.js

@@ -0,0 +1,114 @@
+import { BaseImportAdapter } from './base-adapter.js';
+import fs from 'node:fs';
+import os from 'node:os';
+/**
+ * Pattern for lines that start with a date prefix.
+ * Claude memory entries sometimes have: [2026-03-15] - User prefers TypeScript
+ */
+const DATE_PREFIX_RE = /^\[(\d{4}-\d{2}-\d{2})\]\s*[-:]\s*/;
+/**
+ * Pattern for bullet-prefixed lines.
+ */
+const BULLET_PREFIX_RE = /^[-*\u2022]\s+/;
+/**
+ * Pattern for numbered list lines.
+ */
+const NUMBERED_PREFIX_RE = /^\d+[.)]\s+/;
+/** Maximum messages per conversation chunk for LLM extraction. */
+const CHUNK_SIZE = 20;
+export class ClaudeAdapter extends BaseImportAdapter {
+    source = 'claude';
+    displayName = 'Claude';
+    async parse(input, onProgress) {
+        const warnings = [];
+        const errors = [];
+        let content;
+        if (input.content) {
+            content = input.content;
+        }
+        else if (input.file_path) {
+            try {
+                const resolvedPath = input.file_path.replace(/^~/, os.homedir());
+                content = fs.readFileSync(resolvedPath, 'utf-8');
+            }
+            catch (e) {
+                errors.push(`Failed to read file: ${e instanceof Error ? e.message : 'Unknown error'}`);
+                return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
+            }
+        }
+        else {
+            errors.push('Claude import requires either content (pasted text) or file_path. ' +
+                'Copy your memories from Claude: Settings -> Memory -> select all and copy.');
+            return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
+        }
+        // Claude memory export is plain text, one fact per line.
+        return this.parseMemoriesText(content.trim(), warnings, errors, onProgress);
+    }
+    /**
+     * Parse Claude memories — plain text, one memory per line.
+     * Returns conversation chunks for LLM extraction (no pattern matching).
+     *
+     * Each line is cleaned (date prefixes, bullets, numbers stripped) and
+     * grouped into chunks for the LLM to process.
+     */
+    parseMemoriesText(content, warnings, errors, onProgress) {
+        // Split by newlines and filter
+        const lines = content.split('\n')
+            .map((line) => line.trim())
+            .filter((line) => line.length > 0)
+            // Skip common header lines
+            .filter((line) => !/^(?:memories?|claude memories?|my memories?|saved memories?):?\s*$/i.test(line));
+        if (onProgress) {
+            onProgress({
+                current: 0,
+                total: lines.length,
+                phase: 'parsing',
+                message: `Parsing ${lines.length} Claude memories...`,
+            });
+        }
+        // Clean each line: extract date, strip formatting
+        const cleanedEntries = [];
+        for (const line of lines) {
+            let cleaned = line;
+            let timestamp;
+            // Extract date prefix if present
+            const dateMatch = cleaned.match(DATE_PREFIX_RE);
+            if (dateMatch) {
+                timestamp = dateMatch[1];
+                cleaned = cleaned.replace(DATE_PREFIX_RE, '');
+            }
+            // Strip bullet/numbering markers
+            cleaned = cleaned
+                .replace(BULLET_PREFIX_RE, '')
+                .replace(NUMBERED_PREFIX_RE, '')
+                .trim();
+            if (cleaned.length >= 3) {
+                cleanedEntries.push({ text: cleaned, timestamp });
+            }
+        }
+        // Group memories into chunks of CHUNK_SIZE for efficient LLM extraction
+        const chunks = [];
+        for (let i = 0; i < cleanedEntries.length; i += CHUNK_SIZE) {
+            const batch = cleanedEntries.slice(i, i + CHUNK_SIZE);
+            // Use the timestamp from the first entry in the batch (if available)
+            const batchTimestamp = batch.find((e) => e.timestamp)?.timestamp;
+            chunks.push({
+                title: `Claude memories (${i + 1}-${Math.min(i + CHUNK_SIZE, cleanedEntries.length)})`,
+                messages: batch.map((entry) => ({ role: 'user', text: entry.text })),
+                timestamp: batchTimestamp,
+            });
+        }
+        return {
+            facts: [],
+            chunks,
+            totalMessages: cleanedEntries.length,
+            warnings,
+            errors,
+            source_metadata: {
+                format: 'memories-text',
+                total_lines: lines.length,
+                chunks_count: chunks.length,
+            },
+        };
+    }
+}

package/dist/import-adapters/gemini-adapter.js

@@ -0,0 +1,201 @@
+import { BaseImportAdapter } from './base-adapter.js';
+import fs from 'node:fs';
+import os from 'node:os';
+/** Maximum messages per conversation chunk for LLM extraction. */
+const CHUNK_SIZE = 20;
+/** Gap (in minutes) between entries that starts a new pseudo-session. */
+const SESSION_GAP_MINUTES = 30;
+// ── Timestamp Parsing ────────────────────────────────────────────────────────
+const MONTHS = {
+    Jan: 0, Feb: 1, Mar: 2, Apr: 3, May: 4, Jun: 5,
+    Jul: 6, Aug: 7, Sep: 8, Oct: 9, Nov: 10, Dec: 11,
+};
+/**
+ * Parse Gemini timestamp: "1 Apr 2026, 18:39:35 WEST" → ISO 8601.
+ * Timezone is treated as UTC (all entries use the same TZ, preserving order).
+ */
+function parseTimestamp(raw) {
+    const m = raw.match(/^(\d{1,2})\s+(\w{3})\s+(\d{4}),\s+(\d{2}):(\d{2}):(\d{2})\s+/);
+    if (!m || MONTHS[m[2]] === undefined)
+        return undefined;
+    const d = new Date(Date.UTC(+m[3], MONTHS[m[2]], +m[1], +m[4], +m[5], +m[6]));
+    return d.toISOString();
+}
+// ── HTML Helpers ─────────────────────────────────────────────────────────────
+function decodeEntities(t) {
+    return t.replace(/'/g, "'").replace(/"/g, '"').replace(/&/g, '&')
+        .replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&nbsp;/g, ' ');
+}
+function stripHTML(html) {
+    return html.replace(/<br\s*\/?>/gi, '\n').replace(/<\/p>/gi, '\n')
+        .replace(/<\/li>/gi, '\n').replace(/<\/h[1-6]>/gi, '\n')
+        .replace(/<hr\s*\/?>/gi, '\n---\n').replace(/<[^>]+>/g, '')
+        .replace(/\n{3,}/g, '\n\n').trim();
+}
+// ── Gemini Adapter ───────────────────────────────────────────────────────────
+export class GeminiAdapter extends BaseImportAdapter {
+    source = 'gemini';
+    displayName = 'Google Gemini';
+    async parse(input, onProgress) {
+        const warnings = [];
+        const errors = [];
+        let content;
+        if (input.content) {
+            content = input.content;
+        }
+        else if (input.file_path) {
+            try {
+                const resolved = input.file_path.replace(/^~/, os.homedir());
+                content = fs.readFileSync(resolved, 'utf-8');
+            }
+            catch (e) {
+                errors.push(`Failed to read file: ${e instanceof Error ? e.message : 'Unknown error'}`);
+                return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
+            }
+        }
+        else {
+            errors.push('Gemini import requires either content or file_path. ' +
+                'Export from Google Takeout: takeout.google.com → select Gemini Apps → export. ' +
+                'Provide the "My Activity.html" file path.');
+            return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
+        }
+        if (onProgress) {
+            onProgress({ current: 0, total: 0, phase: 'parsing', message: 'Parsing Gemini HTML...' });
+        }
+        // Parse HTML into entries
+        const entries = this.parseHTML(content);
+        if (entries.length === 0) {
+            warnings.push('No conversation entries found in the HTML file.');
+            return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
+        }
+        // Group into pseudo-sessions by temporal proximity
+        const sessions = this.groupSessions(entries);
+        if (onProgress) {
+            onProgress({
+                current: 0,
+                total: sessions.length,
+                phase: 'parsing',
+                message: `Parsed ${entries.length} entries into ${sessions.length} sessions`,
+            });
+        }
+        // Build conversation chunks from sessions
+        const chunks = [];
+        let totalMessages = 0;
+        for (const session of sessions) {
+            const messages = [];
+            for (const entry of session) {
+                if (entry.userPrompt)
+                    messages.push({ role: 'user', text: entry.userPrompt });
+                if (entry.aiResponse)
+                    messages.push({ role: 'assistant', text: entry.aiResponse });
+            }
+            if (messages.length === 0)
+                continue;
+            totalMessages += messages.length;
+            const timestamp = session[0].timestampISO;
+            // Sub-chunk large sessions
+            for (let i = 0; i < messages.length; i += CHUNK_SIZE) {
+                const batch = messages.slice(i, i + CHUNK_SIZE);
+                const chunkIdx = Math.floor(i / CHUNK_SIZE) + 1;
+                const totalChunks = Math.ceil(messages.length / CHUNK_SIZE);
+                const title = totalChunks > 1
+                    ? `Gemini session (part ${chunkIdx}/${totalChunks})`
+                    : 'Gemini session';
+                chunks.push({ title, messages: batch, timestamp });
+            }
+        }
+        return {
+            facts: [],
+            chunks,
+            totalMessages,
+            warnings,
+            errors,
+            source_metadata: {
+                format: 'gemini-takeout-html',
+                total_entries: entries.length,
+                sessions_count: sessions.length,
+                chunks_count: chunks.length,
+                total_messages: totalMessages,
+                date_range: {
+                    earliest: entries[0]?.timestampISO,
+                    latest: entries[entries.length - 1]?.timestampISO,
+                },
+            },
+        };
+    }
+    /**
+     * Parse Gemini Takeout HTML into structured entries.
+     *
+     * Each outer-cell div contains: "Prompted USER_TEXT<br>TIMESTAMP<br>RESPONSE_HTML"
+     * all within one content-cell.
+     */
+    parseHTML(html) {
+        const entries = [];
+        const cellPattern = /<div class="outer-cell[^"]*">([\s\S]*?)(?=<div class="outer-cell|$)/g;
+        let match;
+        while ((match = cellPattern.exec(html)) !== null) {
+            const cell = match[1];
+            // Only process "Prompted" entries (skip canvas, feedback)
+            const promptedIdx = cell.indexOf('Prompted\u00a0');
+            if (promptedIdx === -1)
+                continue;
+            // Extract timestamp
+            const tsMatch = cell.match(/(\d{1,2}\s+\w{3}\s+\d{4},\s+\d{2}:\d{2}:\d{2}\s+\w+)/);
+            if (!tsMatch)
+                continue;
+            const timestampISO = parseTimestamp(tsMatch[1]);
+            if (!timestampISO)
+                continue;
+            // Split on timestamp to separate user prompt from AI response
+            const afterPrompted = cell.substring(promptedIdx + 'Prompted\u00a0'.length);
+            const tsPattern = /(\d{1,2}\s+\w{3}\s+\d{4},\s+\d{2}:\d{2}:\d{2}\s+\w+)/;
+            const tsIdx = afterPrompted.search(tsPattern);
+            let userPrompt = '';
+            let aiResponse = '';
+            if (tsIdx > 0) {
+                userPrompt = stripHTML(decodeEntities(afterPrompted.substring(0, tsIdx))).trim();
+                const tsInner = afterPrompted.match(tsPattern);
+                if (tsInner) {
+                    const afterTs = afterPrompted.substring(tsIdx + tsInner[0].length)
+                        .replace(/^\s*<br\s*\/?>\s*/i, '');
+                    const endDiv = afterTs.search(/<\/div>\s*<div class="content-cell/);
+                    const rawResp = endDiv !== -1 ? afterTs.substring(0, endDiv) : afterTs;
+                    aiResponse = stripHTML(decodeEntities(rawResp)).trim();
+                }
+            }
+            if (userPrompt.length < 3 && aiResponse.length < 3)
+                continue;
+            entries.push({
+                userPrompt,
+                aiResponse,
+                timestampISO,
+                timestampUnix: Math.floor(new Date(timestampISO).getTime() / 1000),
+            });
+        }
+        // Sort chronologically (HTML is newest-first)
+        entries.sort((a, b) => a.timestampUnix - b.timestampUnix);
+        return entries;
+    }
+    /**
+     * Group entries into pseudo-sessions by temporal proximity.
+     */
+    groupSessions(entries) {
+        if (entries.length === 0)
+            return [];
+        const sessions = [];
+        let current = [entries[0]];
+        for (let i = 1; i < entries.length; i++) {
+            const gap = entries[i].timestampUnix - entries[i - 1].timestampUnix;
+            if (gap > SESSION_GAP_MINUTES * 60) {
+                sessions.push(current);
+                current = [entries[i]];
+            }
+            else {
+                current.push(entries[i]);
+            }
+        }
+        if (current.length > 0)
+            sessions.push(current);
+        return sessions;
+    }
+}

package/dist/import-adapters/index.js

@@ -0,0 +1,26 @@
+export { BaseImportAdapter } from './base-adapter.js';
+export * from './types.js';
+export { Mem0Adapter } from './mem0-adapter.js';
+export { MCPMemoryAdapter } from './mcp-memory-adapter.js';
+export { ChatGPTAdapter } from './chatgpt-adapter.js';
+export { ClaudeAdapter } from './claude-adapter.js';
+export { GeminiAdapter } from './gemini-adapter.js';
+import { Mem0Adapter } from './mem0-adapter.js';
+import { MCPMemoryAdapter } from './mcp-memory-adapter.js';
+import { ChatGPTAdapter } from './chatgpt-adapter.js';
+import { ClaudeAdapter } from './claude-adapter.js';
+import { GeminiAdapter } from './gemini-adapter.js';
+const ADAPTERS = {
+    'mem0': () => new Mem0Adapter(),
+    'mcp-memory': () => new MCPMemoryAdapter(),
+    'chatgpt': () => new ChatGPTAdapter(),
+    'claude': () => new ClaudeAdapter(),
+    'gemini': () => new GeminiAdapter(),
+};
+export function getAdapter(source) {
+    const factory = ADAPTERS[source];
+    if (!factory) {
+        throw new Error(`Unknown import source: ${source}. Valid sources: ${Object.keys(ADAPTERS).join(', ')}`);
+    }
+    return factory();
+}

package/dist/import-adapters/mcp-memory-adapter.js

@@ -0,0 +1,219 @@
+import { BaseImportAdapter } from './base-adapter.js';
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+/**
+ * Entity type mapping to TotalReclaw fact types.
+ */
+const ENTITY_TYPE_MAP = {
+    person: 'fact',
+    project: 'fact',
+    organization: 'fact',
+    tool: 'preference',
+    technology: 'preference',
+    preference: 'preference',
+    goal: 'goal',
+    event: 'episodic',
+    decision: 'decision',
+};
+export class MCPMemoryAdapter extends BaseImportAdapter {
+    source = 'mcp-memory';
+    displayName = 'MCP Memory Server';
+    async parse(input, onProgress) {
+        const warnings = [];
+        const errors = [];
+        let content;
+        if (input.content) {
+            content = input.content;
+        }
+        else if (input.file_path) {
+            try {
+                const resolvedPath = input.file_path.replace(/^~/, os.homedir());
+                content = fs.readFileSync(resolvedPath, 'utf-8');
+            }
+            catch (e) {
+                errors.push(`Failed to read file: ${e instanceof Error ? e.message : 'Unknown error'}`);
+                return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
+            }
+        }
+        else {
+            // Try default MCP memory path
+            const defaultPath = path.join(os.homedir(), '.mcp-memory', 'memory.jsonl');
+            try {
+                content = fs.readFileSync(defaultPath, 'utf-8');
+                warnings.push(`Using default MCP memory path: ${defaultPath}`);
+            }
+            catch {
+                errors.push('No content, file_path, or file at default path (~/.mcp-memory/memory.jsonl). ' +
+                    'Provide the memory.jsonl content or file path.');
+                return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
+            }
+        }
+        // Parse JSONL records
+        const records = this.parseJSONL(content, errors);
+        if (onProgress) {
+            onProgress({
+                current: 0,
+                total: records.length,
+                phase: 'parsing',
+                message: `Parsing ${records.length} MCP Memory records...`,
+            });
+        }
+        // Separate entities and relations
+        const entities = new Map();
+        const relations = [];
+        for (const record of records) {
+            if (record.type === 'entity') {
+                // Later entities override earlier ones (append-only file)
+                entities.set(record.name, record);
+            }
+            else if (record.type === 'relation') {
+                relations.push(record);
+            }
+        }
+        // Convert entities to facts
+        const rawFacts = [];
+        let entityIndex = 0;
+        for (const [name, entity] of entities) {
+            const factType = ENTITY_TYPE_MAP[entity.entityType.toLowerCase()] || 'fact';
+            for (const observation of entity.observations) {
+                // Prefix observation with entity name for context
+                // "Works at Acme Corp" -> "John works at Acme Corp"
+                const text = this.contextualizeObservation(name, observation);
+                rawFacts.push({
+                    text,
+                    type: factType,
+                    importance: 6,
+                    source: 'mcp-memory',
+                    sourceId: `${name}:${entityIndex}`,
+                    tags: [entity.entityType],
+                });
+                entityIndex++;
+            }
+            if (onProgress) {
+                onProgress({
+                    current: rawFacts.length,
+                    total: rawFacts.length + relations.length,
+                    phase: 'parsing',
+                    message: `Parsed ${rawFacts.length} facts from entities...`,
+                });
+            }
+        }
+        // Convert relations to facts
+        for (const rel of relations) {
+            // Only create a fact if both entities exist
+            if (!entities.has(rel.from) || !entities.has(rel.to)) {
+                warnings.push(`Relation references unknown entity: ${rel.from} -> ${rel.to}`);
+                continue;
+            }
+            const text = `${rel.from} ${this.humanizeRelationType(rel.relationType)} ${rel.to}`;
+            rawFacts.push({
+                text,
+                type: 'fact',
+                importance: 5,
+                source: 'mcp-memory',
+                sourceId: `rel:${rel.from}:${rel.relationType}:${rel.to}`,
+                tags: ['relation'],
+            });
+        }
+        const { facts, invalidCount } = this.validateFacts(rawFacts);
+        if (invalidCount > 0) {
+            warnings.push(`${invalidCount} observations had invalid/empty text and were skipped`);
+        }
+        return {
+            facts,
+            chunks: [],
+            totalMessages: 0,
+            warnings,
+            errors,
+            source_metadata: {
+                entities_count: entities.size,
+                relations_count: relations.length,
+                observations_total: rawFacts.length,
+            },
+        };
+    }
+    /**
+     * Parse JSONL content into records.
+     */
+    parseJSONL(content, errors) {
+        const records = [];
+        const lines = content.split('\n').filter((line) => line.trim().length > 0);
+        for (let i = 0; i < lines.length; i++) {
+            try {
+                const record = JSON.parse(lines[i]);
+                if (record.type === 'entity' || record.type === 'relation') {
+                    records.push(record);
+                }
+                else {
+                    // Unknown record type — skip silently (future-proofing)
+                }
+            }
+            catch {
+                errors.push(`Line ${i + 1}: Invalid JSON — skipped`);
+            }
+        }
+        return records;
+    }
+    /**
+     * Add entity context to an observation.
+     *
+     * Heuristic: if the observation already starts with the entity name
+     * (case-insensitive) or a pronoun, return as-is. Otherwise prefix.
+     *
+     * Examples:
+     *  - ("John", "Works at Acme Corp") -> "John works at Acme Corp"
+     *  - ("John", "John likes TypeScript") -> "John likes TypeScript" (no double prefix)
+     *  - ("John", "He prefers React") -> "John prefers React"
+     */
+    contextualizeObservation(entityName, observation) {
+        const obsLower = observation.toLowerCase().trim();
+        const nameLower = entityName.toLowerCase();
+        // Already starts with the entity name
+        if (obsLower.startsWith(nameLower)) {
+            return observation.trim();
+        }
+        // Starts with a pronoun — replace it
+        const pronouns = ['he ', 'she ', 'they ', 'it ', 'his ', 'her ', 'their ', 'its '];
+        for (const pronoun of pronouns) {
+            if (obsLower.startsWith(pronoun)) {
+                return entityName + ' ' + observation.trim().slice(pronoun.length);
+            }
+        }
+        // Starts with a verb (lowercase first word) — prefix entity name
+        const firstChar = observation.trim()[0];
+        if (firstChar && firstChar === firstChar.toLowerCase()) {
+            return `${entityName} ${observation.trim()}`;
+        }
+        // Observation is a standalone sentence — prefix with "About {entity}: "
+        return `${entityName}: ${observation.trim()}`;
+    }
+    /**
+     * Convert relation type slug to human-readable text.
+     *
+     * "works_on" -> "works on"
+     * "MEMBER_OF" -> "is a member of"
+     */
+    humanizeRelationType(relationType) {
+        const slug = relationType.toLowerCase().replace(/_/g, ' ');
+        // Common relation type mappings
+        const mappings = {
+            'works on': 'works on',
+            'works at': 'works at',
+            'member of': 'is a member of',
+            'belongs to': 'belongs to',
+            'created by': 'was created by',
+            'depends on': 'depends on',
+            'uses': 'uses',
+            'knows': 'knows',
+            'related to': 'is related to',
+            'part of': 'is part of',
+            'friend of': 'is friends with',
+            'manages': 'manages',
+            'reports to': 'reports to',
+            'located in': 'is located in',
+            'lives in': 'lives in',
+        };
+        return mappings[slug] || slug;
+    }
+}