@totalreclaw/totalreclaw 3.3.12-rc.2 → 3.3.12-rc.21

package/dist/import-adapters/gemini-adapter.js

@@ -1,38 +1,19 @@
 import { BaseImportAdapter } from './base-adapter.js';
 import fs from 'node:fs';
 import os from 'node:os';
-/** Maximum messages per conversation chunk for LLM extraction. */
-const CHUNK_SIZE = 20;
-/** Gap (in minutes) between entries that starts a new pseudo-session. */
-const SESSION_GAP_MINUTES = 30;
-// ── Timestamp Parsing ────────────────────────────────────────────────────────
-const MONTHS = {
-    Jan: 0, Feb: 1, Mar: 2, Apr: 3, May: 4, Jun: 5,
-    Jul: 6, Aug: 7, Sep: 8, Oct: 9, Nov: 10, Dec: 11,
-};
-/**
- * Parse Gemini timestamp: "1 Apr 2026, 18:39:35 WEST" → ISO 8601.
- * Timezone is treated as UTC (all entries use the same TZ, preserving order).
- */
-function parseTimestamp(raw) {
-    const m = raw.match(/^(\d{1,2})\s+(\w{3})\s+(\d{4}),\s+(\d{2}):(\d{2}):(\d{2})\s+/);
-    if (!m || MONTHS[m[2]] === undefined)
-        return undefined;
-    const d = new Date(Date.UTC(+m[3], MONTHS[m[2]], +m[1], +m[4], +m[5], +m[6]));
-    return d.toISOString();
+import { createRequire } from 'node:module';
+// All Gemini format parsing (MyActivity.json, legacy HTML, Saved-info paste)
+// lives in the shared Rust core (`@totalreclaw/core` WASM) so the logic —
+// including the locale-robust, lossless timestamp handling — is identical across
+// every client (Python/Hermes via PyO3, TS via WASM). This adapter is a thin
+// shim: it owns only file I/O + the size/RAM preflight, then delegates parsing.
+const requireWasm = createRequire(import.meta.url);
+let _wasm = null;
+function getWasm() {
+    if (!_wasm)
+        _wasm = requireWasm('@totalreclaw/core');
+    return _wasm;
 }
-// ── HTML Helpers ─────────────────────────────────────────────────────────────
-function decodeEntities(t) {
-    return t.replace(/'/g, "'").replace(/"/g, '"').replace(/&/g, '&')
-        .replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&nbsp;/g, ' ');
-}
-function stripHTML(html) {
-    return html.replace(/<br\s*\/?>/gi, '\n').replace(/<\/p>/gi, '\n')
-        .replace(/<\/li>/gi, '\n').replace(/<\/h[1-6]>/gi, '\n')
-        .replace(/<hr\s*\/?>/gi, '\n---\n').replace(/<[^>]+>/g, '')
-        .replace(/\n{3,}/g, '\n\n').trim();
-}
-// ── Gemini Adapter ───────────────────────────────────────────────────────────
 export class GeminiAdapter extends BaseImportAdapter {
     source = 'gemini';
     displayName = 'Google Gemini';
@@ -46,6 +27,20 @@ export class GeminiAdapter extends BaseImportAdapter {
         else if (input.file_path) {
             try {
                 const resolved = input.file_path.replace(/^~/, os.homedir());
+                const fileStat = fs.statSync(resolved);
+                const fileSizeMB = fileStat.size / (1024 * 1024);
+                if (fileSizeMB > 500) {
+                    errors.push(`File is too large to import: ${fileSizeMB.toFixed(1)}MB exceeds the 500MB cap. ` +
+                        'Split the file into smaller chunks and import each separately.');
+                    return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
+                }
+                const freeMem = os.freemem();
+                if (freeMem < fileStat.size * 2) {
+                    errors.push(`Not enough free memory: ${(freeMem / (1024 * 1024)).toFixed(0)}MB available, ` +
+                        `~${Math.ceil(fileStat.size * 2 / (1024 * 1024))}MB needed (2× file size). ` +
+                        'Close other applications or split the file.');
+                    return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
+                }
                 content = fs.readFileSync(resolved, 'utf-8');
             }
             catch (e) {
@@ -55,147 +50,36 @@ export class GeminiAdapter extends BaseImportAdapter {
         }
         else {
             errors.push('Gemini import requires either content or file_path. ' +
-                'Export from Google Takeout: takeout.google.com → select Gemini Apps → export. ' +
-                'Provide the "My Activity.html" file path.');
+                'Export from Google Takeout: takeout.google.com → "My Activity" → "Gemini Apps". ' +
+                'Provide the "My Activity.html" (or MyActivity.json) file path, or paste your Saved info.');
             return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
         }
         if (onProgress) {
-            onProgress({ current: 0, total: 0, phase: 'parsing', message: 'Parsing Gemini HTML...' });
+            onProgress({ current: 0, total: 0, phase: 'parsing', message: 'Parsing Gemini export...' });
         }
-        // Parse HTML into entries
-        const entries = this.parseHTML(content);
-        if (entries.length === 0) {
-            warnings.push('No conversation entries found in the HTML file.');
-            return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
-        }
-        // Group into pseudo-sessions by temporal proximity
-        const sessions = this.groupSessions(entries);
+        // Delegate ALL format parsing to the shared core.
+        const result = getWasm().parseGemini(content);
         if (onProgress) {
             onProgress({
-                current: 0,
-                total: sessions.length,
+                current: result.chunks.length,
+                total: result.chunks.length,
                 phase: 'parsing',
-                message: `Parsed ${entries.length} entries into ${sessions.length} sessions`,
+                message: `Parsed ${result.total_messages} messages into ${result.chunks.length} chunks`,
             });
         }
-        // Build conversation chunks from sessions
-        const chunks = [];
-        let totalMessages = 0;
-        for (const session of sessions) {
-            const messages = [];
-            for (const entry of session) {
-                if (entry.userPrompt)
-                    messages.push({ role: 'user', text: entry.userPrompt });
-                if (entry.aiResponse)
-                    messages.push({ role: 'assistant', text: entry.aiResponse });
-            }
-            if (messages.length === 0)
-                continue;
-            totalMessages += messages.length;
-            const timestamp = session[0].timestampISO;
-            // Sub-chunk large sessions
-            for (let i = 0; i < messages.length; i += CHUNK_SIZE) {
-                const batch = messages.slice(i, i + CHUNK_SIZE);
-                const chunkIdx = Math.floor(i / CHUNK_SIZE) + 1;
-                const totalChunks = Math.ceil(messages.length / CHUNK_SIZE);
-                const title = totalChunks > 1
-                    ? `Gemini session (part ${chunkIdx}/${totalChunks})`
-                    : 'Gemini session';
-                chunks.push({ title, messages: batch, timestamp });
-            }
-        }
         return {
             facts: [],
-            chunks,
-            totalMessages,
-            warnings,
-            errors,
+            chunks: result.chunks,
+            totalMessages: result.total_messages,
+            warnings: [...warnings, ...result.warnings],
+            errors: [...errors, ...result.errors],
             source_metadata: {
-                format: 'gemini-takeout-html',
-                total_entries: entries.length,
-                sessions_count: sessions.length,
-                chunks_count: chunks.length,
-                total_messages: totalMessages,
-                date_range: {
-                    earliest: entries[0]?.timestampISO,
-                    latest: entries[entries.length - 1]?.timestampISO,
-                },
+                format: result.format,
+                chunks_count: result.chunks.length,
+                total_messages: result.total_messages,
+                ...(result.records_count ? { records_count: result.records_count } : {}),
+                ...(result.skipped ? { skipped_non_gemini: result.skipped } : {}),
             },
         };
     }
-    /**
-     * Parse Gemini Takeout HTML into structured entries.
-     *
-     * Each outer-cell div contains: "Prompted USER_TEXT<br>TIMESTAMP<br>RESPONSE_HTML"
-     * all within one content-cell.
-     */
-    parseHTML(html) {
-        const entries = [];
-        const cellPattern = /<div class="outer-cell[^"]*">([\s\S]*?)(?=<div class="outer-cell|$)/g;
-        let match;
-        while ((match = cellPattern.exec(html)) !== null) {
-            const cell = match[1];
-            // Only process "Prompted" entries (skip canvas, feedback)
-            const promptedIdx = cell.indexOf('Prompted\u00a0');
-            if (promptedIdx === -1)
-                continue;
-            // Extract timestamp
-            const tsMatch = cell.match(/(\d{1,2}\s+\w{3}\s+\d{4},\s+\d{2}:\d{2}:\d{2}\s+\w+)/);
-            if (!tsMatch)
-                continue;
-            const timestampISO = parseTimestamp(tsMatch[1]);
-            if (!timestampISO)
-                continue;
-            // Split on timestamp to separate user prompt from AI response
-            const afterPrompted = cell.substring(promptedIdx + 'Prompted\u00a0'.length);
-            const tsPattern = /(\d{1,2}\s+\w{3}\s+\d{4},\s+\d{2}:\d{2}:\d{2}\s+\w+)/;
-            const tsIdx = afterPrompted.search(tsPattern);
-            let userPrompt = '';
-            let aiResponse = '';
-            if (tsIdx > 0) {
-                userPrompt = stripHTML(decodeEntities(afterPrompted.substring(0, tsIdx))).trim();
-                const tsInner = afterPrompted.match(tsPattern);
-                if (tsInner) {
-                    const afterTs = afterPrompted.substring(tsIdx + tsInner[0].length)
-                        .replace(/^\s*<br\s*\/?>\s*/i, '');
-                    const endDiv = afterTs.search(/<\/div>\s*<div class="content-cell/);
-                    const rawResp = endDiv !== -1 ? afterTs.substring(0, endDiv) : afterTs;
-                    aiResponse = stripHTML(decodeEntities(rawResp)).trim();
-                }
-            }
-            if (userPrompt.length < 3 && aiResponse.length < 3)
-                continue;
-            entries.push({
-                userPrompt,
-                aiResponse,
-                timestampISO,
-                timestampUnix: Math.floor(new Date(timestampISO).getTime() / 1000),
-            });
-        }
-        // Sort chronologically (HTML is newest-first)
-        entries.sort((a, b) => a.timestampUnix - b.timestampUnix);
-        return entries;
-    }
-    /**
-     * Group entries into pseudo-sessions by temporal proximity.
-     */
-    groupSessions(entries) {
-        if (entries.length === 0)
-            return [];
-        const sessions = [];
-        let current = [entries[0]];
-        for (let i = 1; i < entries.length; i++) {
-            const gap = entries[i].timestampUnix - entries[i - 1].timestampUnix;
-            if (gap > SESSION_GAP_MINUTES * 60) {
-                sessions.push(current);
-                current = [entries[i]];
-            }
-            else {
-                current.push(entries[i]);
-            }
-        }
-        if (current.length > 0)
-            sessions.push(current);
-        return sessions;
-    }
 }

package/dist/import-adapters/mcp-memory-adapter.js

@@ -29,6 +29,20 @@ export class MCPMemoryAdapter extends BaseImportAdapter {
         else if (input.file_path) {
             try {
                 const resolvedPath = input.file_path.replace(/^~/, os.homedir());
+                const fileStat = fs.statSync(resolvedPath);
+                const fileSizeMB = fileStat.size / (1024 * 1024);
+                if (fileSizeMB > 500) {
+                    errors.push(`File is too large to import: ${fileSizeMB.toFixed(1)}MB exceeds the 500MB cap. ` +
+                        'Split the file into smaller chunks and import each separately.');
+                    return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
+                }
+                const freeMem = os.freemem();
+                if (freeMem < fileStat.size * 2) {
+                    errors.push(`Not enough free memory: ${(freeMem / (1024 * 1024)).toFixed(0)}MB available, ` +
+                        `~${Math.ceil(fileStat.size * 2 / (1024 * 1024))}MB needed (2× file size). ` +
+                        'Close other applications or split the file.');
+                    return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
+                }
                 content = fs.readFileSync(resolvedPath, 'utf-8');
             }
             catch (e) {

package/dist/import-state-manager.js

@@ -0,0 +1,100 @@
+/**
+ * import-state-manager — persists import progress to ~/.totalreclaw/import-state/
+ *
+ * Intentionally kept free of any outbound-request tokens or network imports so
+ * the OpenClaw exfiltration scanner does not flag it. Do not add network-call
+ * or remote-request imports here.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+// ---------------------------------------------------------------------------
+// Paths
+// ---------------------------------------------------------------------------
+export let IMPORT_STATE_DIR = path.join(os.homedir(), '.totalreclaw', 'import-state');
+const STALE_THRESHOLD_MS = 60 * 60 * 1000; // 1 hour
+/** Only call from tests. Redirects state I/O to a temp directory. */
+export function setImportStateDirForTests(dir) {
+    IMPORT_STATE_DIR = dir;
+}
+export function getImportStatePath(importId) {
+    return path.join(IMPORT_STATE_DIR, `${importId}.json`);
+}
+// ---------------------------------------------------------------------------
+// Read / write
+// ---------------------------------------------------------------------------
+export function writeImportState(state) {
+    fs.mkdirSync(IMPORT_STATE_DIR, { recursive: true });
+    state.last_updated = new Date().toISOString();
+    fs.writeFileSync(getImportStatePath(state.import_id), JSON.stringify(state, null, 2), 'utf-8');
+}
+export function readImportState(importId) {
+    try {
+        const raw = fs.readFileSync(getImportStatePath(importId), 'utf-8');
+        return JSON.parse(raw);
+    }
+    catch {
+        return null;
+    }
+}
+// ---------------------------------------------------------------------------
+// Freshness
+// ---------------------------------------------------------------------------
+export function isImportStale(state) {
+    const lastUpdated = new Date(state.last_updated).getTime();
+    return Date.now() - lastUpdated > STALE_THRESHOLD_MS;
+}
+// ---------------------------------------------------------------------------
+// Most-recent active import
+// ---------------------------------------------------------------------------
+/**
+ * Returns the most recently started import whose status is running/pending,
+ * or null if none found.
+ */
+export function readMostRecentActiveImport() {
+    try {
+        const files = fs.readdirSync(IMPORT_STATE_DIR).filter((f) => f.endsWith('.json'));
+        let mostRecent = null;
+        for (const file of files) {
+            try {
+                const raw = fs.readFileSync(path.join(IMPORT_STATE_DIR, file), 'utf-8');
+                const state = JSON.parse(raw);
+                if (state.status === 'running' || state.status === 'pending') {
+                    if (!mostRecent || state.started_at > mostRecent.started_at) {
+                        mostRecent = state;
+                    }
+                }
+            }
+            catch {
+                // skip corrupted files
+            }
+        }
+        return mostRecent;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Returns all import states sorted newest-first, regardless of status.
+ * Used for resume and audit.
+ */
+export function listAllImportStates() {
+    try {
+        const files = fs.readdirSync(IMPORT_STATE_DIR).filter((f) => f.endsWith('.json'));
+        const states = [];
+        for (const file of files) {
+            try {
+                const raw = fs.readFileSync(path.join(IMPORT_STATE_DIR, file), 'utf-8');
+                states.push(JSON.parse(raw));
+            }
+            catch {
+                // skip corrupted
+            }
+        }
+        return states.sort((a, b) => b.started_at.localeCompare(a.started_at));
+    }
+    catch {
+        return [];
+    }
+}