@totalreclaw/totalreclaw 1.5.0 → 3.0.6

package/import-adapters/gemini-adapter.ts

@@ -0,0 +1,243 @@
+import { BaseImportAdapter } from './base-adapter.js';
+import type {
+  ImportSource,
+  AdapterParseResult,
+  ConversationChunk,
+  ProgressCallback,
+} from './types.js';
+import fs from 'node:fs';
+import os from 'node:os';
+
+/** Maximum messages per conversation chunk for LLM extraction. */
+const CHUNK_SIZE = 20;
+
+/** Gap (in minutes) between entries that starts a new pseudo-session. */
+const SESSION_GAP_MINUTES = 30;
+
+// ── Timestamp Parsing ────────────────────────────────────────────────────────
+
+const MONTHS: Record<string, number> = {
+  Jan: 0, Feb: 1, Mar: 2, Apr: 3, May: 4, Jun: 5,
+  Jul: 6, Aug: 7, Sep: 8, Oct: 9, Nov: 10, Dec: 11,
+};
+
+/**
+ * Parse Gemini timestamp: "1 Apr 2026, 18:39:35 WEST" → ISO 8601.
+ * Timezone is treated as UTC (all entries use the same TZ, preserving order).
+ */
+function parseTimestamp(raw: string): string | undefined {
+  const m = raw.match(/^(\d{1,2})\s+(\w{3})\s+(\d{4}),\s+(\d{2}):(\d{2}):(\d{2})\s+/);
+  if (!m || MONTHS[m[2]] === undefined) return undefined;
+  const d = new Date(Date.UTC(+m[3], MONTHS[m[2]], +m[1], +m[4], +m[5], +m[6]));
+  return d.toISOString();
+}
+
+// ── HTML Helpers ─────────────────────────────────────────────────────────────
+
+function decodeEntities(t: string): string {
+  return t.replace(/&#39;/g, "'").replace(/&quot;/g, '"').replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&nbsp;/g, ' ');
+}
+
+function stripHTML(html: string): string {
+  return html.replace(/<br\s*\/?>/gi, '\n').replace(/<\/p>/gi, '\n')
+    .replace(/<\/li>/gi, '\n').replace(/<\/h[1-6]>/gi, '\n')
+    .replace(/<hr\s*\/?>/gi, '\n---\n').replace(/<[^>]+>/g, '')
+    .replace(/\n{3,}/g, '\n\n').trim();
+}
+
+// ── Entry Types ──────────────────────────────────────────────────────────────
+
+interface GeminiEntry {
+  userPrompt: string;
+  aiResponse: string;
+  timestampISO: string;
+  timestampUnix: number;
+}
+
+// ── Gemini Adapter ───────────────────────────────────────────────────────────
+
+export class GeminiAdapter extends BaseImportAdapter {
+  readonly source: ImportSource = 'gemini';
+  readonly displayName = 'Google Gemini';
+
+  async parse(
+    input: { content?: string; file_path?: string },
+    onProgress?: ProgressCallback,
+  ): Promise<AdapterParseResult> {
+    const warnings: string[] = [];
+    const errors: string[] = [];
+
+    let content: string;
+
+    if (input.content) {
+      content = input.content;
+    } else if (input.file_path) {
+      try {
+        const resolved = input.file_path.replace(/^~/, os.homedir());
+        content = fs.readFileSync(resolved, 'utf-8');
+      } catch (e) {
+        errors.push(`Failed to read file: ${e instanceof Error ? e.message : 'Unknown error'}`);
+        return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
+      }
+    } else {
+      errors.push(
+        'Gemini import requires either content or file_path. ' +
+        'Export from Google Takeout: takeout.google.com → select Gemini Apps → export. ' +
+        'Provide the "My Activity.html" file path.',
+      );
+      return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
+    }
+
+    if (onProgress) {
+      onProgress({ current: 0, total: 0, phase: 'parsing', message: 'Parsing Gemini HTML...' });
+    }
+
+    // Parse HTML into entries
+    const entries = this.parseHTML(content);
+    if (entries.length === 0) {
+      warnings.push('No conversation entries found in the HTML file.');
+      return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
+    }
+
+    // Group into pseudo-sessions by temporal proximity
+    const sessions = this.groupSessions(entries);
+
+    if (onProgress) {
+      onProgress({
+        current: 0,
+        total: sessions.length,
+        phase: 'parsing',
+        message: `Parsed ${entries.length} entries into ${sessions.length} sessions`,
+      });
+    }
+
+    // Build conversation chunks from sessions
+    const chunks: ConversationChunk[] = [];
+    let totalMessages = 0;
+
+    for (const session of sessions) {
+      const messages: Array<{ role: 'user' | 'assistant'; text: string }> = [];
+      for (const entry of session) {
+        if (entry.userPrompt) messages.push({ role: 'user', text: entry.userPrompt });
+        if (entry.aiResponse) messages.push({ role: 'assistant', text: entry.aiResponse });
+      }
+      if (messages.length === 0) continue;
+
+      totalMessages += messages.length;
+      const timestamp = session[0].timestampISO;
+
+      // Sub-chunk large sessions
+      for (let i = 0; i < messages.length; i += CHUNK_SIZE) {
+        const batch = messages.slice(i, i + CHUNK_SIZE);
+        const chunkIdx = Math.floor(i / CHUNK_SIZE) + 1;
+        const totalChunks = Math.ceil(messages.length / CHUNK_SIZE);
+        const title = totalChunks > 1
+          ? `Gemini session (part ${chunkIdx}/${totalChunks})`
+          : 'Gemini session';
+
+        chunks.push({ title, messages: batch, timestamp });
+      }
+    }
+
+    return {
+      facts: [],
+      chunks,
+      totalMessages,
+      warnings,
+      errors,
+      source_metadata: {
+        format: 'gemini-takeout-html',
+        total_entries: entries.length,
+        sessions_count: sessions.length,
+        chunks_count: chunks.length,
+        total_messages: totalMessages,
+        date_range: {
+          earliest: entries[0]?.timestampISO,
+          latest: entries[entries.length - 1]?.timestampISO,
+        },
+      },
+    };
+  }
+
+  /**
+   * Parse Gemini Takeout HTML into structured entries.
+   *
+   * Each outer-cell div contains: "Prompted USER_TEXT<br>TIMESTAMP<br>RESPONSE_HTML"
+   * all within one content-cell.
+   */
+  private parseHTML(html: string): GeminiEntry[] {
+    const entries: GeminiEntry[] = [];
+    const cellPattern = /<div class="outer-cell[^"]*">([\s\S]*?)(?=<div class="outer-cell|$)/g;
+    let match: RegExpExecArray | null;
+
+    while ((match = cellPattern.exec(html)) !== null) {
+      const cell = match[1];
+
+      // Only process "Prompted" entries (skip canvas, feedback)
+      const promptedIdx = cell.indexOf('Prompted\u00a0');
+      if (promptedIdx === -1) continue;
+
+      // Extract timestamp
+      const tsMatch = cell.match(/(\d{1,2}\s+\w{3}\s+\d{4},\s+\d{2}:\d{2}:\d{2}\s+\w+)/);
+      if (!tsMatch) continue;
+      const timestampISO = parseTimestamp(tsMatch[1]);
+      if (!timestampISO) continue;
+
+      // Split on timestamp to separate user prompt from AI response
+      const afterPrompted = cell.substring(promptedIdx + 'Prompted\u00a0'.length);
+      const tsPattern = /(\d{1,2}\s+\w{3}\s+\d{4},\s+\d{2}:\d{2}:\d{2}\s+\w+)/;
+      const tsIdx = afterPrompted.search(tsPattern);
+
+      let userPrompt = '';
+      let aiResponse = '';
+
+      if (tsIdx > 0) {
+        userPrompt = stripHTML(decodeEntities(afterPrompted.substring(0, tsIdx))).trim();
+
+        const tsInner = afterPrompted.match(tsPattern);
+        if (tsInner) {
+          const afterTs = afterPrompted.substring(tsIdx + tsInner[0].length)
+            .replace(/^\s*<br\s*\/?>\s*/i, '');
+          const endDiv = afterTs.search(/<\/div>\s*<div class="content-cell/);
+          const rawResp = endDiv !== -1 ? afterTs.substring(0, endDiv) : afterTs;
+          aiResponse = stripHTML(decodeEntities(rawResp)).trim();
+        }
+      }
+
+      if (userPrompt.length < 3 && aiResponse.length < 3) continue;
+
+      entries.push({
+        userPrompt,
+        aiResponse,
+        timestampISO,
+        timestampUnix: Math.floor(new Date(timestampISO).getTime() / 1000),
+      });
+    }
+
+    // Sort chronologically (HTML is newest-first)
+    entries.sort((a, b) => a.timestampUnix - b.timestampUnix);
+    return entries;
+  }
+
+  /**
+   * Group entries into pseudo-sessions by temporal proximity.
+   */
+  private groupSessions(entries: GeminiEntry[]): GeminiEntry[][] {
+    if (entries.length === 0) return [];
+    const sessions: GeminiEntry[][] = [];
+    let current: GeminiEntry[] = [entries[0]];
+
+    for (let i = 1; i < entries.length; i++) {
+      const gap = entries[i].timestampUnix - entries[i - 1].timestampUnix;
+      if (gap > SESSION_GAP_MINUTES * 60) {
+        sessions.push(current);
+        current = [entries[i]];
+      } else {
+        current.push(entries[i]);
+      }
+    }
+    if (current.length > 0) sessions.push(current);
+    return sessions;
+  }
+}

package/import-adapters/index.ts

@@ -2,15 +2,24 @@ export { BaseImportAdapter } from './base-adapter.js';
 export * from './types.js';
 export { Mem0Adapter } from './mem0-adapter.js';
 export { MCPMemoryAdapter } from './mcp-memory-adapter.js';
+export { ChatGPTAdapter } from './chatgpt-adapter.js';
+export { ClaudeAdapter } from './claude-adapter.js';
+export { GeminiAdapter } from './gemini-adapter.js';
 
 import type { ImportSource } from './types.js';
 import { Mem0Adapter } from './mem0-adapter.js';
 import { MCPMemoryAdapter } from './mcp-memory-adapter.js';
+import { ChatGPTAdapter } from './chatgpt-adapter.js';
+import { ClaudeAdapter } from './claude-adapter.js';
+import { GeminiAdapter } from './gemini-adapter.js';
 import type { BaseImportAdapter } from './base-adapter.js';
 
 const ADAPTERS: Partial<Record<ImportSource, () => BaseImportAdapter>> = {
   'mem0': () => new Mem0Adapter(),
   'mcp-memory': () => new MCPMemoryAdapter(),
+  'chatgpt': () => new ChatGPTAdapter(),
+  'claude': () => new ClaudeAdapter(),
+  'gemini': () => new GeminiAdapter(),
 };
 
 export function getAdapter(source: ImportSource): BaseImportAdapter {

package/import-adapters/mcp-memory-adapter.ts

@@ -67,7 +67,7 @@ export class MCPMemoryAdapter extends BaseImportAdapter {
         content = fs.readFileSync(resolvedPath, 'utf-8');
       } catch (e) {
         errors.push(`Failed to read file: ${e instanceof Error ? e.message : 'Unknown error'}`);
-        return { facts: [], warnings, errors };
+        return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
       }
     } else {
       // Try default MCP memory path
@@ -80,7 +80,7 @@ export class MCPMemoryAdapter extends BaseImportAdapter {
           'No content, file_path, or file at default path (~/.mcp-memory/memory.jsonl). ' +
           'Provide the memory.jsonl content or file path.',
         );
-        return { facts: [], warnings, errors };
+        return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
       }
     }
 
@@ -170,6 +170,8 @@ export class MCPMemoryAdapter extends BaseImportAdapter {
 
     return {
       facts,
+      chunks: [],
+      totalMessages: 0,
       warnings,
       errors,
       source_metadata: {

package/import-adapters/mem0-adapter.ts

@@ -81,7 +81,7 @@ export class Mem0Adapter extends BaseImportAdapter {
       );
     } else {
       errors.push('Mem0 import requires either content (export file) or api_key');
-      return { facts: [], warnings, errors };
+      return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
     }
 
     if (onProgress) {
@@ -110,7 +110,7 @@ export class Mem0Adapter extends BaseImportAdapter {
       warnings.push(`${invalidCount} memories had invalid/empty text and were skipped`);
     }
 
-    return { facts, warnings, errors, source_metadata: { total_from_source: memories.length } };
+    return { facts, chunks: [], totalMessages: 0, warnings, errors, source_metadata: { total_from_source: memories.length } };
   }
 
   /**

package/import-adapters/types.ts

@@ -19,7 +19,7 @@ export interface NormalizedFact {
   tags?: string[];
 }
 
-export type ImportSource = 'mem0' | 'mcp-memory' | 'memoclaw' | 'generic-json' | 'generic-csv';
+export type ImportSource = 'mem0' | 'mcp-memory' | 'chatgpt' | 'claude' | 'gemini' | 'memoclaw' | 'generic-json' | 'generic-csv';
 
 /**
  * What the user passes to the import tool.
@@ -73,15 +73,38 @@ export interface ImportResult {
 export type ProgressCallback = (progress: {
   current: number;
   total: number;
-  phase: 'fetching' | 'parsing' | 'storing';
+  phase: 'fetching' | 'parsing' | 'storing' | 'extracting';
   message: string;
 }) => void;
 
+/**
+ * A chunk of conversation messages for LLM-based fact extraction.
+ * Adapters that parse conversation data (ChatGPT, Claude) return these
+ * instead of pre-extracted facts, delegating extraction to the LLM.
+ */
+export interface ConversationChunk {
+  /** Human-readable title for progress reporting */
+  title: string;
+  /** Ordered messages in this chunk */
+  messages: Array<{ role: 'user' | 'assistant'; text: string }>;
+  /** Original timestamp (ISO 8601) if available */
+  timestamp?: string;
+}
+
 /**
  * Adapter parse result — returned by each adapter's parse method.
+ *
+ * Adapters return EITHER `facts` (pre-structured sources like Mem0, MCP Memory)
+ * OR `chunks` (conversation-based sources like ChatGPT, Claude) that need
+ * LLM extraction. The caller checks which field is populated.
  */
 export interface AdapterParseResult {
+  /** Pre-structured facts (Mem0, MCP Memory adapters) */
   facts: NormalizedFact[];
+  /** Conversation chunks needing LLM extraction (ChatGPT, Claude adapters) */
+  chunks: ConversationChunk[];
+  /** Total message count across all chunks */
+  totalMessages: number;
   warnings: string[];
   errors: string[];
   /** Metadata about the source (for logging) */