@totalreclaw/totalreclaw 1.4.0 → 1.6.0

package/import-adapters/base-adapter.ts

@@ -8,12 +8,11 @@ import type {
 /**
  * Abstract base class for import adapters.
  *
- * Each adapter:
- * 1. Fetches or reads source data
- * 2. Parses into NormalizedFact[]
- * 3. Validates each fact
+ * Adapters are PARSERS only — they convert raw export data into either:
+ * - Pre-structured facts (Mem0, MCP Memory — facts are already atomic)
+ * - Conversation chunks (ChatGPT, Claude — need LLM extraction)
  *
- * The caller (import tool) handles encryption + storage.
+ * The caller (import tool) handles LLM extraction, encryption, and storage.
  */
 export abstract class BaseImportAdapter {
   abstract readonly source: ImportSource;

package/import-adapters/chatgpt-adapter.ts

@@ -0,0 +1,323 @@
+import { BaseImportAdapter } from './base-adapter.js';
+import type {
+  ImportSource,
+  AdapterParseResult,
+  ConversationChunk,
+  ProgressCallback,
+} from './types.js';
+import fs from 'node:fs';
+import os from 'node:os';
+
+// ── ChatGPT conversations.json types ────────────────────────────────────────
+
+interface ChatGPTMessage {
+  id: string;
+  author: { role: 'user' | 'assistant' | 'system' | 'tool'; name?: string };
+  content: {
+    content_type: string;
+    parts?: (string | null | Record<string, unknown>)[];
+  };
+  create_time?: number;
+  metadata?: Record<string, unknown>;
+}
+
+interface ChatGPTMappingNode {
+  id: string;
+  message?: ChatGPTMessage | null;
+  parent?: string | null;
+  children: string[];
+}
+
+interface ChatGPTConversation {
+  id?: string;
+  title?: string;
+  create_time?: number;
+  update_time?: number;
+  mapping: Record<string, ChatGPTMappingNode>;
+}
+
+/** Maximum messages per conversation chunk for LLM extraction. */
+const CHUNK_SIZE = 20;
+
+// ── ChatGPT Adapter ─────────────────────────────────────────────────────────
+
+export class ChatGPTAdapter extends BaseImportAdapter {
+  readonly source: ImportSource = 'chatgpt';
+  readonly displayName = 'ChatGPT';
+
+  async parse(
+    input: { content?: string; file_path?: string },
+    onProgress?: ProgressCallback,
+  ): Promise<AdapterParseResult> {
+    const warnings: string[] = [];
+    const errors: string[] = [];
+
+    let content: string;
+
+    if (input.content) {
+      content = input.content;
+    } else if (input.file_path) {
+      try {
+        const resolvedPath = input.file_path.replace(/^~/, os.homedir());
+        content = fs.readFileSync(resolvedPath, 'utf-8');
+      } catch (e) {
+        errors.push(`Failed to read file: ${e instanceof Error ? e.message : 'Unknown error'}`);
+        return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
+      }
+    } else {
+      errors.push(
+        'ChatGPT import requires either content (pasted text or JSON) or file_path. ' +
+        'Export from ChatGPT: Settings -> Data Controls -> Export Data (conversations.json), ' +
+        'or copy from Settings -> Personalization -> Memory -> Manage.',
+      );
+      return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
+    }
+
+    // Detect format: JSON array = conversations.json, plain text = memories
+    const trimmed = content.trim();
+
+    if (trimmed.startsWith('[') || trimmed.startsWith('{')) {
+      // Try to parse as JSON (conversations.json or memory list)
+      return this.parseConversationsJson(trimmed, warnings, errors, onProgress);
+    }
+
+    // Plain text: ChatGPT memories (one per line)
+    return this.parseMemoriesText(trimmed, warnings, errors, onProgress);
+  }
+
+  /**
+   * Parse ChatGPT conversations.json — full export with mapping tree.
+   * Returns conversation chunks for LLM extraction (no pattern matching).
+   */
+  private parseConversationsJson(
+    content: string,
+    warnings: string[],
+    errors: string[],
+    onProgress?: ProgressCallback,
+  ): AdapterParseResult {
+    let conversations: ChatGPTConversation[];
+
+    try {
+      const data = JSON.parse(content);
+
+      if (Array.isArray(data)) {
+        conversations = data;
+      } else if (data.conversations && Array.isArray(data.conversations)) {
+        conversations = data.conversations;
+      } else if (data.mapping) {
+        // Single conversation object
+        conversations = [data];
+      } else {
+        errors.push(
+          'Unrecognized ChatGPT format. Expected an array of conversation objects (conversations.json) ' +
+          'or plain text (ChatGPT memories).',
+        );
+        return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
+      }
+    } catch (e) {
+      errors.push(`Failed to parse ChatGPT JSON: ${e instanceof Error ? e.message : 'Unknown error'}`);
+      return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
+    }
+
+    if (onProgress) {
+      onProgress({
+        current: 0,
+        total: conversations.length,
+        phase: 'parsing',
+        message: `Parsing ${conversations.length} ChatGPT conversations...`,
+      });
+    }
+
+    const chunks: ConversationChunk[] = [];
+    let totalMessages = 0;
+    let convIndex = 0;
+
+    for (const conv of conversations) {
+      if (!conv.mapping) {
+        warnings.push(`Conversation "${conv.title || 'untitled'}" has no mapping — skipped`);
+        continue;
+      }
+
+      // Extract user + assistant messages in chronological order
+      const messages = this.extractMessages(conv.mapping);
+      if (messages.length === 0) continue;
+
+      totalMessages += messages.length;
+
+      // Determine timestamp from first message or conversation
+      const timestamp = conv.create_time
+        ? new Date(conv.create_time * 1000).toISOString()
+        : undefined;
+
+      const title = conv.title || 'Untitled Conversation';
+
+      // Chunk into batches of CHUNK_SIZE messages
+      for (let i = 0; i < messages.length; i += CHUNK_SIZE) {
+        const batch = messages.slice(i, i + CHUNK_SIZE);
+        const chunkIndex = Math.floor(i / CHUNK_SIZE) + 1;
+        const totalChunks = Math.ceil(messages.length / CHUNK_SIZE);
+
+        chunks.push({
+          title: totalChunks > 1 ? `${title} (part ${chunkIndex}/${totalChunks})` : title,
+          messages: batch,
+          timestamp,
+        });
+      }
+
+      convIndex++;
+      if (onProgress && convIndex % 50 === 0) {
+        onProgress({
+          current: convIndex,
+          total: conversations.length,
+          phase: 'parsing',
+          message: `Parsed ${convIndex}/${conversations.length} conversations (${chunks.length} chunks, ${totalMessages} messages)...`,
+        });
+      }
+    }
+
+    if (chunks.length === 0 && conversations.length > 0) {
+      warnings.push(
+        `Parsed ${conversations.length} conversations but found no messages with text content.`,
+      );
+    }
+
+    return {
+      facts: [],
+      chunks,
+      totalMessages,
+      warnings,
+      errors,
+      source_metadata: {
+        format: 'conversations.json',
+        conversations_count: conversations.length,
+        chunks_count: chunks.length,
+        total_messages: totalMessages,
+      },
+    };
+  }
+
+  /**
+   * Parse ChatGPT memories — plain text, one memory per line.
+   * Users copy this from Settings -> Personalization -> Memory -> Manage.
+   *
+   * Each line becomes a single-message conversation chunk for LLM extraction.
+   */
+  private parseMemoriesText(
+    content: string,
+    warnings: string[],
+    errors: string[],
+    onProgress?: ProgressCallback,
+  ): AdapterParseResult {
+    // Split by newlines and filter empty lines
+    const lines = content.split('\n')
+      .map((line) => line.trim())
+      .filter((line) => line.length > 0)
+      // Skip common header lines
+      .filter((line) => !/^(?:memories?|chatgpt memories?|my memories?|saved memories?):?\s*$/i.test(line));
+
+    if (onProgress) {
+      onProgress({
+        current: 0,
+        total: lines.length,
+        phase: 'parsing',
+        message: `Parsing ${lines.length} ChatGPT memories...`,
+      });
+    }
+
+    // Clean lines: strip bullet/dash/number markers
+    const cleanedLines = lines.map((line) =>
+      line
+        .replace(/^[-*\u2022]\s+/, '')        // bullet points
+        .replace(/^\d+[.)]\s+/, '')            // numbered lists
+        .trim(),
+    ).filter((line) => line.length >= 3);
+
+    // Group all memories into chunks of CHUNK_SIZE for efficient LLM extraction
+    const chunks: ConversationChunk[] = [];
+    for (let i = 0; i < cleanedLines.length; i += CHUNK_SIZE) {
+      const batch = cleanedLines.slice(i, i + CHUNK_SIZE);
+      chunks.push({
+        title: `ChatGPT memories (${i + 1}-${Math.min(i + CHUNK_SIZE, cleanedLines.length)})`,
+        messages: batch.map((text) => ({ role: 'user' as const, text })),
+      });
+    }
+
+    return {
+      facts: [],
+      chunks,
+      totalMessages: cleanedLines.length,
+      warnings,
+      errors,
+      source_metadata: {
+        format: 'memories-text',
+        total_lines: lines.length,
+        chunks_count: chunks.length,
+      },
+    };
+  }
+
+  /**
+   * Traverse the mapping tree and extract user + assistant messages in chronological order.
+   * Both roles are included because the assistant's response often provides context
+   * that helps the LLM understand what the user meant.
+   */
+  private extractMessages(
+    mapping: Record<string, ChatGPTMappingNode>,
+  ): Array<{ role: 'user' | 'assistant'; text: string }> {
+    // Find the root node (the one with no parent or parent not in mapping)
+    let rootId: string | undefined;
+    for (const [id, node] of Object.entries(mapping)) {
+      if (!node.parent || !mapping[node.parent]) {
+        rootId = id;
+        break;
+      }
+    }
+
+    if (!rootId) return [];
+
+    // Walk the tree breadth-first, following children in order (main thread)
+    const messages: Array<{ role: 'user' | 'assistant'; text: string }> = [];
+    const visited = new Set<string>();
+    const queue: string[] = [rootId];
+
+    while (queue.length > 0) {
+      const nodeId = queue.shift()!;
+      if (visited.has(nodeId)) continue;
+      visited.add(nodeId);
+
+      const node = mapping[nodeId];
+      if (!node) continue;
+
+      const role = node.message?.author?.role;
+      // Only collect user and assistant messages (skip system, tool)
+      if (role === 'user' || role === 'assistant') {
+        const textParts = this.extractTextFromParts(node.message?.content?.parts);
+        if (textParts && textParts.length >= 3) {
+          messages.push({ role, text: textParts });
+        }
+      }
+
+      // Follow children (add them to queue in order)
+      for (const childId of node.children || []) {
+        queue.push(childId);
+      }
+    }
+
+    return messages;
+  }
+
+  /**
+   * Extract plain text from message content parts.
+   * Parts can be strings, null, or complex objects (images, etc.) -- we only want strings.
+   */
+  private extractTextFromParts(parts?: (string | null | Record<string, unknown>)[]): string | null {
+    if (!parts || parts.length === 0) return null;
+
+    const textParts = parts
+      .filter((p): p is string => typeof p === 'string' && p.trim().length > 0);
+
+    if (textParts.length === 0) return null;
+
+    return textParts.join(' ').trim();
+  }
+}

package/import-adapters/claude-adapter.ts

@@ -0,0 +1,146 @@
+import { BaseImportAdapter } from './base-adapter.js';
+import type {
+  ImportSource,
+  AdapterParseResult,
+  ConversationChunk,
+  ProgressCallback,
+} from './types.js';
+import fs from 'node:fs';
+import os from 'node:os';
+
+/**
+ * Pattern for lines that start with a date prefix.
+ * Claude memory entries sometimes have: [2026-03-15] - User prefers TypeScript
+ */
+const DATE_PREFIX_RE = /^\[(\d{4}-\d{2}-\d{2})\]\s*[-:]\s*/;
+
+/**
+ * Pattern for bullet-prefixed lines.
+ */
+const BULLET_PREFIX_RE = /^[-*\u2022]\s+/;
+
+/**
+ * Pattern for numbered list lines.
+ */
+const NUMBERED_PREFIX_RE = /^\d+[.)]\s+/;
+
+/** Maximum messages per conversation chunk for LLM extraction. */
+const CHUNK_SIZE = 20;
+
+export class ClaudeAdapter extends BaseImportAdapter {
+  readonly source: ImportSource = 'claude';
+  readonly displayName = 'Claude';
+
+  async parse(
+    input: { content?: string; file_path?: string },
+    onProgress?: ProgressCallback,
+  ): Promise<AdapterParseResult> {
+    const warnings: string[] = [];
+    const errors: string[] = [];
+
+    let content: string;
+
+    if (input.content) {
+      content = input.content;
+    } else if (input.file_path) {
+      try {
+        const resolvedPath = input.file_path.replace(/^~/, os.homedir());
+        content = fs.readFileSync(resolvedPath, 'utf-8');
+      } catch (e) {
+        errors.push(`Failed to read file: ${e instanceof Error ? e.message : 'Unknown error'}`);
+        return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
+      }
+    } else {
+      errors.push(
+        'Claude import requires either content (pasted text) or file_path. ' +
+        'Copy your memories from Claude: Settings -> Memory -> select all and copy.',
+      );
+      return { facts: [], chunks: [], totalMessages: 0, warnings, errors };
+    }
+
+    // Claude memory export is plain text, one fact per line.
+    return this.parseMemoriesText(content.trim(), warnings, errors, onProgress);
+  }
+
+  /**
+   * Parse Claude memories — plain text, one memory per line.
+   * Returns conversation chunks for LLM extraction (no pattern matching).
+   *
+   * Each line is cleaned (date prefixes, bullets, numbers stripped) and
+   * grouped into chunks for the LLM to process.
+   */
+  private parseMemoriesText(
+    content: string,
+    warnings: string[],
+    errors: string[],
+    onProgress?: ProgressCallback,
+  ): AdapterParseResult {
+    // Split by newlines and filter
+    const lines = content.split('\n')
+      .map((line) => line.trim())
+      .filter((line) => line.length > 0)
+      // Skip common header lines
+      .filter((line) => !/^(?:memories?|claude memories?|my memories?|saved memories?):?\s*$/i.test(line));
+
+    if (onProgress) {
+      onProgress({
+        current: 0,
+        total: lines.length,
+        phase: 'parsing',
+        message: `Parsing ${lines.length} Claude memories...`,
+      });
+    }
+
+    // Clean each line: extract date, strip formatting
+    const cleanedEntries: Array<{ text: string; timestamp?: string }> = [];
+    for (const line of lines) {
+      let cleaned = line;
+      let timestamp: string | undefined;
+
+      // Extract date prefix if present
+      const dateMatch = cleaned.match(DATE_PREFIX_RE);
+      if (dateMatch) {
+        timestamp = dateMatch[1];
+        cleaned = cleaned.replace(DATE_PREFIX_RE, '');
+      }
+
+      // Strip bullet/numbering markers
+      cleaned = cleaned
+        .replace(BULLET_PREFIX_RE, '')
+        .replace(NUMBERED_PREFIX_RE, '')
+        .trim();
+
+      if (cleaned.length >= 3) {
+        cleanedEntries.push({ text: cleaned, timestamp });
+      }
+    }
+
+    // Group memories into chunks of CHUNK_SIZE for efficient LLM extraction
+    const chunks: ConversationChunk[] = [];
+    for (let i = 0; i < cleanedEntries.length; i += CHUNK_SIZE) {
+      const batch = cleanedEntries.slice(i, i + CHUNK_SIZE);
+
+      // Use the timestamp from the first entry in the batch (if available)
+      const batchTimestamp = batch.find((e) => e.timestamp)?.timestamp;
+
+      chunks.push({
+        title: `Claude memories (${i + 1}-${Math.min(i + CHUNK_SIZE, cleanedEntries.length)})`,
+        messages: batch.map((entry) => ({ role: 'user' as const, text: entry.text })),
+        timestamp: batchTimestamp,
+      });
+    }
+
+    return {
+      facts: [],
+      chunks,
+      totalMessages: cleanedEntries.length,
+      warnings,
+      errors,
+      source_metadata: {
+        format: 'memories-text',
+        total_lines: lines.length,
+        chunks_count: chunks.length,
+      },
+    };
+  }
+}