@totalreclaw/totalreclaw 1.0.4 → 1.1.0

package/import-adapters/mcp-memory-adapter.ts

@@ -0,0 +1,274 @@
+import { BaseImportAdapter } from './base-adapter.js';
+import type {
+  ImportSource,
+  NormalizedFact,
+  AdapterParseResult,
+  ProgressCallback,
+} from './types.js';
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+
+/**
+ * MCP Memory Server entity record.
+ */
+interface MCPEntity {
+  type: 'entity';
+  name: string;
+  entityType: string;
+  observations: string[];
+}
+
+/**
+ * MCP Memory Server relation record.
+ */
+interface MCPRelation {
+  type: 'relation';
+  from: string;
+  to: string;
+  relationType: string;
+}
+
+type MCPRecord = MCPEntity | MCPRelation;
+
+/**
+ * Entity type mapping to TotalReclaw fact types.
+ */
+const ENTITY_TYPE_MAP: Record<string, NormalizedFact['type']> = {
+  person: 'fact',
+  project: 'fact',
+  organization: 'fact',
+  tool: 'preference',
+  technology: 'preference',
+  preference: 'preference',
+  goal: 'goal',
+  event: 'episodic',
+  decision: 'decision',
+};
+
+export class MCPMemoryAdapter extends BaseImportAdapter {
+  readonly source: ImportSource = 'mcp-memory';
+  readonly displayName = 'MCP Memory Server';
+
+  async parse(
+    input: { content?: string; file_path?: string },
+    onProgress?: ProgressCallback,
+  ): Promise<AdapterParseResult> {
+    const warnings: string[] = [];
+    const errors: string[] = [];
+
+    let content: string;
+
+    if (input.content) {
+      content = input.content;
+    } else if (input.file_path) {
+      try {
+        const resolvedPath = input.file_path.replace(/^~/, os.homedir());
+        content = fs.readFileSync(resolvedPath, 'utf-8');
+      } catch (e) {
+        errors.push(`Failed to read file: ${e instanceof Error ? e.message : 'Unknown error'}`);
+        return { facts: [], warnings, errors };
+      }
+    } else {
+      // Try default MCP memory path
+      const defaultPath = path.join(os.homedir(), '.mcp-memory', 'memory.jsonl');
+      try {
+        content = fs.readFileSync(defaultPath, 'utf-8');
+        warnings.push(`Using default MCP memory path: ${defaultPath}`);
+      } catch {
+        errors.push(
+          'No content, file_path, or file at default path (~/.mcp-memory/memory.jsonl). ' +
+          'Provide the memory.jsonl content or file path.',
+        );
+        return { facts: [], warnings, errors };
+      }
+    }
+
+    // Parse JSONL records
+    const records = this.parseJSONL(content, errors);
+
+    if (onProgress) {
+      onProgress({
+        current: 0,
+        total: records.length,
+        phase: 'parsing',
+        message: `Parsing ${records.length} MCP Memory records...`,
+      });
+    }
+
+    // Separate entities and relations
+    const entities = new Map<string, MCPEntity>();
+    const relations: MCPRelation[] = [];
+
+    for (const record of records) {
+      if (record.type === 'entity') {
+        // Later entities override earlier ones (append-only file)
+        entities.set(record.name, record);
+      } else if (record.type === 'relation') {
+        relations.push(record);
+      }
+    }
+
+    // Convert entities to facts
+    const rawFacts: Partial<NormalizedFact>[] = [];
+    let entityIndex = 0;
+
+    for (const [name, entity] of entities) {
+      const factType = ENTITY_TYPE_MAP[entity.entityType.toLowerCase()] || 'fact';
+
+      for (const observation of entity.observations) {
+        // Prefix observation with entity name for context
+        // "Works at Acme Corp" -> "John works at Acme Corp"
+        const text = this.contextualizeObservation(name, observation);
+
+        rawFacts.push({
+          text,
+          type: factType,
+          importance: 6,
+          source: 'mcp-memory',
+          sourceId: `${name}:${entityIndex}`,
+          tags: [entity.entityType],
+        });
+        entityIndex++;
+      }
+
+      if (onProgress) {
+        onProgress({
+          current: rawFacts.length,
+          total: rawFacts.length + relations.length,
+          phase: 'parsing',
+          message: `Parsed ${rawFacts.length} facts from entities...`,
+        });
+      }
+    }
+
+    // Convert relations to facts
+    for (const rel of relations) {
+      // Only create a fact if both entities exist
+      if (!entities.has(rel.from) || !entities.has(rel.to)) {
+        warnings.push(`Relation references unknown entity: ${rel.from} -> ${rel.to}`);
+        continue;
+      }
+
+      const text = `${rel.from} ${this.humanizeRelationType(rel.relationType)} ${rel.to}`;
+
+      rawFacts.push({
+        text,
+        type: 'fact',
+        importance: 5,
+        source: 'mcp-memory',
+        sourceId: `rel:${rel.from}:${rel.relationType}:${rel.to}`,
+        tags: ['relation'],
+      });
+    }
+
+    const { facts, invalidCount } = this.validateFacts(rawFacts);
+
+    if (invalidCount > 0) {
+      warnings.push(`${invalidCount} observations had invalid/empty text and were skipped`);
+    }
+
+    return {
+      facts,
+      warnings,
+      errors,
+      source_metadata: {
+        entities_count: entities.size,
+        relations_count: relations.length,
+        observations_total: rawFacts.length,
+      },
+    };
+  }
+
+  /**
+   * Parse JSONL content into records.
+   */
+  private parseJSONL(content: string, errors: string[]): MCPRecord[] {
+    const records: MCPRecord[] = [];
+    const lines = content.split('\n').filter((line) => line.trim().length > 0);
+
+    for (let i = 0; i < lines.length; i++) {
+      try {
+        const record = JSON.parse(lines[i]);
+        if (record.type === 'entity' || record.type === 'relation') {
+          records.push(record as MCPRecord);
+        } else {
+          // Unknown record type — skip silently (future-proofing)
+        }
+      } catch {
+        errors.push(`Line ${i + 1}: Invalid JSON — skipped`);
+      }
+    }
+
+    return records;
+  }
+
+  /**
+   * Add entity context to an observation.
+   *
+   * Heuristic: if the observation already starts with the entity name
+   * (case-insensitive) or a pronoun, return as-is. Otherwise prefix.
+   *
+   * Examples:
+   *  - ("John", "Works at Acme Corp") -> "John works at Acme Corp"
+   *  - ("John", "John likes TypeScript") -> "John likes TypeScript" (no double prefix)
+   *  - ("John", "He prefers React") -> "John prefers React"
+   */
+  private contextualizeObservation(entityName: string, observation: string): string {
+    const obsLower = observation.toLowerCase().trim();
+    const nameLower = entityName.toLowerCase();
+
+    // Already starts with the entity name
+    if (obsLower.startsWith(nameLower)) {
+      return observation.trim();
+    }
+
+    // Starts with a pronoun — replace it
+    const pronouns = ['he ', 'she ', 'they ', 'it ', 'his ', 'her ', 'their ', 'its '];
+    for (const pronoun of pronouns) {
+      if (obsLower.startsWith(pronoun)) {
+        return entityName + ' ' + observation.trim().slice(pronoun.length);
+      }
+    }
+
+    // Starts with a verb (lowercase first word) — prefix entity name
+    const firstChar = observation.trim()[0];
+    if (firstChar && firstChar === firstChar.toLowerCase()) {
+      return `${entityName} ${observation.trim()}`;
+    }
+
+    // Observation is a standalone sentence — prefix with "About {entity}: "
+    return `${entityName}: ${observation.trim()}`;
+  }
+
+  /**
+   * Convert relation type slug to human-readable text.
+   *
+   * "works_on" -> "works on"
+   * "MEMBER_OF" -> "is a member of"
+   */
+  private humanizeRelationType(relationType: string): string {
+    const slug = relationType.toLowerCase().replace(/_/g, ' ');
+
+    // Common relation type mappings
+    const mappings: Record<string, string> = {
+      'works on': 'works on',
+      'works at': 'works at',
+      'member of': 'is a member of',
+      'belongs to': 'belongs to',
+      'created by': 'was created by',
+      'depends on': 'depends on',
+      'uses': 'uses',
+      'knows': 'knows',
+      'related to': 'is related to',
+      'part of': 'is part of',
+      'friend of': 'is friends with',
+      'manages': 'manages',
+      'reports to': 'reports to',
+      'located in': 'is located in',
+      'lives in': 'lives in',
+    };
+
+    return mappings[slug] || slug;
+  }
+}

package/import-adapters/mem0-adapter.ts

@@ -0,0 +1,233 @@
+import { BaseImportAdapter } from './base-adapter.js';
+import type {
+  ImportSource,
+  NormalizedFact,
+  AdapterParseResult,
+  ProgressCallback,
+} from './types.js';
+
+/**
+ * Mem0 memory structure from their API/export.
+ */
+interface Mem0Memory {
+  id: string;
+  memory: string;
+  hash?: string;
+  metadata?: {
+    category?: string;
+    created_at?: string;
+    updated_at?: string;
+  };
+  user_id?: string;
+  agent_id?: string;
+  categories?: string[];
+}
+
+interface Mem0ApiResponse {
+  results: Mem0Memory[];
+  next?: string; // pagination cursor
+  total?: number;
+}
+
+interface Mem0ExportFile {
+  export_date?: string;
+  user_id?: string;
+  memories: Mem0Memory[];
+}
+
+/**
+ * Category mapping from Mem0 categories to TotalReclaw types.
+ */
+const CATEGORY_MAP: Record<string, NormalizedFact['type']> = {
+  preference: 'preference',
+  preferences: 'preference',
+  like: 'preference',
+  dislike: 'preference',
+  fact: 'fact',
+  personal: 'fact',
+  biographical: 'fact',
+  decision: 'decision',
+  goal: 'goal',
+  objective: 'goal',
+  experience: 'episodic',
+  event: 'episodic',
+  memory: 'episodic',
+};
+
+export class Mem0Adapter extends BaseImportAdapter {
+  readonly source: ImportSource = 'mem0';
+  readonly displayName = 'Mem0';
+
+  async parse(
+    input: { content?: string; api_key?: string; source_user_id?: string; api_url?: string },
+    onProgress?: ProgressCallback,
+  ): Promise<AdapterParseResult> {
+    const warnings: string[] = [];
+    const errors: string[] = [];
+
+    let memories: Mem0Memory[];
+
+    if (input.content) {
+      // Parse from pasted content or export file
+      memories = this.parseExportContent(input.content, errors);
+    } else if (input.api_key) {
+      // Fetch from Mem0 API
+      memories = await this.fetchFromApi(
+        input.api_key,
+        input.source_user_id,
+        input.api_url,
+        onProgress,
+        errors,
+      );
+    } else {
+      errors.push('Mem0 import requires either content (export file) or api_key');
+      return { facts: [], warnings, errors };
+    }
+
+    if (onProgress) {
+      onProgress({
+        current: 0,
+        total: memories.length,
+        phase: 'parsing',
+        message: `Parsing ${memories.length} Mem0 memories...`,
+      });
+    }
+
+    // Convert Mem0 memories to NormalizedFacts
+    const rawFacts: Partial<NormalizedFact>[] = memories.map((mem) => ({
+      text: mem.memory,
+      type: this.mapCategory(mem.categories, mem.metadata?.category),
+      importance: 6, // Mem0 doesn't provide importance — default to 6 (above threshold)
+      source: 'mem0' as ImportSource,
+      sourceId: mem.id,
+      sourceTimestamp: mem.metadata?.updated_at || mem.metadata?.created_at,
+      tags: mem.categories || [],
+    }));
+
+    const { facts, invalidCount } = this.validateFacts(rawFacts);
+
+    if (invalidCount > 0) {
+      warnings.push(`${invalidCount} memories had invalid/empty text and were skipped`);
+    }
+
+    return { facts, warnings, errors, source_metadata: { total_from_source: memories.length } };
+  }
+
+  /**
+   * Parse Mem0 export file or pasted JSON.
+   */
+  private parseExportContent(content: string, errors: string[]): Mem0Memory[] {
+    try {
+      const data = JSON.parse(content.trim());
+
+      // Handle export file format: { memories: [...] }
+      if (data.memories && Array.isArray(data.memories)) {
+        return data.memories;
+      }
+
+      // Handle API response format: { results: [...] }
+      if (data.results && Array.isArray(data.results)) {
+        return data.results;
+      }
+
+      // Handle bare array
+      if (Array.isArray(data)) {
+        return data;
+      }
+
+      errors.push('Unrecognized Mem0 format. Expected { memories: [...] }, { results: [...] }, or bare array.');
+      return [];
+    } catch (e) {
+      errors.push(`Failed to parse Mem0 JSON: ${e instanceof Error ? e.message : 'Unknown error'}`);
+      return [];
+    }
+  }
+
+  /**
+   * Fetch all memories from Mem0 API with pagination.
+   */
+  private async fetchFromApi(
+    apiKey: string,
+    sourceUserId?: string,
+    apiUrl?: string,
+    onProgress?: ProgressCallback,
+    errors?: string[],
+  ): Promise<Mem0Memory[]> {
+    const baseUrl = apiUrl || 'https://api.mem0.ai';
+    const allMemories: Mem0Memory[] = [];
+    let page = 1;
+    const pageSize = 100;
+    let hasMore = true;
+
+    while (hasMore) {
+      try {
+        const url = new URL(`${baseUrl}/v1/memories/`);
+        url.searchParams.set('page', String(page));
+        url.searchParams.set('page_size', String(pageSize));
+        if (sourceUserId) {
+          url.searchParams.set('user_id', sourceUserId);
+        }
+
+        const response = await fetch(url.toString(), {
+          headers: {
+            Authorization: `Token ${apiKey}`,
+            'Content-Type': 'application/json',
+          },
+        });
+
+        if (!response.ok) {
+          const errorText = await response.text();
+          errors?.push(`Mem0 API error (${response.status}): ${errorText.slice(0, 200)}`);
+          break;
+        }
+
+        const data: Mem0ApiResponse = await response.json();
+        const memories = data.results || [];
+        allMemories.push(...memories);
+
+        if (onProgress) {
+          onProgress({
+            current: allMemories.length,
+            total: data.total || allMemories.length,
+            phase: 'fetching',
+            message: `Fetched ${allMemories.length} memories from Mem0...`,
+          });
+        }
+
+        hasMore = memories.length === pageSize;
+        page++;
+
+        // Safety limit: 10,000 memories max
+        if (allMemories.length >= 10_000) {
+          errors?.push('Reached 10,000 memory limit. Some memories may not have been fetched.');
+          break;
+        }
+      } catch (e) {
+        errors?.push(`Mem0 API fetch error: ${e instanceof Error ? e.message : 'Unknown error'}`);
+        break;
+      }
+    }
+
+    return allMemories;
+  }
+
+  /**
+   * Map Mem0 category to TotalReclaw fact type.
+   */
+  private mapCategory(
+    categories?: string[],
+    singleCategory?: string,
+  ): NormalizedFact['type'] {
+    const allCategories = [
+      ...(categories || []),
+      ...(singleCategory ? [singleCategory] : []),
+    ];
+
+    for (const cat of allCategories) {
+      const mapped = CATEGORY_MAP[cat.toLowerCase()];
+      if (mapped) return mapped;
+    }
+
+    return 'fact'; // default
+  }
+}

package/import-adapters/types.ts

@@ -0,0 +1,89 @@
+/**
+ * Normalized fact — the common format all adapters produce.
+ * Maps directly to what storeExtractedFacts() / client.remember() expect.
+ */
+export interface NormalizedFact {
+  /** The atomic fact text (max 512 chars) */
+  text: string;
+  /** Fact type matching TotalReclaw's taxonomy */
+  type: 'fact' | 'preference' | 'decision' | 'episodic' | 'goal';
+  /** Importance score 1-10 */
+  importance: number;
+  /** Original source system */
+  source: ImportSource;
+  /** Original ID in the source system (for traceability) */
+  sourceId?: string;
+  /** Original timestamp from source */
+  sourceTimestamp?: string;
+  /** Additional tags */
+  tags?: string[];
+}
+
+export type ImportSource = 'mem0' | 'mcp-memory' | 'memoclaw' | 'generic-json' | 'generic-csv';
+
+/**
+ * What the user passes to the import tool.
+ */
+export interface ImportFromInput {
+  /** Which source to import from */
+  source: ImportSource;
+  /** For API-based sources: the API key or auth token */
+  api_key?: string;
+  /** For API-based sources: the user/agent ID in the source system */
+  source_user_id?: string;
+  /** For file-based sources: the file content (pasted or read) */
+  content?: string;
+  /** For file-based sources: path to the file on disk */
+  file_path?: string;
+  /** Optional: target namespace in TotalReclaw */
+  namespace?: string;
+  /** Optional: dry run — parse and report without storing */
+  dry_run?: boolean;
+  /** Optional: API base URL override (for self-hosted instances) */
+  api_url?: string;
+}
+
+/**
+ * Result of an import operation.
+ */
+export interface ImportResult {
+  success: boolean;
+  source: ImportSource;
+  /** Total facts found in source */
+  total_found: number;
+  /** Facts successfully imported */
+  imported: number;
+  /** Facts skipped (duplicates via content fingerprint) */
+  skipped_duplicate: number;
+  /** Facts skipped (validation errors) */
+  skipped_invalid: number;
+  /** Individual errors */
+  errors: Array<{ index: number; text_preview: string; error: string }>;
+  /** Warnings */
+  warnings: string[];
+  /** Unique import run ID */
+  import_id: string;
+  /** Duration in ms */
+  duration_ms: number;
+}
+
+/**
+ * Progress callback for long-running imports.
+ */
+export type ProgressCallback = (progress: {
+  current: number;
+  total: number;
+  phase: 'fetching' | 'parsing' | 'storing';
+  message: string;
+}) => void;
+
+/**
+ * Adapter parse result — returned by each adapter's parse method.
+ */
+export interface AdapterParseResult {
+  facts: NormalizedFact[];
+  warnings: string[];
+  errors: string[];
+  /** Metadata about the source (for logging) */
+  source_metadata?: Record<string, unknown>;
+}