@askexenow/exe-os 0.9.267 → 0.9.268

package/dist/chunk-XSRBNOLY.js

@@ -0,0 +1,377 @@
+import {
+  extractMemoryGraph
+} from "./chunk-E2FPTCL5.js";
+import {
+  flushBatch,
+  writeMemory
+} from "./chunk-5MOYMGS5.js";
+import {
+  extractKeywords
+} from "./chunk-CHCA3ZM2.js";
+import {
+  getActiveAgent
+} from "./chunk-44MXQ47E.js";
+
+// src/mcp/tools/import-conversations.ts
+import { z } from "zod";
+import crypto from "crypto";
+import { readdir, readFile, stat } from "fs/promises";
+import path, { resolve, normalize } from "path";
+import os from "os";
+
+// src/lib/conversation-parser.ts
+function detectFormat(content, filePath) {
+  const lower = filePath.toLowerCase();
+  if (lower.endsWith(".md") || lower.endsWith(".markdown")) return "markdown";
+  const trimmed = content.trimStart();
+  if (!trimmed.startsWith("[") && !trimmed.startsWith("{")) return "markdown";
+  try {
+    const parsed = JSON.parse(trimmed);
+    if (Array.isArray(parsed) && parsed.length > 0) {
+      const first = parsed[0];
+      if (first && typeof first === "object") {
+        if ("mapping" in first && typeof first.mapping === "object") return "chatgpt";
+        if ("chat_messages" in first) return "claude";
+        if ("uuid" in first && "name" in first) return "claude";
+      }
+    }
+    return "generic";
+  } catch {
+    return "markdown";
+  }
+}
+function extractChatGPTContent(node) {
+  const parts = node.message?.content?.parts;
+  if (!Array.isArray(parts)) return "";
+  return parts.filter((p) => typeof p === "string").join("\n").trim();
+}
+function parseChatGPT(content, sourceFile = "conversations.json") {
+  const data = JSON.parse(content);
+  if (!Array.isArray(data)) return [];
+  return data.map((conv) => {
+    const title = conv.title ?? "Untitled";
+    const messages = [];
+    if (conv.mapping) {
+      const nodes = Object.values(conv.mapping).filter((n) => n.message?.content?.parts && extractChatGPTContent(n).length > 0).sort((a, b) => (a.message?.create_time ?? 0) - (b.message?.create_time ?? 0));
+      for (const node of nodes) {
+        const role = node.message?.author?.role ?? "unknown";
+        const text = extractChatGPTContent(node);
+        if (!text) continue;
+        const createTime = node.message?.create_time;
+        messages.push({
+          role,
+          content: text,
+          timestamp: createTime ? new Date(createTime * 1e3).toISOString() : void 0
+        });
+      }
+    }
+    return { title, messages, sourceFile };
+  }).filter((c) => c.messages.length > 0);
+}
+function parseClaude(content, sourceFile = "conversations.json") {
+  const data = JSON.parse(content);
+  if (!Array.isArray(data)) return [];
+  return data.map((conv) => {
+    const title = conv.name ?? conv.uuid ?? "Untitled";
+    const messages = [];
+    if (conv.chat_messages) {
+      for (const msg of conv.chat_messages) {
+        let text = msg.text ?? "";
+        if (!text && Array.isArray(msg.content)) {
+          text = msg.content.filter((c) => c.type === "text" && c.text).map((c) => c.text).join("\n");
+        }
+        if (!text.trim()) continue;
+        messages.push({
+          role: msg.sender ?? "unknown",
+          content: text.trim(),
+          timestamp: msg.created_at
+        });
+      }
+    }
+    return { title, messages, sourceFile };
+  }).filter((c) => c.messages.length > 0);
+}
+function parseGeneric(content, sourceFile = "conversations.json") {
+  const trimmed = content.trimStart();
+  try {
+    const data = JSON.parse(trimmed);
+    if (Array.isArray(data)) {
+      if (data.length > 0 && typeof data[0] === "object" && ("role" in data[0] || "content" in data[0])) {
+        const messages = data.map((msg) => ({
+          role: msg.role ?? "unknown",
+          content: (msg.content ?? msg.text ?? "").trim(),
+          timestamp: msg.timestamp ?? msg.created_at
+        })).filter((m) => m.content.length > 0);
+        if (messages.length > 0) {
+          return [{ title: sourceFile, messages, sourceFile }];
+        }
+      }
+    }
+  } catch {
+  }
+  return parseMarkdown(content, sourceFile);
+}
+function parseMarkdown(content, sourceFile = "conversation.md") {
+  const lines = content.split("\n");
+  const conversations = [];
+  let currentTitle = sourceFile;
+  let currentMessages = [];
+  let currentRole = "user";
+  let currentContent = [];
+  function flushMessage() {
+    const text = currentContent.join("\n").trim();
+    if (text) {
+      currentMessages.push({ role: currentRole, content: text });
+    }
+    currentContent = [];
+  }
+  function flushConversation() {
+    flushMessage();
+    if (currentMessages.length > 0) {
+      conversations.push({ title: currentTitle, messages: [...currentMessages], sourceFile });
+    }
+    currentMessages = [];
+  }
+  for (const line of lines) {
+    const h1Match = line.match(/^# (.+)$/);
+    if (h1Match) {
+      flushConversation();
+      currentTitle = h1Match[1];
+      continue;
+    }
+    const roleMatch = line.match(/^(?:##\s+|\*\*)(user|assistant|human|system|ai|claude|gpt)(?:\*\*)?:?\s*/i);
+    if (roleMatch) {
+      flushMessage();
+      const raw = roleMatch[1].toLowerCase();
+      currentRole = raw === "human" || raw === "user" ? "user" : raw === "assistant" || raw === "ai" || raw === "claude" || raw === "gpt" ? "assistant" : raw;
+      continue;
+    }
+    currentContent.push(line);
+  }
+  flushConversation();
+  return conversations;
+}
+function conversationsToMemories(convos, format = "generic") {
+  const memories = [];
+  for (const convo of convos) {
+    for (const msg of convo.messages) {
+      if (msg.content.length < 10) continue;
+      memories.push({
+        text: msg.content,
+        kind: "conversation",
+        metadata: {
+          source_type: "import",
+          source_format: format,
+          source_path: convo.sourceFile,
+          conversation_title: convo.title,
+          role: msg.role,
+          original_timestamp: msg.timestamp
+        }
+      });
+    }
+  }
+  return memories;
+}
+
+// src/mcp/tools/import-conversations.ts
+var DEFAULT_CAP = 5e4;
+var YIELD_INTERVAL = 50;
+async function readInputFiles(inputPath) {
+  const files = [];
+  const info = await stat(inputPath);
+  if (info.isDirectory()) {
+    const entries = await readdir(inputPath);
+    for (const entry of entries) {
+      const fullPath = path.join(inputPath, entry);
+      const entryStat = await stat(fullPath);
+      if (entryStat.isFile() && (entry.endsWith(".json") || entry.endsWith(".md") || entry.endsWith(".txt"))) {
+        files.push({ path: fullPath, content: await readFile(fullPath, "utf-8") });
+      }
+    }
+  } else {
+    files.push({ path: inputPath, content: await readFile(inputPath, "utf-8") });
+  }
+  return files;
+}
+function parseFile(content, filePath, requestedFormat) {
+  const format = requestedFormat && requestedFormat !== "auto" ? requestedFormat : detectFormat(content, filePath);
+  let conversations;
+  switch (format) {
+    case "chatgpt":
+      conversations = parseChatGPT(content, filePath);
+      break;
+    case "claude":
+      conversations = parseClaude(content, filePath);
+      break;
+    case "markdown":
+      conversations = parseMarkdown(content, filePath);
+      break;
+    case "generic":
+    default:
+      conversations = parseGeneric(content, filePath);
+      break;
+  }
+  return { format, conversations };
+}
+function registerImportConversations(server) {
+  server.registerTool(
+    "import_conversations",
+    {
+      title: "Import Conversations",
+      description: "Import ChatGPT, Claude, or generic conversation exports into memory. Parses, extracts keywords + graph entities, stores with proper metadata. Deduplicates via content_hash. Supports JSON, markdown, and directory input.",
+      inputSchema: {
+        path: z.string().describe("File or directory path to import"),
+        format: z.enum(["chatgpt", "claude", "generic", "markdown", "auto"]).default("auto").describe("Source format (default: auto-detect)"),
+        project_name: z.string().optional().describe("Project name for imported memories"),
+        dry_run: z.boolean().default(false).describe("Parse and report counts without storing"),
+        force: z.boolean().default(false).describe("Override 50K memory cap"),
+        agent_id: z.string().optional().describe("Agent to attribute memories to (default: current agent)")
+      }
+    },
+    async ({ path: inputPath, format: requestedFormat, project_name, dry_run, force, agent_id: overrideAgentId }) => {
+      const { agentId: currentAgentId, agentRole } = getActiveAgent();
+      const agentId = overrideAgentId ?? currentAgentId;
+      const projectName = project_name ?? "imported";
+      const resolvedPath = resolve(normalize(inputPath));
+      const home = process.env.HOME || os.homedir();
+      const blocked = ["/etc", "/root", "/var", "/usr", "/sys", "/proc"];
+      const sensitiveHome = [".ssh", ".gnupg", ".aws", ".config/gcloud"];
+      for (const dir of blocked) {
+        if (resolvedPath.startsWith(dir + "/") || resolvedPath === dir) {
+          return {
+            content: [{ type: "text", text: `Access denied: cannot read files from ${dir}` }],
+            isError: true
+          };
+        }
+      }
+      for (const rel of sensitiveHome) {
+        const full = resolve(home, rel);
+        if (resolvedPath.startsWith(full + "/") || resolvedPath === full) {
+          return {
+            content: [{ type: "text", text: `Access denied: cannot read files from ~/${rel}` }],
+            isError: true
+          };
+        }
+      }
+      let files;
+      try {
+        files = await readInputFiles(inputPath);
+      } catch (err) {
+        return {
+          content: [{ type: "text", text: `Error reading ${inputPath}: ${err instanceof Error ? err.message : String(err)}` }],
+          isError: true
+        };
+      }
+      if (files.length === 0) {
+        return {
+          content: [{ type: "text", text: `No importable files found at ${inputPath}` }],
+          isError: true
+        };
+      }
+      const allMemories = [];
+      let conversationCount = 0;
+      const formatCounts = {};
+      for (const file of files) {
+        const { format, conversations } = parseFile(file.content, file.path, requestedFormat);
+        formatCounts[format] = (formatCounts[format] ?? 0) + conversations.length;
+        conversationCount += conversations.length;
+        const memories = conversationsToMemories(conversations, format);
+        allMemories.push(...memories);
+      }
+      if (allMemories.length > DEFAULT_CAP && !force) {
+        return {
+          content: [{
+            type: "text",
+            text: `Import would create ${allMemories.length} memories (cap: ${DEFAULT_CAP}). Use force=true to override. Files: ${files.length}, Conversations: ${conversationCount}`
+          }],
+          isError: true
+        };
+      }
+      if (dry_run) {
+        const formatSummary2 = Object.entries(formatCounts).map(([f, c]) => `${f}: ${c}`).join(", ");
+        return {
+          content: [{
+            type: "text",
+            text: `[DRY RUN] Would import:
+  Files: ${files.length}
+  Conversations: ${conversationCount} (${formatSummary2})
+  Memories: ${allMemories.length}
+  Agent: ${agentId}
+  Project: ${projectName}`
+          }]
+        };
+      }
+      let stored = 0;
+      let skipped = 0;
+      for (let i = 0; i < allMemories.length; i++) {
+        const memory = allMemories[i];
+        try {
+          const keywords = extractKeywords(memory.text);
+          const keywordsStr = keywords.length > 0 ? keywords.join(" ") : null;
+          let entities = null;
+          try {
+            const graph = extractMemoryGraph(memory.text, agentId, projectName, "import_conversations");
+            if (graph.entities.length > 0) {
+              entities = JSON.stringify(graph.entities.map((e) => e.name));
+            }
+          } catch {
+          }
+          const id = crypto.randomUUID();
+          await writeMemory({
+            id,
+            agent_id: agentId,
+            agent_role: agentRole,
+            session_id: process.env.SESSION_ID ?? "import",
+            timestamp: memory.metadata.original_timestamp ?? (/* @__PURE__ */ new Date()).toISOString(),
+            tool_name: "import_conversations",
+            project_name: projectName,
+            has_error: false,
+            raw_text: memory.text,
+            vector: null,
+            // Backfill later
+            source_path: memory.metadata.source_path,
+            source_type: "import",
+            memory_type: "conversation",
+            tier: 3,
+            // Passive ingestion
+            importance: memory.metadata.role === "assistant" ? 5 : 4,
+            intent: "observation",
+            domain: projectName,
+            referenced_entities: entities,
+            keywords: keywordsStr,
+            valid_from: memory.metadata.original_timestamp ?? void 0
+          });
+          stored++;
+        } catch {
+          skipped++;
+        }
+        if ((i + 1) % YIELD_INTERVAL === 0) {
+          await flushBatch();
+          await new Promise((r) => setTimeout(r, 0));
+          if ((i + 1) % (YIELD_INTERVAL * 10) === 0) {
+            process.stderr.write(`[import] ${i + 1}/${allMemories.length} memories processed, ${stored} stored
+`);
+          }
+        }
+      }
+      await flushBatch();
+      const formatSummary = Object.entries(formatCounts).map(([f, c]) => `${f}: ${c}`).join(", ");
+      return {
+        content: [{
+          type: "text",
+          text: `Import complete.
+  Files: ${files.length}
+  Conversations: ${conversationCount} (${formatSummary})
+  Memories stored: ${stored}
+  Duplicates skipped: ${skipped}
+  Agent: ${agentId}
+  Project: ${projectName}`
+        }]
+      };
+    }
+  );
+}
+
+export {
+  registerImportConversations
+};