@yugabytedb/meko-mcp 1.0.3

package/lib/migrate/mcp-client.mjs

@@ -0,0 +1,224 @@
+/**
+ * mcp-client.mjs — Thin wrapper over mcp-transport.mjs for the migrator.
+ *
+ * Adds:
+ *   - session management (initialize once, reuse sessionId)
+ *   - a concurrency semaphore on outgoing tool calls
+ *   - exponential-backoff retries on transient errors
+ *   - a small helper that parses the JSON text returned by Meko tools
+ */
+
+import {
+  jsonRpcRequest,
+  classifyError,
+  mcpInitialize,
+  DEFAULT_REQUEST_TIMEOUT_MS,
+} from "../mcp-transport.mjs";
+
+/**
+ * @typedef {Object} CallOptions
+ * @property {number} [timeoutMs]
+ * @property {number} [maxRetries]  Default 3. 0 disables retry.
+ * @property {boolean} [fireAndForget]  If true, failures log+return null.
+ */
+
+export class McpClient {
+  /**
+   * @param {object} opts
+   * @param {string} opts.url
+   * @param {string|null} opts.apiKey
+   * @param {number} [opts.concurrency]  Default 4.
+   * @param {number} [opts.timeoutMs]
+   * @param {{info?: Function, warn?: Function, error?: Function}} [opts.log]
+   */
+  constructor(opts) {
+    this.url = opts.url;
+    this.apiKey = opts.apiKey ?? null;
+    // Defense in depth: NaN or anything non-positive would leave the
+    // semaphore permanently blocked (queued waiters never fire because
+    // `inflight < NaN` is always false). Fall back to the default.
+    const concurrency = Number(opts.concurrency);
+    this.concurrency = Number.isFinite(concurrency) && concurrency >= 1
+      ? Math.floor(concurrency)
+      : 4;
+    this.timeoutMs = opts.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
+    this.log = opts.log ?? {};
+    this.sessionId = null;
+    this._reqId = 100;
+    this._inflight = 0;
+    this._waiters = [];
+  }
+
+  async connect() {
+    const res = await mcpInitialize(this.url, this.apiKey, {
+      clientName: "meko-migrate",
+      timeoutMs: this.timeoutMs,
+    });
+    if (!res.ok) {
+      const err = new Error(res.message || "initialize failed");
+      err.statusCode = res.statusCode;
+      throw err;
+    }
+    this.sessionId = res.sessionId;
+  }
+
+  /**
+   * Call a tool over the open session.
+   *
+   * @param {string} name
+   * @param {object} args
+   * @param {CallOptions} [opts]
+   * @returns {Promise<{ok: boolean, result?: object, text?: string, error?: object, statusCode?: number}>}
+   */
+  async callTool(name, args, opts = {}) {
+    await this._acquire();
+    try {
+      return await this._callWithRetry(name, args, opts);
+    } finally {
+      this._release();
+    }
+  }
+
+  async _callWithRetry(name, args, opts) {
+    const maxRetries = opts.maxRetries ?? 3;
+    let lastErr = null;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+      try {
+        const out = await this._callOnce(name, args, opts);
+        // HTTP-level failures inside a structured response go through
+        // _callOnce too; it throws on non-2xx. A tool-level error surfaces
+        // as `{ok:false, error:{...}}` and we return it without retry.
+        return out;
+      } catch (err) {
+        lastErr = err;
+        const code = err.statusCode;
+        if (code === 401 || code === 403) {
+          // Auth is fatal; surface immediately.
+          throw err;
+        }
+        if (attempt < maxRetries) {
+          const delay = 1000 * 2 ** attempt;
+          this.log.warn?.("mcp_retry", {
+            tool: name,
+            attempt: attempt + 1,
+            delay_ms: delay,
+            error: err.message,
+          });
+          await sleep(delay);
+          continue;
+        }
+      }
+    }
+    if (opts.fireAndForget) {
+      this.log.warn?.("mcp_fire_and_forget_failed", {
+        tool: name,
+        error: lastErr?.message,
+      });
+      return { ok: false, error: { message: lastErr?.message ?? "unknown" } };
+    }
+    throw lastErr;
+  }
+
+  async _callOnce(name, args, opts) {
+    const payload = {
+      jsonrpc: "2.0",
+      id: this._reqId++,
+      method: "tools/call",
+      params: { name, arguments: args },
+    };
+
+    let res;
+    try {
+      res = await jsonRpcRequest(this.url, payload, this.apiKey, this.sessionId, {
+        timeoutMs: opts.timeoutMs ?? this.timeoutMs,
+      });
+    } catch (err) {
+      const e = new Error(classifyError(err, this.timeoutMs));
+      e.cause = err;
+      throw e;
+    }
+
+    if (res.statusCode === 401 || res.statusCode === 403) {
+      const e = new Error(
+        `Auth rejected by server (HTTP ${res.statusCode}). Check API_KEY.`,
+      );
+      e.statusCode = res.statusCode;
+      throw e;
+    }
+
+    if (res.statusCode < 200 || res.statusCode >= 300) {
+      const e = new Error(
+        `tools/call ${name} returned HTTP ${res.statusCode}: ${res.body.slice(0, 200)}`,
+      );
+      e.statusCode = res.statusCode;
+      throw e;
+    }
+
+    let parsed;
+    try {
+      parsed = JSON.parse(res.body);
+    } catch {
+      const e = new Error(
+        `tools/call ${name} returned non-JSON: ${res.body.slice(0, 200)}`,
+      );
+      e.statusCode = res.statusCode;
+      throw e;
+    }
+
+    if (parsed.error) {
+      return {
+        ok: false,
+        error: parsed.error,
+        statusCode: res.statusCode,
+      };
+    }
+
+    const text = extractFirstText(parsed.result);
+    return {
+      ok: true,
+      result: parsed.result,
+      text,
+      statusCode: res.statusCode,
+    };
+  }
+
+  // --- Semaphore -----------------------------------------------------------
+
+  _acquire() {
+    if (this._inflight < this.concurrency) {
+      this._inflight++;
+      return Promise.resolve();
+    }
+    return new Promise((resolve) => {
+      this._waiters.push(() => {
+        this._inflight++;
+        resolve();
+      });
+    });
+  }
+
+  _release() {
+    this._inflight--;
+    const next = this._waiters.shift();
+    if (next) next();
+  }
+}
+
+/**
+ * Meko tool results come back as `{content: [{type:"text", text: "..."}], ...}`.
+ * Pull the first text block out (or null).
+ */
+function extractFirstText(result) {
+  const content = result?.content;
+  if (!Array.isArray(content)) return null;
+  for (const block of content) {
+    if (block && block.type === "text" && typeof block.text === "string") {
+      return block.text;
+    }
+  }
+  return null;
+}
+
+function sleep(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}

package/lib/migrate/orchestrator.mjs

@@ -0,0 +1,443 @@
+/**
+ * orchestrator.mjs — Drives a migration run end-to-end.
+ *
+ * Reads state, acquires a lock, enumerates sources, walks the MCP tools,
+ * flushes progress/state, and returns a summary. All I/O goes through
+ * the lower-level modules (`state`, `progress`, `lockfile`, `mcp-client`).
+ */
+
+import path from "node:path";
+import fs from "node:fs";
+import * as state from "./state.mjs";
+import * as progress from "./progress.mjs";
+import * as lockfile from "./lockfile.mjs";
+import { McpClient } from "./mcp-client.mjs";
+import { createLogger, nullLogger } from "./logger.mjs";
+import { seedFor } from "./id.mjs";
+import * as claudeCodeAdapter from "./adapters/claude-code.mjs";
+import * as cursorAdapter from "./adapters/cursor.mjs";
+import * as memoryMdAdapter from "./adapters/memory-md.mjs";
+
+/**
+ * @param {object} opts
+ * @param {string} opts.url
+ * @param {string|null} opts.apiKey
+ * @param {string} opts.runId
+ * @param {"user"|"project"} opts.scope
+ * @param {string} opts.projectCwd
+ * @param {string} opts.runDir
+ * @param {string} opts.statePath
+ * @param {string} opts.lockPath
+ * @param {boolean} [opts.dryRun]
+ * @param {boolean} [opts.noExtract]
+ * @param {boolean} [opts.force]              Bypass state-level skip-if-done.
+ * @param {boolean} [opts.lockAlreadyHeld]    Caller (e.g. detached child) says the lock is already owned by this process; skip acquire.
+ * @param {number}  [opts.concurrency]
+ * @param {string}  [opts.home]
+ * @returns {Promise<{ok: boolean, summary: object, counts: object}>}
+ */
+export async function runMigration(opts) {
+  fs.mkdirSync(opts.runDir, { recursive: true });
+
+  const log = opts.dryRun
+    ? nullLogger()
+    : createLogger(path.join(opts.runDir, "log.ndjson"), { runId: opts.runId });
+
+  const prog = progress.initial(opts.runId, opts.scope);
+  const progressPath = path.join(opts.runDir, "progress.json");
+  const writeProgress = () => {
+    try { progress.write(progressPath, prog); } catch { /* best effort */ }
+  };
+
+  let st = state.load(opts.statePath);
+  let opsSinceFlush = 0;
+  const FLUSH_EVERY = 25;
+  const flushState = () => {
+    try { state.save(opts.statePath, st); } catch { /* best effort */ }
+    opsSinceFlush = 0;
+  };
+
+  // Lock acquisition. On the production path the detached-child runner has
+  // already marked the lock with its own PID, so we MUST NOT try to acquire
+  // again — that would self-deadlock. Tests that call runMigration()
+  // directly default to acquiring here.
+  let lockAcquiredHere = false;
+  if (!opts.dryRun && !opts.lockAlreadyHeld) {
+    const acq = lockfile.acquire(opts.lockPath, { runId: opts.runId }, { force: opts.force });
+    if (!acq.acquired) {
+      log.error("lock_busy", { holder: acq.holder });
+      return { ok: false, summary: { error: "lock_busy", holder: acq.holder }, counts: prog.counts };
+    }
+    lockAcquiredHere = true;
+  }
+
+  // Plan the work.
+  const claudeSessions = await claudeCodeAdapter.listSessions({
+    scope: opts.scope,
+    projectCwd: opts.projectCwd,
+    home: opts.home,
+  });
+  const cursorSessions = cursorAdapter.listSessions({
+    scope: opts.scope,
+    projectCwd: opts.projectCwd,
+    home: opts.home,
+  });
+  const memoryItems = await memoryMdAdapter.enumerateMemoryItems({
+    scope: opts.scope,
+    projectCwd: opts.projectCwd,
+    home: opts.home,
+  });
+
+  prog.counts.sessionsPlanned = claudeSessions.length + cursorSessions.length;
+  writeProgress();
+
+  log.info("plan", {
+    scope: opts.scope,
+    claudeSessions: claudeSessions.length,
+    cursorSessions: cursorSessions.length,
+    memoryItems: memoryItems.length,
+    dryRun: !!opts.dryRun,
+    noExtract: !!opts.noExtract,
+  });
+
+  if (opts.dryRun) {
+    // Peek-parse every session so we can report planned turn counts and
+    // the total expected MCP call volume — helps users estimate
+    // LLM-extraction cost before firing a real run.
+    let pairedTurns = 0;
+    for (const ref of claudeSessions) {
+      const parsed = await claudeCodeAdapter.parseSession(ref.path);
+      pairedTurns += parsed.exchanges.length;
+    }
+    for (const ref of cursorSessions) {
+      const parsed = cursorAdapter.parseSession(ref.path);
+      pairedTurns += parsed.exchanges.length;
+    }
+    const plannedConversationCreates = claudeSessions.length + cursorSessions.length
+      + (memoryItems.length > 0 ? countDistinctMemoryAgents(memoryItems) : 0);
+    const plannedConversationAddMessage = pairedTurns;
+    const plannedMemoryAddExtract = opts.noExtract ? 0 : pairedTurns;
+    const plannedMemoryAddText = memoryItems.length;
+
+    const summary = {
+      scope: opts.scope,
+      claudeSessions: claudeSessions.length,
+      cursorSessions: cursorSessions.length,
+      memoryItems: memoryItems.length,
+      pairedTurns,
+      plannedCalls: {
+        conversation_create: plannedConversationCreates,
+        conversation_add_message: plannedConversationAddMessage,
+        memory_add_messages: plannedMemoryAddExtract,
+        memory_add_text: plannedMemoryAddText,
+        total:
+          plannedConversationCreates +
+          plannedConversationAddMessage +
+          plannedMemoryAddExtract +
+          plannedMemoryAddText,
+      },
+    };
+    return { ok: true, summary, counts: prog.counts };
+  }
+
+  // Open MCP client.
+  const client = new McpClient({
+    url: opts.url,
+    apiKey: opts.apiKey,
+    concurrency: opts.concurrency ?? 4,
+    log,
+  });
+
+  try {
+    await client.connect();
+    log.info("mcp_connected", { sessionId: client.sessionId });
+  } catch (err) {
+    log.error("mcp_connect_failed", { error: err.message, code: err.statusCode });
+    releaseLock();
+    prog.status = "failed";
+    prog.finishedAt = new Date().toISOString();
+    writeProgress();
+    return { ok: false, summary: { error: "mcp_connect_failed", detail: err.message }, counts: prog.counts };
+  }
+
+  // -- Conversations --
+  for (const ref of [...claudeSessions, ...cursorSessions]) {
+    let existing = state.getSession(st, ref.agentId, ref.sessionUuid);
+    if (existing?.status === "done" && !opts.force) {
+      log.info("session_skip_done", { agentId: ref.agentId, sessionUuid: ref.sessionUuid });
+      prog.counts.sessionsDone++;
+      writeProgress();
+      continue;
+    }
+    // --force re-ingests: wipe resume state so we don't skip messages.
+    if (existing && opts.force) {
+      state.markSession(st, ref.agentId, ref.sessionUuid, {
+        status: "pending",
+        messagesSent: 0,
+        memoryPairsSent: 0,
+        lastUuid: null,
+        completedAt: null,
+        lastError: null,
+      });
+      existing = state.getSession(st, ref.agentId, ref.sessionUuid);
+    }
+
+    prog.currentAgent = ref.agentId;
+    prog.currentSession = ref.sessionUuid;
+    writeProgress();
+
+    const parsed = (ref.path && ref.path.endsWith(".jsonl"))
+      ? await claudeCodeAdapter.parseSession(ref.path)
+      : cursorAdapter.parseSession(ref.path);
+
+    if (parsed.partial) {
+      log.warn("session_partial", { path: ref.path });
+    }
+
+    // Ensure a conversation_id exists.
+    let conversationId = existing?.conversationId || "";
+    if (!conversationId) {
+      const createRes = await safeCall(client, "conversation_create", {
+        scope: "write",
+        agent_id: ref.agentId,
+        session_id: ref.sessionUuid,
+        title: (parsed.firstUserText || ref.sessionUuid).slice(0, 120),
+        metadata: JSON.stringify({
+          source: ref.path.endsWith(".jsonl") ? "claude-code" : "cursor",
+          cwd: ref.cwd || null,
+          gitBranch: ref.slug ? (parsed.gitBranch || null) : null,
+          startedAt: parsed.startedAt || null,
+        }),
+      }, log);
+
+      if (!createRes.ok) {
+        log.error("conversation_create_failed", { agentId: ref.agentId, sessionUuid: ref.sessionUuid, error: createRes.error?.message });
+        state.markSession(st, ref.agentId, ref.sessionUuid, {
+          status: "failed",
+          lastError: createRes.error?.message ?? "conversation_create failed",
+        });
+        prog.counts.warnings++;
+        if (++opsSinceFlush >= FLUSH_EVERY) flushState();
+        writeProgress();
+        continue;
+      }
+
+      conversationId = extractId(createRes.text) || "";
+      state.markSession(st, ref.agentId, ref.sessionUuid, {
+        conversationId,
+        status: "partial",
+      });
+      if (++opsSinceFlush >= FLUSH_EVERY) flushState();
+    }
+
+    // Replay turns.
+    let messagesSent = existing?.messagesSent ?? 0;
+    let memoryPairsSent = existing?.memoryPairsSent ?? 0;
+    let lastUuid = existing?.lastUuid ?? null;
+
+    // Resume from lastUuid if present.
+    let skipping = !!lastUuid;
+    for (const ex of parsed.exchanges) {
+      if (skipping) {
+        if (ex.user_uuid === lastUuid) skipping = false;
+        continue;
+      }
+      const seed = seedFor(ref.agentId, ref.sessionUuid, ex.user_uuid);
+
+      // (a) conversation_add_message — fire-and-forget; server batches.
+      const addMsgPromise = client.callTool("conversation_add_message", {
+        scope: "write",
+        conversation_id: conversationId,
+        agent_id: ref.agentId,
+        input: ex.input,
+        output: ex.output,
+        reasoning: ex.reasoning || undefined,
+        seed,
+        metadata: JSON.stringify({
+          source: "migrate",
+          sessionUuid: ref.sessionUuid,
+          userUuid: ex.user_uuid,
+          timestamp: ex.timestamp,
+        }),
+      }, { fireAndForget: true });
+
+      // (b) memory_add(messages=...) — invokes Mem0 extractor server-side.
+      let memPromise = Promise.resolve(null);
+      if (!opts.noExtract && (ex.input || ex.output)) {
+        memPromise = client.callTool("memory_add", {
+          scope: "write",
+          agent_id: ref.agentId,
+          conversation_id: conversationId,
+          messages: [
+            { role: "user", content: ex.input || "" },
+            { role: "assistant", content: ex.output || "" },
+          ],
+          metadata: JSON.stringify({
+            source: "migrate",
+            sessionUuid: ref.sessionUuid,
+            userUuid: ex.user_uuid,
+          }),
+        }, { fireAndForget: true });
+      }
+
+      const [, memRes] = await Promise.all([addMsgPromise, memPromise]);
+      messagesSent++;
+      if (!opts.noExtract) {
+        if (memRes && memRes.ok) {
+          memoryPairsSent++;
+          prog.counts.memoryPairsSent++;
+        } else if (memRes) {
+          prog.counts.memoryErrors++;
+        }
+      }
+      lastUuid = ex.user_uuid;
+      prog.counts.messagesSent++;
+
+      state.markSession(st, ref.agentId, ref.sessionUuid, {
+        conversationId,
+        status: "partial",
+        messagesSent,
+        memoryPairsSent,
+        lastUuid,
+      });
+      if (++opsSinceFlush >= FLUSH_EVERY) flushState();
+      writeProgress();
+    }
+
+    state.markSession(st, ref.agentId, ref.sessionUuid, {
+      conversationId,
+      status: parsed.partial ? "partial" : "done",
+      messagesSent,
+      memoryPairsSent,
+      lastUuid,
+      completedAt: new Date().toISOString(),
+      lastError: null,
+    });
+    flushState();
+    if (!parsed.partial) prog.counts.sessionsDone++;
+    writeProgress();
+    log.info("session_done", {
+      agentId: ref.agentId,
+      sessionUuid: ref.sessionUuid,
+      messagesSent,
+      memoryPairsSent,
+      partial: parsed.partial,
+    });
+  }
+
+  // -- Memory MD files --
+  const umbrellaPerAgent = new Map();
+  for (const item of memoryItems) {
+    if (state.isMemoryDone(st, item.agentId, item.hash) && !opts.force) continue;
+
+    let convId = umbrellaPerAgent.get(item.agentId);
+    if (!convId) {
+      const existingUmbrella = state.ensureAgent(st, item.agentId).memoryConversationId;
+      if (existingUmbrella) {
+        convId = existingUmbrella;
+      } else {
+        const res = await safeCall(client, "conversation_create", {
+          scope: "write",
+          agent_id: item.agentId,
+          title: "memory-migration",
+          metadata: JSON.stringify({ source: "migrate", purpose: "memory-umbrella" }),
+        }, log);
+        if (!res.ok) {
+          log.error("memory_umbrella_failed", { agentId: item.agentId, error: res.error?.message });
+          prog.counts.warnings++;
+          continue;
+        }
+        convId = extractId(res.text) || "";
+        state.ensureAgent(st, item.agentId).memoryConversationId = convId;
+      }
+      umbrellaPerAgent.set(item.agentId, convId);
+    }
+
+    const res = await client.callTool("memory_add", {
+      scope: "write",
+      agent_id: item.agentId,
+      conversation_id: convId,
+      text: item.text,
+      metadata: JSON.stringify(item.metadata),
+    }, { fireAndForget: true });
+
+    if (res.ok) {
+      state.markMemory(st, item.agentId, item.hash, {
+        memoryId: extractId(res.text) || "",
+        name: item.name,
+        addedAt: new Date().toISOString(),
+      });
+      prog.counts.memoriesFromMD++;
+    } else {
+      prog.counts.memoryErrors++;
+      log.warn("memory_md_failed", { agentId: item.agentId, name: item.name, error: res.error?.message });
+    }
+    if (++opsSinceFlush >= FLUSH_EVERY) flushState();
+    writeProgress();
+  }
+
+  flushState();
+  prog.status = "done";
+  prog.finishedAt = new Date().toISOString();
+  prog.currentAgent = null;
+  prog.currentSession = null;
+  writeProgress();
+
+  st.runs.push({
+    runId: opts.runId,
+    startedAt: prog.startedAt,
+    finishedAt: prog.finishedAt,
+    scope: opts.scope,
+    status: "done",
+    counts: prog.counts,
+  });
+  flushState();
+
+  log.info("done", { summary: prog.counts });
+  log.close();
+  releaseLock();
+
+  return { ok: true, summary: prog.counts, counts: prog.counts };
+
+  function releaseLock() {
+    if (lockAcquiredHere) {
+      try { lockfile.release(opts.lockPath); } catch { /* ignore */ }
+    }
+  }
+}
+
+/**
+ * How many umbrella memory-migration conversations the real run would open.
+ */
+function countDistinctMemoryAgents(items) {
+  const set = new Set();
+  for (const item of items) set.add(item.agentId);
+  return set.size;
+}
+
+/**
+ * Call a tool with basic error-surfacing. Does not retry beyond the
+ * client's own retry logic. Returns `{ok: boolean, text?: string, error?: any}`.
+ */
+async function safeCall(client, name, args, log) {
+  try {
+    return await client.callTool(name, args);
+  } catch (err) {
+    log.error("mcp_call_threw", { tool: name, error: err.message });
+    return { ok: false, error: { message: err.message } };
+  }
+}
+
+/**
+ * Meko tools return JSON text in their content blocks. Parse that and
+ * return an `id` field if present.
+ */
+function extractId(text) {
+  if (!text) return null;
+  try {
+    const obj = JSON.parse(text);
+    return obj.id || obj.conversation_id || obj.memory_id || null;
+  } catch {
+    return null;
+  }
+}