@yugabytedb/meko-mcp 1.0.3

package/lib/migrate/progress.mjs

@@ -0,0 +1,76 @@
+/**
+ * progress.mjs — Lightweight progress snapshots for `--migrate-status`.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+
+/**
+ * @typedef {Object} Progress
+ * @property {string} runId
+ * @property {string} scope
+ * @property {string} startedAt
+ * @property {string|null} finishedAt
+ * @property {"running"|"done"|"failed"|"interrupted"} status
+ * @property {string|null} currentAgent
+ * @property {string|null} currentSession
+ * @property {Counts} counts
+ *
+ * @typedef {Object} Counts
+ * @property {number} sessionsPlanned
+ * @property {number} sessionsDone
+ * @property {number} messagesSent
+ * @property {number} memoryPairsSent
+ * @property {number} memoriesFromMD
+ * @property {number} warnings
+ * @property {number} memoryErrors
+ */
+
+/**
+ * @returns {Progress}
+ */
+export function initial(runId, scope) {
+  return {
+    runId,
+    scope,
+    startedAt: new Date().toISOString(),
+    finishedAt: null,
+    status: "running",
+    currentAgent: null,
+    currentSession: null,
+    counts: {
+      sessionsPlanned: 0,
+      sessionsDone: 0,
+      messagesSent: 0,
+      memoryPairsSent: 0,
+      memoriesFromMD: 0,
+      warnings: 0,
+      memoryErrors: 0,
+    },
+  };
+}
+
+/**
+ * Atomic write.
+ *
+ * @param {string} progressPath
+ * @param {Progress} progress
+ */
+export function write(progressPath, progress) {
+  fs.mkdirSync(path.dirname(progressPath), { recursive: true });
+  const tmp = `${progressPath}.tmp.${process.pid}.${Date.now()}`;
+  fs.writeFileSync(tmp, JSON.stringify(progress, null, 2));
+  fs.renameSync(tmp, progressPath);
+}
+
+/**
+ * @param {string} progressPath
+ * @returns {Progress | null}
+ */
+export function read(progressPath) {
+  try {
+    return JSON.parse(fs.readFileSync(progressPath, "utf8"));
+  } catch {
+    return null;
+  }
+}

package/lib/migrate/state.mjs

@@ -0,0 +1,177 @@
+/**
+ * state.mjs — Durable cross-run state for the migrator.
+ *
+ * File layout documented in the feature plan. Writes are atomic
+ * (tmp + rename) and debounced by the caller via `flushNow()` / periodic.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+
+export const STATE_VERSION = 1;
+
+/**
+ * @typedef {Object} State
+ * @property {number} version
+ * @property {string} updatedAt
+ * @property {Array<object>} runs
+ * @property {Record<string, AgentState>} agents
+ *
+ * @typedef {Object} AgentState
+ * @property {string|null} memoryConversationId
+ * @property {Record<string, ConversationState>} conversations
+ * @property {Record<string, MemoryState>} memories
+ *
+ * @typedef {Object} ConversationState
+ * @property {string} conversationId
+ * @property {"pending"|"partial"|"done"|"failed"} status
+ * @property {number} messagesSent
+ * @property {number} memoryPairsSent
+ * @property {string|null} lastUuid
+ * @property {string|null} completedAt
+ * @property {string|null} lastError
+ *
+ * @typedef {Object} MemoryState
+ * @property {string} memoryId
+ * @property {string} name
+ * @property {string} addedAt
+ */
+
+/**
+ * Load state from disk. Returns an empty shell if the file is missing.
+ *
+ * @param {string} statePath
+ * @returns {State}
+ */
+export function load(statePath) {
+  try {
+    const raw = fs.readFileSync(statePath, "utf8");
+    const parsed = JSON.parse(raw);
+    if (parsed && typeof parsed === "object") {
+      return {
+        version: parsed.version ?? STATE_VERSION,
+        updatedAt: parsed.updatedAt ?? new Date().toISOString(),
+        runs: Array.isArray(parsed.runs) ? parsed.runs : [],
+        agents: parsed.agents && typeof parsed.agents === "object" ? parsed.agents : {},
+      };
+    }
+  } catch {
+    /* fall through to empty */
+  }
+  return empty();
+}
+
+/**
+ * @returns {State}
+ */
+export function empty() {
+  return {
+    version: STATE_VERSION,
+    updatedAt: new Date().toISOString(),
+    runs: [],
+    agents: {},
+  };
+}
+
+/**
+ * Atomic write: writes to `${statePath}.tmp` then renames.
+ *
+ * @param {string} statePath
+ * @param {State} state
+ */
+export function save(statePath, state) {
+  fs.mkdirSync(path.dirname(statePath), { recursive: true });
+  state.updatedAt = new Date().toISOString();
+  const tmp = `${statePath}.tmp.${process.pid}.${Date.now()}`;
+  fs.writeFileSync(tmp, JSON.stringify(state, null, 2));
+  fs.renameSync(tmp, statePath);
+}
+
+/**
+ * Ensure an agent bucket exists and return it (mutable reference).
+ *
+ * @param {State} state
+ * @param {string} agentId
+ * @returns {AgentState}
+ */
+export function ensureAgent(state, agentId) {
+  if (!state.agents[agentId]) {
+    state.agents[agentId] = {
+      memoryConversationId: null,
+      conversations: {},
+      memories: {},
+    };
+  }
+  return state.agents[agentId];
+}
+
+/**
+ * Has this session already been fully ingested?
+ *
+ * @param {State} state
+ * @param {string} agentId
+ * @param {string} sessionUuid
+ * @returns {boolean}
+ */
+export function isSessionDone(state, agentId, sessionUuid) {
+  return state.agents[agentId]?.conversations?.[sessionUuid]?.status === "done";
+}
+
+/**
+ * Return the stored conversation record for (agentId, sessionUuid), or null.
+ * Callers use this to resume a `partial` ingestion.
+ *
+ * @param {State} state
+ * @param {string} agentId
+ * @param {string} sessionUuid
+ * @returns {ConversationState | null}
+ */
+export function getSession(state, agentId, sessionUuid) {
+  return state.agents[agentId]?.conversations?.[sessionUuid] ?? null;
+}
+
+/**
+ * Upsert a session record. Returns the mutated record.
+ *
+ * @param {State} state
+ * @param {string} agentId
+ * @param {string} sessionUuid
+ * @param {Partial<ConversationState>} patch
+ * @returns {ConversationState}
+ */
+export function markSession(state, agentId, sessionUuid, patch) {
+  const agent = ensureAgent(state, agentId);
+  const prev = agent.conversations[sessionUuid] ?? {
+    conversationId: "",
+    status: "pending",
+    messagesSent: 0,
+    memoryPairsSent: 0,
+    lastUuid: null,
+    completedAt: null,
+    lastError: null,
+  };
+  const next = { ...prev, ...patch };
+  agent.conversations[sessionUuid] = next;
+  return next;
+}
+
+/**
+ * @param {State} state
+ * @param {string} agentId
+ * @param {string} hash
+ * @returns {boolean}
+ */
+export function isMemoryDone(state, agentId, hash) {
+  return Boolean(state.agents[agentId]?.memories?.[hash]);
+}
+
+/**
+ * @param {State} state
+ * @param {string} agentId
+ * @param {string} hash
+ * @param {MemoryState} record
+ */
+export function markMemory(state, agentId, hash, record) {
+  const agent = ensureAgent(state, agentId);
+  agent.memories[hash] = record;
+}

package/lib/migrate.mjs

@@ -0,0 +1,328 @@
+/**
+ * migrate.mjs — CLI-facing entry points for the `migrate` subcommand.
+ *
+ * Three externally-visible actions:
+ *   - startMigration(...) — foreground wrapper that forks a detached child
+ *   - readStatus(...)     — print latest run's progress
+ *   - stopMigration(...)  — SIGTERM the running child, wait briefly, SIGKILL
+ *
+ * The detached child re-enters this module via `runChildMigration()`
+ * (invoked from bin/create-meko-setup.mjs when --__migrate-child is set).
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import { runMigration } from "./migrate/orchestrator.mjs";
+import { createLogger } from "./migrate/logger.mjs";
+import { generateRunId, forkDetached, migrateRoot } from "./migrate/daemon.mjs";
+import * as lockfile from "./migrate/lockfile.mjs";
+import * as progress from "./migrate/progress.mjs";
+
+function home() {
+  return process.env.HOME || "";
+}
+
+function paths(opts = {}) {
+  const root = migrateRoot({ home: opts.home ?? home() });
+  return {
+    root,
+    statePath: path.join(root, "state.json"),
+    lockPath: path.join(root, "lock"),
+    runDir: (runId) => path.join(root, runId),
+    logFile: (runId) => path.join(root, runId, "stdout.log"),
+    progressFile: (runId) => path.join(root, runId, "progress.json"),
+  };
+}
+
+function findLatestRunId(root) {
+  try {
+    const dirs = fs.readdirSync(root, { withFileTypes: true })
+      .filter((d) => d.isDirectory())
+      .map((d) => d.name)
+      .filter((n) => /^\d{8}-\d{6}-[0-9a-f]{6}$/.test(n))
+      .sort();
+    return dirs.at(-1) ?? null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Foreground wrapper: acquire lock, fork detached child, print run-id.
+ *
+ * @param {object} opts
+ * @param {string} opts.url
+ * @param {string|null} opts.apiKey
+ * @param {"user"|"project"} opts.scope
+ * @param {string} opts.projectRoot
+ * @param {string} opts.scriptPath       Absolute path to create-meko-setup.mjs.
+ * @param {boolean} [opts.dryRun]
+ * @param {boolean} [opts.noExtract]
+ * @param {boolean} [opts.force]
+ * @param {number}  [opts.rate]
+ * @returns {Promise<{ok: boolean, runId?: string, message: string}>}
+ */
+export async function startMigration(opts) {
+  const P = paths();
+  fs.mkdirSync(P.root, { recursive: true });
+
+  // Dry-run is a local-only planning pass: no HTTP, no lock, no fork.
+  // Don't gate it on URL/apiKey.
+  if (opts.dryRun) {
+    const runId = generateRunId();
+    const runDir = P.runDir(runId);
+    const summary = await runMigration({
+      url: opts.url ?? "",
+      apiKey: opts.apiKey ?? null,
+      runId,
+      scope: opts.scope,
+      projectCwd: opts.projectRoot,
+      runDir,
+      statePath: P.statePath,
+      lockPath: P.lockPath,
+      dryRun: true,
+      noExtract: opts.noExtract,
+      concurrency: opts.rate,
+    });
+    const s = summary.summary;
+    const planned = s.plannedCalls ?? {};
+    return {
+      ok: summary.ok,
+      runId,
+      message:
+        (opts.url ? "" : "(no --url provided — planning only)\n") +
+        `Dry run:\n` +
+        `  Claude Code sessions: ${s.claudeSessions}\n` +
+        `  Cursor sessions:      ${s.cursorSessions}\n` +
+        `  Paired user↔asst turns: ${s.pairedTurns}\n` +
+        `  Memory items (MD):    ${s.memoryItems}\n` +
+        `  Planned MCP calls:\n` +
+        `    conversation_create:      ${planned.conversation_create ?? 0}\n` +
+        `    conversation_add_message: ${planned.conversation_add_message ?? 0}\n` +
+        `    memory_add (messages=):   ${planned.memory_add_messages ?? 0}\n` +
+        `    memory_add (text=):       ${planned.memory_add_text ?? 0}\n` +
+        `    total:                    ${planned.total ?? 0}`,
+    };
+  }
+
+  // Real run — now URL/apiKey are required (dry-run branch already returned).
+  if (!opts.url) {
+    return { ok: false, message: "--url (or MEKO_MCP_URL) is required for --migrate." };
+  }
+  if (!opts.apiKey && !isLocalUrl(opts.url)) {
+    return { ok: false, message: "--api-key (or MEKO_API_KEY) is required for cloud --migrate." };
+  }
+
+  // Reserve the lock slot in the foreground so a concurrent --migrate fails
+  // fast. We use a two-step dance: first, O_EXCL-create the lock with the
+  // foreground PID (just to claim the slot), then — after spawning the
+  // child — rewrite it in place with the child's PID. The orchestrator in
+  // the child is told the lock is already held so it doesn't try to
+  // re-acquire (which would self-deadlock, since it would see its own PID
+  // as a live holder).
+  const runId = generateRunId();
+  const acq = lockfile.acquire(P.lockPath, { runId }, { force: opts.force });
+  if (!acq.acquired) {
+    return {
+      ok: false,
+      message:
+        `Another migration is running (pid=${acq.holder?.pid}, run-id=${acq.holder?.runId}). ` +
+        `Use --migrate-status, --migrate-stop, or --force to override.`,
+    };
+  }
+
+  fs.mkdirSync(P.runDir(runId), { recursive: true });
+  fs.writeFileSync(
+    path.join(P.runDir(runId), "cmd.json"),
+    JSON.stringify(
+      {
+        runId,
+        scope: opts.scope,
+        url: opts.url,
+        apiKeyPresent: !!opts.apiKey,
+        projectRoot: opts.projectRoot,
+        noExtract: !!opts.noExtract,
+        force: !!opts.force,
+        rate: opts.rate ?? null,
+        startedAt: new Date().toISOString(),
+      },
+      null,
+      2,
+    ),
+  );
+
+  const passthrough = ["--scope", opts.scope, "--url", opts.url];
+  if (opts.noExtract) passthrough.push("--no-extract");
+  if (opts.force) passthrough.push("--force");
+  if (opts.rate) passthrough.push("--rate", String(opts.rate));
+
+  const env = { ...process.env };
+  if (opts.apiKey) env.MEKO_API_KEY = opts.apiKey;
+
+  const pid = forkDetached({
+    scriptPath: opts.scriptPath,
+    runId,
+    args: passthrough,
+    logFilePath: P.logFile(runId),
+    env,
+    cwd: opts.projectRoot,
+  });
+
+  // Hand lock ownership to the child. The rewrite is the same file that
+  // was O_EXCL-created above, so no race with a competing --migrate.
+  try {
+    fs.writeFileSync(
+      P.lockPath,
+      JSON.stringify({ pid, runId, startedAt: new Date().toISOString() }),
+    );
+  } catch { /* best effort */ }
+
+  return {
+    ok: true,
+    runId,
+    message:
+      `Migration started. run-id=${runId}, pid=${pid}\n` +
+      `Logs:   ${P.logFile(runId)}\n` +
+      `Status: create-meko-setup --migrate-status`,
+  };
+}
+
+/**
+ * Invoked by the detached child via bin/create-meko-setup.mjs.
+ *
+ * The foreground wrapper has already taken the lock and written the
+ * child's PID into it; we MUST NOT re-acquire (that would self-deadlock
+ * because lockfile.isStale(our-own-pid) returns false).
+ *
+ * @param {object} opts
+ * @param {string} opts.runId
+ * @param {"user"|"project"} opts.scope
+ * @param {string} opts.url
+ * @param {string|null} opts.apiKey
+ * @param {string} opts.projectRoot
+ * @param {boolean} [opts.noExtract]
+ * @param {boolean} [opts.force]
+ * @param {number} [opts.rate]
+ * @returns {Promise<number>}  Exit code.
+ */
+export async function runChildMigration(opts) {
+  const P = paths();
+  const runDir = P.runDir(opts.runId);
+  fs.mkdirSync(runDir, { recursive: true });
+
+  const log = createLogger(path.join(runDir, "log.ndjson"), { runId: opts.runId });
+
+  let shuttingDown = false;
+  const onSignal = (sig) => {
+    if (shuttingDown) return;
+    shuttingDown = true;
+    log.warn("interrupted", { signal: sig });
+    try { lockfile.release(P.lockPath); } catch { /* ignore */ }
+    log.close();
+    process.exit(130);
+  };
+  process.on("SIGTERM", () => onSignal("SIGTERM"));
+  process.on("SIGINT", () => onSignal("SIGINT"));
+
+  try {
+    const result = await runMigration({
+      url: opts.url,
+      apiKey: opts.apiKey,
+      runId: opts.runId,
+      scope: opts.scope,
+      projectCwd: opts.projectRoot,
+      runDir,
+      statePath: P.statePath,
+      lockPath: P.lockPath,
+      noExtract: opts.noExtract,
+      force: opts.force,
+      lockAlreadyHeld: true,
+      concurrency: opts.rate,
+    });
+    log.info("child_done", result.summary);
+    log.close();
+    try { lockfile.release(P.lockPath); } catch { /* ignore */ }
+    return result.ok ? 0 : 1;
+  } catch (err) {
+    log.error("child_crashed", { error: err.message, stack: err.stack });
+    log.close();
+    try { lockfile.release(P.lockPath); } catch { /* ignore */ }
+    return 1;
+  }
+}
+
+/**
+ * Print and return the status of the latest (or requested) migration run.
+ *
+ * @param {object} opts
+ * @param {string} [opts.runId]
+ * @returns {{ok: boolean, runId: string|null, progress: object|null, message: string}}
+ */
+export function readStatus(opts = {}) {
+  const P = paths();
+  const runId = opts.runId ?? findLatestRunId(P.root);
+  if (!runId) {
+    return { ok: false, runId: null, progress: null, message: "No migration runs found." };
+  }
+  const prog = progress.read(P.progressFile(runId));
+  if (!prog) {
+    return { ok: false, runId, progress: null, message: `No progress file for run-id=${runId}.` };
+  }
+  const lines = [
+    `run-id:       ${prog.runId}`,
+    `status:       ${prog.status}`,
+    `scope:        ${prog.scope}`,
+    `started:      ${prog.startedAt}`,
+    `finished:     ${prog.finishedAt ?? "(in progress)"}`,
+    `current:      ${prog.currentAgent ?? "-"} / ${prog.currentSession ?? "-"}`,
+    `sessions:     ${prog.counts.sessionsDone} / ${prog.counts.sessionsPlanned}`,
+    `messages:     ${prog.counts.messagesSent}`,
+    `memory pairs: ${prog.counts.memoryPairsSent}`,
+    `memory (MD):  ${prog.counts.memoriesFromMD}`,
+    `warnings:     ${prog.counts.warnings}`,
+    `memory errs:  ${prog.counts.memoryErrors}`,
+  ];
+  return { ok: true, runId, progress: prog, message: lines.join("\n") };
+}
+
+/**
+ * SIGTERM the holder, wait briefly, SIGKILL. Best-effort.
+ *
+ * @returns {{ok: boolean, message: string}}
+ */
+export async function stopMigration() {
+  const P = paths();
+  const holder = lockfile.read(P.lockPath);
+  if (!holder) return { ok: false, message: "No migration lock found." };
+  if (lockfile.isStale(holder)) {
+    try { lockfile.release(P.lockPath); } catch { /* ignore */ }
+    return { ok: true, message: `Stale lock removed (pid=${holder.pid}).` };
+  }
+  try {
+    process.kill(holder.pid, "SIGTERM");
+  } catch (err) {
+    return { ok: false, message: `Failed to signal pid=${holder.pid}: ${err.message}` };
+  }
+
+  // Wait up to 2s.
+  const deadline = Date.now() + 2000;
+  while (Date.now() < deadline) {
+    await new Promise((r) => setTimeout(r, 100));
+    if (lockfile.isStale(lockfile.read(P.lockPath) ?? holder)) {
+      return { ok: true, message: `Stopped pid=${holder.pid} (run-id=${holder.runId}).` };
+    }
+  }
+
+  try {
+    process.kill(holder.pid, "SIGKILL");
+  } catch { /* ignore */ }
+  try { lockfile.release(P.lockPath); } catch { /* ignore */ }
+  return {
+    ok: true,
+    message: `Force-killed pid=${holder.pid} (run-id=${holder.runId}).`,
+  };
+}
+
+function isLocalUrl(url) {
+  return /localhost|127\.0\.0\.1/.test(url ?? "");
+}