open-memex 0.4.0-alpha.7 → 0.4.1

package/src/export.ts

@@ -0,0 +1,206 @@
+/**
+ * Export / import: portable memory archives (§9, D40).
+ *
+ * `export` bundles selected memories (markdown source of truth + manifest)
+ * into a single .tar.gz for moving to another machine or another app.
+ * `import` restores a bundle: personal memories go to the personal dir,
+ * project memories are re-keyed to the current project and land in the
+ * outbox as drafts (submit moves them into the repo).
+ *
+ * D40: export excludes `visibility: private` by default; `--all` / `-a`
+ * includes everything — the full-migration escape hatch.
+ */
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { execFileSync } from "node:child_process";
+import { db } from "./store/db.ts";
+import {
+  readMemoryFile,
+  writeMemoryFile,
+  normalizeFrontmatter,
+} from "./store/markdown.ts";
+import { upsertFromFile } from "./store/sync.ts";
+import { contentHash } from "./store/lifecycle.ts";
+
+const EXPORT_FORMAT = "open-memex-export/1";
+
+function fail(msg: string): never {
+  throw new Error(`[open-memex] ${msg}`);
+}
+
+function checkTar(): void {
+  try {
+    execFileSync("tar", ["--version"], { stdio: ["ignore", "pipe", "ignore"] });
+  } catch {
+    fail("the `tar` command is required for export/import but was not found on PATH");
+  }
+}
+
+export interface ExportOptions {
+  /** scope keys to include (resolved by the caller) */
+  scopeKeys: string[];
+  type?: string;
+  tag?: string;
+  /** D40: include visibility:private memories */
+  includePrivate?: boolean;
+  outFile?: string;
+}
+
+export interface ExportResult {
+  file: string;
+  exported: number;
+  skippedPrivate: number;
+  includePrivate: boolean;
+}
+
+interface ExportRow {
+  id: string;
+  scope_key: string;
+  scope: string;
+  visibility: string;
+  type: string;
+  file_path: string;
+}
+
+export function exportMemories(opts: ExportOptions): ExportResult {
+  checkTar();
+  const includePrivate = opts.includePrivate ?? false;
+
+  let sql = `SELECT id, scope_key, scope, visibility, type, file_path FROM memories WHERE scope_key IN (${opts.scopeKeys.map(() => "?").join(",")})`;
+  const params: unknown[] = [...opts.scopeKeys];
+  if (!includePrivate) sql += ` AND visibility != 'private'`;
+  if (opts.type) {
+    sql += ` AND type = ?`;
+    params.push(opts.type);
+  }
+  if (opts.tag) {
+    sql += ` AND (',' || tags || ',' LIKE ?)`;
+    params.push(`%,${opts.tag},%`);
+  }
+  sql += ` ORDER BY updated_at DESC`;
+  const rows = db().prepare(sql).all(...(params as any[])) as ExportRow[];
+
+  const skippedPrivate = includePrivate
+    ? 0
+    : (db().prepare(
+        `SELECT COUNT(*) AS n FROM memories WHERE scope_key IN (${opts.scopeKeys.map(() => "?").join(",")}) AND visibility = 'private'`,
+      ).get(...(opts.scopeKeys as any[])) as { n: number }).n;
+
+  const stage = fs.mkdtempSync(path.join(os.tmpdir(), "open-memex-export-"));
+  try {
+    const memDir = path.join(stage, "memories");
+    fs.mkdirSync(memDir, { recursive: true });
+    const manifestEntries: Array<{ id: string; scope: string; file: string }> = [];
+    let exported = 0;
+    for (const r of rows) {
+      const mf = readMemoryFile(r.file_path);
+      if (!mf) continue; // stale index row — skip, don't fail the export
+      const rel = path.join(r.scope, `${r.id}.md`);
+      const dest = path.join(memDir, rel);
+      fs.mkdirSync(path.dirname(dest), { recursive: true });
+      fs.copyFileSync(r.file_path, dest);
+      manifestEntries.push({ id: r.id, scope: r.scope, file: path.join("memories", rel) });
+      exported++;
+    }
+    const manifest = {
+      format: EXPORT_FORMAT,
+      exported_at: new Date().toISOString(),
+      open_memex_version: readPackageVersion(),
+      include_private: includePrivate,
+      filters: {
+        scope_keys: opts.scopeKeys,
+        type: opts.type ?? null,
+        tag: opts.tag ?? null,
+      },
+      memories: manifestEntries,
+    };
+    fs.writeFileSync(path.join(stage, "manifest.json"), JSON.stringify(manifest, null, 2), "utf8");
+
+    const stamp = new Date().toISOString().replace(/[:.]/g, "").slice(0, 15);
+    const outFile = opts.outFile ?? path.resolve(`open-memex-export-${stamp}.tar.gz`);
+    execFileSync("tar", ["-czf", outFile, "-C", stage, "manifest.json", "memories"], {
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    return { file: outFile, exported, skippedPrivate, includePrivate };
+  } finally {
+    fs.rmSync(stage, { recursive: true, force: true });
+  }
+}
+
+function readPackageVersion(): string {
+  try {
+    const here = new URL(import.meta.url);
+    const pkg = path.join(path.dirname(here.pathname), "..", "package.json");
+    return (JSON.parse(fs.readFileSync(pkg, "utf8")) as { version: string }).version;
+  } catch {
+    return "unknown";
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Import
+// ---------------------------------------------------------------------------
+
+export interface ImportOptions {
+  /** current project scope key — project memories are re-keyed to it */
+  projectScopeKey: string;
+  dryRun?: boolean;
+}
+
+export interface ImportResult {
+  imported: number;
+  skippedIdentical: number;
+  skippedConflict: { id: string; file: string }[];
+}
+
+export function importBundle(bundlePath: string, opts: ImportOptions): ImportResult {
+  checkTar();
+  if (!fs.existsSync(bundlePath)) fail(`bundle not found: ${bundlePath}`);
+  const stage = fs.mkdtempSync(path.join(os.tmpdir(), "open-memex-import-"));
+  try {
+    execFileSync("tar", ["-xzf", path.resolve(bundlePath), "-C", stage], {
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    const manifestPath = path.join(stage, "manifest.json");
+    if (!fs.existsSync(manifestPath)) fail("not an open-memex export bundle (manifest.json missing)");
+    const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")) as {
+      format: string;
+      memories: Array<{ id: string; scope: string; file: string }>;
+    };
+    if (manifest.format !== EXPORT_FORMAT) {
+      fail(`unsupported bundle format: ${manifest.format} (expected ${EXPORT_FORMAT})`);
+    }
+
+    const result: ImportResult = { imported: 0, skippedIdentical: 0, skippedConflict: [] };
+    const existingStmt = db().prepare(`SELECT content_hash FROM memories WHERE id = ?`);
+    for (const entry of manifest.memories ?? []) {
+      const src = path.join(stage, entry.file);
+      const mf = readMemoryFile(src);
+      if (!mf) continue;
+      const fm = normalizeFrontmatter({ ...mf.fm } as unknown as Record<string, unknown>);
+      // Re-key to this machine: personal stays personal; project memories
+      // adopt the current project's scope key (keys embed a path hash).
+      if (fm.scope === "project") fm.scope_key = opts.projectScopeKey;
+      else if (fm.scope === "personal") fm.scope_key = "personal";
+
+      const existing = existingStmt.get(fm.id) as { content_hash: string } | undefined;
+      if (existing) {
+        if (existing.content_hash === contentHash(mf.body)) result.skippedIdentical++;
+        else result.skippedConflict.push({ id: fm.id, file: entry.file });
+        continue;
+      }
+      if (opts.dryRun) {
+        result.imported++;
+        continue;
+      }
+      const { filePath } = writeMemoryFile(fm, mf.body);
+      const written = readMemoryFile(filePath);
+      if (written) upsertFromFile(written);
+      result.imported++;
+    }
+    return result;
+  } finally {
+    fs.rmSync(stage, { recursive: true, force: true });
+  }
+}

package/src/mcp.ts

@@ -137,6 +137,22 @@ export async function runMcpServer() {
   syncScope(PERSONAL_SCOPE.key, "session");
   console.error(`[open-memex] MCP server up. scope=${scope.key}`);
 
+  // D12: pulls are explicit by default — session start never touches the
+  // network. With sync.autoPull, one best-effort pull; a failure never
+  // blocks the session, it just logs and continues.
+  if (cfg.sync?.autoPull) {
+    try {
+      const { GitProvider } = await import("./providers/git.ts");
+      const r = new GitProvider().pull(process.cwd());
+      syncScope(scope.key, "pull");
+      console.error(
+        `[open-memex] auto-pull: ${r.branch} ${r.fastForwarded ? "fast-forwarded" : "already up to date"}`,
+      );
+    } catch (e) {
+      console.error(`[open-memex] auto-pull skipped: ${(e as Error).message}`);
+    }
+  }
+
   // D26: re-sync on every request, not just at startup. The in-repo dir
   // follows the current git branch, so a branch switch mid-session would
   // otherwise leave the index pointing at files that no longer exist.

package/src/providers/git.ts

@@ -0,0 +1,191 @@
+/**
+ * GitProvider — the git transport for repo-synced shared scopes (§9, D12).
+ *
+ * Git is a transport, not the product boundary: this provider moves the
+ * in-repo memory dir (`.ai/open-memex/`) between the local checkout and the
+ * remote. It never touches the appdata outbox, never force-pushes, never
+ * auto-merges a divergence — those are human decisions.
+ *
+ * Hard rules (from §9 / D12 / D36):
+ * - pull is explicit (`open-memex pull`); pull = fetch + fast-forward only.
+ * - push is explicit (`open-memex push`); the tool never pushes on its own.
+ * - a failed pull/push fails with a clear message and leaves no broken state.
+ */
+import { execFileSync } from "node:child_process";
+
+function fail(msg: string): never {
+  throw new Error(`[open-memex] ${msg}`);
+}
+
+/** Run git, raising a readable error. `timeoutMs` guards network calls. */
+function git(root: string, args: string[], timeoutMs = 0): string {
+  try {
+    return execFileSync("git", args, {
+      cwd: root,
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "pipe"],
+      ...(timeoutMs > 0 ? { timeout: timeoutMs } : {}),
+    }).trim();
+  } catch (e) {
+    const err = e as { stderr?: string; message?: string; code?: string };
+    if (err.code === "ETIMEDOUT") fail(`git ${args.join(" ")} timed out — remote unreachable?`);
+    const detail = (err.stderr ?? err.message ?? "").trim().split("\n")[0];
+    fail(`git ${args.join(" ")} failed${detail ? `: ${detail}` : ""}`);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Provider capability model (§2) — every provider declares what it can do.
+// ---------------------------------------------------------------------------
+
+export interface MemoryProviderCapabilities {
+  read: boolean;
+  write: boolean;
+  delete: boolean;
+  /** version / audit trail */
+  history: boolean;
+  sync: "none" | "pull" | "push" | "bidirectional";
+}
+
+export interface MemoryProvider {
+  readonly name: string;
+  readonly capabilities: MemoryProviderCapabilities;
+}
+
+export interface PullReceipt {
+  at: string; // RFC 3339
+  branch: string;
+  remote: string;
+  before: string; // HEAD sha before the pull
+  after: string; // HEAD sha after the pull
+  fastForwarded: boolean;
+}
+
+export interface PushReceipt {
+  at: string; // RFC 3339
+  branch: string;
+  remote: string;
+  head: string; // sha pushed
+}
+
+export interface GitSyncStatus {
+  branch: string;
+  upstream: string | null;
+  remote: string | null;
+  ahead: number;
+  behind: number;
+}
+
+const FETCH_TIMEOUT_MS = 30_000;
+
+export class GitProvider implements MemoryProvider {
+  readonly name = "git";
+  readonly capabilities: MemoryProviderCapabilities = {
+    read: true,
+    write: true,
+    delete: false,
+    history: true,
+    sync: "bidirectional",
+  };
+
+  /** Current branch + upstream, or a clear failure when git can't answer. */
+  status(root: string): GitSyncStatus {
+    git(root, ["rev-parse", "--git-dir"]);
+    const branch =
+      git(root, ["branch", "--show-current"]) ||
+      git(root, ["rev-parse", "--abbrev-ref", "HEAD"]);
+    let upstream: string | null = null;
+    try {
+      upstream = git(root, ["rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"]);
+    } catch {
+      upstream = null;
+    }
+    let ahead = 0;
+    let behind = 0;
+    if (upstream) {
+      const counts = git(root, ["rev-list", "--left-right", "--count", `HEAD...${upstream}`]);
+      const [a, b] = counts.split(/\s+/).map((n) => parseInt(n, 10));
+      ahead = Number.isFinite(a) ? a : 0;
+      behind = Number.isFinite(b) ? b : 0;
+    }
+    const remote = upstream ? upstream.split("/")[0] : null;
+    return { branch, upstream, remote, ahead, behind };
+  }
+
+  /**
+   * Explicit pull: fetch + fast-forward only (§9).
+   * Diverged branches are NOT merged — the user resolves them by hand.
+   */
+  pull(root: string): PullReceipt {
+    const st = this.status(root);
+    if (!st.upstream || !st.remote) {
+      fail(
+        `branch ${st.branch} has no upstream — set one with ` +
+          `\`git push -u <remote> ${st.branch}\`, then pull again`,
+      );
+    }
+    git(root, ["fetch", st.remote], FETCH_TIMEOUT_MS);
+    const before = git(root, ["rev-parse", "HEAD"]);
+    const remoteSha = git(root, ["rev-parse", st.upstream]);
+    if (before === remoteSha) {
+      return {
+        at: new Date().toISOString(),
+        branch: st.branch,
+        remote: st.remote,
+        before,
+        after: before,
+        fastForwarded: false,
+      };
+    }
+    // Fast-forward is possible iff HEAD is an ancestor of the upstream.
+    let ffPossible = false;
+    try {
+      git(root, ["merge-base", "--is-ancestor", "HEAD", st.upstream]);
+      ffPossible = true;
+    } catch {
+      ffPossible = false;
+    }
+    if (!ffPossible) {
+      fail(
+        `branch ${st.branch} has diverged from ${st.upstream} — ` +
+          `open-memex never force-merges; resolve it by hand ` +
+          `(rebase or merge), then pull again`,
+      );
+    }
+    git(root, ["merge", "--ff-only", st.upstream]);
+    const after = git(root, ["rev-parse", "HEAD"]);
+    return {
+      at: new Date().toISOString(),
+      branch: st.branch,
+      remote: st.remote,
+      before,
+      after,
+      fastForwarded: true,
+    };
+  }
+
+  /** Explicit push of the current branch. Never called automatically. */
+  push(root: string): PushReceipt {
+    const st = this.status(root);
+    if (!st.remote) {
+      // No upstream yet: push explicitly sets it (-u), still user-invoked.
+      const remotes = git(root, ["remote"]);
+      const remote = remotes.split("\n").map((r) => r.trim()).filter(Boolean)[0];
+      if (!remote) fail("no git remote configured — add one before pushing");
+      git(root, ["push", "-u", remote, st.branch], FETCH_TIMEOUT_MS);
+      return {
+        at: new Date().toISOString(),
+        branch: st.branch,
+        remote,
+        head: git(root, ["rev-parse", "HEAD"]),
+      };
+    }
+    git(root, ["push", st.remote, st.branch], FETCH_TIMEOUT_MS);
+    return {
+      at: new Date().toISOString(),
+      branch: st.branch,
+      remote: st.remote,
+      head: git(root, ["rev-parse", "HEAD"]),
+    };
+  }
+}

package/src/store/sync.ts

@@ -153,7 +153,7 @@ export function syncScope(scopeKey: string, kind: SyncKind = "auto"): SyncStats
 // ---------------------------------------------------------------------------
 
 /** What triggered the sync — shown in sync-status so the user can see it. */
-export type SyncKind = "session" | "request" | "cli" | "submit" | "auto";
+export type SyncKind = "session" | "request" | "cli" | "submit" | "pull" | "push" | "auto";
 
 export interface SyncStateEntry {
   lastSyncAt: string; // RFC 3339

package/src/store/v2migrate.ts

@@ -118,23 +118,48 @@ export interface MigrateV2Result {
 }
 
 /**
- * Merge a legacy `my-o-memory` data root into the current `open-memex` root.
- * The old dir is renamed to a dated backup — never deleted. Returns the
- * backup path, or null when no legacy dir exists.
+ * Locate a legacy `my-o-memory` data root next to the current root.
+ * Pure lookup — no disk writes.
  */
-function migrateLegacyDataRoot(dryRun: boolean): string | null {
+export function findLegacyDataRoot(): string | null {
   const { root } = paths();
   const legacy = path.join(path.dirname(root), "my-o-memory");
   if (!fs.existsSync(legacy) || !fs.statSync(legacy).isDirectory()) return null;
+  return legacy;
+}
 
+/** Dated backup path for a legacy data root. Pure — no disk writes. */
+export function legacyBackupPath(legacy: string): string {
   const stamp = new Date().toISOString().slice(0, 10);
-  const backup = `${legacy}.backup-${stamp}`;
-  if (!dryRun) {
-    const legacyMem = path.join(legacy, "memories");
-    if (fs.existsSync(legacyMem)) {
-      for (const entry of fs.readdirSync(legacyMem)) {
-        const src = path.join(legacyMem, entry);
-        const dst = path.join(root, "memories", entry);
+  return `${legacy}.backup-${stamp}`;
+}
+
+/**
+ * Merge a legacy `my-o-memory` data root into the current `open-memex` root.
+ * The old dir is renamed to a dated backup — never deleted.
+ * Failures throw an actionable Error (no raw syscall dump): on Windows the
+ * backup rename typically fails with EPERM when another program holds the
+ * folder open.
+ */
+function mergeLegacyDataRoot(legacy: string, backup: string): void {
+  const { memories } = paths();
+  const fail = (where: string, err: unknown): never => {
+    const code = (err as NodeJS.ErrnoException)?.code;
+    throw new Error(
+      `could not ${where} (${legacy})` +
+        (code ? ` [${code}]` : "") +
+        `. Another program may be holding the folder open (e.g. a running MCP server, editor, or antivirus). ` +
+        `Your memories are safe — nothing was deleted. Close the program and re-run ` +
+        `\`open-memex migrate --to-v2\`, or rename the folder to ${backup} yourself and re-run.`,
+    );
+  };
+  const legacyMem = path.join(legacy, "memories");
+  if (fs.existsSync(legacyMem)) {
+    fs.mkdirSync(memories, { recursive: true });
+    for (const entry of fs.readdirSync(legacyMem)) {
+      const src = path.join(legacyMem, entry);
+      const dst = path.join(memories, entry);
+      try {
         if (!fs.existsSync(dst)) {
           fs.renameSync(src, dst);
         } else {
@@ -145,46 +170,75 @@ function migrateLegacyDataRoot(dryRun: boolean): string | null {
             if (!fs.existsSync(d)) fs.renameSync(s, d);
           }
         }
+      } catch (err) {
+        fail(`move memories from the legacy data dir`, err);
       }
     }
+  }
+  try {
     fs.renameSync(legacy, backup);
+  } catch (err) {
+    fail(`back up the legacy data dir`, err);
   }
-  return backup;
 }
 
-export function migrateV2(opts: { dryRun: boolean }): MigrateV2Result {
-  const { memories: memoriesRoot } = paths();
-  const result: MigrateV2Result = {
-    scanned: 0,
-    converted: 0,
-    skippedV2: 0,
-    plans: [],
-    legacyBackup: null,
-  };
-
-  result.legacyBackup = migrateLegacyDataRoot(opts.dryRun);
-
-  if (!fs.existsSync(memoriesRoot)) return result;
-  for (const entry of fs.readdirSync(memoriesRoot, { withFileTypes: true })) {
+/** Plan v1 → v2 conversion for every `.md` file under `dir`. Read-only. */
+function planDir(
+  dir: string,
+  memoriesRoot: string,
+  result: MigrateV2Result,
+): void {
+  if (!fs.existsSync(dir)) return;
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
     if (!entry.isDirectory()) continue;
-    const dir = path.join(memoriesRoot, entry.name);
-    for (const name of fs.readdirSync(dir)) {
+    const sub = path.join(dir, entry.name);
+    for (const name of fs.readdirSync(sub)) {
       if (!name.endsWith(".md")) continue;
-      const filePath = path.join(dir, name);
+      const filePath = path.join(sub, name);
       result.scanned++;
       const plan = planConversion(filePath, memoriesRoot);
       if (!plan) {
         result.skippedV2++;
         continue;
       }
-      if (!opts.dryRun) {
-        fs.mkdirSync(path.dirname(plan.toPath), { recursive: true });
-        fs.writeFileSync(plan.toPath, serialize(plan.fm, plan.body), "utf8");
-        if (plan.toPath !== plan.fromPath) fs.unlinkSync(plan.fromPath);
-      }
       result.converted++;
       result.plans.push(plan);
     }
   }
+}
+
+/** Apply conversion plans to disk (real run only — never in dry-run). */
+function applyPlans(result: MigrateV2Result): void {
+  for (const plan of result.plans) {
+    fs.mkdirSync(path.dirname(plan.toPath), { recursive: true });
+    fs.writeFileSync(plan.toPath, serialize(plan.fm, plan.body), "utf8");
+    if (plan.toPath !== plan.fromPath) fs.unlinkSync(plan.fromPath);
+  }
+}
+
+export function migrateV2(opts: { dryRun: boolean }): MigrateV2Result {
+  const { memories: memoriesRoot } = paths();
+  const result: MigrateV2Result = {
+    scanned: 0,
+    converted: 0,
+    skippedV2: 0,
+    plans: [],
+    legacyBackup: null,
+  };
+
+  const legacy = findLegacyDataRoot();
+  if (legacy) {
+    result.legacyBackup = legacyBackupPath(legacy);
+    if (opts.dryRun) {
+      // D44: dry-run previews the legacy files in place — nothing is moved,
+      // so the preview actually shows what would convert (issue #7).
+      planDir(path.join(legacy, "memories"), memoriesRoot, result);
+    } else {
+      mergeLegacyDataRoot(legacy, result.legacyBackup);
+    }
+  }
+
+  planDir(memoriesRoot, memoriesRoot, result);
+  if (!opts.dryRun) applyPlans(result);
   return result;
 }