open-memex 0.4.0-alpha.7 → 0.4.0

package/src/export.ts

@@ -0,0 +1,206 @@
+/**
+ * Export / import: portable memory archives (§9, D40).
+ *
+ * `export` bundles selected memories (markdown source of truth + manifest)
+ * into a single .tar.gz for moving to another machine or another app.
+ * `import` restores a bundle: personal memories go to the personal dir,
+ * project memories are re-keyed to the current project and land in the
+ * outbox as drafts (submit moves them into the repo).
+ *
+ * D40: export excludes `visibility: private` by default; `--all` / `-a`
+ * includes everything — the full-migration escape hatch.
+ */
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { execFileSync } from "node:child_process";
+import { db } from "./store/db.ts";
+import {
+  readMemoryFile,
+  writeMemoryFile,
+  normalizeFrontmatter,
+} from "./store/markdown.ts";
+import { upsertFromFile } from "./store/sync.ts";
+import { contentHash } from "./store/lifecycle.ts";
+
+const EXPORT_FORMAT = "open-memex-export/1";
+
+function fail(msg: string): never {
+  throw new Error(`[open-memex] ${msg}`);
+}
+
+function checkTar(): void {
+  try {
+    execFileSync("tar", ["--version"], { stdio: ["ignore", "pipe", "ignore"] });
+  } catch {
+    fail("the `tar` command is required for export/import but was not found on PATH");
+  }
+}
+
+export interface ExportOptions {
+  /** scope keys to include (resolved by the caller) */
+  scopeKeys: string[];
+  type?: string;
+  tag?: string;
+  /** D40: include visibility:private memories */
+  includePrivate?: boolean;
+  outFile?: string;
+}
+
+export interface ExportResult {
+  file: string;
+  exported: number;
+  skippedPrivate: number;
+  includePrivate: boolean;
+}
+
+interface ExportRow {
+  id: string;
+  scope_key: string;
+  scope: string;
+  visibility: string;
+  type: string;
+  file_path: string;
+}
+
+export function exportMemories(opts: ExportOptions): ExportResult {
+  checkTar();
+  const includePrivate = opts.includePrivate ?? false;
+
+  let sql = `SELECT id, scope_key, scope, visibility, type, file_path FROM memories WHERE scope_key IN (${opts.scopeKeys.map(() => "?").join(",")})`;
+  const params: unknown[] = [...opts.scopeKeys];
+  if (!includePrivate) sql += ` AND visibility != 'private'`;
+  if (opts.type) {
+    sql += ` AND type = ?`;
+    params.push(opts.type);
+  }
+  if (opts.tag) {
+    sql += ` AND (',' || tags || ',' LIKE ?)`;
+    params.push(`%,${opts.tag},%`);
+  }
+  sql += ` ORDER BY updated_at DESC`;
+  const rows = db().prepare(sql).all(...(params as any[])) as ExportRow[];
+
+  const skippedPrivate = includePrivate
+    ? 0
+    : (db().prepare(
+        `SELECT COUNT(*) AS n FROM memories WHERE scope_key IN (${opts.scopeKeys.map(() => "?").join(",")}) AND visibility = 'private'`,
+      ).get(...(opts.scopeKeys as any[])) as { n: number }).n;
+
+  const stage = fs.mkdtempSync(path.join(os.tmpdir(), "open-memex-export-"));
+  try {
+    const memDir = path.join(stage, "memories");
+    fs.mkdirSync(memDir, { recursive: true });
+    const manifestEntries: Array<{ id: string; scope: string; file: string }> = [];
+    let exported = 0;
+    for (const r of rows) {
+      const mf = readMemoryFile(r.file_path);
+      if (!mf) continue; // stale index row — skip, don't fail the export
+      const rel = path.join(r.scope, `${r.id}.md`);
+      const dest = path.join(memDir, rel);
+      fs.mkdirSync(path.dirname(dest), { recursive: true });
+      fs.copyFileSync(r.file_path, dest);
+      manifestEntries.push({ id: r.id, scope: r.scope, file: path.join("memories", rel) });
+      exported++;
+    }
+    const manifest = {
+      format: EXPORT_FORMAT,
+      exported_at: new Date().toISOString(),
+      open_memex_version: readPackageVersion(),
+      include_private: includePrivate,
+      filters: {
+        scope_keys: opts.scopeKeys,
+        type: opts.type ?? null,
+        tag: opts.tag ?? null,
+      },
+      memories: manifestEntries,
+    };
+    fs.writeFileSync(path.join(stage, "manifest.json"), JSON.stringify(manifest, null, 2), "utf8");
+
+    const stamp = new Date().toISOString().replace(/[:.]/g, "").slice(0, 15);
+    const outFile = opts.outFile ?? path.resolve(`open-memex-export-${stamp}.tar.gz`);
+    execFileSync("tar", ["-czf", outFile, "-C", stage, "manifest.json", "memories"], {
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    return { file: outFile, exported, skippedPrivate, includePrivate };
+  } finally {
+    fs.rmSync(stage, { recursive: true, force: true });
+  }
+}
+
+function readPackageVersion(): string {
+  try {
+    const here = new URL(import.meta.url);
+    const pkg = path.join(path.dirname(here.pathname), "..", "package.json");
+    return (JSON.parse(fs.readFileSync(pkg, "utf8")) as { version: string }).version;
+  } catch {
+    return "unknown";
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Import
+// ---------------------------------------------------------------------------
+
+export interface ImportOptions {
+  /** current project scope key — project memories are re-keyed to it */
+  projectScopeKey: string;
+  dryRun?: boolean;
+}
+
+export interface ImportResult {
+  imported: number;
+  skippedIdentical: number;
+  skippedConflict: { id: string; file: string }[];
+}
+
+export function importBundle(bundlePath: string, opts: ImportOptions): ImportResult {
+  checkTar();
+  if (!fs.existsSync(bundlePath)) fail(`bundle not found: ${bundlePath}`);
+  const stage = fs.mkdtempSync(path.join(os.tmpdir(), "open-memex-import-"));
+  try {
+    execFileSync("tar", ["-xzf", path.resolve(bundlePath), "-C", stage], {
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    const manifestPath = path.join(stage, "manifest.json");
+    if (!fs.existsSync(manifestPath)) fail("not an open-memex export bundle (manifest.json missing)");
+    const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")) as {
+      format: string;
+      memories: Array<{ id: string; scope: string; file: string }>;
+    };
+    if (manifest.format !== EXPORT_FORMAT) {
+      fail(`unsupported bundle format: ${manifest.format} (expected ${EXPORT_FORMAT})`);
+    }
+
+    const result: ImportResult = { imported: 0, skippedIdentical: 0, skippedConflict: [] };
+    const existingStmt = db().prepare(`SELECT content_hash FROM memories WHERE id = ?`);
+    for (const entry of manifest.memories ?? []) {
+      const src = path.join(stage, entry.file);
+      const mf = readMemoryFile(src);
+      if (!mf) continue;
+      const fm = normalizeFrontmatter({ ...mf.fm } as unknown as Record<string, unknown>);
+      // Re-key to this machine: personal stays personal; project memories
+      // adopt the current project's scope key (keys embed a path hash).
+      if (fm.scope === "project") fm.scope_key = opts.projectScopeKey;
+      else if (fm.scope === "personal") fm.scope_key = "personal";
+
+      const existing = existingStmt.get(fm.id) as { content_hash: string } | undefined;
+      if (existing) {
+        if (existing.content_hash === contentHash(mf.body)) result.skippedIdentical++;
+        else result.skippedConflict.push({ id: fm.id, file: entry.file });
+        continue;
+      }
+      if (opts.dryRun) {
+        result.imported++;
+        continue;
+      }
+      const { filePath } = writeMemoryFile(fm, mf.body);
+      const written = readMemoryFile(filePath);
+      if (written) upsertFromFile(written);
+      result.imported++;
+    }
+    return result;
+  } finally {
+    fs.rmSync(stage, { recursive: true, force: true });
+  }
+}

package/src/mcp.ts

@@ -137,6 +137,22 @@ export async function runMcpServer() {
   syncScope(PERSONAL_SCOPE.key, "session");
   console.error(`[open-memex] MCP server up. scope=${scope.key}`);
 
+  // D12: pulls are explicit by default — session start never touches the
+  // network. With sync.autoPull, one best-effort pull; a failure never
+  // blocks the session, it just logs and continues.
+  if (cfg.sync?.autoPull) {
+    try {
+      const { GitProvider } = await import("./providers/git.ts");
+      const r = new GitProvider().pull(process.cwd());
+      syncScope(scope.key, "pull");
+      console.error(
+        `[open-memex] auto-pull: ${r.branch} ${r.fastForwarded ? "fast-forwarded" : "already up to date"}`,
+      );
+    } catch (e) {
+      console.error(`[open-memex] auto-pull skipped: ${(e as Error).message}`);
+    }
+  }
+
   // D26: re-sync on every request, not just at startup. The in-repo dir
   // follows the current git branch, so a branch switch mid-session would
   // otherwise leave the index pointing at files that no longer exist.

package/src/providers/git.ts

@@ -0,0 +1,191 @@
+/**
+ * GitProvider — the git transport for repo-synced shared scopes (§9, D12).
+ *
+ * Git is a transport, not the product boundary: this provider moves the
+ * in-repo memory dir (`.ai/open-memex/`) between the local checkout and the
+ * remote. It never touches the appdata outbox, never force-pushes, never
+ * auto-merges a divergence — those are human decisions.
+ *
+ * Hard rules (from §9 / D12 / D36):
+ * - pull is explicit (`open-memex pull`); pull = fetch + fast-forward only.
+ * - push is explicit (`open-memex push`); the tool never pushes on its own.
+ * - a failed pull/push fails with a clear message and leaves no broken state.
+ */
+import { execFileSync } from "node:child_process";
+
+function fail(msg: string): never {
+  throw new Error(`[open-memex] ${msg}`);
+}
+
+/** Run git, raising a readable error. `timeoutMs` guards network calls. */
+function git(root: string, args: string[], timeoutMs = 0): string {
+  try {
+    return execFileSync("git", args, {
+      cwd: root,
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "pipe"],
+      ...(timeoutMs > 0 ? { timeout: timeoutMs } : {}),
+    }).trim();
+  } catch (e) {
+    const err = e as { stderr?: string; message?: string; code?: string };
+    if (err.code === "ETIMEDOUT") fail(`git ${args.join(" ")} timed out — remote unreachable?`);
+    const detail = (err.stderr ?? err.message ?? "").trim().split("\n")[0];
+    fail(`git ${args.join(" ")} failed${detail ? `: ${detail}` : ""}`);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Provider capability model (§2) — every provider declares what it can do.
+// ---------------------------------------------------------------------------
+
+export interface MemoryProviderCapabilities {
+  read: boolean;
+  write: boolean;
+  delete: boolean;
+  /** version / audit trail */
+  history: boolean;
+  sync: "none" | "pull" | "push" | "bidirectional";
+}
+
+export interface MemoryProvider {
+  readonly name: string;
+  readonly capabilities: MemoryProviderCapabilities;
+}
+
+export interface PullReceipt {
+  at: string; // RFC 3339
+  branch: string;
+  remote: string;
+  before: string; // HEAD sha before the pull
+  after: string; // HEAD sha after the pull
+  fastForwarded: boolean;
+}
+
+export interface PushReceipt {
+  at: string; // RFC 3339
+  branch: string;
+  remote: string;
+  head: string; // sha pushed
+}
+
+export interface GitSyncStatus {
+  branch: string;
+  upstream: string | null;
+  remote: string | null;
+  ahead: number;
+  behind: number;
+}
+
+const FETCH_TIMEOUT_MS = 30_000;
+
+export class GitProvider implements MemoryProvider {
+  readonly name = "git";
+  readonly capabilities: MemoryProviderCapabilities = {
+    read: true,
+    write: true,
+    delete: false,
+    history: true,
+    sync: "bidirectional",
+  };
+
+  /** Current branch + upstream, or a clear failure when git can't answer. */
+  status(root: string): GitSyncStatus {
+    git(root, ["rev-parse", "--git-dir"]);
+    const branch =
+      git(root, ["branch", "--show-current"]) ||
+      git(root, ["rev-parse", "--abbrev-ref", "HEAD"]);
+    let upstream: string | null = null;
+    try {
+      upstream = git(root, ["rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"]);
+    } catch {
+      upstream = null;
+    }
+    let ahead = 0;
+    let behind = 0;
+    if (upstream) {
+      const counts = git(root, ["rev-list", "--left-right", "--count", `HEAD...${upstream}`]);
+      const [a, b] = counts.split(/\s+/).map((n) => parseInt(n, 10));
+      ahead = Number.isFinite(a) ? a : 0;
+      behind = Number.isFinite(b) ? b : 0;
+    }
+    const remote = upstream ? upstream.split("/")[0] : null;
+    return { branch, upstream, remote, ahead, behind };
+  }
+
+  /**
+   * Explicit pull: fetch + fast-forward only (§9).
+   * Diverged branches are NOT merged — the user resolves them by hand.
+   */
+  pull(root: string): PullReceipt {
+    const st = this.status(root);
+    if (!st.upstream || !st.remote) {
+      fail(
+        `branch ${st.branch} has no upstream — set one with ` +
+          `\`git push -u <remote> ${st.branch}\`, then pull again`,
+      );
+    }
+    git(root, ["fetch", st.remote], FETCH_TIMEOUT_MS);
+    const before = git(root, ["rev-parse", "HEAD"]);
+    const remoteSha = git(root, ["rev-parse", st.upstream]);
+    if (before === remoteSha) {
+      return {
+        at: new Date().toISOString(),
+        branch: st.branch,
+        remote: st.remote,
+        before,
+        after: before,
+        fastForwarded: false,
+      };
+    }
+    // Fast-forward is possible iff HEAD is an ancestor of the upstream.
+    let ffPossible = false;
+    try {
+      git(root, ["merge-base", "--is-ancestor", "HEAD", st.upstream]);
+      ffPossible = true;
+    } catch {
+      ffPossible = false;
+    }
+    if (!ffPossible) {
+      fail(
+        `branch ${st.branch} has diverged from ${st.upstream} — ` +
+          `open-memex never force-merges; resolve it by hand ` +
+          `(rebase or merge), then pull again`,
+      );
+    }
+    git(root, ["merge", "--ff-only", st.upstream]);
+    const after = git(root, ["rev-parse", "HEAD"]);
+    return {
+      at: new Date().toISOString(),
+      branch: st.branch,
+      remote: st.remote,
+      before,
+      after,
+      fastForwarded: true,
+    };
+  }
+
+  /** Explicit push of the current branch. Never called automatically. */
+  push(root: string): PushReceipt {
+    const st = this.status(root);
+    if (!st.remote) {
+      // No upstream yet: push explicitly sets it (-u), still user-invoked.
+      const remotes = git(root, ["remote"]);
+      const remote = remotes.split("\n").map((r) => r.trim()).filter(Boolean)[0];
+      if (!remote) fail("no git remote configured — add one before pushing");
+      git(root, ["push", "-u", remote, st.branch], FETCH_TIMEOUT_MS);
+      return {
+        at: new Date().toISOString(),
+        branch: st.branch,
+        remote,
+        head: git(root, ["rev-parse", "HEAD"]),
+      };
+    }
+    git(root, ["push", st.remote, st.branch], FETCH_TIMEOUT_MS);
+    return {
+      at: new Date().toISOString(),
+      branch: st.branch,
+      remote: st.remote,
+      head: git(root, ["rev-parse", "HEAD"]),
+    };
+  }
+}

package/src/store/sync.ts

@@ -153,7 +153,7 @@ export function syncScope(scopeKey: string, kind: SyncKind = "auto"): SyncStats
 // ---------------------------------------------------------------------------
 
 /** What triggered the sync — shown in sync-status so the user can see it. */
-export type SyncKind = "session" | "request" | "cli" | "submit" | "auto";
+export type SyncKind = "session" | "request" | "cli" | "submit" | "pull" | "push" | "auto";
 
 export interface SyncStateEntry {
   lastSyncAt: string; // RFC 3339