open-memex 0.3.0 → 0.4.0-alpha.10

package/dist/store/sync.js

@@ -1,11 +1,14 @@
 import fs from "node:fs";
+import path from "node:path";
 import { db } from "./db.js";
 import { cjkIndexText } from "../retrieve/cjk.js";
 import { contentHash, repairChain } from "./lifecycle.js";
-import { iterMemoryFiles, readMemoryFile, timeToMs, } from "./markdown.js";
+import { iterMemoryFiles, iterInRepoMemoryFiles, readMemoryFile, timeToMs, } from "./markdown.js";
+import { projectRoot, paths } from "../paths.js";
+import { loadConfig } from "../config.js";
 const UPSERT_SQL = `
-  INSERT INTO memories (id, scope_key, scope, visibility, project_name, type, role, importance, status, tags, content, cjk, content_hash, superseded_by, source, file_path, mtime_ms, created_at, updated_at)
-  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  INSERT INTO memories (id, scope_key, scope, visibility, project_name, type, role, importance, status, tags, content, cjk, content_hash, superseded_by, source, file_path, mtime_ms, created_at, updated_at, review_state)
+  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
   ON CONFLICT(id) DO UPDATE SET
     scope_key    = excluded.scope_key,
     scope        = excluded.scope,
@@ -21,6 +24,7 @@ const UPSERT_SQL = `
     content_hash = excluded.content_hash,
     superseded_by = excluded.superseded_by,
     source       = excluded.source,
+    review_state = excluded.review_state,
     file_path    = excluded.file_path,
     mtime_ms     = excluded.mtime_ms,
     updated_at   = excluded.updated_at
@@ -49,12 +53,12 @@ export function upsertFromFile(mf) {
 function writeRow(mf, mtimeMs) {
     const { fm, body, filePath } = mf;
     const tags = (fm.tags ?? []).join(",");
-    db().prepare(UPSERT_SQL).run(fm.id, fm.scope_key, fm.scope, fm.visibility, fm.project_name, fm.type, fm.role, fm.importance, fm.status, tags, body, cjkIndexText(body + "\n" + tags), contentHash(body), fm.superseded_by ?? null, fm.source ?? "", filePath, mtimeMs, timeToMs(fm.created_at), timeToMs(fm.updated_at));
+    db().prepare(UPSERT_SQL).run(fm.id, fm.scope_key, fm.scope, fm.visibility, fm.project_name, fm.type, fm.role, fm.importance, fm.status, tags, body, cjkIndexText(body + "\n" + tags), contentHash(body), fm.superseded_by ?? null, fm.source ?? "", filePath, mtimeMs, timeToMs(fm.created_at), timeToMs(fm.updated_at), fm.review_state ?? "draft");
 }
 export function deleteFromIndex(id) {
     db().prepare(`DELETE FROM memories WHERE id = ?`).run(id);
 }
-export function syncScope(scopeKey) {
+export function syncScope(scopeKey, kind = "auto") {
     const d = db();
     const stats = { added: 0, updated: 0, removed: 0, scanned: 0 };
     const existing = d
@@ -62,7 +66,25 @@ export function syncScope(scopeKey) {
         .all(scopeKey);
     const existingById = new Map(existing.map((r) => [r.id, r]));
     const seen = new Set();
-    for (const fp of iterMemoryFiles(scopeKey)) {
+    // 2B/D26: project scopes have two homes — the appdata outbox (drafts,
+    // branch-independent) and the in-repo dir (submitted memories, following
+    // the current branch). No migration: appdata files stay put until an
+    // explicit `submit` moves them. On the near-impossible id collision the
+    // in-repo (submitted) copy wins, so scan appdata first.
+    const filePaths = [];
+    if (scopeKey.startsWith("project__")) {
+        const root = projectRoot();
+        const memoryDir = loadConfig().memoryDir;
+        for (const fp of iterMemoryFiles(scopeKey))
+            filePaths.push(fp);
+        for (const fp of iterInRepoMemoryFiles(root, memoryDir))
+            filePaths.push(fp);
+    }
+    else {
+        for (const fp of iterMemoryFiles(scopeKey))
+            filePaths.push(fp);
+    }
+    for (const fp of filePaths) {
         const mf = readMemoryFile(fp);
         if (!mf)
             continue;
@@ -84,5 +106,51 @@ export function syncScope(scopeKey) {
             stats.removed++;
         }
     }
+    recordSync(scopeKey, kind, stats);
     return stats;
 }
+function syncStatePath() {
+    return path.join(paths().root, "sync-state.json");
+}
+/** Best-effort: a failed write never breaks the sync itself. */
+export function recordSync(scopeKey, kind, stats) {
+    try {
+        const p = syncStatePath();
+        let all = {};
+        if (fs.existsSync(p)) {
+            try {
+                all = JSON.parse(fs.readFileSync(p, "utf8"));
+            }
+            catch {
+                /* corrupt — start fresh */
+            }
+        }
+        all[scopeKey] = {
+            lastSyncAt: new Date().toISOString(),
+            lastSyncKind: kind,
+            added: stats.added,
+            updated: stats.updated,
+            removed: stats.removed,
+            scanned: stats.scanned,
+        };
+        fs.writeFileSync(p, JSON.stringify(all, null, 2), "utf8");
+    }
+    catch {
+        /* visibility is advisory — never fail a sync over it */
+    }
+}
+export function readSyncState(scopeKey) {
+    try {
+        const p = syncStatePath();
+        if (!fs.existsSync(p))
+            return null;
+        const all = JSON.parse(fs.readFileSync(p, "utf8"));
+        const e = all?.[scopeKey];
+        if (!e || typeof e.lastSyncAt !== "string")
+            return null;
+        return e;
+    }
+    catch {
+        return null;
+    }
+}

package/dist/submit.js

@@ -0,0 +1,347 @@
+/**
+ * 2B/D26: submit — move outbox drafts into a git branch for review.
+ *
+ * The AI-driven sync loop:
+ *   1. `open-memex sync-status` — what is waiting (outbox drafts, in-review, published)
+ *   2. user picks memories to sync
+ *   3. `open-memex submit <ids>` — new branch (default `mem/sync-*`), copy
+ *      drafts into the repo dir as `proposed`, commit locally, move the
+ *      outbox originals out (copy + verify + delete)
+ *   4. push + open PR — done by the agent holding the user's approval, or by
+ *      the human; the tool only prints the exact commands (D27)
+ *
+ * Git automation line (D27): the tool automates LOCAL git
+ * only (branch, copy, commit). Nothing leaves the machine without the user's
+ * explicit approval — push/PR are the agent's or the human's job.
+ *
+ * `submit` is idempotent: re-submitting an id whose file is already on the
+ * branch with identical content just completes the outbox move. A same-id
+ * file with DIFFERENT content is a conflict — abort, the human judges.
+ */
+import { execFileSync } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+import { loadConfig } from "./config.js";
+import { inRepoMemoriesDir, memoriesDirPath, projectRoot } from "./paths.js";
+import { resolveProjectScope } from "./scope.js";
+import { contentHash } from "./store/lifecycle.js";
+import { msToRfc3339, parse as parseMemory, readMemoryFile, serialize, } from "./store/markdown.js";
+import { upsertFromFile, recordSync, readSyncState } from "./store/sync.js";
+import { db } from "./store/db.js";
+import { currentAuthor } from "./review.js";
+function fail(msg) {
+    throw new Error(`[open-memex] ${msg}`);
+}
+function git(root, args) {
+    try {
+        return execFileSync("git", args, {
+            cwd: root,
+            encoding: "utf8",
+            stdio: ["ignore", "pipe", "pipe"],
+        }).trim();
+    }
+    catch (e) {
+        const err = e;
+        const detail = (err.stderr ?? err.message ?? "").trim().split("\n")[0];
+        fail(`git ${args.join(" ")} failed${detail ? `: ${detail}` : ""}`);
+    }
+}
+function snippetOf(body) {
+    const line = body.split("\n").map((l) => l.trim()).find((l) => l.length > 0) ?? "";
+    return line.length > 60 ? line.slice(0, 60) + "…" : line;
+}
+/** Files under .ai/open-memex that git sees as new/modified/untracked. */
+function uncommittedRepoFiles(root, repoDir) {
+    const out = new Set();
+    try {
+        const porcelain = git(root, ["status", "--porcelain", "--", repoDir]);
+        for (const line of porcelain.split("\n")) {
+            const m = line.match(/^.{2}\s+(.*)$/);
+            if (!m)
+                continue;
+            let p = m[1].trim();
+            // handle renames: "R  old -> new"
+            if (p.includes(" -> "))
+                p = p.split(" -> ")[1];
+            // strip quotes git adds around unusual names
+            if (p.startsWith('"') && p.endsWith('"'))
+                p = p.slice(1, -1);
+            out.add(path.resolve(root, p));
+        }
+    }
+    catch {
+        /* not a git repo or git unavailable — no git state */
+    }
+    return out;
+}
+export function getSyncStatus() {
+    const root = projectRoot();
+    const scope = resolveProjectScope(root);
+    const cfg = loadConfig();
+    const outboxDir = memoriesDirPath(scope.key);
+    const repoDir = path.join(root, cfg.memoryDir);
+    const rows = db()
+        .prepare(`SELECT id, review_state, file_path, content FROM memories WHERE scope_key = ? ORDER BY updated_at DESC`)
+        .all(scope.key);
+    const uncommitted = fs.existsSync(repoDir) ? uncommittedRepoFiles(root, repoDir) : new Set();
+    const outbox = [];
+    const inRepo = [];
+    const uncommittedList = [];
+    for (const r of rows) {
+        const inOutbox = r.file_path.startsWith(outboxDir + path.sep);
+        const entry = {
+            id: r.id,
+            title: snippetOf(r.content ?? ""),
+            reviewState: (r.review_state ?? "draft"),
+            location: inOutbox ? "outbox" : "repo",
+            gitState: "n/a",
+        };
+        if (inOutbox) {
+            outbox.push(entry);
+        }
+        else {
+            entry.gitState = uncommitted.has(path.resolve(r.file_path)) ? "uncommitted" : "committed";
+            inRepo.push(entry);
+            if (entry.gitState === "uncommitted")
+                uncommittedList.push(entry);
+        }
+    }
+    return {
+        scopeKey: scope.key,
+        projectName: scope.projectName,
+        outbox,
+        inRepo,
+        uncommitted: uncommittedList,
+    };
+}
+export function formatSyncStatus(st) {
+    const lines = [];
+    lines.push(`project: ${st.projectName} (${st.scopeKey})`);
+    // D31: when the index was last synced and what triggered it.
+    const last = readSyncState(st.scopeKey);
+    if (last) {
+        lines.push(`last sync: ${last.lastSyncAt} (${last.lastSyncKind}) — +${last.added} ~${last.updated} -${last.removed} (scanned ${last.scanned})`);
+    }
+    else {
+        lines.push(`last sync: never`);
+    }
+    lines.push(`outbox (appdata, pending sync): ${st.outbox.length}`);
+    for (const e of st.outbox) {
+        lines.push(`  ${e.id}  [${e.reviewState}] ${e.title}`);
+    }
+    const byState = new Map();
+    for (const e of st.inRepo)
+        byState.set(e.reviewState, (byState.get(e.reviewState) ?? 0) + 1);
+    const breakdown = [...byState.entries()].map(([s, n]) => `${n} ${s}`).join(", ") || "none";
+    lines.push(`repo .ai/open-memex: ${st.inRepo.length} (${breakdown})`);
+    for (const e of st.inRepo) {
+        const u = e.gitState === "uncommitted" ? " (uncommitted)" : "";
+        lines.push(`  ${e.id}  [${e.reviewState}]${u} ${e.title}`);
+    }
+    if (st.uncommitted.length > 0) {
+        lines.push(`note: ${st.uncommitted.length} repo file(s) not yet committed — they ride with the working tree until you commit.`);
+    }
+    if (st.outbox.length > 0) {
+        lines.push(`next: open-memex submit ${st.outbox.map((e) => e.id).join(" ")}`);
+    }
+    return lines.join("\n");
+}
+export function submitMemories(ids, opts = {}) {
+    if (ids.length === 0)
+        fail("submit needs at least one memory id");
+    const root = projectRoot();
+    // submit needs a git repo — the memories land in .ai/open-memex/ and are
+    // committed locally. Branch creation is never automatic (D36).
+    git(root, ["rev-parse", "--git-dir"]);
+    const scope = resolveProjectScope(root);
+    const cfg = loadConfig();
+    const outboxDir = memoriesDirPath(scope.key);
+    const repoDir = inRepoMemoriesDir(root, cfg.memoryDir);
+    // 1. Validate everything first (all-or-nothing — nothing is written yet).
+    //    Read the outbox files directly from disk: the index resolves same-id
+    //    collisions in favor of the repo copy, which would hide a genuine
+    //    outbox-vs-branch conflict from us.
+    const problems = [];
+    const validated = [];
+    for (const id of ids) {
+        if (!/^[A-Za-z0-9_-]+$/.test(id)) {
+            problems.push(`${id}: not a valid memory id`);
+            continue;
+        }
+        const srcPath = path.join(outboxDir, `${id}.md`);
+        if (!fs.existsSync(srcPath)) {
+            problems.push(`${id}: not in the outbox (never staged, or already submitted? see sync-status)`);
+            continue;
+        }
+        const mf = readMemoryFile(srcPath);
+        if (!mf) {
+            problems.push(`${id}: unreadable outbox file`);
+            continue;
+        }
+        if (mf.fm.id !== id) {
+            problems.push(`${id}: outbox file's frontmatter id is ${mf.fm.id} — not submitting a mislabeled file`);
+            continue;
+        }
+        if (mf.fm.scope !== "project") {
+            problems.push(`${id}: scope is ${mf.fm.scope}, only project memories can be submitted`);
+            continue;
+        }
+        const rs = (mf.fm.review_state ?? "draft");
+        // draft/rejected move to proposed; a local-approved copy keeps its approval.
+        if (rs !== "draft" && rs !== "rejected" && rs !== "approved") {
+            problems.push(`${id}: review_state is ${rs}, only draft/rejected/approved can be submitted`);
+            continue;
+        }
+        validated.push({ id, srcPath, body: mf.body, reviewState: rs });
+    }
+    if (problems.length > 0) {
+        fail(`submit aborted — nothing was written:\n  ${problems.join("\n  ")}`);
+    }
+    // 2. Resolve the target branch.
+    //    D36: submit never creates a branch on its own — it works on the branch
+    //    you're already on. Pass --branch <name> (explicitly) only when the user
+    //    approved the full chain (branch + push + PR).
+    const startBranch = git(root, ["branch", "--show-current"]) || git(root, ["rev-parse", "--abbrev-ref", "HEAD"]);
+    let branch = startBranch;
+    let createdBranch = false;
+    if (opts.branch) {
+        branch = opts.branch;
+        let n = 0;
+        let candidate = branch;
+        while (true) {
+            const exists = git(root, ["branch", "--list", candidate]);
+            if (!exists)
+                break;
+            n++;
+            candidate = `${branch}-${n}`;
+        }
+        branch = candidate;
+        git(root, ["checkout", "-b", branch]);
+        createdBranch = true;
+    }
+    const base = opts.base ?? startBranch;
+    const author = currentAuthor();
+    // 3. Copy drafts into the repo dir as proposed (same id — one memory, one id).
+    //    Same id + different content on the branch = conflict → abort, human judges.
+    const submitted = [];
+    const skippedIdentical = [];
+    const now = msToRfc3339(Date.now());
+    const written = [];
+    const conflicts = [];
+    for (const v of validated) {
+        const destPath = path.join(repoDir, `${v.id}.md`);
+        const raw = fs.existsSync(destPath) ? fs.readFileSync(destPath, "utf8") : null;
+        if (raw !== null) {
+            const existing = parseMemory(raw);
+            const sameContent = existing !== null && contentHash(existing.body) === contentHash(v.body);
+            if (sameContent) {
+                skippedIdentical.push(v.id);
+                written.push({ id: v.id, srcPath: v.srcPath, destPath, hash: contentHash(v.body) });
+                continue;
+            }
+            conflicts.push(`${v.id}: already on branch ${branch} with different content — resolve it manually, then re-submit`);
+            continue;
+        }
+        const parsed = parseMemory(fs.readFileSync(v.srcPath, "utf8"));
+        if (!parsed)
+            fail(`${v.id}: unreadable source file`);
+        // draft/rejected → proposed; a local-approved copy arrives already
+        // approved and keeps its approval (and its approved_by).
+        const destState = v.reviewState === "approved" ? "approved" : "proposed";
+        const fm = {
+            ...parsed.fm,
+            review_state: destState,
+            updated_at: now,
+            proposed_by: parsed.fm.proposed_by ?? author,
+            // D29: the outbox → repo move is part of the audit trail.
+            review_history: [
+                ...(parsed.fm.review_history ?? []),
+                {
+                    at: now,
+                    by: author,
+                    from: parsed.fm.review_state ?? "draft",
+                    to: destState,
+                    note: `submitted to branch ${branch}`,
+                },
+            ],
+        };
+        const text = serialize(fm, parsed.body);
+        fs.writeFileSync(destPath, text, "utf8");
+        written.push({ id: v.id, srcPath: v.srcPath, destPath, hash: contentHash(parsed.body) });
+        submitted.push({ id: v.id, filePath: destPath, reviewState: destState });
+    }
+    if (conflicts.length > 0) {
+        // Roll back the copies we just made — nothing half-submitted.
+        for (const w of written) {
+            if (submitted.some((s) => s.id === w.id) && fs.existsSync(w.destPath)) {
+                fs.unlinkSync(w.destPath);
+            }
+        }
+        // If we created the branch and never committed, remove it too — an
+        // aborted submit leaves no trace.
+        if (createdBranch) {
+            try {
+                git(root, ["checkout", "--quiet", startBranch]);
+                git(root, ["branch", "--quiet", "-D", branch]);
+            }
+            catch {
+                /* best effort — the error below is what matters */
+            }
+        }
+        fail(`submit aborted — nothing was committed:\n  ${conflicts.join("\n  ")}`);
+    }
+    // 4. Commit locally (tool automates local git only — D27 line).
+    //    Only our own files are committed: anything else the user staged stays
+    //    staged and untouched.
+    const destPaths = written.map((w) => w.destPath);
+    const relPaths = destPaths.map((p) => path.relative(root, p));
+    git(root, ["add", "--", ...destPaths]);
+    const staged = git(root, ["diff", "--cached", "--name-only", "--", ...relPaths]);
+    let committed = false;
+    if (staged.trim()) {
+        const short = submitted.map((s) => s.id.slice(0, 8)).join(" ");
+        git(root, ["commit", "-m", `mem: submit ${submitted.length} ${submitted.length === 1 ? "memory" : "memories"} for review (${short})`, "--", ...relPaths]);
+        committed = true;
+    }
+    // 5. Verify the committed copies, then move the outbox originals out
+    //    (copy + verify + delete — never a bare rename across the move).
+    for (const w of written) {
+        const back = fs.readFileSync(w.destPath, "utf8");
+        const bParsed = parseMemory(back);
+        if (!bParsed || contentHash(bParsed.body) !== w.hash) {
+            fail(`${w.id}: verification failed after commit — outbox copy kept at ${w.srcPath}`);
+        }
+        // Same id, new home: ON CONFLICT(id) rewrites the row to the repo path.
+        upsertFromFile({
+            fm: bParsed.fm,
+            body: bParsed.body,
+            filePath: w.destPath,
+            mtimeMs: fs.statSync(w.destPath).mtimeMs,
+        });
+        fs.unlinkSync(w.srcPath);
+    }
+    const ids8 = validated.map((v) => v.id.slice(0, 8)).join(" ");
+    // D31: the submit itself is a sync event — visible in sync-status.
+    recordSync(scope.key, "submit", {
+        added: submitted.length,
+        updated: skippedIdentical.length,
+        removed: 0,
+        scanned: validated.length,
+    });
+    // D36: no auto-branch — the commit sits on the branch you were already on.
+    // Next steps are printed, not run. For a separate memory PR, create a branch
+    // first (the commit comes along), then push + open the PR.
+    const branchCmd = `git checkout -b mem/sync-YYYYMMDD`;
+    return {
+        branch,
+        base,
+        submitted,
+        skippedIdentical,
+        committed,
+        createdBranch,
+        pushCommand: `git push -u origin ${branch}`,
+        prCommand: createdBranch
+            ? `gh pr create --base ${base} --title "mem: review ${validated.length} ${validated.length === 1 ? "memory" : "memories"}" --body "Submitted from the open-memex outbox: ${ids8}."`
+            : `${branchCmd}  # if you want a separate memory PR (else push ${branch} directly)\n  git push -u origin <new-branch> && gh pr create --base ${base} --title "mem: review ${validated.length}" --body "Submitted from the open-memex outbox: ${ids8}."`,
+    };
+}