open-memex 0.1.0 → 0.3.0-alpha

package/src/store/sync.ts

@@ -1,17 +1,31 @@
 import fs from "node:fs";
 import { db } from "./db.ts";
-import { iterMemoryFiles, readMemoryFile, type MemoryFile } from "./markdown.ts";
+import { cjkIndexText } from "../retrieve/cjk.ts";
+import { contentHash, repairChain } from "./lifecycle.ts";
+import {
+  iterMemoryFiles,
+  readMemoryFile,
+  timeToMs,
+  type MemoryFile,
+} from "./markdown.ts";
 
 const UPSERT_SQL = `
-  INSERT INTO memories (id, scope_key, scope_kind, project_name, type, tags, content, source, file_path, mtime_ms, created_at, updated_at)
-  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  INSERT INTO memories (id, scope_key, scope, visibility, project_name, type, role, importance, status, tags, content, cjk, content_hash, superseded_by, source, file_path, mtime_ms, created_at, updated_at)
+  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
   ON CONFLICT(id) DO UPDATE SET
     scope_key    = excluded.scope_key,
-    scope_kind   = excluded.scope_kind,
+    scope        = excluded.scope,
+    visibility   = excluded.visibility,
     project_name = excluded.project_name,
     type         = excluded.type,
+    role         = excluded.role,
+    importance   = excluded.importance,
+    status       = excluded.status,
     tags         = excluded.tags,
     content      = excluded.content,
+    cjk          = excluded.cjk,
+    content_hash = excluded.content_hash,
+    superseded_by = excluded.superseded_by,
     source       = excluded.source,
     file_path    = excluded.file_path,
     mtime_ms     = excluded.mtime_ms,
@@ -19,20 +33,50 @@ const UPSERT_SQL = `
 `;
 
 export function upsertFromFile(mf: MemoryFile): void {
-  const { fm, body, filePath, mtimeMs } = mf;
+  // Chain integrity (§3.3): every write path through Core validates the
+  // supersedes/superseded_by pair and auto-completes a missing side.
+  const { warnings, repaired } = repairChain(mf);
+  for (const w of warnings) console.warn(`[open-memex] chain: ${w}`);
+  const mtimeOf = (m: MemoryFile): number => {
+    try {
+      return fs.statSync(m.filePath).mtimeMs;
+    } catch {
+      return m.mtimeMs; // file vanished mid-repair; keep the old mtime
+    }
+  };
+  writeRow(
+    mf,
+    repaired.some((r) => r.filePath === mf.filePath) ? mtimeOf(mf) : mf.mtimeMs,
+  );
+  for (const r of repaired) {
+    if (r.filePath === mf.filePath) continue;
+    writeRow(r, mtimeOf(r));
+  }
+}
+
+function writeRow(mf: MemoryFile, mtimeMs: number): void {
+  const { fm, body, filePath } = mf;
+  const tags = (fm.tags ?? []).join(",");
   db().prepare(UPSERT_SQL).run(
     fm.id,
     fm.scope_key,
-    fm.scope_kind,
+    fm.scope,
+    fm.visibility,
     fm.project_name,
     fm.type,
-    (fm.tags ?? []).join(","),
+    fm.role,
+    fm.importance,
+    fm.status,
+    tags,
     body,
+    cjkIndexText(body + "\n" + tags),
+    contentHash(body),
+    fm.superseded_by ?? null,
     fm.source ?? "",
     filePath,
     mtimeMs,
-    fm.created_at,
-    fm.updated_at,
+    timeToMs(fm.created_at),
+    timeToMs(fm.updated_at),
   );
 }
 

package/src/store/v2migrate.ts

@@ -0,0 +1,190 @@
+import fs from "node:fs";
+import path from "node:path";
+import { paths } from "../paths.ts";
+import {
+  parseRawFrontmatter,
+  normalizeFrontmatter,
+  isTaxonomyType,
+  serialize,
+  type Frontmatter,
+} from "./markdown.ts";
+
+/**
+ * v1 → v2 file migration (V2-DESIGN §19). Explicit only — run via
+ * `open-memex migrate --to-v2`. Converts frontmatter in place:
+ *
+ *   - scope rename: v1 `user` → v2 `personal` (scope, scope_key, and the
+ *     storage dir `memories/user/` → `memories/personal/`)
+ *   - times: epoch ms → RFC 3339
+ *   - `priority: N` → `importance: low|normal|high`
+ *   - `type: instruction` → `type: knowledge` + `role: instruction`
+ *   - unknown v1 types → `knowledge` (v2 taxonomy, §3.1)
+ *   - adds `schema_version: 2`, `visibility`, `role`, `status`
+ *
+ * Legacy `my-o-memory` data roots (§19/D13: `.my-o-memory/` →
+ * `.open-memex/`, `~/.my-o-memory/` → `~/.open-memex/`) are merged into the
+ * current root; the old dir is kept as a dated backup.
+ */
+
+const V2_MARKER = /^schema_version:\s*2(\s|$)/m;
+
+export function isV2File(raw: string): boolean {
+  return V2_MARKER.test(raw);
+}
+
+export interface ConversionPlan {
+  fm: Frontmatter;
+  body: string;
+  fromPath: string;
+  toPath: string;
+  changes: string[];
+}
+
+/**
+ * Plan the v1 → v2 conversion of one file. Returns null when the file is
+ * already v2 (nothing to do). Pure — no disk writes.
+ */
+export function planConversion(
+  filePath: string,
+  memoriesRoot: string,
+): ConversionPlan | null {
+  const raw = fs.readFileSync(filePath, "utf8");
+  if (isV2File(raw)) return null;
+  const parsed = parseRawFrontmatter(raw);
+  if (!parsed) return null;
+  const { rawFm, body } = parsed;
+
+  const fm = normalizeFrontmatter(rawFm);
+  const changes: string[] = [];
+
+  // scope rename
+  if (rawFm.scope_kind === "user" || rawFm.scope_key === "user") {
+    changes.push("scope user → personal");
+  } else if (!rawFm.scope) {
+    changes.push(`scope → ${fm.scope} (default)`);
+  }
+
+  // storage dir move follows scope_key
+  const toPath = path.join(memoriesRoot, fm.scope_key, path.basename(filePath));
+  if (toPath !== filePath) {
+    changes.push(
+      `file moved: ${path.basename(path.dirname(filePath))}/ → ${fm.scope_key}/`,
+    );
+  }
+
+  // visibility default
+  if (!rawFm.visibility) {
+    changes.push(`visibility → ${fm.visibility} (default)`);
+  }
+
+  // type taxonomy
+  const rawType = rawFm.type;
+  if (rawType === "instruction") {
+    changes.push("type instruction → knowledge, role → instruction (D11)");
+  } else if (typeof rawType === "string" && rawType && !isTaxonomyType(rawType)) {
+    changes.push(`type "${rawType}" → knowledge (not in v2 taxonomy)`);
+    fm.type = "knowledge";
+  } else if (!rawType) {
+    changes.push(`type → ${fm.type} (default)`);
+  }
+
+  // priority → importance
+  if (typeof rawFm.priority === "number") {
+    changes.push(`priority ${rawFm.priority} → importance ${fm.importance}`);
+  } else if (!rawFm.importance) {
+    changes.push(`importance → ${fm.importance} (default)`);
+  }
+
+  // times
+  if (typeof rawFm.created_at === "number") {
+    changes.push("created_at/updated_at: epoch ms → RFC 3339");
+  }
+
+  // status default
+  if (!rawFm.status) {
+    changes.push(`status → ${fm.status} (default)`);
+  }
+
+  changes.push("schema_version → 2");
+  return { fm, body, fromPath: filePath, toPath, changes };
+}
+
+export interface MigrateV2Result {
+  scanned: number;
+  converted: number;
+  skippedV2: number;
+  plans: ConversionPlan[];
+  legacyBackup: string | null;
+}
+
+/**
+ * Merge a legacy `my-o-memory` data root into the current `open-memex` root.
+ * The old dir is renamed to a dated backup — never deleted. Returns the
+ * backup path, or null when no legacy dir exists.
+ */
+function migrateLegacyDataRoot(dryRun: boolean): string | null {
+  const { root } = paths();
+  const legacy = path.join(path.dirname(root), "my-o-memory");
+  if (!fs.existsSync(legacy) || !fs.statSync(legacy).isDirectory()) return null;
+
+  const stamp = new Date().toISOString().slice(0, 10);
+  const backup = `${legacy}.backup-${stamp}`;
+  if (!dryRun) {
+    const legacyMem = path.join(legacy, "memories");
+    if (fs.existsSync(legacyMem)) {
+      for (const entry of fs.readdirSync(legacyMem)) {
+        const src = path.join(legacyMem, entry);
+        const dst = path.join(root, "memories", entry);
+        if (!fs.existsSync(dst)) {
+          fs.renameSync(src, dst);
+        } else {
+          // merge file-by-file, never overwrite
+          for (const name of fs.readdirSync(src)) {
+            const s = path.join(src, name);
+            const d = path.join(dst, name);
+            if (!fs.existsSync(d)) fs.renameSync(s, d);
+          }
+        }
+      }
+    }
+    fs.renameSync(legacy, backup);
+  }
+  return backup;
+}
+
+export function migrateV2(opts: { dryRun: boolean }): MigrateV2Result {
+  const { memories: memoriesRoot } = paths();
+  const result: MigrateV2Result = {
+    scanned: 0,
+    converted: 0,
+    skippedV2: 0,
+    plans: [],
+    legacyBackup: null,
+  };
+
+  result.legacyBackup = migrateLegacyDataRoot(opts.dryRun);
+
+  if (!fs.existsSync(memoriesRoot)) return result;
+  for (const entry of fs.readdirSync(memoriesRoot, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    const dir = path.join(memoriesRoot, entry.name);
+    for (const name of fs.readdirSync(dir)) {
+      if (!name.endsWith(".md")) continue;
+      const filePath = path.join(dir, name);
+      result.scanned++;
+      const plan = planConversion(filePath, memoriesRoot);
+      if (!plan) {
+        result.skippedV2++;
+        continue;
+      }
+      if (!opts.dryRun) {
+        fs.mkdirSync(path.dirname(plan.toPath), { recursive: true });
+        fs.writeFileSync(plan.toPath, serialize(plan.fm, plan.body), "utf8");
+        if (plan.toPath !== plan.fromPath) fs.unlinkSync(plan.fromPath);
+      }
+      result.converted++;
+      result.plans.push(plan);
+    }
+  }
+  return result;
+}

package/src/tools/memory.ts

@@ -1,174 +1,60 @@
 import { tool } from "@opencode-ai/plugin/tool";
 import type { Scope } from "../scope.ts";
 import type { MyOMemoryConfig } from "../config.ts";
-import { USER_SCOPE } from "../scope.ts";
-import { search, list } from "../retrieve/search.ts";
 import {
-  writeMemoryFile,
-  deleteMemoryFile,
-  readMemoryFile,
-  ulid,
-  type Frontmatter,
-} from "../store/markdown.ts";
-import { upsertFromFile, deleteFromIndex } from "../store/sync.ts";
-import { db } from "../store/db.ts";
-import { redact } from "../redact.ts";
-
-const z = tool.schema;
-
-const MEMORY_TYPES = [
-  "note",
-  "preference",
-  "project-config",
-  "architecture",
-  "error-solution",
-  "learned-pattern",
-  "conversation",
-] as const;
+  addMemory,
+  searchMemories,
+  listMemories,
+  supersedeMemory,
+  forgetMemory,
+  memoryAddArgs,
+  memorySearchArgs,
+  memoryListArgs,
+  memorySupersedeArgs,
+  memoryForgetArgs,
+  TOOL_DESCRIPTIONS,
+} from "./ops.ts";
 
 export function makeTools(getScope: () => Scope, cfg: MyOMemoryConfig) {
-  const scopeArg = z
-    .enum(["project", "user"])
-    .optional()
-    .describe(
-      "Memory scope. `project` = tied to this repo. `user` = global across all your projects. Default: project.",
-    );
-
-  function resolveScope(kind?: "project" | "user"): Scope {
-    return kind === "user" ? USER_SCOPE : getScope();
-  }
-
   const memory_add = tool({
-    description:
-      "Save a fact, preference, decision, or note to persistent local memory. Call this whenever the user tells you something you should remember in future sessions (project conventions, tool choices, personal preferences, error fixes). Keep each memory to one self-contained statement.",
-    args: {
-      content: z.string().min(1).describe("The fact to remember. One idea per memory."),
-      type: z
-        .enum(MEMORY_TYPES)
-        .optional()
-        .describe("Category of memory. Default: note."),
-      scope: scopeArg,
-      tags: z.array(z.string()).optional().describe("Optional tags for filtering."),
-    },
+    description: TOOL_DESCRIPTIONS.memory_add,
+    args: memoryAddArgs,
     async execute(args) {
-      const { content: redacted, hadSecret, matchedPattern } = redact(
-        args.content,
-        cfg.redactPatterns,
-      );
-      if (hadSecret) {
-        return {
-          title: "memory: rejected (secret detected)",
-          output: `Refused to save: content matched a secret pattern (${matchedPattern}). Wrap the sensitive part in <private>...</private> tags or paraphrase, then try again.`,
-        };
-      }
-      const s = resolveScope(args.scope);
-      const now = Date.now();
-      const fm: Frontmatter = {
-        id: ulid(),
-        scope_key: s.key,
-        scope_kind: s.kind,
-        project_name: s.projectName,
-        type: args.type ?? "note",
-        tags: args.tags ?? [],
-        source: "tool",
-        created_at: now,
-        updated_at: now,
-      };
-      const { filePath } = writeMemoryFile(fm, redacted);
-      const mf = readMemoryFile(filePath);
-      if (mf) upsertFromFile(mf);
-      return {
-        title: `memory: saved ${fm.id}`,
-        output: `Saved to ${s.key} as ${fm.type}. id=${fm.id}`,
-      };
+      return addMemory(getScope, cfg, args);
     },
   });
 
   const memory_search = tool({
-    description:
-      "Search persistent memory by keyword (BM25 full-text). Returns matching memories from the current project and/or user scope. Use before asking the user something they may have told you before.",
-    args: {
-      query: z
-        .string()
-        .min(1)
-        .describe("Free-text query. File paths, error strings, identifiers work well."),
-      scope: z
-        .enum(["project", "user", "both"])
-        .optional()
-        .describe("Which scope(s) to search. Default: both."),
-      type: z.string().optional().describe("Restrict to memories of this type."),
-      limit: z.number().int().min(1).max(50).optional(),
-    },
+    description: TOOL_DESCRIPTIONS.memory_search,
+    args: memorySearchArgs,
     async execute(args) {
-      const project = getScope();
-      const scopeKeys =
-        args.scope === "user"
-          ? [USER_SCOPE.key]
-          : args.scope === "project"
-            ? [project.key]
-            : [project.key, USER_SCOPE.key];
-      const hits = search(args.query, {
-        scopeKeys,
-        limit: args.limit,
-        type: args.type,
-      });
-      if (hits.length === 0) {
-        return {
-          title: "memory: 0 results",
-          output: `No memories matched "${args.query}".`,
-        };
-      }
-      const lines = hits.map(
-        (h) =>
-          `- [${h.scope_key === USER_SCOPE.key ? "user" : "project"}/${h.type}] id=${h.id}\n  ${h.snippet.replace(/\s+/g, " ").trim()}`,
-      );
-      return {
-        title: `memory: ${hits.length} result(s)`,
-        output: lines.join("\n"),
-      };
+      return searchMemories(getScope, args);
     },
   });
 
   const memory_list = tool({
-    description: "List memories in a scope, newest first. Useful for browsing.",
-    args: {
-      scope: scopeArg,
-      type: z.string().optional(),
-      limit: z.number().int().min(1).max(100).optional(),
+    description: TOOL_DESCRIPTIONS.memory_list,
+    args: memoryListArgs,
+    async execute(args) {
+      return listMemories(getScope, args);
     },
+  });
+
+  const memory_supersede = tool({
+    description: TOOL_DESCRIPTIONS.memory_supersede,
+    args: memorySupersedeArgs,
     async execute(args) {
-      const s = resolveScope(args.scope);
-      const hits = list(s.key, { type: args.type, limit: args.limit });
-      if (hits.length === 0) {
-        return { title: "memory: empty", output: `No memories in scope ${s.key}.` };
-      }
-      const lines = hits.map(
-        (h) => `- [${h.type}] id=${h.id} — ${h.snippet.replace(/\s+/g, " ").trim()}`,
-      );
-      return {
-        title: `memory: ${hits.length} in ${s.key}`,
-        output: lines.join("\n"),
-      };
+      return supersedeMemory(cfg, args);
     },
   });
 
   const memory_forget = tool({
-    description: "Delete a memory by id. Use when the user asks to forget something.",
-    args: {
-      id: z.string().min(1),
-    },
+    description: TOOL_DESCRIPTIONS.memory_forget,
+    args: memoryForgetArgs,
     async execute(args) {
-      const row = db()
-        .prepare(`SELECT scope_key FROM memories WHERE id = ?`)
-        .get(args.id) as { scope_key: string } | undefined;
-      if (!row) {
-        return { title: "memory: not found", output: `No memory with id ${args.id}.` };
-      }
-      deleteMemoryFile(row.scope_key, args.id);
-      deleteFromIndex(args.id);
-      return { title: "memory: forgotten", output: `Deleted ${args.id}.` };
+      return forgetMemory(args);
     },
   });
 
-  return { memory_add, memory_search, memory_list, memory_forget };
+  return { memory_add, memory_search, memory_list, memory_forget, memory_supersede };
 }