open-memex 0.1.0 → 0.3.0-alpha

package/src/store/db.ts

@@ -23,18 +23,25 @@ function loadDatabase(): AnyDatabaseCtor {
 
 let _db: AnyDatabase | null = null;
 
-const SCHEMA = `
+const TABLE_SCHEMA = `
 CREATE TABLE IF NOT EXISTS memories (
   id           TEXT PRIMARY KEY,
   scope_key    TEXT NOT NULL,
-  scope_kind   TEXT NOT NULL,
-  project_name TEXT NOT NULL,
-  type         TEXT NOT NULL DEFAULT 'note',
+  scope        TEXT NOT NULL,
+  visibility   TEXT NOT NULL DEFAULT 'private',
+  project_name TEXT NOT NULL DEFAULT '',
+  type         TEXT NOT NULL DEFAULT 'fact',
+  role         TEXT NOT NULL DEFAULT 'knowledge',
+  importance   TEXT NOT NULL DEFAULT 'normal',
+  status       TEXT NOT NULL DEFAULT 'active',
   tags         TEXT NOT NULL DEFAULT '',
   content      TEXT NOT NULL,
+  cjk          TEXT NOT NULL DEFAULT '',
+  content_hash TEXT NOT NULL DEFAULT '',
+  superseded_by TEXT,
   source       TEXT NOT NULL DEFAULT '',
   file_path    TEXT NOT NULL,
-  mtime_ms     INTEGER NOT NULL,
+  mtime_ms     REAL NOT NULL,
   created_at   INTEGER NOT NULL,
   updated_at   INTEGER NOT NULL
 );
@@ -42,11 +49,20 @@ CREATE TABLE IF NOT EXISTS memories (
 CREATE INDEX IF NOT EXISTS idx_memories_scope_updated
   ON memories(scope_key, updated_at DESC);
 CREATE INDEX IF NOT EXISTS idx_memories_type ON memories(type);
+CREATE INDEX IF NOT EXISTS idx_memories_status ON memories(status);
+`;
 
+// The `cjk` column holds pre-tokenized CJK unigrams+bigrams (see
+// src/retrieve/cjk.ts). FTS5's unicode61 treats a CJK run as one token, so
+// without this column CJK substring search cannot work. Kept as a separate
+// column (rather than a custom tokenizer) so both runtimes stay on stock
+// SQLite with zero native dependencies.
+const FTS_SCHEMA = `
 CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
   content,
   tags,
   type,
+  cjk,
   scope_key UNINDEXED,
   content='memories',
   content_rowid='rowid',
@@ -54,23 +70,53 @@ CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
 );
 
 CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
-  INSERT INTO memories_fts(rowid, content, tags, type, scope_key)
-  VALUES (new.rowid, new.content, new.tags, new.type, new.scope_key);
+  INSERT INTO memories_fts(rowid, content, tags, type, cjk, scope_key)
+  VALUES (new.rowid, new.content, new.tags, new.type, new.cjk, new.scope_key);
 END;
 
 CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN
-  INSERT INTO memories_fts(memories_fts, rowid, content, tags, type, scope_key)
-  VALUES ('delete', old.rowid, old.content, old.tags, old.type, old.scope_key);
+  INSERT INTO memories_fts(memories_fts, rowid, content, tags, type, cjk, scope_key)
+  VALUES ('delete', old.rowid, old.content, old.tags, old.type, old.cjk, old.scope_key);
 END;
 
 CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN
-  INSERT INTO memories_fts(memories_fts, rowid, content, tags, type, scope_key)
-  VALUES ('delete', old.rowid, old.content, old.tags, old.type, old.scope_key);
-  INSERT INTO memories_fts(rowid, content, tags, type, scope_key)
-  VALUES (new.rowid, new.content, new.tags, new.type, new.scope_key);
+  INSERT INTO memories_fts(memories_fts, rowid, content, tags, type, cjk, scope_key)
+  VALUES ('delete', old.rowid, old.content, old.tags, old.type, old.cjk, old.scope_key);
+  INSERT INTO memories_fts(rowid, content, tags, type, cjk, scope_key)
+  VALUES (new.rowid, new.content, new.tags, new.type, new.cjk, new.scope_key);
 END;
 `;
 
+/** Current index schema version. Bump when TABLE_SCHEMA/FTS_SCHEMA change. */
+const SCHEMA_VERSION = 5;
+
+function userVersion(d: AnyDatabase): number {
+  const row = d.prepare("PRAGMA user_version").get() as { user_version: number };
+  return row.user_version;
+}
+
+/**
+ * Any schema change: the query layer is fully derived from markdown (D1),
+ * so wipe it and let the next syncScope() repopulate. The markdown files
+ * themselves are untouched — `migrate --to-v2` handles the file format.
+ * NOTE ordering matters: triggers are dropped BEFORE the wipe, and the FTS
+ * table is rebuilt after. FTS5's 'delete' command corrupts
+ * (SQLITE_CORRUPT_VTAB) when it targets a rowid that was never indexed, so
+ * the wipe must not fire any FTS trigger while the index is out of sync
+ * with the table (found 2026-09-26).
+ */
+function rebuildIndexSchema(d: AnyDatabase): void {
+  d.exec(`DROP TRIGGER IF EXISTS memories_ai;
+          DROP TRIGGER IF EXISTS memories_ad;
+          DROP TRIGGER IF EXISTS memories_au;`);
+  d.exec("DELETE FROM memories");
+  d.exec(`DROP TABLE IF EXISTS memories_fts;`);
+  d.exec(`DROP TABLE IF EXISTS memories;`);
+  d.exec(TABLE_SCHEMA);
+  d.exec(FTS_SCHEMA);
+  d.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`);
+}
+
 export function db(): AnyDatabase {
   if (_db) return _db;
   const { indexDb } = paths();
@@ -79,7 +125,9 @@ export function db(): AnyDatabase {
   d.exec("PRAGMA journal_mode = WAL;");
   d.exec("PRAGMA synchronous = NORMAL;");
   d.exec("PRAGMA foreign_keys = ON;");
-  d.exec(SCHEMA);
+  d.exec(TABLE_SCHEMA);
+  d.exec(FTS_SCHEMA);
+  if (userVersion(d) < SCHEMA_VERSION) rebuildIndexSchema(d);
   _db = d;
   return d;
 }

package/src/store/lifecycle.ts

@@ -0,0 +1,280 @@
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { db } from "./db.ts";
+import { paths, memoriesDirPath } from "../paths.ts";
+import { cjkIndexText } from "../retrieve/cjk.ts";
+import {
+  parse,
+  serialize,
+  ulid,
+  msToRfc3339,
+  type Frontmatter,
+  type MemoryFile,
+  type MemoryStatus,
+} from "./markdown.ts";
+
+/**
+ * Dedup + lifecycle (V2-DESIGN §3.3, §3.4).
+ *
+ * - Dedup on write: exact content hash → idempotent add; fuzzy token
+ *   overlap (Jaccard ≥ 0.8, CJK-bigram aware) → near-duplicate notice.
+ * - Lifecycle: active → superseded | deprecated | retracted | archived.
+ *   Superseding never overwrites: the old memory keeps its history with a
+ *   bidirectional supersedes/superseded_by chain. Retrieval (§3.3) returns
+ *   only the newest of a chain and excludes retracted/archived.
+ */
+
+/** sha256 of whitespace-normalized body. Stored in the index (derived). */
+export function contentHash(body: string): string {
+  const norm = body.replace(/\s+/g, " ").trim();
+  return createHash("sha256").update(norm, "utf8").digest("hex");
+}
+
+function tokenSet(text: string): Set<string> {
+  const latin = text.toLowerCase().match(/[a-z0-9_\-]+/g) ?? [];
+  const cjk = cjkIndexText(text).split(" ").filter(Boolean);
+  return new Set([...latin, ...cjk]);
+}
+
+/** Jaccard similarity over latin tokens + CJK bigrams. 1 = identical. */
+export function similarity(a: string, b: string): number {
+  const sa = tokenSet(a);
+  const sb = tokenSet(b);
+  if (sa.size === 0 || sb.size === 0) return 0;
+  let inter = 0;
+  for (const t of sa) if (sb.has(t)) inter++;
+  const union = sa.size + sb.size - inter;
+  return union === 0 ? 0 : inter / union;
+}
+
+export interface DuplicateCandidate {
+  id: string;
+  score: number;
+  snippet: string;
+}
+
+export interface DupResult {
+  exact?: DuplicateCandidate;
+  near: DuplicateCandidate[];
+}
+
+export const NEAR_DUP_THRESHOLD = 0.8;
+
+interface IndexRow {
+  id: string;
+  content_hash: string;
+  content: string;
+}
+
+/** Find exact + near duplicates of `body` among ACTIVE memories in scope. */
+export function findDuplicates(
+  scopeKey: string,
+  body: string,
+  excludeId?: string,
+): DupResult {
+  const hash = contentHash(body);
+  const rows = db()
+    .prepare(
+      `SELECT id, content_hash, content FROM memories
+       WHERE scope_key = ? AND status = 'active'`,
+    )
+    .all(scopeKey) as IndexRow[];
+
+  let exact: DuplicateCandidate | undefined;
+  const near: DuplicateCandidate[] = [];
+  for (const r of rows) {
+    if (r.id === excludeId) continue;
+    const snippet = r.content.replace(/\s+/g, " ").trim().slice(0, 120);
+    if (r.content_hash && r.content_hash === hash) {
+      exact = { id: r.id, score: 1, snippet };
+      continue;
+    }
+    const score = similarity(body, r.content);
+    if (score >= NEAR_DUP_THRESHOLD) near.push({ id: r.id, score, snippet });
+  }
+  near.sort((x, y) => y.score - x.score);
+  return { exact, near: near.slice(0, 3) };
+}
+
+/** Locate a memory file by id: index first, then a full dir scan fallback. */
+export function findMemoryFile(id: string): MemoryFile | null {
+  const { memories } = paths();
+  let filePath: string | null = null;
+  try {
+    const row = db()
+      .prepare(`SELECT scope_key FROM memories WHERE id = ?`)
+      .get(id) as { scope_key: string } | undefined;
+    if (row) {
+      const p = path.join(memoriesDirPath(row.scope_key), `${id}.md`);
+      if (fs.existsSync(p)) filePath = p;
+    }
+  } catch {
+    // index unavailable — fall through to scan
+  }
+  if (!filePath && fs.existsSync(memories)) {
+    for (const entry of fs.readdirSync(memories, { withFileTypes: true })) {
+      if (!entry.isDirectory()) continue;
+      const p = path.join(memories, entry.name, `${id}.md`);
+      if (fs.existsSync(p)) {
+        filePath = p;
+        break;
+      }
+    }
+  }
+  if (!filePath) return null;
+  const raw = fs.readFileSync(filePath, "utf8");
+  const parsed = parse(raw);
+  if (!parsed) return null;
+  const st = fs.statSync(filePath);
+  return { fm: parsed.fm, body: parsed.body, filePath, mtimeMs: st.mtimeMs };
+}
+
+export function rewriteMemoryFile(mf: MemoryFile): void {
+  fs.writeFileSync(mf.filePath, serialize(mf.fm, mf.body), "utf8");
+}
+
+export interface ChainRepairResult {
+  warnings: string[];
+  /** Linked memories whose files were rewritten; caller should re-index them. */
+  repaired: MemoryFile[];
+}
+
+/**
+ * Chain integrity (§3.3): supersedes/superseded_by must be pairwise
+ * consistent. A missing side is auto-completed and reported — a broken chain
+ * must never silently degrade retrieval. Dangling pointers (target file
+ * gone) can only be reported.
+ */
+export function repairChain(mf: MemoryFile): ChainRepairResult {
+  const warnings: string[] = [];
+  const repaired: MemoryFile[] = [];
+  const fm = mf.fm;
+
+  if (fm.supersedes) {
+    const prev = findMemoryFile(fm.supersedes);
+    if (!prev) {
+      warnings.push(
+        `${fm.id}: supersedes target ${fm.supersedes} not found (dangling)`,
+      );
+    } else if (prev.fm.superseded_by !== fm.id) {
+      prev.fm.superseded_by = fm.id;
+      if (prev.fm.status === "active") prev.fm.status = "superseded";
+      prev.fm.updated_at = msToRfc3339(Date.now());
+      rewriteMemoryFile(prev);
+      repaired.push(prev);
+      warnings.push(
+        `${fm.id}: auto-completed ${prev.fm.id}.superseded_by → ${fm.id}`,
+      );
+    }
+  }
+
+  if (fm.superseded_by) {
+    const next = findMemoryFile(fm.superseded_by);
+    if (!next) {
+      warnings.push(
+        `${fm.id}: superseded_by target ${fm.superseded_by} not found (dangling)`,
+      );
+    } else if (next.fm.supersedes !== fm.id) {
+      next.fm.supersedes = fm.id;
+      next.fm.updated_at = msToRfc3339(Date.now());
+      rewriteMemoryFile(next);
+      repaired.push(next);
+      warnings.push(
+        `${fm.id}: auto-completed ${next.fm.id}.supersedes → ${fm.id}`,
+      );
+    }
+  }
+
+  return { warnings, repaired };
+}
+
+export interface SupersedeInput {
+  body: string;
+  type?: string;
+  tags?: string[];
+  source?: string;
+}
+
+/**
+ * Replace an active memory with a new one. The old memory is NOT overwritten:
+ * it becomes `status: superseded` with a forward pointer; the new memory
+ * points back. History preserved; retrieval returns the newest (§3.4).
+ */
+export function supersede(
+  oldId: string,
+  input: SupersedeInput,
+): { oldMf: MemoryFile; newMf: MemoryFile } {
+  const oldMf = findMemoryFile(oldId);
+  if (!oldMf) throw new Error(`no memory with id ${oldId}`);
+  if (oldMf.fm.status !== "active") {
+    throw new Error(
+      `cannot supersede memory with status '${oldMf.fm.status}' (id ${oldId}); only active memories can be superseded`,
+    );
+  }
+
+  const now = msToRfc3339(Date.now());
+  const newFm: Frontmatter = {
+    ...oldMf.fm,
+    id: ulid(),
+    type: input.type ?? oldMf.fm.type,
+    tags: input.tags ?? oldMf.fm.tags,
+    source: input.source ?? oldMf.fm.source,
+    status: "active",
+    created_at: now,
+    updated_at: now,
+    supersedes: oldId,
+    superseded_by: null,
+  };
+  const dir = path.dirname(oldMf.filePath);
+  const newPath = path.join(dir, `${newFm.id}.md`);
+  fs.writeFileSync(newPath, serialize(newFm, input.body), "utf8");
+  const st = fs.statSync(newPath);
+  const newMf: MemoryFile = {
+    fm: newFm,
+    body: input.body,
+    filePath: newPath,
+    mtimeMs: st.mtimeMs,
+  };
+
+  oldMf.fm.status = "superseded";
+  oldMf.fm.superseded_by = newFm.id;
+  oldMf.fm.updated_at = now;
+  rewriteMemoryFile(oldMf);
+
+  return { oldMf, newMf };
+}
+
+const SETTABLE_STATUSES: ReadonlyArray<MemoryStatus> = [
+  "active",
+  "deprecated",
+  "retracted",
+  "archived",
+];
+
+export function isSettableStatus(s: string): s is MemoryStatus {
+  return (SETTABLE_STATUSES as ReadonlyArray<string>).includes(s);
+}
+
+/**
+ * Direct lifecycle transition. `superseded` is NOT settable here — it is
+ * managed exclusively by `supersede()` so the chain stays consistent.
+ */
+export function setStatus(id: string, status: MemoryStatus): MemoryFile {
+  if (!isSettableStatus(status)) {
+    throw new Error(
+      `invalid status '${status}'; use one of: ${SETTABLE_STATUSES.join(", ")}`,
+    );
+  }
+  const mf = findMemoryFile(id);
+  if (!mf) throw new Error(`no memory with id ${id}`);
+  if (mf.fm.status === "superseded") {
+    throw new Error(
+      `memory ${id} is superseded (chain-managed); supersede it again instead of changing status directly`,
+    );
+  }
+  mf.fm.status = status;
+  mf.fm.updated_at = msToRfc3339(Date.now());
+  rewriteMemoryFile(mf);
+  return mf;
+}

package/src/store/markdown.ts

@@ -4,16 +4,54 @@ import { randomBytes } from "node:crypto";
 import yaml from "js-yaml";
 import { memoriesDirFor, memoriesDirPath } from "../paths.ts";
 
+/** v2 content-kind taxonomy (V2-DESIGN §3.1). `type` = what the memory IS. */
+export const MEMORY_TYPE_TAXONOMY = [
+  "preference",
+  "fact",
+  "decision",
+  "lesson",
+  "warning",
+  "workflow",
+  "architecture",
+  "constraint",
+  "todo",
+  "knowledge",
+  "observation",
+] as const;
+
+const TAXONOMY = new Set<string>(MEMORY_TYPE_TAXONOMY);
+
+export type ScopeKind = "personal" | "project" | "org";
+export type Visibility = "private" | "internal" | "shared";
+export type MemoryRole = "knowledge" | "instruction";
+export type Importance = "low" | "normal" | "high";
+export type MemoryStatus =
+  | "active"
+  | "superseded"
+  | "deprecated"
+  | "retracted"
+  | "archived";
+
+/** v2 frontmatter (V2-DESIGN §3). Markdown is the source of truth; the
+ *  SQLite index is derived and rebuildable (D1). `scope_key` is kept as the
+ *  local storage address; `scope` is the semantic ownership. */
 export interface Frontmatter {
   id: string;
+  schema_version: 2;
   scope_key: string;
-  scope_kind: "user" | "project";
+  scope: ScopeKind;
+  visibility: Visibility;
   project_name: string;
   type: string;
+  role: MemoryRole;
+  importance: Importance;
+  status: MemoryStatus;
   tags: string[];
   source: string;
-  created_at: number;
-  updated_at: number;
+  created_at: string; // RFC 3339, never bare epoch (§3)
+  updated_at: string; // RFC 3339
+  supersedes: string | null; // on the NEW memory → points BACK (§3.3)
+  superseded_by: string | null; // on the OLD memory → points FORWARD
 }
 
 export interface MemoryFile {
@@ -39,28 +77,142 @@ export function ulid(): string {
   return ts + rand;
 }
 
+/** epoch ms → RFC 3339 (v2 times). */
+export function msToRfc3339(ms: number): string {
+  return new Date(ms).toISOString().replace(/\.000Z$/, "Z");
+}
+
+/** RFC 3339 (or epoch ms) → epoch ms. Falls back to now on garbage. */
+export function timeToMs(v: unknown): number {
+  if (typeof v === "number" && Number.isFinite(v)) return Math.round(v);
+  if (typeof v === "string") {
+    const t = Date.parse(v);
+    if (Number.isFinite(t)) return t;
+  }
+  return Date.now();
+}
+
+function asScopeKind(v: unknown, scopeKind: unknown): ScopeKind {
+  if (v === "personal" || v === "project" || v === "org") return v;
+  if (scopeKind === "user") return "personal"; // v1 → v2 (§19)
+  if (scopeKind === "project") return "project";
+  return "personal";
+}
+
+function asVisibility(v: unknown, scope: ScopeKind): Visibility {
+  if (v === "private" || v === "internal" || v === "shared") return v;
+  // v2 defaults (§4): personal stays local, project is internal.
+  return scope === "personal" ? "private" : "internal";
+}
+
+function asRole(v: unknown): MemoryRole {
+  if (v === "knowledge" || v === "instruction") return v;
+  return "knowledge";
+}
+
+function asImportance(v: unknown, priority: unknown): Importance {
+  if (v === "low" || v === "normal" || v === "high") return v;
+  // v1 `priority: N` → importance (§19): 1–3 low, 8–10 high, else normal.
+  if (typeof priority === "number" && Number.isFinite(priority)) {
+    if (priority <= 3) return "low";
+    if (priority >= 8) return "high";
+  }
+  return "normal";
+}
+
+function asStatus(v: unknown): MemoryStatus {
+  if (
+    v === "active" ||
+    v === "superseded" ||
+    v === "deprecated" ||
+    v === "retracted" ||
+    v === "archived"
+  )
+    return v;
+  return "active";
+}
+
+/**
+ * Normalize raw (possibly v1) frontmatter into v2 shape. Lenient on read:
+ * v1 files (epoch times, scope_kind, priority, type: instruction) are mapped
+ * per §19 so old files keep working even before `migrate --to-v2` rewrites
+ * them. Use `planConversion` in v2migrate.ts for the explicit, reporting
+ * migration path.
+ */
+export function normalizeFrontmatter(
+  raw: Record<string, unknown>,
+): Frontmatter {
+  const scope = asScopeKind(raw.scope, raw.scope_kind);
+  let scopeKey =
+    typeof raw.scope_key === "string" && raw.scope_key ? raw.scope_key : scope;
+  if (scopeKey === "user") scopeKey = "personal"; // v1 storage dir → v2
+
+  let type = typeof raw.type === "string" && raw.type ? raw.type : "fact";
+  let role = asRole(raw.role);
+  if (type === "instruction") {
+    // §19: v1 `type: instruction` → content-kind + role split (D11).
+    type = "knowledge";
+    role = "instruction";
+  }
+
+  return {
+    id: String(raw.id),
+    schema_version: 2,
+    scope_key: scopeKey,
+    scope,
+    visibility: asVisibility(raw.visibility, scope),
+    project_name:
+      typeof raw.project_name === "string" ? raw.project_name : scopeKey,
+    type,
+    role,
+    importance: asImportance(raw.importance, raw.priority),
+    status: asStatus(raw.status),
+    tags: Array.isArray(raw.tags)
+      ? raw.tags.filter((t): t is string => typeof t === "string")
+      : [],
+    source: typeof raw.source === "string" ? raw.source : "",
+    created_at: msToRfc3339(timeToMs(raw.created_at)),
+    updated_at: msToRfc3339(timeToMs(raw.updated_at)),
+    supersedes:
+      typeof raw.supersedes === "string" && raw.supersedes ? raw.supersedes : null,
+    superseded_by:
+      typeof raw.superseded_by === "string" && raw.superseded_by
+        ? raw.superseded_by
+        : null,
+  };
+}
+
+export function isTaxonomyType(t: string): boolean {
+  return TAXONOMY.has(t);
+}
+
 export function serialize(fm: Frontmatter, body: string): string {
   const yml = yaml.dump(fm, { lineWidth: -1, quotingType: '"' });
   return `---\n${yml}---\n\n${body.trimEnd()}\n`;
 }
 
-export function parse(raw: string): { fm: Frontmatter; body: string } | null {
+/** Raw frontmatter parse (no normalization) — for migration tooling. */
+export function parseRawFrontmatter(
+  raw: string,
+): { rawFm: Record<string, unknown>; body: string } | null {
   const m = raw.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/);
   if (!m) return null;
   try {
-    const fm = yaml.load(m[1]!) as Frontmatter;
-    if (!fm || typeof fm !== "object" || !fm.id) return null;
-    // Normalize
-    fm.tags = Array.isArray(fm.tags) ? fm.tags : [];
-    fm.type = fm.type ?? "note";
-    fm.source = fm.source ?? "";
+    const rawFm = yaml.load(m[1]!) as Record<string, unknown>;
+    if (!rawFm || typeof rawFm !== "object" || !rawFm.id) return null;
     const body = (m[2] ?? "").replace(/^\n+/, "");
-    return { fm, body };
+    return { rawFm, body };
   } catch {
     return null;
   }
 }
 
+export function parse(raw: string): { fm: Frontmatter; body: string } | null {
+  const parsed = parseRawFrontmatter(raw);
+  if (!parsed) return null;
+  return { fm: normalizeFrontmatter(parsed.rawFm), body: parsed.body };
+}
+
 export function writeMemoryFile(
   fm: Frontmatter,
   body: string,