open-memex 0.1.0 → 0.3.0-alpha

package/src/tools/ops.ts

@@ -0,0 +1,259 @@
+/**
+ * Framework-agnostic memory operations.
+ *
+ * The same functions back both the opencode plugin tools
+ * (`src/tools/memory.ts`) and the generic MCP server (`src/mcp.ts`).
+ * They take plain validated args and return a plain { title, output }
+ * result; each host adapts that to its own tool-result shape.
+ */
+import { z } from "zod";
+import type { Scope } from "../scope.ts";
+import type { MyOMemoryConfig } from "../config.ts";
+import { PERSONAL_SCOPE } from "../scope.ts";
+import { search, list } from "../retrieve/search.ts";
+import {
+  writeMemoryFile,
+  deleteMemoryFile,
+  readMemoryFile,
+  ulid,
+  msToRfc3339,
+  MEMORY_TYPE_TAXONOMY,
+  type Frontmatter,
+} from "../store/markdown.ts";
+import { upsertFromFile, deleteFromIndex } from "../store/sync.ts";
+import { findDuplicates, supersede } from "../store/lifecycle.ts";
+import { db } from "../store/db.ts";
+import { redact } from "../redact.ts";
+
+export interface ToolResult {
+  title: string;
+  output: string;
+}
+
+/** LLM-facing tool descriptions, shared by the opencode plugin and the MCP server. */
+export const TOOL_DESCRIPTIONS = {
+  memory_add:
+    "Save a fact, preference, decision, or note to persistent local memory. Call this PROACTIVELY whenever the user shares something worth remembering across sessions — project conventions, tool choices, personal preferences, decisions made, error fixes and their causes. Do not wait to be asked. Keep each memory to one self-contained statement. Default scope is the current project; use the personal scope for facts about the user that apply across all projects.",
+  memory_search:
+    "Search persistent memory by keyword (BM25 full-text). Returns matching memories from the current project and/or personal scope. Call before asking the user about past decisions, conventions, or preferences they may have told you before — try a few keyword variants, including the user's own language, when the first search comes up empty.",
+  memory_list:
+    "List memories in a scope, newest first. Useful for browsing what is remembered, or verifying that a save landed.",
+  memory_supersede:
+    "Replace an existing memory with a newer version. The old memory is kept as history (status: superseded) and retrieval returns the new one. Use when a saved fact becomes outdated and should be replaced rather than duplicated.",
+  memory_forget: "Delete a memory by id. Use when the user asks to forget something.",
+} as const;
+
+/** Shared zod input shapes (raw shape, not z.object — hosts wrap as needed). */
+export const scopeArg = z
+  .enum(["project", "personal", "user"])
+  .optional()
+  .describe(
+    "Memory scope. `project` = tied to this repo. `personal` = global across all your projects. `user` is a deprecated alias of `personal`. Default: project.",
+  );
+
+export const memoryAddArgs = {
+  content: z.string().min(1).describe("The fact to remember. One idea per memory."),
+  type: z
+    .enum(MEMORY_TYPE_TAXONOMY)
+    .optional()
+    .describe("Category of memory. Default: note."),
+  scope: scopeArg,
+  tags: z.array(z.string()).optional().describe("Optional tags for filtering."),
+};
+export type MemoryAddArgs = z.infer<z.ZodObject<typeof memoryAddArgs>>;
+
+export const memorySearchArgs = {
+  query: z
+    .string()
+    .min(1)
+    .describe("Free-text query. File paths, error strings, identifiers work well."),
+  scope: z
+    .enum(["project", "personal", "user", "both"])
+    .optional()
+    .describe("Which scope(s) to search. Default: both."),
+  type: z.string().optional().describe("Restrict to memories of this type."),
+  limit: z.number().int().min(1).max(50).optional(),
+};
+export type MemorySearchArgs = z.infer<z.ZodObject<typeof memorySearchArgs>>;
+
+export const memoryListArgs = {
+  scope: scopeArg,
+  type: z.string().optional(),
+  limit: z.number().int().min(1).max(100).optional(),
+};
+export type MemoryListArgs = z.infer<z.ZodObject<typeof memoryListArgs>>;
+
+export const memorySupersedeArgs = {
+  id: z.string().min(1).describe("ID of the memory being replaced."),
+  content: z.string().min(1).describe("The new, corrected content."),
+  type: z
+    .enum(MEMORY_TYPE_TAXONOMY)
+    .optional()
+    .describe("Category of the new memory. Defaults to the old memory's type."),
+  tags: z.array(z.string()).optional().describe("Optional tags for the new memory."),
+};
+export type MemorySupersedeArgs = z.infer<z.ZodObject<typeof memorySupersedeArgs>>;
+
+export const memoryForgetArgs = {
+  id: z.string().min(1),
+};
+export type MemoryForgetArgs = z.infer<z.ZodObject<typeof memoryForgetArgs>>;
+
+function resolveScope(
+  getScope: () => Scope,
+  kind?: "project" | "personal" | "user",
+): Scope {
+  return kind === "personal" || kind === "user" ? PERSONAL_SCOPE : getScope();
+}
+
+function buildFrontmatter(
+  s: Scope,
+  body: {
+    type: Frontmatter["type"];
+    tags: string[];
+    source: Frontmatter["source"];
+  },
+): Frontmatter {
+  const rfc = msToRfc3339(Date.now());
+  return {
+    id: ulid(),
+    schema_version: 2,
+    scope_key: s.key,
+    scope: s.kind === "project" ? "project" : "personal",
+    visibility: s.kind === "project" ? "internal" : "private",
+    project_name: s.projectName,
+    type: body.type,
+    role: "knowledge",
+    importance: "normal",
+    status: "active",
+    tags: body.tags,
+    source: body.source,
+    created_at: rfc,
+    updated_at: rfc,
+    supersedes: null,
+    superseded_by: null,
+  };
+}
+
+export async function addMemory(
+  getScope: () => Scope,
+  cfg: MyOMemoryConfig,
+  args: MemoryAddArgs,
+): Promise<ToolResult> {
+  const { content: redacted, hadSecret, matchedPattern } = redact(
+    args.content,
+    cfg.redactPatterns,
+  );
+  const secretNote = hadSecret
+    ? `\nNote: content matched a secret pattern (${matchedPattern}); it was saved with the secret masked (first 4 chars kept, rest replaced with x).`
+    : "";
+  const s = resolveScope(getScope, args.scope);
+  // Dedup on write (§3.4): identical content is idempotent; near-duplicates
+  // are reported so the caller can supersede instead of duplicating.
+  const dups = findDuplicates(s.key, redacted);
+  if (dups.exact) {
+    return {
+      title: "memory: already exists",
+      output: `Identical memory already exists: id=${dups.exact.id}. Not duplicated.`,
+    };
+  }
+  const fm = buildFrontmatter(s, {
+    type: args.type ?? "fact",
+    tags: args.tags ?? [],
+    source: "tool",
+  });
+  const { filePath } = writeMemoryFile(fm, redacted);
+  const mf = readMemoryFile(filePath);
+  if (mf) upsertFromFile(mf);
+  let output = `Saved to ${s.kind} scope (${s.key}) as ${fm.type}. id=${fm.id}${secretNote}`;
+  for (const n of dups.near) {
+    output += `\nNote: similar memory exists (score ${n.score.toFixed(2)}): id=${n.id} — ${n.snippet}. Use memory_supersede if this replaces it.`;
+  }
+  return { title: `memory: saved ${fm.id}`, output };
+}
+
+export async function searchMemories(
+  getScope: () => Scope,
+  args: MemorySearchArgs,
+): Promise<ToolResult> {
+  const project = getScope();
+  const scopeKeys =
+    args.scope === "personal" || args.scope === "user"
+      ? [PERSONAL_SCOPE.key]
+      : args.scope === "project"
+        ? [project.key]
+        : [project.key, PERSONAL_SCOPE.key];
+  const hits = search(args.query, {
+    scopeKeys,
+    limit: args.limit,
+    type: args.type,
+  });
+  if (hits.length === 0) {
+    return {
+      title: "memory: 0 results",
+      output: `No memories matched "${args.query}".`,
+    };
+  }
+  const lines = hits.map(
+    (h) =>
+      `- [${h.scope_key === PERSONAL_SCOPE.key ? "personal" : "project"}/${h.type}] id=${h.id}\n  ${h.snippet.replace(/\s+/g, " ").trim()}`,
+  );
+  return { title: `memory: ${hits.length} result(s)`, output: lines.join("\n") };
+}
+
+export async function listMemories(
+  getScope: () => Scope,
+  args: MemoryListArgs,
+): Promise<ToolResult> {
+  const s = resolveScope(getScope, args.scope);
+  const hits = list(s.key, { type: args.type, limit: args.limit });
+  if (hits.length === 0) {
+    return { title: "memory: empty", output: `No memories in scope ${s.key}.` };
+  }
+  const lines = hits.map(
+    (h) => `- [${h.type}] id=${h.id} — ${h.snippet.replace(/\s+/g, " ").trim()}`,
+  );
+  return { title: `memory: ${hits.length} in ${s.key}`, output: lines.join("\n") };
+}
+
+export async function supersedeMemory(
+  cfg: MyOMemoryConfig,
+  args: MemorySupersedeArgs,
+): Promise<ToolResult> {
+  const { content: redacted, hadSecret, matchedPattern } = redact(
+    args.content,
+    cfg.redactPatterns,
+  );
+  const secretNote = hadSecret ? ` (secret masked: ${matchedPattern})` : "";
+  try {
+    const { oldMf, newMf } = supersede(args.id, {
+      body: redacted,
+      type: args.type,
+      tags: args.tags,
+      source: "tool",
+    });
+    upsertFromFile(oldMf);
+    upsertFromFile(newMf);
+    return {
+      title: `memory: superseded ${oldMf.fm.id}`,
+      output: `Replaced ${oldMf.fm.id} with ${newMf.fm.id} (old kept as history).${secretNote}`,
+    };
+  } catch (e) {
+    return {
+      title: "memory: supersede failed",
+      output: (e as Error).message,
+    };
+  }
+}
+
+export async function forgetMemory(args: MemoryForgetArgs): Promise<ToolResult> {
+  const row = db()
+    .prepare(`SELECT scope_key FROM memories WHERE id = ?`)
+    .get(args.id) as { scope_key: string } | undefined;
+  if (!row) {
+    return { title: "memory: not found", output: `No memory with id ${args.id}.` };
+  }
+  deleteMemoryFile(row.scope_key, args.id);
+  deleteFromIndex(args.id);
+  return { title: "memory: forgotten", output: `Deleted ${args.id}.` };
+}

package/PLAN.md

@@ -1,168 +0,0 @@
-# open-memex — Design & Roadmap
-
-## Goal
-
-A local-only persistent memory plugin for opencode. No SaaS, no account, no
-network calls. Memories live as human-readable markdown files, indexed by
-SQLite FTS5 for BM25 keyword search.
-
-## Non-goals (MVP)
-
-- No cloud sync
-- No embeddings / vector search (v2)
-- No LLM-driven memory extraction (v2)
-- No knowledge graph / entity linking
-- No multi-user collaboration
-
-## Architecture
-
-```
-+-----------------------------+
-|         opencode (Bun)      |
-|                             |
-|  chat.message hook  ------> keyword capture ---+
-|  system.transform   ------> context inject <---|--- memory files
-|  tool.memory_*      ------> add/search/list/forget    (source of truth)
-+-----------------------------+                  |
-                                                 v
-                                        +----------------+
-                                        |  SQLite index  |
-                                        | (FTS5, BM25)   |
-                                        +----------------+
-```
-
-- `memories` — one file per memory. YAML frontmatter + body.
-- `index.db` — SQLite (via `better-sqlite3`) with FTS5 virtual table over content/tags/type.
-- Loads under opencode's embedded Bun runtime (better-sqlite3 works via N-API compat).
-- CLI runs under Node 22+ with `--experimental-strip-types` — no Bun required, no build step.
-
-## Data model
-
-### File on disk
-
-`memories/<scope_key>/<id>.md`
-
-```markdown
----
-id: 01HZ...
-scope_key: project__open-memex__a1b2c3d4e5f6
-scope_kind: project
-project_name: open-memex
-type: project-config
-tags: [bun, sqlite]
-source: tool
-created_at: 1770000000000
-updated_at: 1770000000000
----
-
-This project runs under Bun and uses bun:sqlite. Do not add better-sqlite3.
-```
-
-### SQLite tables
-
-- `memories(id PK, scope_key, scope_kind, project_name, type, tags CSV, content, source, file_path, mtime_ms, created_at, updated_at)`
-- `memories_fts` — FTS5 virtual table over `content`, `tags`, `type`, external-content mode pointing at `memories.rowid`.
-- Triggers keep `memories_fts` in sync on insert/update/delete.
-
-## Scoping
-
-- **user** scope key: literal `"user"`
-- **project** scope key: `project__<sanitized-name>__<12-hex-of-sha256>`
-  - Seed: normalized git origin URL if present, else absolute cwd path (lowercased)
-  - Same repo across machines -> same key, so memories can be `git`-committed (future)
-
-### Scope-key drift and migration
-
-Because the scope key changes when a repo's git origin appears or changes,
-memories captured *before* a remote was added end up orphaned under the
-cwd-based key. The plugin logs a one-shot warning at load when it detects
-this, and ships two CLI commands to resolve it:
-
-- `cli scopes` — list every project scope directory with a file count.
-- `cli migrate --from <old> [--to <current>] [--dry-run] [--on-conflict newer|overwrite|skip]`
-  — rewrites `scope_key` in each file's frontmatter, moves the files, and
-  reindexes. Default conflict strategy is "keep the newer `updated_at`".
-
-Auto-migration on load is intentionally *not* done: two unrelated repos at
-the same cwd would silently merge.
-
-## Capture mechanisms
-
-MVP:
-1. **Explicit tool call** — agent calls `memory_add`.
-2. **Keyword trigger** — regex against the user message parts. Captured group 1 becomes the memory body. Defaults: `remember`, `note that`, `TIL`, `save this`.
-
-Not in MVP:
-3. Auto-capture every N turns via LLM summary.
-4. LLM-decided "should I recall?" per turn (supermemory's reasoned-recall).
-
-## Retrieval
-
-MVP: **BM25 via SQLite FTS5.** Query is tokenized, punctuation stripped, each token becomes a prefix match. `bm25(memories_fts)` used as the score. Snippets via `snippet(memories_fts, ...)`.
-
-Not in MVP:
-- Local embeddings (Transformers.js `Xenova/bge-small-en-v1.5`) + sqlite-vec.
-- Reciprocal Rank Fusion across BM25 + vector lanes.
-- Cross-encoder rerank (FastEmbed jina-reranker-tiny).
-
-## Context injection
-
-On the first `experimental.chat.system.transform` for each session, push a
-`[OPEN-MEMEX]` block containing:
-- User profile / preferences (top N from `user` scope)
-- Project knowledge (top N from current project scope, newest first)
-
-Injected once per session per plugin load (in-memory set keyed by sessionID).
-
-## Redaction
-
-Applied to any content on the way in (tool + keyword capture):
-1. Strip `<private>...</private>` -> `[REDACTED]`.
-2. Match remaining text against secret regex list (OpenAI, GitHub, AWS, Slack...). If any hit -> refuse write, tell agent to redact.
-
-## Config surface
-
-`~/.config/opencode/open-memex.jsonc`:
-
-```jsonc
-{
-  "maxProjectMemories": 8,
-  "maxProfileItems": 5,
-  "injectOnFirstTurn": true,
-  "keywordCaptureEnabled": true,
-  "keywordPatterns": ["^\\s*remember...", ...],
-  "redactPatterns": ["sk-[A-Za-z0-9]{20,}", ...],
-  "logLevel": "info"
-}
-```
-
-Env overrides:
-- `MY_O_MEMORY_HOME` — override storage root
-- `MY_O_MEMORY_CONFIG` — override config file path
-
-## v2 roadmap (in rough priority)
-
-1. **Local embeddings** — Xenova/bge-small ONNX via Transformers.js; sqlite-vec table; RRF fuse with BM25.
-2. **Auto-capture** every N turns using the host LLM (opt-in).
-3. **Preemptive compaction hook** — inject memories into `experimental.session.compacting` context.
-4. **Deduplication** on write (hash + fuzzy).
-5. **`/memory-init`** slash command — walk the repo, summarize modules, save.
-6. **Recall receipts** — return which lane matched (BM25 / vector / recency) for debuggability.
-7. **Bitemporal** valid_from/valid_to for facts that supersede each other.
-8. **Admission barrier** — reject retrieved memories from being re-ingested, reject assistant self-writes, block prompt-injection in file content.
-
-## v3+
-
-- Ollama detection -> use `nomic-embed-text` when present.
-- Cross-encoder rerank.
-- Memory decay / consolidation.
-- Import/export to markdown archives.
-- Optional knowledge-graph layer for entity linking.
-
-## Prior art referenced
-
-- `opencode-supermemory` — plugin lifecycle, capture heuristics
-- `doobidoo/mcp-memory-service` — local sqlite-vec + BM25 hybrid, decay
-- `memoripy` — RRF, admission barrier, dependency-free hash embeddings
-- `basic-memory` — markdown-first store, obsidian-compatible
-- Anthropic client-managed memory tool pattern — filesystem primitives