@moxxy/plugin-memory 0.27.0

package/src/store.ts

@@ -0,0 +1,292 @@
+import { promises as fs } from 'node:fs';
+import * as path from 'node:path';
+import { createMutex, type EmbeddingProvider, type Mutex } from '@moxxy/sdk';
+import { moxxyPath, writeFileAtomic } from '@moxxy/sdk/server';
+import { renderFrontmatter } from './parse.js';
+import { TfIdfEmbedder } from './tfidf.js';
+import { EmbeddingIndex } from './embedding-cache.js';
+import {
+  memoryFrontmatterSchema,
+  type MemoryEntry,
+  type MemoryFrontmatter,
+  type MemoryType,
+  type RecallMode,
+} from './store/types.js';
+import { isEnoent, listEntries, readEntry, safeRead, writeIndex, type IndexRow } from './store/io.js';
+import { rankByKeywords, recallVector, type RankedMemory } from './store/search.js';
+
+export {
+  memoryTypeSchema,
+  memoryFrontmatterSchema,
+  type MemoryEntry,
+  type MemoryFrontmatter,
+  type MemoryType,
+  type RecallMode,
+} from './store/types.js';
+export type { RankedMemory } from './store/search.js';
+
+export interface MemoryStoreOptions {
+  readonly dir?: string;
+  /**
+   * Optional embedding provider. When supplied, `recall()` uses cosine
+   * similarity over dense vectors. When omitted, the built-in TF-IDF
+   * embedder is used. Pass `embedder: null` to force keyword-only recall.
+   *
+   * May also be a lazy resolver `() => EmbeddingProvider | null`, resolved once
+   * on first recall — so the host can wire the registry-selected embedder
+   * (`() => session.embedders.tryGetActive()`) that isn't known until plugins
+   * have loaded, without forcing the store to be built after the session.
+   */
+  readonly embedder?: EmbeddingProvider | null | (() => EmbeddingProvider | null);
+  /**
+   * Cache computed embeddings on disk (`<dir>/.embeddings.json`) so unchanged
+   * memories aren't re-embedded on every recall. Defaults to true for all
+   * embedders EXCEPT TF-IDF (which derives vocab from the whole corpus, so
+   * per-entry caching doesn't help).
+   */
+  readonly persistEmbeddings?: boolean;
+  /**
+   * Soft cap on the number of stored memories. WARN-ONLY by design: memories
+   * are deliberately-saved user knowledge consumed both via `recall` and via
+   * the MEMORY.md index agents read directly, so silent oldest-eviction would
+   * be silent data loss. When a save pushes the store past this cap, the save
+   * still succeeds but a warning is logged and surfaced through
+   * {@link MemoryStore.capStatus} (the `memory_save` tool relays it to the
+   * model so it can consolidate or `memory_forget` stale entries).
+   */
+  readonly maxMemories?: number;
+}
+
+/** Default soft cap — see {@link MemoryStoreOptions.maxMemories}. */
+export const DEFAULT_MAX_MEMORIES = 500;
+
+export function defaultMemoryDir(): string {
+  return moxxyPath('memory');
+}
+
+export class MemoryStore {
+  readonly dir: string;
+  private readonly resolveEmbedder: () => EmbeddingProvider | null;
+  private readonly persistOpt: boolean | undefined;
+  // Embedder + index are resolved LAZILY on first recall (memoized), so a host
+  // that selects the embedder from a registry after plugins load can pass a
+  // resolver here without the store needing to be built after the session.
+  private embedderCache: EmbeddingProvider | null | undefined;
+  private indexCache: EmbeddingIndex | null | undefined;
+  // Per-instance mutex. save/update/forget/recall each read-modify-write the
+  // entry file, MEMORY.md, and the embedding index; without serialization two
+  // overlapping calls clobber MEMORY.md and race the embedding cache.
+  private readonly mutex: Mutex = createMutex();
+  private readonly maxMemories: number;
+  // In-memory MEMORY.md rows (name → frontmatter+path), hydrated lazily from
+  // disk ONCE, then maintained incrementally on save/update/forget — so a
+  // write no longer re-reads + re-parses every memory file to rebuild the
+  // index (the old O(N)-per-write rebuild). Entries written to the dir by
+  // other processes appear after the next hydration (new store instance);
+  // within one process this store is the only writer, same assumption the
+  // mutex already makes.
+  private indexRows: Map<string, IndexRow> | null = null;
+
+  constructor(opts: MemoryStoreOptions = {}) {
+    this.dir = opts.dir ?? defaultMemoryDir();
+    this.persistOpt = opts.persistEmbeddings;
+    this.maxMemories = opts.maxMemories ?? DEFAULT_MAX_MEMORIES;
+    const e = opts.embedder;
+    if (typeof e === 'function') {
+      this.resolveEmbedder = e;
+    } else if (e === null) {
+      this.resolveEmbedder = () => null;
+    } else if (e !== undefined) {
+      this.resolveEmbedder = () => e;
+    } else {
+      const tfidf = new TfIdfEmbedder();
+      this.resolveEmbedder = () => tfidf;
+    }
+  }
+
+  private getEmbedder(): EmbeddingProvider | null {
+    if (this.embedderCache === undefined) this.embedderCache = this.resolveEmbedder();
+    return this.embedderCache;
+  }
+
+  private getIndex(): EmbeddingIndex | null {
+    if (this.indexCache === undefined) {
+      const emb = this.getEmbedder();
+      // TF-IDF's vocab depends on the whole corpus, so per-entry caching is
+      // useless — recompute every recall. For neural embedders, caching is
+      // a big win since each entry's vector is corpus-independent.
+      const isTfIdf = emb instanceof TfIdfEmbedder;
+      const persist = this.persistOpt ?? (emb !== null && !isTfIdf);
+      this.indexCache = persist && emb ? new EmbeddingIndex(this.dir, emb.name, emb.dim) : null;
+    }
+    return this.indexCache;
+  }
+
+  get embedderName(): string {
+    return this.getEmbedder()?.name ?? 'keyword';
+  }
+
+  list(filterType?: MemoryType): Promise<ReadonlyArray<MemoryEntry>> {
+    return listEntries(this.dir, filterType);
+  }
+
+  get(name: string): Promise<MemoryEntry | null> {
+    // fileFor() throws synchronously on a traversal/separator name; surface it
+    // as a rejected promise so callers see one consistent (async) failure mode
+    // instead of an exception thrown before the promise is even constructed.
+    let file: string;
+    try {
+      file = this.fileFor(name);
+    } catch (err) {
+      return Promise.reject(err);
+    }
+    return readEntry(file);
+  }
+
+  save(
+    input: Omit<MemoryFrontmatter, 'createdAt' | 'updatedAt'> & { body: string },
+  ): Promise<MemoryEntry> {
+    return this.mutex.run(() => this.writeEntry(input));
+  }
+
+  update(
+    name: string,
+    patch: { body?: string; description?: string; tags?: ReadonlyArray<string> },
+  ): Promise<MemoryEntry | null> {
+    // Read-modify-write under the mutex; calls the internal (unserialized)
+    // writer so it doesn't deadlock on its own chain.
+    return this.mutex.run(async () => {
+      const existing = await readEntry(this.fileFor(name));
+      if (!existing) return null;
+      const mergedTags = patch.tags ?? existing.frontmatter.tags;
+      return this.writeEntry({
+        name: existing.frontmatter.name,
+        type: existing.frontmatter.type,
+        description: patch.description ?? existing.frontmatter.description,
+        ...(mergedTags ? { tags: [...mergedTags] } : {}),
+        body: patch.body ?? existing.body,
+      });
+    });
+  }
+
+  forget(name: string): Promise<boolean> {
+    return this.mutex.run(async () => {
+      const filePath = this.fileFor(name);
+      try {
+        await fs.unlink(filePath);
+        const rows = await this.rows();
+        rows.delete(name);
+        await this.writeIndexFromRows(rows);
+        return true;
+      } catch (err) {
+        if (isEnoent(err)) return false;
+        throw err;
+      }
+    });
+  }
+
+  /**
+   * Soft-cap status (see {@link MemoryStoreOptions.maxMemories}). `over` means
+   * the store holds more entries than the cap; nothing is ever evicted.
+   */
+  async capStatus(): Promise<{ count: number; max: number; over: boolean }> {
+    // Read the index-row cache under the mutex: writeEntry/forget mutate it
+    // while holding the same lock, so a concurrent capStatus() outside the lock
+    // could observe a half-updated map or trigger a redundant disk hydration.
+    const size = await this.mutex.run(async () => (await this.rows()).size);
+    return { count: size, max: this.maxMemories, over: size > this.maxMemories };
+  }
+
+  /** The actual entry write. NOT serialized — callers hold the mutex. */
+  private async writeEntry(
+    input: Omit<MemoryFrontmatter, 'createdAt' | 'updatedAt'> & { body: string },
+  ): Promise<MemoryEntry> {
+    await fs.mkdir(this.dir, { recursive: true });
+    const filePath = this.fileFor(input.name);
+    const existing = await safeRead(filePath);
+    const now = new Date().toISOString();
+    const createdAt = existing?.frontmatter.createdAt ?? now;
+    const frontmatter = memoryFrontmatterSchema.parse({
+      name: input.name,
+      type: input.type,
+      description: input.description,
+      ...(input.tags !== undefined ? { tags: input.tags } : {}),
+      createdAt,
+      updatedAt: now,
+    });
+    const content = `${renderFrontmatter(frontmatter)}\n\n${input.body.trimEnd()}\n`;
+    await writeFileAtomic(filePath, content);
+    // Incremental index maintenance: update THIS entry's row and re-render
+    // MEMORY.md from the in-memory rows — no per-write re-read of every file.
+    const rows = await this.rows();
+    rows.set(frontmatter.name, { frontmatter, path: filePath });
+    await this.writeIndexFromRows(rows);
+    if (rows.size > this.maxMemories) {
+      // Warn-only soft cap — eviction would silently destroy saved knowledge.
+      console.warn(
+        `[plugin-memory] memory store holds ${rows.size} entries (soft cap ${this.maxMemories}). ` +
+          `Nothing is evicted; consider consolidating or forgetting stale memories.`,
+      );
+    }
+    return { frontmatter, body: input.body.trimEnd(), path: filePath };
+  }
+
+  /**
+   * Search memories by a free-text query. Uses vector cosine similarity when
+   * an EmbeddingProvider is configured (the default is the built-in TF-IDF
+   * embedder); falls back to keyword scoring when `mode: 'keyword'` or when
+   * no embedder is wired.
+   */
+  async recall(
+    query: string,
+    opts: { limit?: number; type?: MemoryType; mode?: RecallMode } = {},
+  ): Promise<ReadonlyArray<RankedMemory>> {
+    const limit = opts.limit ?? 5;
+    const mode = opts.mode ?? 'auto';
+    const all = await this.list(opts.type);
+    if (all.length === 0) return [];
+
+    const embedder = this.getEmbedder();
+    const useVector = mode === 'vector' || (mode === 'auto' && embedder !== null);
+    if (useVector && embedder) {
+      return recallVector(all, query, limit, embedder, this.getIndex(), this.mutex);
+    }
+    return rankByKeywords(all, query, limit);
+  }
+
+  private fileFor(name: string): string {
+    // Belt-and-suspenders containment: the tool schemas restrict `name` to a
+    // slug, but the store is a public API and the inproc isolator can't enforce
+    // the fs glob — so refuse any name that escapes the memory dir (traversal,
+    // absolute path, or embedded separators) rather than unlink/read outside it.
+    const root = path.resolve(this.dir);
+    const resolved = path.resolve(root, `${name}.md`);
+    // The resolved file must live DIRECTLY in `root` (no nested or escaped path).
+    if (path.dirname(resolved) !== root) {
+      throw new Error(`invalid memory name: ${name}`);
+    }
+    return resolved;
+  }
+
+  /** Hydrate the index-row cache from disk once, then serve it from memory.
+   *  Callers that mutate it hold the store mutex (writeEntry/forget). */
+  private async rows(): Promise<Map<string, IndexRow>> {
+    if (!this.indexRows) {
+      const entries = await listEntries(this.dir);
+      this.indexRows = new Map(
+        entries.map((e) => [e.frontmatter.name, { frontmatter: e.frontmatter, path: e.path }]),
+      );
+    }
+    return this.indexRows;
+  }
+
+  /** Render MEMORY.md from the cached rows, name-sorted so the output is
+   *  deterministic (readdir order, which the old full rebuild inherited,
+   *  was platform-dependent anyway). */
+  private writeIndexFromRows(rows: Map<string, IndexRow>): Promise<void> {
+    const sorted = [...rows.values()].sort((a, b) =>
+      a.frontmatter.name.localeCompare(b.frontmatter.name),
+    );
+    return writeIndex(this.dir, sorted);
+  }
+}

package/src/tfidf.test.ts

@@ -0,0 +1,59 @@
+import { describe, expect, it } from 'vitest';
+import { TfIdfEmbedder, cosineSimilarity, tokenize } from './tfidf.js';
+
+describe('tokenize', () => {
+  it('lowercases, splits on non-alphanumerics, drops short tokens + stopwords', () => {
+    expect(tokenize('The QUICK brown fox')).toEqual(['quick', 'brown', 'fox']);
+    expect(tokenize('foo-bar baz_qux 123abc')).toEqual(['foo-bar', 'baz_qux', '123abc']);
+  });
+
+  it('strips diacritics for stable matching', () => {
+    expect(tokenize('café naïve')).toEqual(['cafe', 'naive']);
+  });
+});
+
+describe('TfIdfEmbedder', () => {
+  it('embed() before fit() returns empty vectors', async () => {
+    const e = new TfIdfEmbedder();
+    const [v] = await e.embed(['hello world']);
+    expect(v).toEqual([]);
+  });
+
+  it('produces vectors over a fitted vocab', async () => {
+    const corpus = [
+      'team prefers tRPC over REST',
+      'production runs Postgres 16',
+      'sentry alerts go to slack',
+    ];
+    const e = new TfIdfEmbedder();
+    e.fit(corpus);
+    const vectors = await e.embed(corpus);
+    expect(vectors).toHaveLength(3);
+    expect(vectors[0]!.length).toBeGreaterThan(0);
+    expect(vectors[0]!.length).toBe(vectors[1]!.length);
+  });
+
+  it('ranks a relevant query higher than an irrelevant one', async () => {
+    const corpus = [
+      'team prefers tRPC over REST for endpoints',
+      'production database is Postgres 16',
+      'feature flags live in GrowthBook',
+    ];
+    const e = new TfIdfEmbedder();
+    e.fit([...corpus, 'what API style does the team use', 'database flavor in prod']);
+    const v = await e.embed([...corpus, 'what API style does the team use']);
+    const queryVec = v[v.length - 1]!;
+    const scores = corpus.map((_, i) => cosineSimilarity(v[i]!, queryVec));
+    // tRPC entry should be most similar to "API style" query
+    expect(scores[0]).toBeGreaterThan(scores[1]!);
+    expect(scores[0]).toBeGreaterThan(scores[2]!);
+  });
+
+  it('cosineSimilarity returns 1 for identical normalized vectors', () => {
+    expect(cosineSimilarity([0.6, 0.8], [0.6, 0.8])).toBeCloseTo(1, 5);
+  });
+
+  it('cosineSimilarity returns 0 for orthogonal vectors', () => {
+    expect(cosineSimilarity([1, 0], [0, 1])).toBeCloseTo(0, 5);
+  });
+});

package/src/tfidf.ts

@@ -0,0 +1,105 @@
+import type { EmbeddingProvider } from '@moxxy/sdk';
+
+/**
+ * Built-in zero-dependency TF-IDF embedder. Computes a sparse-but-stored-dense
+ * vector per text. Builds vocabulary lazily from the corpus you provide via
+ * `fit()`. Once fit, `embed()` produces vectors over that fixed vocabulary.
+ *
+ * Quality is meh by neural-embedding standards but: deterministic, runs in
+ * <1ms for hundreds of entries, no network, no model download, no API key.
+ * Real semantic embeddings can be wired by passing a different EmbeddingProvider
+ * to MemoryStore.
+ */
+export class TfIdfEmbedder implements EmbeddingProvider {
+  readonly name = 'tfidf';
+  private vocab: ReadonlyArray<string> = [];
+  private vocabIndex = new Map<string, number>();
+  private idf: ReadonlyArray<number> = [];
+
+  get dim(): number | 'dynamic' {
+    return this.vocab.length || 'dynamic';
+  }
+
+  /**
+   * Compute vocabulary + IDF weights from the corpus. Call this whenever the
+   * source corpus changes; embed() must be called only after fit().
+   */
+  fit(corpus: ReadonlyArray<string>): void {
+    const docs = corpus.map(tokenize);
+    const df = new Map<string, number>();
+    for (const doc of docs) {
+      const seen = new Set<string>();
+      for (const t of doc) {
+        if (seen.has(t)) continue;
+        seen.add(t);
+        df.set(t, (df.get(t) ?? 0) + 1);
+      }
+    }
+    // Filter: keep tokens appearing at least once and at most in 95% of docs.
+    const N = Math.max(1, docs.length);
+    const maxDf = Math.max(1, Math.ceil(N * 0.95));
+    const kept: Array<[string, number]> = [];
+    for (const [token, count] of df) {
+      if (count <= maxDf) kept.push([token, count]);
+    }
+    kept.sort((a, b) => a[0].localeCompare(b[0]));
+    this.vocab = kept.map((x) => x[0]);
+    this.vocabIndex = new Map(this.vocab.map((t, i) => [t, i]));
+    this.idf = kept.map(([, dfi]) => Math.log((N + 1) / (dfi + 1)) + 1);
+  }
+
+  embedSync(text: string): number[] {
+    if (this.vocab.length === 0) return [];
+    const tokens = tokenize(text);
+    const tf = new Map<number, number>();
+    for (const t of tokens) {
+      const idx = this.vocabIndex.get(t);
+      if (idx === undefined) continue;
+      tf.set(idx, (tf.get(idx) ?? 0) + 1);
+    }
+    const vec = new Array<number>(this.vocab.length).fill(0);
+    const docLen = tokens.length || 1;
+    for (const [idx, count] of tf) {
+      vec[idx] = (count / docLen) * (this.idf[idx] ?? 1);
+    }
+    return l2Normalize(vec);
+  }
+
+  async embed(texts: ReadonlyArray<string>): Promise<ReadonlyArray<ReadonlyArray<number>>> {
+    return texts.map((t) => this.embedSync(t));
+  }
+}
+
+export function cosineSimilarity(a: ReadonlyArray<number>, b: ReadonlyArray<number>): number {
+  const n = Math.min(a.length, b.length);
+  let dot = 0;
+  for (let i = 0; i < n; i++) dot += (a[i] ?? 0) * (b[i] ?? 0);
+  return dot;
+}
+
+export function l2Normalize(v: number[]): number[] {
+  let sum = 0;
+  for (const x of v) sum += x * x;
+  if (sum === 0) return v;
+  const norm = Math.sqrt(sum);
+  return v.map((x) => x / norm);
+}
+
+export function tokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .normalize('NFKD')
+    .replace(/[̀-ͯ]/g, '')
+    .split(/[^a-z0-9_-]+/)
+    .filter((t) => t.length >= 2 && !STOPWORDS.has(t));
+}
+
+const STOPWORDS = new Set([
+  'the', 'a', 'an', 'and', 'or', 'but', 'is', 'are', 'was', 'were', 'be',
+  'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
+  'should', 'could', 'may', 'might', 'must', 'shall', 'to', 'of', 'in', 'for',
+  'on', 'at', 'by', 'with', 'from', 'as', 'into', 'through', 'about', 'between',
+  'this', 'that', 'these', 'those', 'it', 'its', 'they', 'them', 'their', 'we',
+  'us', 'our', 'you', 'your', 'he', 'she', 'his', 'her', 'i', 'me', 'my',
+  'so', 'than', 'too', 'very', 'just', 'not', 'no', 'yes', 'if', 'then',
+]);

package/src/vector-recall.test.ts

@@ -0,0 +1,88 @@
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { promises as fs } from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import type { EmbeddingProvider } from '@moxxy/sdk';
+import { MemoryStore } from './store.js';
+
+let tmp: string;
+beforeEach(async () => {
+  tmp = await fs.mkdtemp(path.join(os.tmpdir(), 'mox-vec-'));
+});
+afterEach(async () => {
+  await fs.rm(tmp, { recursive: true, force: true });
+});
+
+const sampleCorpus = async (store: MemoryStore): Promise<void> => {
+  await store.save({
+    name: 'trpc-preference',
+    type: 'preference',
+    description: 'Team prefers tRPC API over REST API.',
+    body: 'When generating new API endpoints scaffold a tRPC procedure rather than a REST controller. tRPC API style is the standard.',
+  });
+  await store.save({
+    name: 'postgres-prod',
+    type: 'project',
+    description: 'Production database is Postgres 16.',
+    body: 'All migrations target Postgres 16. Run pg_dump for backups every night.',
+  });
+  await store.save({
+    name: 'growthbook-flags',
+    type: 'reference',
+    description: 'Feature flags live in GrowthBook.',
+    body: 'GrowthBook owns flag definitions. The SDK is initialized in app/bootstrap.ts.',
+  });
+};
+
+describe('MemoryStore vector recall (TF-IDF default)', () => {
+  it('uses vector mode by default when an embedder is configured', async () => {
+    const store = new MemoryStore({ dir: tmp });
+    expect(store.embedderName).toBe('tfidf');
+    await sampleCorpus(store);
+    const matches = await store.recall('what API style does the team prefer');
+    expect(matches[0]!.entry.frontmatter.name).toBe('trpc-preference');
+  });
+
+  it('vector recall ranks semantically relevant entries above incidental keyword matches', async () => {
+    const store = new MemoryStore({ dir: tmp });
+    await sampleCorpus(store);
+    const matches = await store.recall('postgres database backups', { limit: 3 });
+    expect(matches[0]!.entry.frontmatter.name).toBe('postgres-prod');
+  });
+
+  it('mode: "keyword" forces the legacy scorer', async () => {
+    const store = new MemoryStore({ dir: tmp });
+    await sampleCorpus(store);
+    const matches = await store.recall('postgres', { mode: 'keyword' });
+    expect(matches[0]!.entry.frontmatter.name).toBe('postgres-prod');
+  });
+
+  it('embedder: null disables vector entirely and forces keyword', async () => {
+    const store = new MemoryStore({ dir: tmp, embedder: null });
+    expect(store.embedderName).toBe('keyword');
+    await sampleCorpus(store);
+    const matches = await store.recall('database flavor in prod');
+    // Falls through to keyword: "database" + "prod" only appear in postgres-prod
+    expect(matches[0]!.entry.frontmatter.name).toBe('postgres-prod');
+  });
+
+  it('accepts a custom EmbeddingProvider via the constructor', async () => {
+    const stub: EmbeddingProvider = {
+      name: 'stub',
+      dim: 3,
+      async embed(texts) {
+        return texts.map((t) => (t.includes('postgres') ? [1, 0, 0] : [0, 1, 0]));
+      },
+    };
+    const store = new MemoryStore({ dir: tmp, embedder: stub });
+    expect(store.embedderName).toBe('stub');
+    await sampleCorpus(store);
+    const matches = await store.recall('postgres');
+    expect(matches[0]!.entry.frontmatter.name).toBe('postgres-prod');
+  });
+
+  it('returns empty list when corpus is empty', async () => {
+    const store = new MemoryStore({ dir: tmp });
+    expect(await store.recall('anything')).toEqual([]);
+  });
+});