@crewhaus/memory-store 0.1.0

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@crewhaus/memory-store",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "M4.2 — persistent cross-session memory store. File-backed JSONL with simple BM25-style text search. Per-spec scoped.",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "scripts": {
+    "test": "bun test src"
+  },
+  "dependencies": {
+    "@crewhaus/errors": "0.0.0"
+  },
+  "license": "Apache-2.0",
+  "author": {
+    "name": "Max Meier",
+    "email": "max@studiomax.io",
+    "url": "https://studiomax.io"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/crewhaus/factory.git",
+    "directory": "packages/memory-store"
+  },
+  "homepage": "https://github.com/crewhaus/factory/tree/main/packages/memory-store#readme",
+  "bugs": {
+    "url": "https://github.com/crewhaus/factory/issues"
+  },
+  "publishConfig": {
+    "access": "restricted"
+  },
+  "files": [
+    "src",
+    "README.md",
+    "LICENSE",
+    "NOTICE"
+  ]
+}

package/src/index.test.ts

@@ -0,0 +1,132 @@
+/**
+ * Tests for the persistent memory store. Covers basic remember/recall,
+ * BM25 ranking sanity, file persistence across instances, and graceful
+ * handling of malformed lines.
+ */
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { appendFileSync, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { MemoryStoreError, createMemoryStore } from "./index";
+
+let tmp: string;
+
+beforeEach(() => {
+  tmp = mkdtempSync(join(tmpdir(), "memory-store-"));
+});
+
+afterEach(() => {
+  rmSync(tmp, { recursive: true, force: true });
+});
+
+describe("createMemoryStore", () => {
+  test("rejects empty specName", () => {
+    expect(() => createMemoryStore({ specName: "", rootDir: tmp })).toThrow(MemoryStoreError);
+  });
+
+  test("rejects specName with bad characters (path traversal etc.)", () => {
+    expect(() => createMemoryStore({ specName: "../etc/passwd", rootDir: tmp })).toThrow(
+      MemoryStoreError,
+    );
+    expect(() => createMemoryStore({ specName: "foo bar", rootDir: tmp })).toThrow(
+      MemoryStoreError,
+    );
+  });
+
+  test("path() returns the expected file location", () => {
+    const store = createMemoryStore({ specName: "my-spec", rootDir: tmp });
+    expect(store.path()).toBe(join(tmp, "my-spec.jsonl"));
+  });
+});
+
+describe("remember + recall", () => {
+  test("size() returns 0 before any remember", async () => {
+    const store = createMemoryStore({ specName: "s", rootDir: tmp });
+    expect(await store.size()).toBe(0);
+  });
+
+  test("remember() persists and recall() finds it", async () => {
+    const store = createMemoryStore({ specName: "s", rootDir: tmp });
+    await store.remember("the user prefers TypeScript over JavaScript");
+    const results = await store.recall("TypeScript");
+    expect(results.length).toBe(1);
+    expect(results[0]?.entry.text).toBe("the user prefers TypeScript over JavaScript");
+    expect(results[0]?.score).toBeGreaterThan(0);
+  });
+
+  test("remember assigns ids with the mem_ prefix", async () => {
+    const store = createMemoryStore({ specName: "s", rootDir: tmp });
+    const e1 = await store.remember("a");
+    const e2 = await store.remember("b");
+    expect(e1.id).toMatch(/^mem_[0-9a-f]{16}$/);
+    expect(e2.id).toMatch(/^mem_[0-9a-f]{16}$/);
+    expect(e1.id).not.toBe(e2.id);
+  });
+
+  test("recall() ranks closer matches higher", async () => {
+    const store = createMemoryStore({ specName: "s", rootDir: tmp });
+    await store.remember("the user prefers TypeScript");
+    await store.remember("we discussed Python yesterday");
+    await store.remember("TypeScript is a typed superset of JavaScript");
+    const results = await store.recall("TypeScript", 3);
+    expect(results.length).toBe(2);
+    // Both TypeScript-mentioning entries should outrank the Python one,
+    // which is excluded entirely (zero score).
+    expect(results[0]?.score).toBeGreaterThanOrEqual(results[1]?.score);
+    expect(results.find((r) => r.entry.text.includes("Python"))).toBeUndefined();
+  });
+
+  test("tags are indexed for recall", async () => {
+    const store = createMemoryStore({ specName: "s", rootDir: tmp });
+    await store.remember("personal note", ["family", "anniversary"]);
+    const results = await store.recall("anniversary");
+    expect(results.length).toBe(1);
+    expect(results[0]?.entry.tags).toEqual(["family", "anniversary"]);
+  });
+
+  test("recall returns empty array when nothing matches", async () => {
+    const store = createMemoryStore({ specName: "s", rootDir: tmp });
+    await store.remember("apples and oranges");
+    const results = await store.recall("kumquat");
+    expect(results).toEqual([]);
+  });
+
+  test("k parameter caps the result count", async () => {
+    const store = createMemoryStore({ specName: "s", rootDir: tmp });
+    for (let i = 0; i < 5; i++) {
+      await store.remember(`memory entry number ${i} about banana`);
+    }
+    const results = await store.recall("banana", 2);
+    expect(results.length).toBe(2);
+  });
+
+  test("persists across store instances (file-backed)", async () => {
+    const store1 = createMemoryStore({ specName: "persist", rootDir: tmp });
+    await store1.remember("durable claim");
+    const store2 = createMemoryStore({ specName: "persist", rootDir: tmp });
+    expect(await store2.size()).toBe(1);
+    const results = await store2.recall("durable");
+    expect(results.length).toBe(1);
+  });
+
+  test("malformed lines in the JSONL are skipped (not raised)", async () => {
+    const store = createMemoryStore({ specName: "broken", rootDir: tmp });
+    await store.remember("good entry");
+    // Corrupt the file by appending a broken line.
+    appendFileSync(store.path(), "{not json\n");
+    await store.remember("another good entry");
+    expect(await store.size()).toBe(2);
+    const results = await store.recall("entry");
+    expect(results.length).toBe(2);
+  });
+
+  test("recall rejects empty queries", async () => {
+    const store = createMemoryStore({ specName: "s", rootDir: tmp });
+    await expect(store.recall("")).rejects.toThrow(MemoryStoreError);
+  });
+
+  test("remember rejects empty text", async () => {
+    const store = createMemoryStore({ specName: "s", rootDir: tmp });
+    await expect(store.remember("")).rejects.toThrow(MemoryStoreError);
+  });
+});

package/src/index.ts

@@ -0,0 +1,216 @@
+/**
+ * Catalog R7 sibling — `memory-store` (M4.2 of the heavy-hitter plan).
+ *
+ * Persistent cross-session memory: a file-backed JSONL store of facts
+ * the user wants the agent to remember across sessions. Each entry
+ * carries text + optional tags + a created-at timestamp. Recall does a
+ * BM25-style ranking over the tokenized text.
+ *
+ * Why JSONL: matches event-log and session-store; trivial to inspect
+ * with `tail`/`jq`; append-only writes are crash-safe; small enough
+ * working sets (< 10k entries per spec) that an in-memory load on each
+ * `recall()` call is fine.
+ *
+ * Why simple BM25 instead of embeddings: zero deps, deterministic for
+ * tests, and the working set is small. When a user grows into a much
+ * larger memory bank, swap the search backend behind the `MemoryStore`
+ * interface — `recall()` is the only consumer-visible signature.
+ *
+ * File path: `<rootDir>/<specName>.jsonl` where rootDir defaults to
+ * `.crewhaus/memories/`. One file per spec keeps memories scoped.
+ */
+import { existsSync, mkdirSync } from "node:fs";
+import { appendFile, readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { CrewhausError } from "@crewhaus/errors";
+
+export const DEFAULT_ROOT_DIR = ".crewhaus/memories";
+
+export type MemoryEntry = {
+  readonly id: string;
+  readonly text: string;
+  readonly tags: readonly string[];
+  readonly createdAt: string;
+};
+
+export type MemoryRecallResult = {
+  readonly entry: MemoryEntry;
+  readonly score: number;
+};
+
+export interface MemoryStoreOptions {
+  readonly rootDir?: string;
+  readonly specName: string;
+  readonly now?: () => Date;
+}
+
+export interface MemoryStore {
+  /** Append a new memory. Returns the assigned entry. */
+  remember(text: string, tags?: readonly string[]): Promise<MemoryEntry>;
+  /** Top-k matches for the query, ranked by BM25-style score. */
+  recall(query: string, k?: number): Promise<readonly MemoryRecallResult[]>;
+  /** Diagnostic: how many entries are stored. */
+  size(): Promise<number>;
+  /** Diagnostic: where on disk this store writes. */
+  path(): string;
+}
+
+export class MemoryStoreError extends CrewhausError {
+  override readonly name = "MemoryStoreError";
+  constructor(message: string, cause?: unknown) {
+    super("config", message, cause);
+  }
+}
+
+const DEFAULT_K = 5;
+const ID_PREFIX = "mem_";
+
+/**
+ * Construct a memory store for a given spec. The store is lazy — the
+ * underlying file is created on the first `remember()` call.
+ */
+export function createMemoryStore(opts: MemoryStoreOptions): MemoryStore {
+  if (!opts.specName) {
+    throw new MemoryStoreError("specName is required");
+  }
+  if (!/^[a-zA-Z0-9_\-.]+$/.test(opts.specName)) {
+    throw new MemoryStoreError(
+      `invalid specName "${opts.specName}" — must match [a-zA-Z0-9_\\-.]+`,
+    );
+  }
+  const rootDir = opts.rootDir ?? DEFAULT_ROOT_DIR;
+  const now = opts.now ?? (() => new Date());
+  const filePath = join(rootDir, `${opts.specName}.jsonl`);
+
+  async function ensureRootDir(): Promise<void> {
+    if (!existsSync(rootDir)) {
+      mkdirSync(rootDir, { recursive: true });
+    }
+  }
+
+  async function loadAll(): Promise<MemoryEntry[]> {
+    if (!existsSync(filePath)) return [];
+    let raw: string;
+    try {
+      raw = await readFile(filePath, "utf-8");
+    } catch {
+      return [];
+    }
+    const entries: MemoryEntry[] = [];
+    for (const line of raw.split("\n")) {
+      if (!line.trim()) continue;
+      try {
+        const parsed = JSON.parse(line) as unknown;
+        if (isMemoryEntry(parsed)) entries.push(parsed);
+      } catch {
+        // Malformed line — skip. Append-only writes mean partial-write
+        // corruption is the most likely failure mode; one bad line
+        // shouldn't block recall on the others.
+      }
+    }
+    return entries;
+  }
+
+  function mkId(): string {
+    // 8-byte random hex; collision probability with < 10k entries per
+    // spec is negligible. crypto.randomUUID().replace would also work
+    // but we keep a fixed-width id for grep-friendliness in the JSONL.
+    let hex = "";
+    for (let i = 0; i < 16; i++) {
+      hex += Math.floor(Math.random() * 16).toString(16);
+    }
+    return `${ID_PREFIX}${hex}`;
+  }
+
+  return {
+    async remember(text: string, tags: readonly string[] = []): Promise<MemoryEntry> {
+      if (typeof text !== "string" || text.length === 0) {
+        throw new MemoryStoreError("remember(): text must be a non-empty string");
+      }
+      const entry: MemoryEntry = {
+        id: mkId(),
+        text,
+        tags: [...tags],
+        createdAt: now().toISOString(),
+      };
+      await ensureRootDir();
+      await appendFile(filePath, `${JSON.stringify(entry)}\n`, { mode: 0o600 });
+      return entry;
+    },
+
+    async recall(query: string, k: number = DEFAULT_K): Promise<readonly MemoryRecallResult[]> {
+      if (typeof query !== "string" || query.length === 0) {
+        throw new MemoryStoreError("recall(): query must be a non-empty string");
+      }
+      const all = await loadAll();
+      if (all.length === 0) return [];
+      const queryTerms = tokenize(query);
+      if (queryTerms.length === 0) return [];
+
+      // BM25-style scoring:
+      //   - tf  = term frequency in this document
+      //   - idf = log((N - df + 0.5) / (df + 0.5))
+      //   - score = sum over terms of idf * (tf * (k1 + 1)) / (tf + k1 * (1 - b + b * (dl / avgdl)))
+      const k1 = 1.5;
+      const b = 0.75;
+      const docs = all.map((entry) => ({
+        entry,
+        terms: tokenize(`${entry.text} ${entry.tags.join(" ")}`),
+      }));
+      const N = docs.length;
+      const avgdl = docs.reduce((sum, d) => sum + d.terms.length, 0) / Math.max(1, N);
+
+      // Document frequency for each query term.
+      const df = new Map<string, number>();
+      for (const t of new Set(queryTerms)) {
+        df.set(t, docs.filter((d) => d.terms.includes(t)).length);
+      }
+
+      const results: MemoryRecallResult[] = [];
+      for (const d of docs) {
+        let score = 0;
+        for (const t of queryTerms) {
+          const tf = d.terms.filter((x) => x === t).length;
+          if (tf === 0) continue;
+          const dfi = df.get(t) ?? 0;
+          const idf = Math.log((N - dfi + 0.5) / (dfi + 0.5) + 1);
+          const dl = d.terms.length;
+          const norm = tf * (k1 + 1);
+          const denom = tf + k1 * (1 - b + (b * dl) / Math.max(1, avgdl));
+          score += idf * (norm / denom);
+        }
+        if (score > 0) results.push({ entry: d.entry, score });
+      }
+      results.sort((a, b) => b.score - a.score);
+      return results.slice(0, Math.max(0, k));
+    },
+
+    async size(): Promise<number> {
+      const all = await loadAll();
+      return all.length;
+    },
+
+    path(): string {
+      return filePath;
+    },
+  };
+}
+
+function isMemoryEntry(value: unknown): value is MemoryEntry {
+  if (typeof value !== "object" || value === null) return false;
+  const v = value as Record<string, unknown>;
+  return (
+    typeof v["id"] === "string" &&
+    typeof v["text"] === "string" &&
+    Array.isArray(v["tags"]) &&
+    (v["tags"] as unknown[]).every((t) => typeof t === "string") &&
+    typeof v["createdAt"] === "string"
+  );
+}
+
+function tokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter((t) => t.length > 0);
+}