@remnic/core 9.3.703 → 9.3.705

package/src/storage-contract/cache-coherence.test.ts

@@ -0,0 +1,100 @@
+/**
+ * Issue #1533 — Phase A contract test: cache coherence across instances.
+ *
+ * Rule 37 (memory-cache): write via instance A, read via a FRESH instance B
+ * over the same dir → B sees the write. The module-level caches
+ * (readAllMemories in-flight, version sentinels) are keyed by baseDir and
+ * shared/static — this test pins that a write invalidates correctly so a
+ * second instance never serves stale data.
+ */
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { StorageManager } from "../storage.js";
+import { resetStaticCaches } from "./harness.js";
+
+test("cache-coherence: write via instance A, read via fresh instance B → B sees the write", async () => {
+  const { mkdtemp, rm } = await import("node:fs/promises");
+  const os = await import("node:os");
+  const path = await import("node:path");
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-cache-coherence-"));
+  try {
+    resetStaticCaches();
+    const smA = new StorageManager(dir);
+    await smA.ensureDirectories();
+
+    const id = await smA.writeMemory("fact", "written by A");
+
+    // Fresh instance over the same dir
+    const smB = new StorageManager(dir);
+    await smB.ensureDirectories();
+    const found = await smB.getMemoryById(id);
+    assert.ok(found, "instance B must see the memory written by instance A");
+    assert.equal(found!.content, "written by A");
+  } finally {
+    resetStaticCaches();
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("cache-coherence: delete via instance A is reflected in fresh instance B", async () => {
+  const { mkdtemp, rm } = await import("node:fs/promises");
+  const os = await import("node:os");
+  const path = await import("node:path");
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-cache-del-"));
+  try {
+    resetStaticCaches();
+    const smA = new StorageManager(dir);
+    await smA.ensureDirectories();
+    const id = await smA.writeMemory("fact", "to be deleted");
+
+    // Confirm B sees it
+    const smB = new StorageManager(dir);
+    await smB.ensureDirectories();
+    assert.ok(await smB.getMemoryById(id));
+
+    // A deletes
+    await smA.invalidateMemory(id);
+
+    // Fresh instance C must NOT see it
+    const smC = new StorageManager(dir);
+    await smC.ensureDirectories();
+    const found = await smC.getMemoryById(id);
+    assert.equal(found, null, "fresh instance must not see a deleted memory");
+  } finally {
+    resetStaticCaches();
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("cache-coherence: readAllMemories reflects writes across instances after invalidation", async () => {
+  const { mkdtemp, rm } = await import("node:fs/promises");
+  const os = await import("node:os");
+  const path = await import("node:path");
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-cache-list-"));
+  try {
+    resetStaticCaches();
+    const smA = new StorageManager(dir);
+    await smA.ensureDirectories();
+    await smA.writeMemory("fact", "first");
+    await smA.writeMemory("decision", "second");
+
+    const smB = new StorageManager(dir);
+    await smB.ensureDirectories();
+    const all = await smB.readAllMemories();
+    assert.ok(all.length >= 2, "fresh instance must list all written memories");
+
+    // Write more via A
+    await smA.writeMemory("principle", "third");
+
+    const smC = new StorageManager(dir);
+    await smC.ensureDirectories();
+    const allAfter = await smC.readAllMemories();
+    assert.ok(allAfter.length >= 3, "fresh instance must see the additional write");
+    assert.ok(allAfter.some((m) => m.content === "third"));
+  } finally {
+    resetStaticCaches();
+    await rm(dir, { recursive: true, force: true });
+  }
+});

package/src/storage-contract/content-hash.test.ts

@@ -0,0 +1,102 @@
+/**
+ * Issue #1533 — Phase A contract test: content hash identity.
+ *
+ * Rule 23: the write-side and dedup-side MUST hash the same `rawContent`,
+ * never the timestamped/cited form. `writeMemory` persists a `contentHash` on
+ * the frontmatter for facts; `hasFactContentHash` queries the dedup index.
+ * This test pins that the two sides agree so subsequent extractions of the
+ * same logical fact are deduped correctly even when citation timestamps differ.
+ */
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { ContentHashIndex } from "../storage.js";
+import { makeStorage } from "./harness.js";
+
+test("content-hash: ContentHashIndex.computeHash is deterministic for the same input", () => {
+  const a = ContentHashIndex.computeHash("the sky is blue");
+  const b = ContentHashIndex.computeHash("the sky is blue");
+  assert.equal(a, b);
+  assert.notEqual(
+    ContentHashIndex.computeHash("the sky is blue"),
+    ContentHashIndex.computeHash("the sky is green"),
+  );
+});
+
+test("content-hash: writeMemory for fact registers the hash so hasFactContentHash returns true", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    const rawFact = "PostgreSQL is our primary database";
+    await storage.writeMemory("fact", rawFact);
+
+    const has = await storage.hasFactContentHash(rawFact);
+    assert.equal(has, true, "hasFactContentHash must return true for a just-written fact");
+  } finally {
+    await cleanup();
+  }
+});
+
+test("content-hash: contentHashSource overrides the persisted body for dedup (rule 23)", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    const rawFact = "We migrated to MySQL in March";
+    // Persist a citation-annotated body, but index the RAW fact text
+    await storage.writeMemory("fact", `${rawFact} [cited 2026-01-01]`, {
+      contentHashSource: rawFact,
+    });
+
+    // Querying with the RAW fact (not the cited form) must hit
+    assert.equal(
+      await storage.hasFactContentHash(rawFact),
+      true,
+      "hasFactContentHash must match on the raw fact, not the cited body",
+    );
+    // The cited form should NOT be separately indexed (it has the citation suffix)
+    assert.equal(
+      await storage.hasFactContentHash(`${rawFact} [cited 2026-01-01]`),
+      false,
+      "the cited/timestamped form must not be the indexed hash",
+    );
+  } finally {
+    await cleanup();
+  }
+});
+
+test("content-hash: non-fact categories do not register a content hash", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    await storage.writeMemory("decision", "we chose option B");
+    assert.equal(
+      await storage.hasFactContentHash("we chose option B"),
+      false,
+      "non-fact categories must not enter the fact dedup index",
+    );
+  } finally {
+    await cleanup();
+  }
+});
+
+test("content-hash: normalizeContent produces stable text for equivalent inputs", () => {
+  const a = ContentHashIndex.normalizeContent("  Hello   World  ");
+  const b = ContentHashIndex.normalizeContent("Hello World");
+  assert.equal(a, b, "normalizeContent must collapse whitespace consistently");
+});
+
+test("content-hash: the frontmatter contentHash matches computeHash of the raw content", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    const rawFact = "deterministic hash check";
+    const id = await storage.writeMemory("fact", rawFact);
+    const memory = await storage.getMemoryById(id);
+    assert.ok(memory);
+    assert.ok(memory!.frontmatter.contentHash, "frontmatter must carry a contentHash for facts");
+    assert.equal(
+      memory!.frontmatter.contentHash,
+      ContentHashIndex.computeHash(rawFact),
+      "frontmatter contentHash must equal computeHash of the raw content",
+    );
+  } finally {
+    await cleanup();
+  }
+});

package/src/storage-contract/failure-semantics.test.ts

@@ -0,0 +1,108 @@
+/**
+ * Issue #1533 — Phase A contract test: failure semantics.
+ *
+ * Rule 34: reads distinguish empty-dir from unreadable-dir — `{ok:false}` vs
+ * `[]`. Where the current surface already does this, we pin it; where it
+ * doesn't, we record the gap (for the silent-failures issue) rather than
+ * change behavior here.
+ *
+ * Also pins containment: a FILE where a directory is expected is skipped via
+ * lstat().isDirectory(), not existsSync() (rule 24).
+ */
+
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import { StorageManager } from "../storage.js";
+import { makeStorage } from "./harness.js";
+
+test("failure-semantics: readAllMemories on a fresh (empty) dir returns [] not null", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    const result = await storage.readAllMemories();
+    assert.ok(Array.isArray(result), "readAllMemories must return an array");
+    assert.equal(result.length, 0, "empty store must return an empty array");
+  } finally {
+    await cleanup();
+  }
+});
+
+test("failure-semantics: getMemoryById on empty store returns null (not throw)", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    const result = await storage.getMemoryById("anything");
+    assert.equal(result, null);
+  } finally {
+    await cleanup();
+  }
+});
+
+test("failure-semantics: readMemoryByPath on a non-existent file returns null (not throw)", async () => {
+  const { storage, baseDir, cleanup } = await makeStorage();
+  try {
+    const result = await storage.readMemoryByPath(path.join(baseDir, "facts", "2026-01-01", "nope.md"));
+    assert.equal(result, null);
+  } finally {
+    await cleanup();
+  }
+});
+
+test("failure-semantics: readMemoryByPath on a malformed markdown file returns null", async () => {
+  const { storage, baseDir, cleanup } = await makeStorage();
+  try {
+    const dir = path.join(baseDir, "facts", "2026-01-01");
+    await mkdir(dir, { recursive: true });
+    const filePath = path.join(dir, "garbage.md");
+    await writeFile(filePath, "this has no frontmatter at all\njust random text", "utf-8");
+
+    const result = await storage.readMemoryByPath(filePath);
+    assert.equal(result, null, "malformed file (no frontmatter, not entity) must return null");
+  } finally {
+    await cleanup();
+  }
+});
+
+test("failure-semantics: a FILE where a category directory is expected does not crash readAllMemories", async () => {
+  // Create a FILE named "facts" (not a directory) at the base dir, then scan.
+  // collectActiveMemoryPaths uses lstat().isDirectory() (not existsSync) so a
+  // file-as-directory entry is skipped without crashing (rule 24 containment).
+  // We do NOT call ensureDirectories — it would try to mkdir facts/<today> and
+  // fail on the file. The scan path must be robust independently of dir creation.
+  const baseDir = await mkdtemp(path.join(os.tmpdir(), "remnic-fail-file-as-dir-"));
+  try {
+    StorageManager.clearAllStaticCaches();
+    await writeFile(path.join(baseDir, "facts"), "I am a file not a directory", "utf-8");
+    const storage = new StorageManager(baseDir);
+
+    const result = await storage.readAllMemories();
+    assert.ok(Array.isArray(result));
+    assert.equal(result.length, 0, "a file-as-directory must not produce spurious memories");
+  } finally {
+    StorageManager.clearAllStaticCaches();
+    await rm(baseDir, { recursive: true, force: true });
+  }
+});
+
+test("failure-semantics: invalidateMemory on a missing memory returns false (not throw)", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    const result = await storage.invalidateMemory("totally-missing-id-99999");
+    assert.equal(result, false);
+  } finally {
+    await cleanup();
+  }
+});
+
+test("failure-semantics: ensureDirectories is idempotent (safe to call multiple times)", async () => {
+  const { storage, cleanup } = await makeStorage();
+  try {
+    await storage.ensureDirectories();
+    await storage.ensureDirectories();
+    await storage.ensureDirectories();
+  } finally {
+    await cleanup();
+  }
+});

package/src/storage-contract/harness.ts

@@ -0,0 +1,148 @@
+/**
+ * Issue #1533 — Storage contract test harness.
+ *
+ * Shared fixture helpers for the storage public-surface contract suite AND
+ * the #1522 catalog-touch fitness test (both live under `storage-contract/`
+ * per the issue's coordination note: "build both on ONE shared harness so the
+ * write-entry-point enumeration lives in exactly one place").
+ *
+ * Every helper creates a fresh temp directory and returns a `cleanup` that
+ * tears it down — tests never share state. `StorageManager.clearAllStaticCaches()`
+ * is called on setup and teardown so the module-level caches (readAllMemories
+ * in-flight, version sentinels, secure-store entity cache) do not leak between
+ * tests.
+ */
+
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+import { StorageManager } from "../storage.js";
+import {
+  NamespaceStorageRouter,
+} from "../namespaces/storage.js";
+import type { PluginConfig } from "../types.js";
+
+/** Clear every module-level cache so tests start from a known-empty state. */
+export function resetStaticCaches(): void {
+  StorageManager.clearAllStaticCaches();
+}
+
+/**
+ * Create a StorageManager backed by a fresh temp directory.
+ *
+ * Returns the manager, the on-disk base directory, and a `cleanup` that removes
+ * the directory and clears static caches. Callers MUST call `cleanup` in a
+ * `try/finally` (or via `node:test`'s `t.after`).
+ */
+export async function makeStorage(
+  prefix = "remnic-storage-contract-",
+): Promise<{
+  storage: StorageManager;
+  baseDir: string;
+  cleanup: () => Promise<void>;
+}> {
+  resetStaticCaches();
+  const baseDir = await mkdtemp(path.join(os.tmpdir(), prefix));
+  const storage = new StorageManager(baseDir);
+  await storage.ensureDirectories();
+  storage.invalidateAllMemoriesCacheForDir();
+  return {
+    storage,
+    baseDir,
+    cleanup: async () => {
+      resetStaticCaches();
+      await rm(baseDir, { recursive: true, force: true });
+    },
+  };
+}
+
+/**
+ * Build a minimal valid memory markdown body for raw seeding (bypasses
+ * writeMemory so the test controls exact on-disk bytes).
+ */
+export function rawMemoryMarkdown(
+  id: string,
+  category: string,
+  content: string,
+  extra: Record<string, string> = {},
+): string {
+  const now = new Date().toISOString();
+  const lines = [
+    "---",
+    `id: ${id}`,
+    `category: ${category}`,
+    `created: ${now}`,
+    `updated: ${now}`,
+    "source: test",
+    "confidence: 0.9",
+    'tags: ["contract"]',
+  ];
+  for (const [key, value] of Object.entries(extra)) {
+    lines.push(`${key}: ${value}`);
+  }
+  lines.push("---", "", content, "");
+  return lines.join("\n");
+}
+
+/**
+ * Seed a raw markdown file at a path relative to `baseDir`. Used when a test
+ * needs to control exact placement (e.g. a specific category dir + date nest)
+ * without going through `writeMemory`.
+ */
+export async function seedRawFile(
+  baseDir: string,
+  relPath: string,
+  content: string,
+): Promise<string> {
+  const fullPath = path.join(baseDir, relPath);
+  await mkdir(path.dirname(fullPath), { recursive: true });
+  await writeFile(fullPath, content, "utf-8");
+  return fullPath;
+}
+
+/**
+ * Minimal PluginConfig for NamespaceStorageRouter tests. Only the fields the
+ * router reads are populated; the rest default to values that keep the router
+ * in the namespaces-enabled mode the tests exercise.
+ */
+export function makeNamespaceConfig(
+  memoryDir: string,
+  overrides: Record<string, unknown> = {},
+): PluginConfig {
+  return {
+    memoryDir,
+    namespacesEnabled: true,
+    defaultNamespace: "default",
+    entitySchemas: {},
+    inlineSourceAttributionFormat: undefined,
+    ...overrides,
+  } as unknown as PluginConfig;
+}
+
+/**
+ * Create a NamespaceStorageRouter over a fresh temp dir with namespaces
+ * enabled. Returns the router, the memory dir, and a cleanup.
+ */
+export async function makeNamespaceRouter(
+  overrides: Partial<PluginConfig> = {},
+): Promise<{
+  router: NamespaceStorageRouter;
+  memoryDir: string;
+  config: PluginConfig;
+  cleanup: () => Promise<void>;
+}> {
+  resetStaticCaches();
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-ns-contract-"));
+  const config = makeNamespaceConfig(memoryDir, overrides);
+  const router = new NamespaceStorageRouter(config);
+  return {
+    router,
+    memoryDir,
+    config,
+    cleanup: async () => {
+      resetStaticCaches();
+      await rm(memoryDir, { recursive: true, force: true });
+    },
+  };
+}

package/src/storage-contract/namespace-routing.test.ts

@@ -0,0 +1,135 @@
+/**
+ * Issue #1533 — Phase A contract test: namespace routing.
+ *
+ * NamespaceStorageRouter must route the SAME operation through the correct
+ * on-disk root for default vs named namespaces, while rejecting escaping paths
+ * (the containment invariants from #1506 — a FILE at the namespaces root or at
+ * a token dir is rejected via statSync().isDirectory(), not existsSync; rule 24).
+ *
+ * Contracts pinned:
+ *  - default namespace → memoryDir (legacy root)
+ *  - named namespace → memoryDir/namespaces/<token>
+ *  - unsafe namespace segments (/, \, .., absolute) are rejected
+ *  - writes in one namespace are invisible to another
+ */
+
+import assert from "node:assert/strict";
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import test from "node:test";
+
+import { makeStorage, makeNamespaceRouter } from "./harness.js";
+import { resolveNamespaceStorageRoot } from "../namespaces/storage.js";
+import { namespaceIdentityToken } from "../namespaces/identity.js";
+
+test("namespace-routing: default namespace resolves to the legacy memoryDir root", async () => {
+  const { router, memoryDir, config, cleanup } = await makeNamespaceRouter();
+  try {
+    const root = await resolveNamespaceStorageRoot(config, "default");
+    assert.equal(root, memoryDir);
+
+    const sm = await router.storageFor("default");
+    assert.equal(sm.dir, memoryDir);
+  } finally {
+    await cleanup();
+  }
+});
+
+test("namespace-routing: named namespace resolves to a tokenized subdirectory", async () => {
+  const { router, memoryDir, config, cleanup } = await makeNamespaceRouter();
+  try {
+    const ns = "team-alpha";
+    const root = await resolveNamespaceStorageRoot(config, ns);
+    const token = namespaceIdentityToken(ns);
+    assert.ok(
+      root.includes(path.join("namespaces", token)),
+      `named namespace root ${root} should include namespaces/${token}`,
+    );
+
+    const sm = await router.storageFor(ns);
+    assert.ok(sm.dir.startsWith(memoryDir), "named namespace must be under memoryDir");
+    assert.ok(sm.dir.includes("namespaces"), "named namespace must be under namespaces/");
+  } finally {
+    await cleanup();
+  }
+});
+
+test("namespace-routing: two named namespaces resolve to DIFFERENT roots", async () => {
+  const { router, cleanup } = await makeNamespaceRouter();
+  try {
+    const sm1 = await router.storageFor("team-alpha");
+    const sm2 = await router.storageFor("team-beta");
+    assert.notEqual(sm1.dir, sm2.dir, "different namespaces must have different roots");
+  } finally {
+    await cleanup();
+  }
+});
+
+test("namespace-routing: writes in one namespace are invisible to another (isolation)", async () => {
+  const { router, cleanup } = await makeNamespaceRouter();
+  try {
+    const sm1 = await router.storageFor("team-alpha");
+    const id = await sm1.writeMemory("fact", "alpha-only fact");
+
+    const sm2 = await router.storageFor("team-beta");
+    const found = await sm2.getMemoryById(id);
+    assert.equal(found, null, "beta namespace must not see alpha's memories");
+  } finally {
+    await cleanup();
+  }
+});
+
+test("namespace-routing: resolveNamespaceStorageRoot rejects traversal segments", async () => {
+  const { config, cleanup } = await makeNamespaceRouter();
+  try {
+    for (const bad of ["../escape", "a/b", "a\\b", "/abs"]) {
+      await assert.rejects(
+        () => resolveNamespaceStorageRoot(config, bad),
+        /unsafe namespace path segment/,
+        `should reject ${bad}`,
+      );
+    }
+  } finally {
+    await cleanup();
+  }
+});
+
+test("namespace-routing: storageFor rejects unsafe non-default namespaces", async () => {
+  const { router, cleanup } = await makeNamespaceRouter();
+  try {
+    for (const bad of ["../escape", "a/b"]) {
+      await assert.rejects(
+        () => router.storageFor(bad),
+        /unsafe namespace/,
+        `should reject ${bad}`,
+      );
+    }
+  } finally {
+    await cleanup();
+  }
+});
+
+test("namespace-routing: namespaces disabled → all namespaces use memoryDir", async () => {
+  const { router, memoryDir, cleanup } = await makeNamespaceRouter({
+    namespacesEnabled: false,
+  });
+  try {
+    const sm1 = await router.storageFor("default");
+    const sm2 = await router.storageFor("anything");
+    assert.equal(sm1.dir, memoryDir);
+    assert.equal(sm2.dir, memoryDir);
+  } finally {
+    await cleanup();
+  }
+});
+
+test("namespace-routing: router caches StorageManager per namespace (same instance on repeat)", async () => {
+  const { router, cleanup } = await makeNamespaceRouter();
+  try {
+    const sm1 = await router.storageFor("team-alpha");
+    const sm2 = await router.storageFor("team-alpha");
+    assert.equal(sm1, sm2, "storageFor must return the cached instance for the same namespace");
+  } finally {
+    await cleanup();
+  }
+});