nlm-memory 0.4.0

package/tests/unit/llm/ollama-client.test.ts

@@ -0,0 +1,124 @@
+/**
+ * Unit tests for OllamaClient.classify against an injected fake fetch.
+ * No network. Verifies prompt construction, JSON-mode handling, schema
+ * validation, fence stripping, and error mapping.
+ */
+
+import { describe, expect, it } from "vitest";
+import { OllamaClient, ClassifierSchemaError } from "../../../src/llm/ollama-client.js";
+import { LLMUnreachableError } from "../../../src/ports/llm-client.js";
+
+type FakeFetch = typeof fetch;
+
+function jsonResponse(body: unknown, init: ResponseInit = {}): Response {
+  return new Response(JSON.stringify(body), {
+    status: init.status ?? 200,
+    headers: { "content-type": "application/json" },
+  });
+}
+
+function makeFetch(reply: (req: { url: string; body: unknown }) => Response | Promise<Response>): FakeFetch {
+  return (async (input: string | URL | Request, init?: RequestInit) => {
+    const url = typeof input === "string" ? input : input.toString();
+    const body = init?.body ? JSON.parse(init.body.toString()) : null;
+    return reply({ url, body });
+  }) as FakeFetch;
+}
+
+const VALID_PAYLOAD = {
+  label: "Phase C classifier port",
+  summary: "Built OllamaClient.classify with JSON-mode and schema validation against the shared prompt module.",
+  entities: ["NLM", "Ollama", "OllamaClient"],
+  decisions: ["Map schema-invalid output to a typed ClassifierSchemaError instead of returning null"],
+  open: ["Whether to retry once on schema failure or send to inbox immediately"],
+  confidence: 0.85,
+};
+
+describe("OllamaClient.classify", () => {
+  it("parses a valid JSON payload through the Ollama chat envelope", async () => {
+    const fetchImpl = makeFetch(({ url, body }) => {
+      expect(url).toContain("/api/chat");
+      const b = body as { model: string; messages: { role: string; content: string }[]; format: string };
+      expect(b.format).toBe("json");
+      expect(b.messages[0]?.role).toBe("system");
+      expect(b.messages[0]?.content).toContain("session classifier");
+      expect(b.messages[1]?.content).toContain("TRANSCRIPT TO CLASSIFY");
+      return jsonResponse({ message: { content: JSON.stringify(VALID_PAYLOAD) } });
+    });
+    const client = new OllamaClient({ fetchImpl });
+    const result = await client.classify("user: build the classifier\nassistant: done");
+    expect(result.label).toBe(VALID_PAYLOAD.label);
+    expect(result.entities).toEqual(VALID_PAYLOAD.entities);
+    expect(result.decisions).toHaveLength(1);
+    expect(result.confidence).toBeCloseTo(0.85);
+  });
+
+  it("threads priorContext into the user prompt when supplied", async () => {
+    let userContent = "";
+    const fetchImpl = makeFetch(({ body }) => {
+      const b = body as { messages: { role: string; content: string }[] };
+      userContent = b.messages[1]?.content ?? "";
+      return jsonResponse({ message: { content: JSON.stringify(VALID_PAYLOAD) } });
+    });
+    const client = new OllamaClient({ fetchImpl });
+    await client.classify("...transcript...", "earlier session about Hono routing");
+    expect(userContent).toContain("PRIOR CONTEXT (already filed):");
+    expect(userContent).toContain("earlier session about Hono routing");
+  });
+
+  it("strips markdown fences before JSON parse", async () => {
+    const fenced = "```json\n" + JSON.stringify(VALID_PAYLOAD) + "\n```";
+    const fetchImpl = makeFetch(() =>
+      jsonResponse({ message: { content: fenced } }),
+    );
+    const client = new OllamaClient({ fetchImpl });
+    const result = await client.classify("transcript");
+    expect(result.label).toBe(VALID_PAYLOAD.label);
+  });
+
+  it("throws ClassifierSchemaError when required keys are missing", async () => {
+    const bad = { label: "x", summary: "y" };
+    const fetchImpl = makeFetch(() =>
+      jsonResponse({ message: { content: JSON.stringify(bad) } }),
+    );
+    const client = new OllamaClient({ fetchImpl });
+    await expect(client.classify("transcript")).rejects.toBeInstanceOf(ClassifierSchemaError);
+  });
+
+  it("throws ClassifierSchemaError when the model returns non-JSON", async () => {
+    const fetchImpl = makeFetch(() =>
+      jsonResponse({ message: { content: "not even close to json" } }),
+    );
+    const client = new OllamaClient({ fetchImpl });
+    await expect(client.classify("transcript")).rejects.toBeInstanceOf(ClassifierSchemaError);
+  });
+
+  it("maps non-200 HTTP responses to LLMUnreachableError", async () => {
+    const fetchImpl = makeFetch(() =>
+      new Response("server down", { status: 503 }),
+    );
+    const client = new OllamaClient({ fetchImpl });
+    await expect(client.classify("transcript")).rejects.toBeInstanceOf(LLMUnreachableError);
+  });
+
+  it("maps network throws to LLMUnreachableError", async () => {
+    const fetchImpl = (async () => {
+      throw new TypeError("ECONNREFUSED");
+    }) as FakeFetch;
+    const client = new OllamaClient({ fetchImpl });
+    await expect(client.classify("transcript")).rejects.toBeInstanceOf(LLMUnreachableError);
+  });
+
+  it("coerces non-string entities to strings and trims whitespace", async () => {
+    const messy = {
+      ...VALID_PAYLOAD,
+      entities: ["  n8n  ", " ", "Qdrant", 42],
+    };
+    const fetchImpl = makeFetch(() =>
+      jsonResponse({ message: { content: JSON.stringify(messy) } }),
+    );
+    const client = new OllamaClient({ fetchImpl });
+    const result = await client.classify("transcript");
+    expect(result.entities).toEqual(["n8n", "Qdrant", "42"]);
+  });
+});

package/tests/unit/scripts/longmemeval-scorer.test.ts

@@ -0,0 +1,114 @@
+import { describe, expect, it } from "vitest";
+import {
+  aggregate,
+  scoreOne,
+} from "../../../scripts/longmemeval/scorer.js";
+
+describe("scoreOne", () => {
+  it("recallAtK=1 when a gold id is in the top-k", () => {
+    const r = scoreOne({
+      returnedIds: ["a", "b", "gold-1", "d", "e"],
+      goldIds: ["gold-1", "gold-2"],
+      returnedBodies: ["", "", "", "", ""],
+      answer: "anything",
+      k: 5,
+    });
+    expect(r.recallAtK).toBe(1);
+  });
+
+  it("recallAtK=0 when no gold id appears in the top-k", () => {
+    const r = scoreOne({
+      returnedIds: ["a", "b", "c", "d", "e"],
+      goldIds: ["gold-1"],
+      returnedBodies: ["", "", "", "", ""],
+      answer: "z",
+      k: 5,
+    });
+    expect(r.recallAtK).toBe(0);
+  });
+
+  it("recallAtK respects k=3 ceiling even when gold is at position 4", () => {
+    const r = scoreOne({
+      returnedIds: ["a", "b", "c", "gold-1", "e"],
+      goldIds: ["gold-1"],
+      returnedBodies: [],
+      answer: "z",
+      k: 3,
+    });
+    expect(r.recallAtK).toBe(0);
+  });
+
+  it("sessionBodyHit=1 when answer substring appears in any returned body (case/whitespace insensitive)", () => {
+    const r = scoreOne({
+      returnedIds: ["a", "b"],
+      goldIds: ["gold"],
+      returnedBodies: [
+        "irrelevant transcript content",
+        "User: Where did you grow up?\nAssistant: I grew up in   Austin, Texas.",
+      ],
+      answer: "austin texas",
+      k: 5,
+    });
+    expect(r.sessionBodyHit).toBe(1);
+  });
+
+  it("sessionBodyHit=0 when answer is absent from all returned bodies", () => {
+    const r = scoreOne({
+      returnedIds: ["a"],
+      goldIds: ["gold"],
+      returnedBodies: ["nothing matches here"],
+      answer: "Houston",
+      k: 5,
+    });
+    expect(r.sessionBodyHit).toBe(0);
+  });
+
+  it("coerces a numeric answer to string and matches with word boundaries", () => {
+    const hit = scoreOne({
+      returnedIds: ["a"],
+      goldIds: ["gold"],
+      returnedBodies: ["The user mentioned 3 brothers in the conversation."],
+      answer: 3,
+      k: 5,
+    });
+    expect(hit.sessionBodyHit).toBe(1);
+
+    const miss = scoreOne({
+      returnedIds: ["a"],
+      goldIds: ["gold"],
+      returnedBodies: ["They paid $30 for the meal and tipped 20%."],
+      answer: 3,
+      k: 5,
+    });
+    expect(miss.sessionBodyHit).toBe(0);
+  });
+
+  it("sessionBodyHit=0 when answer is empty", () => {
+    const r = scoreOne({
+      returnedIds: ["a"],
+      goldIds: ["gold"],
+      returnedBodies: ["whatever"],
+      answer: "",
+      k: 5,
+    });
+    expect(r.sessionBodyHit).toBe(0);
+  });
+});
+
+describe("aggregate", () => {
+  it("computes mean rates rounded to 3 decimal places", () => {
+    const a = aggregate([
+      { recallAtK: 1, sessionBodyHit: 1 },
+      { recallAtK: 0, sessionBodyHit: 1 },
+      { recallAtK: 1, sessionBodyHit: 0 },
+    ]);
+    expect(a.n).toBe(3);
+    expect(a.recallAtK).toBeCloseTo(0.667, 3);
+    expect(a.sessionBodyHitRate).toBeCloseTo(0.667, 3);
+  });
+
+  it("returns zeros for empty input", () => {
+    const a = aggregate([]);
+    expect(a).toEqual({ n: 0, recallAtK: 0, sessionBodyHitRate: 0 });
+  });
+});

package/tsconfig.json

@@ -0,0 +1,31 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "lib": ["ES2022"],
+    "outDir": "dist",
+    "rootDir": "src",
+    "strict": true,
+    "noUncheckedIndexedAccess": true,
+    "exactOptionalPropertyTypes": true,
+    "noImplicitOverride": true,
+    "noFallthroughCasesInSwitch": true,
+    "forceConsistentCasingInFileNames": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "declaration": true,
+    "sourceMap": true,
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "verbatimModuleSyntax": true,
+    "baseUrl": "src",
+    "paths": {
+      "@core/*": ["./core/*"],
+      "@ports/*": ["./ports/*"],
+      "@shared/*": ["./shared/*"]
+    }
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["node_modules", "dist", "src/ui"]
+}

package/tsconfig.test.json

@@ -0,0 +1,11 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "rootDir": ".",
+    "noEmit": true,
+    "declaration": false,
+    "sourceMap": false
+  },
+  "include": ["src/**/*.ts", "tests/**/*.ts"],
+  "exclude": ["node_modules", "dist", "src/ui"]
+}

package/vitest.config.ts

@@ -0,0 +1,22 @@
+import { defineConfig } from "vitest/config";
+import { fileURLToPath } from "node:url";
+
+export default defineConfig({
+  test: {
+    globals: false,
+    environment: "node",
+    include: ["tests/**/*.test.ts"],
+    coverage: {
+      provider: "v8",
+      include: ["src/core/**", "src/ports/**"],
+      reporter: ["text", "html"],
+    },
+  },
+  resolve: {
+    alias: {
+      "@core": fileURLToPath(new URL("./src/core", import.meta.url)),
+      "@ports": fileURLToPath(new URL("./src/ports", import.meta.url)),
+      "@shared": fileURLToPath(new URL("./src/shared", import.meta.url)),
+    },
+  },
+});