@remnic/core 1.1.12 → 1.1.14

package/src/buffer-surprise.test.ts

@@ -0,0 +1,339 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import {
+  computeSurprise,
+  DEFAULT_SURPRISE_K,
+  type RecentMemoryLike,
+} from "./buffer-surprise.js";
+
+// -----------------------------------------------------------------------------
+// Deterministic mock embedFn
+// -----------------------------------------------------------------------------
+//
+// We use two complementary mock embedders so that tests are fully reproducible
+// and do not depend on any real embedding model.
+//
+// 1. `hashEmbedder(dim)` — a character-hash "bag of codepoints" embedder.
+//    Identical strings produce identical vectors (perfect cosine = 1).
+//    Distinct strings produce vectors whose similarity reflects character
+//    overlap. Deterministic, pure, and independent across calls.
+//
+// 2. `fixtureEmbedder(map)` — returns caller-supplied canned vectors for
+//    specific texts. Used when a test needs exactly orthogonal vectors or
+//    precisely-tuned near-duplicates.
+
+function hashEmbedder(dim = 16) {
+  return async (text: string): Promise<readonly number[]> => {
+    const vec = new Array<number>(dim).fill(0);
+    const normalized = text.toLowerCase();
+    for (let i = 0; i < normalized.length; i += 1) {
+      const code = normalized.charCodeAt(i);
+      // Non-negative bucket, plus a small per-position weight so the embedder
+      // is sensitive to character order (not just character frequency).
+      const bucket = code % dim;
+      vec[bucket] = (vec[bucket] ?? 0) + 1 + (i % 3) * 0.01;
+    }
+    return vec;
+  };
+}
+
+function fixtureEmbedder(
+  map: Record<string, readonly number[]>,
+  fallback: readonly number[] | null = null,
+) {
+  return async (text: string): Promise<readonly number[]> => {
+    if (Object.prototype.hasOwnProperty.call(map, text)) {
+      // Defensive copy so callers can't see mutations across calls.
+      return [...map[text]!];
+    }
+    if (fallback) return [...fallback];
+    throw new Error(`fixtureEmbedder: no fixture for ${JSON.stringify(text)}`);
+  };
+}
+
+function mem(id: string, content: string): RecentMemoryLike {
+  return { id, content };
+}
+
+// -----------------------------------------------------------------------------
+// Tests
+// -----------------------------------------------------------------------------
+
+test("computeSurprise returns 1.0 when recentMemories is empty", async () => {
+  const score = await computeSurprise("anything", [], {
+    embedFn: hashEmbedder(),
+  });
+  assert.equal(score, 1);
+});
+
+test("computeSurprise returns 1.0 when recentMemories is empty (no embedFn call)", async () => {
+  // The empty-memories case should not need to embed the turn; the short
+  // circuit lets callers skip the network round-trip.
+  let calls = 0;
+  const embedFn = async () => {
+    calls += 1;
+    return [1, 0];
+  };
+  const score = await computeSurprise("hello", [], { embedFn });
+  assert.equal(score, 1);
+  assert.equal(calls, 0, "embedFn should not be called for empty corpus");
+});
+
+test("computeSurprise returns ~0 for a single identical memory", async () => {
+  const memories = [mem("a", "user likes espresso")];
+  const score = await computeSurprise("user likes espresso", memories, {
+    embedFn: hashEmbedder(),
+  });
+  // Identical text under the hash embedder → cosine 1 → surprise 0.
+  assert.ok(
+    Math.abs(score - 0) < 1e-9,
+    `expected surprise ~0, got ${score}`,
+  );
+});
+
+test("computeSurprise returns ~1 for orthogonal memories", async () => {
+  // Construct vectors that are exactly orthogonal to the turn.
+  const embed = fixtureEmbedder({
+    TURN: [1, 0, 0, 0],
+    A: [0, 1, 0, 0],
+    B: [0, 0, 1, 0],
+    C: [0, 0, 0, 1],
+  });
+  const memories = [mem("a", "A"), mem("b", "B"), mem("c", "C")];
+  const score = await computeSurprise("TURN", memories, { embedFn: embed });
+  assert.equal(score, 1);
+});
+
+test("computeSurprise is monotonic: adding a more-similar memory decreases the score", async () => {
+  // Fixture: turn points along the x-axis. The "far" memory is orthogonal
+  // (cosine 0). The "near" memory is highly aligned with the turn (cosine
+  // close to 1). Averaging with k=2 over [far, near] should produce a
+  // strictly lower surprise than averaging over [far] alone.
+  const embed = fixtureEmbedder({
+    TURN: [1, 0, 0, 0],
+    FAR: [0, 1, 0, 0],
+    NEAR: [0.99, 0.01, 0, 0],
+  });
+
+  const farOnly = [mem("far", "FAR")];
+  const farAndNear = [mem("far", "FAR"), mem("near", "NEAR")];
+
+  const scoreFarOnly = await computeSurprise("TURN", farOnly, {
+    embedFn: embed,
+    k: 2,
+  });
+  const scoreFarAndNear = await computeSurprise("TURN", farAndNear, {
+    embedFn: embed,
+    k: 2,
+  });
+  assert.ok(
+    scoreFarAndNear < scoreFarOnly,
+    `expected adding a near memory to decrease surprise; farOnly=${scoreFarOnly}, farAndNear=${scoreFarAndNear}`,
+  );
+});
+
+test("computeSurprise clamps k to the number of memories when k > n", async () => {
+  // With 3 memories and k=10, the score must equal the score computed with
+  // k=3 (because the top-k average over 3 similarities is the average of
+  // all of them in both cases).
+  const embed = fixtureEmbedder({
+    TURN: [1, 0, 0, 0],
+    A: [0.9, 0.1, 0, 0],
+    B: [0.5, 0.5, 0, 0],
+    C: [0.1, 0.9, 0, 0],
+  });
+  const memories = [mem("a", "A"), mem("b", "B"), mem("c", "C")];
+
+  const kHuge = await computeSurprise("TURN", memories, {
+    embedFn: embed,
+    k: 10,
+  });
+  const kExact = await computeSurprise("TURN", memories, {
+    embedFn: embed,
+    k: 3,
+  });
+  assert.ok(
+    Math.abs(kHuge - kExact) < 1e-12,
+    `expected k=10 and k=3 to match when n=3; kHuge=${kHuge}, kExact=${kExact}`,
+  );
+});
+
+test("computeSurprise clamps k to >= 1 when caller passes 0 or negative", async () => {
+  // k=0 or k=-1 must not short-circuit to "no neighbors considered";
+  // the docstring says k is clamped to [1, n].
+  const embed = fixtureEmbedder({
+    TURN: [1, 0, 0, 0],
+    A: [1, 0, 0, 0], // identical direction → cos 1 → surprise 0
+  });
+  const memories = [mem("a", "A")];
+
+  const kZero = await computeSurprise("TURN", memories, {
+    embedFn: embed,
+    k: 0,
+  });
+  const kNeg = await computeSurprise("TURN", memories, {
+    embedFn: embed,
+    k: -5,
+  });
+  const kOne = await computeSurprise("TURN", memories, {
+    embedFn: embed,
+    k: 1,
+  });
+  assert.equal(kZero, kOne);
+  assert.equal(kNeg, kOne);
+  assert.equal(kOne, 0);
+});
+
+test("computeSurprise uses top-k nearest, not all memories, when k < n", async () => {
+  // With k=1, the score should reflect ONLY the single most-similar memory.
+  // A corpus of [very-similar, far, far, far] at k=1 must yield ~0 surprise
+  // (because the nearest memory is very close), whereas k=4 over the same
+  // corpus yields a much higher surprise (because the average dilutes the
+  // single near hit with three far ones).
+  const embed = fixtureEmbedder({
+    TURN: [1, 0, 0, 0],
+    NEAR: [0.99, 0.01, 0, 0],
+    FAR1: [0, 1, 0, 0],
+    FAR2: [0, 0, 1, 0],
+    FAR3: [0, 0, 0, 1],
+  });
+  const memories = [
+    mem("near", "NEAR"),
+    mem("far1", "FAR1"),
+    mem("far2", "FAR2"),
+    mem("far3", "FAR3"),
+  ];
+
+  const kOne = await computeSurprise("TURN", memories, {
+    embedFn: embed,
+    k: 1,
+  });
+  const kAll = await computeSurprise("TURN", memories, {
+    embedFn: embed,
+    k: 4,
+  });
+  assert.ok(kOne < 0.05, `k=1 should be ~0, got ${kOne}`);
+  assert.ok(kAll > 0.5, `k=4 should dilute toward ~0.75, got ${kAll}`);
+  assert.ok(kAll > kOne, "larger k should not decrease surprise here");
+});
+
+test("computeSurprise propagates embedFn rejections", async () => {
+  const boom = new Error("embedding service unavailable");
+  const embedFn = async () => {
+    throw boom;
+  };
+  await assert.rejects(
+    () =>
+      computeSurprise("turn", [mem("a", "something")], {
+        embedFn,
+      }),
+    /embedding service unavailable/,
+  );
+});
+
+test("computeSurprise treats zero-norm embeddings as similarity 0", async () => {
+  // A zero vector has no direction; cosine is undefined. We document that
+  // these pairs contribute similarity 0 (maximally surprising), not NaN.
+  const embed = fixtureEmbedder({
+    TURN: [1, 0, 0, 0],
+    ZERO: [0, 0, 0, 0],
+  });
+  const score = await computeSurprise("TURN", [mem("z", "ZERO")], {
+    embedFn: embed,
+  });
+  assert.equal(score, 1);
+});
+
+test("computeSurprise returns 0 when embedding-length mismatch (treated as sim 0)", async () => {
+  // We intentionally do NOT silently truncate to the shorter vector —
+  // mismatched dims almost always mean a config bug. They should surface
+  // as "no similarity" (surprise = 1), not a quietly-wrong score.
+  const embed = fixtureEmbedder({
+    TURN: [1, 0, 0, 0],
+    SHORT: [1, 0],
+  });
+  const score = await computeSurprise("TURN", [mem("s", "SHORT")], {
+    embedFn: embed,
+  });
+  assert.equal(score, 1);
+});
+
+test("computeSurprise default k is 5 (docstring matches constant)", async () => {
+  assert.equal(DEFAULT_SURPRISE_K, 5);
+
+  // Behavioral check: when n=5 and we omit k, we should get the same answer
+  // as passing k=5 explicitly.
+  const embed = hashEmbedder(8);
+  const memories = [
+    mem("a", "alpha content"),
+    mem("b", "beta content"),
+    mem("c", "gamma content"),
+    mem("d", "delta content"),
+    mem("e", "epsilon content"),
+  ];
+  const noK = await computeSurprise("zeta content", memories, {
+    embedFn: embed,
+  });
+  const k5 = await computeSurprise("zeta content", memories, {
+    embedFn: embed,
+    k: 5,
+  });
+  assert.equal(noK, k5);
+});
+
+test("computeSurprise is deterministic for the same inputs and embedFn", async () => {
+  const embed = hashEmbedder();
+  const memories = [
+    mem("a", "deterministic test fixture one"),
+    mem("b", "another canned memory fragment"),
+    mem("c", "third entry with different content"),
+  ];
+  const a = await computeSurprise("query about something new", memories, {
+    embedFn: embed,
+  });
+  const b = await computeSurprise("query about something new", memories, {
+    embedFn: embed,
+  });
+  assert.equal(a, b);
+});
+
+test("computeSurprise result is always in [0, 1]", async () => {
+  const embed = hashEmbedder();
+  const corpora: RecentMemoryLike[][] = [
+    [],
+    [mem("a", "single memory")],
+    [mem("a", "foo"), mem("b", "bar"), mem("c", "foo")],
+  ];
+  for (const memories of corpora) {
+    for (const turn of ["foo", "bar", "novel question", ""]) {
+      const score = await computeSurprise(turn, memories, { embedFn: embed });
+      assert.ok(
+        score >= 0 && score <= 1,
+        `score out of range for turn=${JSON.stringify(turn)} len=${memories.length}: ${score}`,
+      );
+    }
+  }
+});
+
+test("computeSurprise throws TypeError when embedFn is missing", async () => {
+  await assert.rejects(
+    () =>
+      // biome-ignore lint: intentionally passing a malformed options object
+      computeSurprise("turn", [mem("a", "x")], {} as any),
+    /embedFn/,
+  );
+});
+
+test("computeSurprise throws TypeError when turn is not a string", async () => {
+  await assert.rejects(
+    () =>
+      computeSurprise(
+        // biome-ignore lint: intentionally passing wrong type
+        42 as any,
+        [mem("a", "x")],
+        { embedFn: hashEmbedder() },
+      ),
+    /must be a string/,
+  );
+});

package/src/buffer-surprise.ts

@@ -0,0 +1,203 @@
+/**
+ * Surprise helper — D-MEM-style novelty score for a turn against recent memories.
+ *
+ * Used (eventually, in a follow-up PR) by the smart buffer as an additive flush
+ * trigger: a turn that is semantically far from everything already in memory is
+ * "surprising" and worth extracting immediately, even if turn-count and signal
+ * heuristics would otherwise keep buffering.
+ *
+ * This module intentionally contains ONLY the pure scoring function. Wiring
+ * into the buffer, configuration flags, telemetry, and benchmark work live in
+ * later PRs (see issue #563). Keeping the helper in a sibling file instead of
+ * inside `buffer.ts` keeps the buffer module focused and makes the eventual
+ * integration a small, reviewable change.
+ *
+ * # Formula
+ *
+ * Given an input `turn` and a list of `recentMemories`:
+ *
+ * 1. Embed the turn and every candidate memory via the caller-provided
+ *    `embedFn`.
+ * 2. For each candidate, compute cosine similarity `cos(turn, candidate)` in
+ *    `[0, 1]` (negative cosines are clamped to 0 — "opposite directions"
+ *    is treated as maximally diverse, consistent with `recall-mmr.ts`).
+ * 3. Keep the top-`k` *highest* similarities (the turn's nearest neighbors).
+ * 4. Average those top-`k` similarities → `nearestSim ∈ [0, 1]`.
+ * 5. Return `1 − nearestSim` as the surprise score.
+ *
+ * Intuition: a turn that is close to at least one recent memory has a high
+ * `nearestSim` and therefore a *low* surprise (near 0 = redundant). A turn
+ * that is far from all of its k nearest neighbors has a low `nearestSim` and
+ * therefore a *high* surprise (near 1 = novel).
+ *
+ * Using the top-k average instead of a single nearest neighbor makes the
+ * score less sensitive to one outlier duplicate (e.g. an exact restatement of
+ * a stale fact). `k=1` reduces to pure nearest-neighbor distance.
+ *
+ * # Edge cases
+ *
+ * - `recentMemories` empty → returns `1.0` (maximally surprising). There is
+ *   nothing to compare against, so the turn cannot be redundant.
+ * - `k` default is `5`. It is clamped to `[1, recentMemories.length]` so that
+ *   small corpora still produce a meaningful score (e.g. 3 memories with
+ *   `k=5` behaves the same as `k=3`).
+ * - Zero-norm embeddings (all zeros) contribute similarity `0` (they cannot
+ *   be "close" to anything), which is treated as maximally surprising for
+ *   that pair. The rest of the corpus is scored normally.
+ * - Embedding-length mismatches between the turn and a candidate are treated
+ *   as similarity `0` for that pair. We do not silently truncate to the
+ *   shorter vector because that would hide a real configuration bug.
+ * - `embedFn` rejection is allowed to propagate. The caller decides whether
+ *   to catch, fall back, or fail the flush decision — this helper has no
+ *   opinion beyond the pure score.
+ *
+ * # Purity
+ *
+ * No I/O, no hidden globals. Given the same inputs and the same `embedFn`,
+ * the output is deterministic.
+ */
+
+import { cosineSimilarity } from "./semantic-chunking.js";
+
+/** Minimal shape of a recent memory passed to `computeSurprise`. */
+export interface RecentMemoryLike {
+  /** Stable identifier. Only used for debug/logging; not part of the score. */
+  readonly id: string;
+  /** Text to embed and compare against the incoming turn. */
+  readonly content: string;
+}
+
+/** Options accepted by `computeSurprise`. */
+export interface ComputeSurpriseOptions {
+  /**
+   * Embedding function. Called once for the turn and once per candidate
+   * memory. Callers may wrap a provider client, a local model, or a
+   * deterministic hash for tests.
+   */
+  readonly embedFn: (text: string) => Promise<readonly number[]>;
+  /**
+   * Number of nearest neighbors to average over. Defaults to 5. Clamped to
+   * `[1, recentMemories.length]`.
+   */
+  readonly k?: number;
+}
+
+/** Default k (top nearest neighbors to average). */
+export const DEFAULT_SURPRISE_K = 5;
+
+/**
+ * Compute a surprise score in `[0, 1]` for `turn` against `recentMemories`.
+ *
+ * See the module-level docstring for the exact formula, edge cases, and
+ * purity guarantees.
+ *
+ * @param turn           The incoming turn text (caller is responsible for
+ *                       stringifying structured turn content — this helper
+ *                       only needs the text to embed).
+ * @param recentMemories Candidate memories to compare against.
+ * @param options        `embedFn` (required) and optional `k` (defaults to 5).
+ * @returns A scalar in `[0, 1]`. `1.0` = maximally surprising / novel;
+ *          `0.0` = redundant with at least one recent memory.
+ */
+export async function computeSurprise(
+  turn: string,
+  recentMemories: readonly RecentMemoryLike[],
+  options: ComputeSurpriseOptions,
+): Promise<number> {
+  if (typeof turn !== "string") {
+    throw new TypeError("computeSurprise: `turn` must be a string");
+  }
+  if (!options || typeof options.embedFn !== "function") {
+    throw new TypeError(
+      "computeSurprise: `options.embedFn` is required and must be a function",
+    );
+  }
+  const candidates = Array.isArray(recentMemories) ? recentMemories : [];
+
+  // Empty corpus → maximally surprising. Document decision: we prefer
+  // "novelty by default" over "silence" because the surprise score is meant
+  // to *promote* flushing, and an empty buffer of recent memories is the
+  // clearest case of "we have no basis to call this redundant".
+  if (candidates.length === 0) return 1;
+
+  const kRaw = typeof options.k === "number" ? options.k : DEFAULT_SURPRISE_K;
+  // Clamp k to [1, candidates.length]. Non-integer / non-finite values fall
+  // back to the default, then get re-clamped below.
+  const kClamped = clampK(kRaw, candidates.length);
+
+  // Embed the turn and every candidate. Rejections propagate — that's the
+  // caller's decision to handle. Run candidate embeddings in parallel so a
+  // slow embedder does not serialize the whole pass.
+  const turnEmbedding = await options.embedFn(turn);
+  const candidateEmbeddings = await Promise.all(
+    candidates.map((c) => options.embedFn(c.content)),
+  );
+
+  const sims: number[] = [];
+  for (let i = 0; i < candidates.length; i += 1) {
+    const candEmbedding = candidateEmbeddings[i] ?? [];
+    sims.push(safeCosine(turnEmbedding, candEmbedding));
+  }
+
+  // Sort similarities descending and take the top-k (the nearest neighbors).
+  sims.sort((a, b) => b - a);
+  const top = sims.slice(0, kClamped);
+
+  if (top.length === 0) {
+    // Defensive — with `candidates.length >= 1` and `kClamped >= 1`, we
+    // should always have at least one similarity. If we somehow do not,
+    // treat as maximally surprising rather than returning NaN.
+    return 1;
+  }
+
+  let sum = 0;
+  for (const s of top) sum += s;
+  const nearestSim = sum / top.length;
+
+  const surprise = 1 - nearestSim;
+  // Numerical safety: clamp the final value. Cosine is clamped to [0, 1]
+  // per-pair already, so the mean cannot escape that range under normal
+  // arithmetic, but floating-point drift is cheap to guard against.
+  if (!Number.isFinite(surprise)) return 1;
+  if (surprise < 0) return 0;
+  if (surprise > 1) return 1;
+  return surprise;
+}
+
+// -----------------------------------------------------------------------------
+// Internals
+// -----------------------------------------------------------------------------
+
+function clampK(k: number, ceiling: number): number {
+  if (!Number.isFinite(k)) {
+    return Math.min(DEFAULT_SURPRISE_K, Math.max(1, ceiling));
+  }
+  const floored = Math.floor(k);
+  if (floored < 1) return 1;
+  if (floored > ceiling) return ceiling;
+  return floored;
+}
+
+/**
+ * Safe wrapper around `cosineSimilarity` that returns a similarity value in
+ * `[0, 1]` (negative cosines are clamped to 0, matching the convention used
+ * by `recall-mmr.ts`) and tolerates mismatched or empty vectors without
+ * throwing. See `semantic-chunking.ts` for the shared cosine implementation.
+ *
+ * Returns `0` for:
+ * - empty vectors,
+ * - length mismatches (callers likely have a config bug, but surprise
+ *   scoring is a soft signal — do not crash the buffer flush decision),
+ * - non-finite cosine results (zero-norm inputs, arithmetic drift).
+ */
+function safeCosine(a: readonly number[], b: readonly number[]): number {
+  if (!Array.isArray(a) || !Array.isArray(b)) return 0;
+  if (a.length === 0 || b.length === 0) return 0;
+  if (a.length !== b.length) return 0;
+  const raw = cosineSimilarity(a as number[], b as number[]);
+  if (!Number.isFinite(raw)) return 0;
+  if (raw < 0) return 0;
+  if (raw > 1) return 1;
+  return raw;
+}
+