onbuzz 4.8.3 → 4.9.1

package/src/services/embeddings/utilities/textChunker.js

@@ -0,0 +1,290 @@
+/**
+ * textChunker — split text into ~N-token chunks for embedding.
+ *
+ * Why we chunk:
+ *   - Embedding models attend best to inputs in the 200–500 token range.
+ *     A 5K-token system prompt embedded as one vector blurs every concept
+ *     together; chunked, each topic gets its own retrievable point.
+ *   - Provider hard caps: nomic 8K, BGE 512, MiniLM 256. Truncation at
+ *     ingest would drop the tail of any long input silently. We chunk
+ *     so nothing is lost.
+ *
+ * Token counting:
+ *   We use a char/4 proxy throughout. Real tokenization (tiktoken) would
+ *   add a heavy dep just for sizing, and the chunker doesn't need
+ *   precision — overshooting by ~10% is fine because providers have
+ *   their own internal token-level truncation as a safety net.
+ *
+ * Algorithm:
+ *   1. Treat \n\n separators as paragraph atoms — preserve semantic units.
+ *   2. Greedily pack atoms into a chunk up to maxChars.
+ *   3. If a single atom exceeds maxChars (rare — only for tool outputs
+ *      without paragraph breaks), split it on sentence boundaries (". "
+ *      etc.); if still too big, hard-slice at maxChars.
+ *   4. Each chunk after the first carries an `overlapChars` tail from the
+ *      previous chunk so cross-chunk topics aren't split mid-thought.
+ *
+ * Output:
+ *   Array of `{ text, start, end, index }` where:
+ *     - text:  the chunk content (incl. any overlap)
+ *     - start: character offset in the original text where THIS CHUNK's
+ *              NEW content begins (i.e. excludes the overlap region —
+ *              lets the caller hydrate back to the original message)
+ *     - end:   exclusive end offset of THIS CHUNK's new content
+ *     - index: 0-based chunk index
+ *
+ *   Concatenating text[start:end] across all chunks reconstructs the
+ *   original input exactly (modulo the join character — there isn't one,
+ *   chunks are contiguous over the new-content offsets).
+ */
+
+/** Default sizing: ~300-token chunks with ~50-token overlap. Approx char-based. */
+export const DEFAULT_MAX_CHARS = 1200;
+export const DEFAULT_OVERLAP_CHARS = 200;
+
+/**
+ * Chunk text into approximately-token-sized pieces.
+ *
+ * @param {string} text
+ * @param {object} [opts]
+ * @param {number} [opts.maxChars=1200]      Hard cap per chunk (incl. overlap).
+ * @param {number} [opts.overlapChars=200]   Trailing context from prev chunk.
+ * @returns {Array<{text: string, start: number, end: number, index: number}>}
+ */
+export function chunkText(text, opts = {}) {
+  if (typeof text !== 'string' || text.length === 0) return [];
+
+  const maxChars = Math.max(50, opts.maxChars ?? DEFAULT_MAX_CHARS);
+  const overlap  = Math.min(Math.max(0, opts.overlapChars ?? DEFAULT_OVERLAP_CHARS), maxChars - 50);
+
+  // Fast path: single chunk fits.
+  if (text.length <= maxChars) {
+    return [{ text, start: 0, end: text.length, index: 0 }];
+  }
+
+  // Split into paragraph atoms. We DO NOT collapse whitespace — the
+  // chunks must concatenate back to a faithful reconstruction.
+  const atoms = splitWithSeparator(text, '\n\n');
+
+  // Convert atoms whose own length exceeds maxChars into sub-atoms.
+  // First by sentence, then by hard slicing as a last resort.
+  const safeAtoms = [];
+  for (const atom of atoms) {
+    if (atom.text.length <= maxChars) {
+      safeAtoms.push(atom);
+      continue;
+    }
+    const subBySentence = splitLongAtomBySentence(atom, maxChars);
+    for (const sub of subBySentence) {
+      if (sub.text.length <= maxChars) {
+        safeAtoms.push(sub);
+        continue;
+      }
+      // Still too long → hard slice. We pay the semantic cost only here,
+      // and only on pathological inputs (a single sentence > maxChars).
+      safeAtoms.push(...hardSliceAtom(sub, maxChars));
+    }
+  }
+
+  // Greedy pack atoms into chunks ≤ maxChars (treating the overlap budget
+  // as reserved space at the start of each non-first chunk).
+  const chunks = [];
+  let bufferStart = safeAtoms[0]?.start ?? 0;
+  let bufferEnd   = bufferStart;
+  let bufferText  = '';
+  let chunkIndex  = 0;
+  // Tracks the overlap suffix to prepend to the next chunk.
+  let pendingOverlap = '';
+
+  const flush = () => {
+    if (bufferText.length === 0) return;
+    const combined = pendingOverlap + bufferText;
+    chunks.push({
+      text: combined,
+      start: bufferStart,
+      end: bufferEnd,
+      index: chunkIndex++,
+    });
+    // Compute new pending overlap from the tail of the COMBINED text so
+    // the overlap window is what the model will actually see at the
+    // start of the next chunk.
+    pendingOverlap = overlap > 0
+      ? combined.slice(Math.max(0, combined.length - overlap))
+      : '';
+    bufferText = '';
+  };
+
+  for (const atom of safeAtoms) {
+    // The room left in this chunk = maxChars - overlap (if any) - current.
+    const overlapBudget = chunkIndex === 0 ? 0 : overlap;
+    const headroom = maxChars - overlapBudget - bufferText.length;
+
+    if (atom.text.length <= headroom || bufferText.length === 0) {
+      // Either fits, or this is the first atom in a fresh buffer — accept it.
+      if (bufferText.length === 0) bufferStart = atom.start;
+      bufferText += atom.text;
+      bufferEnd   = atom.end;
+    } else {
+      // Doesn't fit — flush current buffer, then start new one with this atom.
+      flush();
+      bufferStart = atom.start;
+      bufferText  = atom.text;
+      bufferEnd   = atom.end;
+    }
+  }
+  flush();
+
+  return chunks;
+}
+
+/**
+ * Split `text` on `sep`, preserving offsets and KEEPING the separator
+ * at the end of each piece except the last (so concatenation reconstructs
+ * the original). Yields `{text, start, end}` per piece.
+ * @private
+ */
+function splitWithSeparator(text, sep) {
+  const atoms = [];
+  let cursor = 0;
+  let i;
+  while ((i = text.indexOf(sep, cursor)) !== -1) {
+    const end = i + sep.length;
+    atoms.push({ text: text.slice(cursor, end), start: cursor, end });
+    cursor = end;
+  }
+  if (cursor < text.length) {
+    atoms.push({ text: text.slice(cursor), start: cursor, end: text.length });
+  }
+  return atoms;
+}
+
+/**
+ * Split a single oversized atom on sentence boundaries (". ", "! ", "? ",
+ * ".\n", or any standalone newline). Same return shape as splitWithSeparator.
+ * @private
+ */
+function splitLongAtomBySentence(atom, maxChars) {
+  const text = atom.text;
+  const out = [];
+  let pieceStart = 0;
+  // Track the most recent sentence boundary that's still WITHIN budget,
+  // so we can flush there when adding the next sentence would exceed it.
+  let lastSafeBoundary = -1;
+
+  let cursor = 0;
+  while (cursor < text.length) {
+    const candidateEnd = findNextSentenceBoundary(text, cursor);
+    if (candidateEnd === -1) break;
+
+    const pieceLen = candidateEnd - pieceStart;
+
+    if (pieceLen > maxChars) {
+      // Adding this sentence would overflow.
+      if (lastSafeBoundary > pieceStart) {
+        // Flush at the last in-budget boundary and re-evaluate THIS
+        // sentence against the new piece (don't advance cursor).
+        out.push({
+          text: text.slice(pieceStart, lastSafeBoundary),
+          start: atom.start + pieceStart,
+          end:   atom.start + lastSafeBoundary,
+        });
+        pieceStart = lastSafeBoundary;
+        lastSafeBoundary = -1;
+        continue;
+      }
+      // No safe boundary AND this single sentence exceeds maxChars. Emit
+      // it anyway — the caller (chunkText) will hard-slice as a last
+      // resort. This branch only fires for pathological sentence-less
+      // inputs like a base64 blob.
+      out.push({
+        text: text.slice(pieceStart, candidateEnd),
+        start: atom.start + pieceStart,
+        end:   atom.start + candidateEnd,
+      });
+      pieceStart = candidateEnd;
+      lastSafeBoundary = -1;
+      cursor = candidateEnd;
+      continue;
+    }
+
+    // Still within budget — remember this boundary as our safe flush point.
+    lastSafeBoundary = candidateEnd;
+    cursor = candidateEnd;
+  }
+
+  // Loop exited because no more sentence boundaries remain. If the
+  // remaining tail exceeds maxChars but we DID find an earlier safe
+  // boundary inside it, flush there first so the final emitted piece
+  // also lands on a sentence break.
+  if (pieceStart < text.length) {
+    const tailLen = text.length - pieceStart;
+    if (tailLen > maxChars && lastSafeBoundary > pieceStart) {
+      out.push({
+        text: text.slice(pieceStart, lastSafeBoundary),
+        start: atom.start + pieceStart,
+        end:   atom.start + lastSafeBoundary,
+      });
+      pieceStart = lastSafeBoundary;
+    }
+    out.push({
+      text: text.slice(pieceStart),
+      start: atom.start + pieceStart,
+      end:   atom.start + text.length,
+    });
+  }
+  return out.length > 0 ? out : [atom];
+}
+
+/**
+ * Find the END of the next sentence-ending boundary in `text`, starting
+ * from `from`. The boundary characters (e.g. ". ", "\n") are INCLUDED in
+ * the returned index, so `text.slice(start, boundaryEnd)` gives a piece
+ * that ends naturally. Returns -1 when no boundary remains.
+ *
+ * Uses native indexOf — no regex.
+ * @private
+ */
+function findNextSentenceBoundary(text, from) {
+  // We look for one of: ". ", "! ", "? ", ".\n", "!\n", "?\n", or a lone "\n".
+  // Multi-char patterns are tried first; we return whichever STARTS earliest
+  // and resolve ties by preferring the longer (more specific) boundary.
+  let bestStart = -1;
+  let bestEnd = -1;
+  for (const candidate of ['. ', '! ', '? ', '.\n', '!\n', '?\n', '\n']) {
+    const idx = text.indexOf(candidate, from);
+    if (idx === -1) continue;
+    const end = idx + candidate.length;
+    if (bestStart === -1 || idx < bestStart || (idx === bestStart && end > bestEnd)) {
+      bestStart = idx;
+      bestEnd = end;
+    }
+  }
+  return bestEnd;
+}
+
+/**
+ * Hard-slice an atom that's still bigger than maxChars even after
+ * sentence splitting (e.g. a single token-less blob like a base64 string).
+ * @private
+ */
+function hardSliceAtom(atom, maxChars) {
+  const out = [];
+  for (let i = 0; i < atom.text.length; i += maxChars) {
+    const end = Math.min(atom.text.length, i + maxChars);
+    out.push({
+      text: atom.text.slice(i, end),
+      start: atom.start + i,
+      end:   atom.start + end,
+    });
+  }
+  return out;
+}
+
+/**
+ * Estimate tokens by chars/4. Same heuristic the orchestrator uses for
+ * budget tracking — exported so consumers can pre-flight the sizing.
+ */
+export function estimateTokensChunked(text) {
+  if (typeof text !== 'string' || !text) return 0;
+  return Math.ceil(text.length / 4);
+}

package/src/services/embeddings/vectorStore/__tests__/scoring.test.js

@@ -0,0 +1,189 @@
+/**
+ * scoring.test.js — multi-signal ranking helpers.
+ *
+ * Pins (against the production-tested values from Mem0 / ChatGPT):
+ *   - Identity scorer when no signals enabled (backwards compat)
+ *   - Sigmoid recency: maxFactor at age=0, midpoint at age=halfLife,
+ *     minFactor at age >> halfLife
+ *   - Access boost saturates (log curve, clamped at maxFactor)
+ *   - Missing metadata → identity (no boost, no crash)
+ *   - Composition is multiplicative, clamped to factor range
+ *   - Tunable via opts (halfLifeDays, weight, etc.)
+ *   - Deterministic via `now` override
+ */
+
+import { describe, test, expect } from '@jest/globals';
+import { makeScoringFn, sigmoidRecencyFactor } from '../scoring.js';
+
+const NOW = Date.parse('2026-05-26T12:00:00Z');
+const DAY = 24 * 60 * 60 * 1000;
+
+function daysAgo(n) {
+  return new Date(NOW - n * DAY).toISOString();
+}
+
+describe('makeScoringFn — identity / no-op cases', () => {
+  test('no opts → returns base score unchanged', () => {
+    const fn = makeScoringFn();
+    expect(fn(0.42, {})).toBe(0.42);
+    expect(fn(0.99, { createdAt: daysAgo(1) })).toBe(0.99);
+  });
+
+  test('missing metadata → returns base score (no crash)', () => {
+    const fn = makeScoringFn({ recency: true, access: true, now: NOW });
+    expect(fn(0.5, null)).toBe(0.5);
+    expect(fn(0.5, undefined)).toBe(0.5);
+  });
+
+  test('metadata without timestamps → no recency boost applied', () => {
+    const fn = makeScoringFn({ recency: true, now: NOW });
+    expect(fn(0.5, { agentId: 'a' })).toBe(0.5);
+  });
+
+  test('metadata.accessCount === 0 → no access boost applied', () => {
+    const fn = makeScoringFn({ access: true });
+    expect(fn(0.5, { accessCount: 0 })).toBe(0.5);
+    expect(fn(0.5, {})).toBe(0.5);
+  });
+});
+
+describe('sigmoidRecencyFactor — shape', () => {
+  const opts = { halfLifeDays: 7, minFactor: 0.3, maxFactor: 1.5 };
+
+  test('age=0 → near maxFactor', () => {
+    expect(sigmoidRecencyFactor(0, opts)).toBeGreaterThan(1.45);
+    expect(sigmoidRecencyFactor(0, opts)).toBeLessThanOrEqual(1.5);
+  });
+
+  test('age=halfLife → midpoint (≈0.9)', () => {
+    const mid = (opts.minFactor + opts.maxFactor) / 2;
+    expect(sigmoidRecencyFactor(7, opts)).toBeCloseTo(mid, 5);
+  });
+
+  test('age=2×halfLife → near minFactor (≈0.3)', () => {
+    expect(sigmoidRecencyFactor(14, opts)).toBeLessThan(0.35);
+    expect(sigmoidRecencyFactor(14, opts)).toBeGreaterThanOrEqual(0.3);
+  });
+
+  test('age=very large → approaches minFactor asymptote', () => {
+    expect(sigmoidRecencyFactor(365, opts)).toBeCloseTo(0.3, 4);
+  });
+
+  test('monotonically decreasing with age', () => {
+    let prev = Infinity;
+    for (const age of [0, 1, 3, 7, 14, 30, 90]) {
+      const f = sigmoidRecencyFactor(age, opts);
+      expect(f).toBeLessThan(prev);
+      prev = f;
+    }
+  });
+
+  test('honours custom halfLifeDays', () => {
+    const fast = { ...opts, halfLifeDays: 1 };
+    const slow = { ...opts, halfLifeDays: 30 };
+    // At age=7, fast decay is deep into damped territory; slow decay near max.
+    expect(sigmoidRecencyFactor(7, fast)).toBeLessThan(0.4);
+    expect(sigmoidRecencyFactor(7, slow)).toBeGreaterThan(1.4);
+  });
+
+  test('honours custom min/max factor range', () => {
+    const custom = { halfLifeDays: 7, minFactor: 0.5, maxFactor: 2.0 };
+    expect(sigmoidRecencyFactor(0, custom)).toBeGreaterThan(1.9);
+    expect(sigmoidRecencyFactor(7, custom)).toBeCloseTo(1.25, 5);   // midpoint
+    expect(sigmoidRecencyFactor(100, custom)).toBeCloseTo(0.5, 4);
+  });
+});
+
+describe('makeScoringFn — recency end-to-end', () => {
+  test('today vs. month-ago: today scores higher', () => {
+    const fn = makeScoringFn({ recency: true, now: NOW });
+    const base = 0.7;
+    const today = fn(base, { createdAt: daysAgo(0) });
+    const monthAgo = fn(base, { createdAt: daysAgo(30) });
+    expect(today).toBeGreaterThan(monthAgo);
+  });
+
+  test('a very-fresh weak match can beat a stale strong match', () => {
+    const fn = makeScoringFn({ recency: true, now: NOW });
+    // Old item with strong cosine (0.85)
+    const oldStrong = fn(0.85, { createdAt: daysAgo(60) });
+    // Fresh item with weaker cosine (0.65)
+    const freshWeak = fn(0.65, { createdAt: daysAgo(0) });
+    // The recency boost matters but shouldn't fully invert massive cosine gaps.
+    // Range factor in [0.3, 1.5] means strongest possible boost is 1.5× max.
+    // 0.85 * 0.3 (oldest) = 0.255; 0.65 * 1.5 = 0.975 → fresh-weak wins big.
+    expect(freshWeak).toBeGreaterThan(oldStrong);
+  });
+
+  test('prefers updatedAt over createdAt when both present', () => {
+    const fn = makeScoringFn({ recency: true, now: NOW });
+    const ancient = fn(0.5, { createdAt: daysAgo(365), updatedAt: daysAgo(1) });
+    const stale   = fn(0.5, { createdAt: daysAgo(1), updatedAt: daysAgo(365) });
+    expect(ancient).toBeGreaterThan(stale);   // updatedAt=1 day ago wins
+  });
+
+  test('falls back to indexedAt when createdAt/updatedAt missing', () => {
+    const fn = makeScoringFn({ recency: true, now: NOW });
+    const recent = fn(0.5, { indexedAt: daysAgo(1) });
+    const old    = fn(0.5, { indexedAt: daysAgo(60) });
+    expect(recent).toBeGreaterThan(old);
+  });
+});
+
+describe('makeScoringFn — access boost', () => {
+  test('higher access count → higher score, monotonic', () => {
+    const fn = makeScoringFn({ access: true });
+    const s0 = fn(0.5, { accessCount: 0 });
+    const s1 = fn(0.5, { accessCount: 1 });
+    const s10 = fn(0.5, { accessCount: 10 });
+    const s100 = fn(0.5, { accessCount: 100 });
+    expect(s1).toBeGreaterThan(s0);
+    expect(s10).toBeGreaterThan(s1);
+    expect(s100).toBeGreaterThan(s10);
+  });
+
+  test('saturation comes from the maxFactor clamp (default 1.5)', () => {
+    const fn = makeScoringFn({ access: true });
+    const s100  = fn(1.0, { accessCount: 100 });    // unclamped boost ~1.46
+    const s10k  = fn(1.0, { accessCount: 10_000 }); // unclamped boost ~1.92 → clamped
+    const s10M  = fn(1.0, { accessCount: 10_000_000 }); // also clamped
+    expect(s100).toBeLessThan(s10k);
+    expect(s10k).toBeCloseTo(s10M, 6);   // both clamped to maxFactor
+    expect(s10k).toBeCloseTo(1.5, 6);
+  });
+
+  test('clamped at maxFactor — pathological access counts cannot blow up', () => {
+    const fn = makeScoringFn({ access: { weight: 1.0, maxFactor: 1.5 } });
+    // ln(1 + huge) is large; without clamping, factor would explode.
+    expect(fn(1.0, { accessCount: 1e9 })).toBeLessThanOrEqual(1.5);
+  });
+
+  test('custom weight tunes the curve', () => {
+    const gentle = makeScoringFn({ access: { weight: 0.01 } });
+    const strong = makeScoringFn({ access: { weight: 1.0 } });
+    expect(strong(0.5, { accessCount: 10 })).toBeGreaterThan(gentle(0.5, { accessCount: 10 }));
+  });
+});
+
+describe('makeScoringFn — composition (recency × access)', () => {
+  test('combines multiplicatively', () => {
+    const fn = makeScoringFn({ recency: true, access: true, now: NOW });
+    const both    = fn(0.5, { createdAt: daysAgo(0), accessCount: 5 });
+    const onlyRec = fn(0.5, { createdAt: daysAgo(0), accessCount: 0 });
+    expect(both).toBeGreaterThan(onlyRec);
+  });
+
+  test('clamping range: composed factor cannot drive score above maxFactor × maxFactor', () => {
+    const fn = makeScoringFn({ recency: true, access: true, now: NOW });
+    // Best-case row: brand new + heavily accessed.
+    const best = fn(1.0, { createdAt: daysAgo(0), accessCount: 1e6 });
+    // Recency caps at ~1.5, access caps at 1.5 → 2.25 ceiling.
+    expect(best).toBeLessThanOrEqual(2.26);
+  });
+
+  test('deterministic via `now` override', () => {
+    const a = makeScoringFn({ recency: true, now: NOW });
+    const b = makeScoringFn({ recency: true, now: NOW });
+    expect(a(0.5, { createdAt: daysAgo(3) })).toBe(b(0.5, { createdAt: daysAgo(3) }));
+  });
+});