@vellumai/assistant 0.5.16 → 0.6.1

package/src/memory/search/formatting.test.ts

@@ -1,140 +0,0 @@
-/**
- * Unit tests for buildMemoryInjection — focused on echoes/serendipity
- * rendering and budget enforcement.
- */
-import { describe, expect, test } from "bun:test";
-
-import { buildMemoryInjection } from "./formatting.js";
-import type { Candidate } from "./types.js";
-
-type CandidateWithLabel = Candidate & { sourceLabel?: string };
-
-function makeCandidate(
-  overrides: Partial<CandidateWithLabel> & { id: string },
-): CandidateWithLabel {
-  return {
-    key: `item:${overrides.id}`,
-    type: "item",
-    source: "semantic",
-    text: overrides.text ?? `Statement for ${overrides.id}`,
-    kind: overrides.kind ?? "fact",
-    confidence: 1,
-    importance: overrides.importance ?? 0.5,
-    createdAt: overrides.createdAt ?? Date.now(),
-    semantic: 0.8,
-    recency: 0.5,
-    finalScore: overrides.finalScore ?? 0.6,
-    ...overrides,
-  };
-}
-
-describe("buildMemoryInjection — echoes section", () => {
-  test("renders <echoes> after <recalled> when serendipity items provided", () => {
-    const candidates = [makeCandidate({ id: "c1", finalScore: 0.8 })];
-    const serendipityItems = [
-      makeCandidate({ id: "s1", finalScore: 0, importance: 0.7 }),
-    ];
-
-    const result = buildMemoryInjection({
-      candidates,
-      serendipityItems,
-      totalBudgetTokens: 2000,
-    });
-
-    expect(result).toContain("<recalled>");
-    expect(result).toContain("</recalled>");
-    expect(result).toContain("<echoes>");
-    expect(result).toContain("</echoes>");
-    // <echoes> comes after </recalled>
-    const recalledEnd = result.indexOf("</recalled>");
-    const echoesStart = result.indexOf("<echoes>");
-    expect(echoesStart).toBeGreaterThan(recalledEnd);
-  });
-
-  test("renders only <echoes> when no recalled candidates but serendipity items exist", () => {
-    const serendipityItems = [
-      makeCandidate({ id: "s1", finalScore: 0, importance: 0.6 }),
-      makeCandidate({ id: "s2", finalScore: 0, importance: 0.4 }),
-    ];
-
-    const result = buildMemoryInjection({
-      candidates: [],
-      serendipityItems,
-      totalBudgetTokens: 2000,
-    });
-
-    expect(result).toContain("<memory_context");
-    expect(result).not.toContain("<recalled>");
-    expect(result).toContain("<echoes>");
-    expect(result).toContain("</echoes>");
-    expect(result).toContain("s1");
-    expect(result).toContain("s2");
-  });
-
-  test("echoes section respects ~400 token cap", () => {
-    // Create serendipity items with very long text to test budget
-    const longText = "word ".repeat(200); // ~200 tokens
-    const serendipityItems = [
-      makeCandidate({ id: "s1", text: longText, finalScore: 0 }),
-      makeCandidate({ id: "s2", text: longText, finalScore: 0 }),
-      makeCandidate({ id: "s3", text: longText, finalScore: 0 }),
-    ];
-
-    const result = buildMemoryInjection({
-      candidates: [],
-      serendipityItems,
-      totalBudgetTokens: 5000, // plenty of total budget
-    });
-
-    // At ~200 tokens each, the 400-token echoes cap should allow at most 2
-    const itemMatches = result.match(/<item /g) ?? [];
-    expect(itemMatches.length).toBeLessThanOrEqual(2);
-  });
-
-  test("no <echoes> section when serendipity array is empty", () => {
-    const candidates = [makeCandidate({ id: "c1", finalScore: 0.8 })];
-
-    const result = buildMemoryInjection({
-      candidates,
-      serendipityItems: [],
-      totalBudgetTokens: 2000,
-    });
-
-    expect(result).toContain("<recalled>");
-    expect(result).not.toContain("<echoes>");
-  });
-
-  test("no <echoes> section when serendipity items omitted", () => {
-    const candidates = [makeCandidate({ id: "c1", finalScore: 0.8 })];
-
-    const result = buildMemoryInjection({
-      candidates,
-      totalBudgetTokens: 2000,
-    });
-
-    expect(result).toContain("<recalled>");
-    expect(result).not.toContain("<echoes>");
-  });
-
-  test("echoes items include importance and kind attributes", () => {
-    const serendipityItems = [
-      makeCandidate({
-        id: "echo1",
-        kind: "preference",
-        importance: 0.85,
-        text: "User likes dark mode",
-        finalScore: 0,
-      }),
-    ];
-
-    const result = buildMemoryInjection({
-      candidates: [],
-      serendipityItems,
-      totalBudgetTokens: 2000,
-    });
-
-    expect(result).toContain('kind="preference"');
-    expect(result).toContain('importance="0.85"');
-    expect(result).toContain("User likes dark mode");
-  });
-});

package/src/memory/search/formatting.ts

@@ -1,262 +0,0 @@
-import { eq } from "drizzle-orm";
-
-import { estimateTextTokens } from "../../context/token-estimator.js";
-import { getLogger } from "../../util/logger.js";
-import { getDb } from "../db.js";
-import { memoryItems } from "../schema.js";
-import type { Candidate } from "./types.js";
-
-const log = getLogger("memory-formatting");
-
-/**
- * Escape XML-like tag sequences in recalled text to prevent delimiter injection.
- * Recalled content is interpolated verbatim inside `<memory>` wrapper tags,
- * so any literal `</memory>` (or similar) in the text could break the wrapper
- * and let recalled content masquerade as top-level prompt instructions.
- *
- * Strategy: replace `<` in any XML-tag-like pattern with the Unicode full-width
- * less-than sign (U+FF1C) which is visually similar but won't be parsed as XML.
- */
-export function escapeXmlTags(text: string): string {
-  // Match anything that looks like an XML tag: <word...> or </word...>
-  return text.replace(
-    /<\/?[a-zA-Z][a-zA-Z0-9_-]*[\s>\/]/g,
-    (match) => "\uFF1C" + match.slice(1),
-  );
-}
-
-/**
- * Convert an epoch-ms timestamp to a timezone-aware absolute time string.
- * Format: "YYYY-MM-DD HH:mm TZ" (e.g. "2025-02-13 15:30 PST").
- */
-export function formatAbsoluteTime(epochMs: number): string {
-  const date = new Date(epochMs);
-  const year = date.getFullYear();
-  const month = String(date.getMonth() + 1).padStart(2, "0");
-  const day = String(date.getDate()).padStart(2, "0");
-  const hours = String(date.getHours()).padStart(2, "0");
-  const minutes = String(date.getMinutes()).padStart(2, "0");
-
-  // Extract short timezone abbreviation (e.g. "PST", "EST", "UTC")
-  const tz =
-    new Intl.DateTimeFormat("en-US", { timeZoneName: "short" })
-      .formatToParts(date)
-      .find((p) => p.type === "timeZoneName")?.value ?? "UTC";
-
-  return `${year}-${month}-${day} ${hours}:${minutes} ${tz}`;
-}
-
-/**
- * Convert an epoch-ms timestamp to a human-readable relative time string.
- */
-export function formatRelativeTime(epochMs: number): string {
-  const elapsed = Math.max(0, Date.now() - epochMs);
-  const hours = elapsed / (1000 * 60 * 60);
-  if (hours < 1) return "just now";
-  if (hours < 24) {
-    const h = Math.floor(hours);
-    return `${h} hour${h === 1 ? "" : "s"} ago`;
-  }
-  const days = hours / 24;
-  if (days < 7) {
-    const d = Math.floor(days);
-    return `${d} day${d === 1 ? "" : "s"} ago`;
-  }
-  if (days < 30) {
-    const w = Math.floor(days / 7);
-    return `${w} week${w === 1 ? "" : "s"} ago`;
-  }
-  if (days < 365) {
-    const m = Math.floor(days / 30);
-    return `${m} month${m === 1 ? "" : "s"} ago`;
-  }
-  const y = Math.floor(days / 365);
-  return `${y} year${y === 1 ? "" : "s"} ago`;
-}
-
-// ---------------------------------------------------------------------------
-// Unified injection format
-// ---------------------------------------------------------------------------
-
-/**
- * Build a unified `<memory_context>` XML injection block from scored candidates.
- *
- * All candidates are rendered in a single `<recalled>` section sorted by
- * `finalScore` descending, with each candidate tagged by type:
- * - items: `<item id="item:ID" kind="KIND" importance="N.NN" timestamp="..." from="...">`
- * - segments: `<segment id="seg:ID" timestamp="..." from="...">`
- * - summaries: `<summary id="sum:ID" timestamp="..." from="...">`
- *
- * An optional `<echoes>` section renders serendipity items — random
- * importance-weighted memories for unexpected connections.
- *
- * Respects token budget: iterates candidates in score order, accumulates
- * token estimates, and stops when the budget is exhausted.
- */
-export function buildMemoryInjection(params: {
-  candidates: Array<Candidate & { sourceLabel?: string; staleness?: string; supersedes?: string }>;
-  serendipityItems?: Array<Candidate & { sourceLabel?: string }>;
-  totalBudgetTokens?: number;
-}): string {
-  const { candidates, serendipityItems, totalBudgetTokens } = params;
-
-  if (candidates.length === 0 && (!serendipityItems || serendipityItems.length === 0)) {
-    return "";
-  }
-
-  // Sort by finalScore descending
-  const sorted = [...candidates].sort((a, b) => b.finalScore - a.finalScore);
-
-  // Reserve tokens for structural overhead
-  const WRAPPER_OVERHEAD_TOKENS = estimateTextTokens(
-    "<memory_context __injected>\n<recalled>\n</recalled>\n</memory_context>",
-  );
-  let remainingTokens = totalBudgetTokens
-    ? Math.max(1, totalBudgetTokens - WRAPPER_OVERHEAD_TOKENS)
-    : Infinity;
-
-  // Render candidates within budget
-  const lines: string[] = [];
-  for (const c of sorted) {
-    if (remainingTokens <= 0) break;
-    const line = renderCandidate(c);
-    const tokens = estimateTextTokens(line);
-    if (tokens > remainingTokens) continue;
-    lines.push(line);
-    remainingTokens -= tokens;
-  }
-
-  if (lines.length === 0 && (!serendipityItems || serendipityItems.length === 0)) {
-    return "";
-  }
-
-  const sections: string[] = [];
-
-  if (lines.length > 0) {
-    sections.push(`<recalled>\n${lines.join("\n")}\n</recalled>`);
-  }
-
-  // Echoes section for serendipity items — capped at ~400 tokens of
-  // the remaining budget after <recalled> items are rendered.
-  if (serendipityItems && serendipityItems.length > 0) {
-    const ECHOES_MAX_TOKENS = 400;
-    let echoesBudget = Math.min(remainingTokens, ECHOES_MAX_TOKENS);
-    const echoLines: string[] = [];
-    for (const c of serendipityItems) {
-      if (echoesBudget <= 0) break;
-      const line = renderCandidate(c);
-      const tokens = estimateTextTokens(line);
-      if (tokens > echoesBudget) continue;
-      echoLines.push(line);
-      echoesBudget -= tokens;
-      remainingTokens -= tokens;
-    }
-    if (echoLines.length > 0) {
-      sections.push(`<echoes>\n${echoLines.join("\n")}\n</echoes>`);
-    }
-  }
-
-  if (sections.length === 0) return "";
-
-  return `<memory_context __injected>\n${sections.join("\n")}\n</memory_context>`;
-}
-
-/**
- * Look up the supersession chain for a given superseded item ID.
- *
- * Returns the immediate predecessor's statement and timestamp, plus the
- * total chain depth (how many items were superseded in sequence).
- * Chain traversal is capped at 10 iterations to prevent infinite loops.
- */
-export function lookupSupersessionChain(supersededId: string): {
-  previousStatement: string;
-  previousTimestamp: number;
-  chainDepth: number;
-} | null {
-  try {
-    const db = getDb();
-
-    // Look up the immediate predecessor
-    const predecessor = db
-      .select({
-        statement: memoryItems.statement,
-        firstSeenAt: memoryItems.firstSeenAt,
-        supersedes: memoryItems.supersedes,
-      })
-      .from(memoryItems)
-      .where(eq(memoryItems.id, supersededId))
-      .get();
-
-    if (!predecessor) return null;
-
-    // Count chain depth by following supersedes links (cap at 10)
-    let chainDepth = 1;
-    let currentSupersedes = predecessor.supersedes;
-    const MAX_CHAIN_DEPTH = 10;
-
-    while (currentSupersedes && chainDepth < MAX_CHAIN_DEPTH) {
-      const ancestor = db
-        .select({ supersedes: memoryItems.supersedes })
-        .from(memoryItems)
-        .where(eq(memoryItems.id, currentSupersedes))
-        .get();
-
-      if (!ancestor) break;
-      chainDepth++;
-      currentSupersedes = ancestor.supersedes;
-    }
-
-    return {
-      previousStatement: predecessor.statement,
-      previousTimestamp: predecessor.firstSeenAt,
-      chainDepth,
-    };
-  } catch (err) {
-    log.warn({ err }, "Failed to look up supersession chain");
-    return null;
-  }
-}
-
-/**
- * Render a single candidate as an XML element based on its type.
- */
-function renderCandidate(c: Candidate & { sourceLabel?: string; supersedes?: string }): string {
-  const text = escapeXmlTags(c.text);
-  const timestamp = formatAbsoluteTime(c.createdAt);
-  const fromAttr = c.sourceLabel
-    ? ` from="${escapeXmlAttr(c.sourceLabel)}"`
-    : "";
-  const pathAttr = c.sourcePath
-    ? ` path="${escapeXmlAttr(c.sourcePath)}"`
-    : "";
-
-  // Build inline supersession suffix for items
-  let supersessionSuffix = "";
-  if (c.type === "item" && c.supersedes) {
-    const chain = lookupSupersessionChain(c.supersedes);
-    if (chain) {
-      const prevTimestamp = formatAbsoluteTime(chain.previousTimestamp);
-      supersessionSuffix = `<supersedes count="${chain.chainDepth}">${escapeXmlTags(chain.previousStatement)} (${prevTimestamp})</supersedes>`;
-    }
-  }
-
-  switch (c.type) {
-    case "item":
-      return `<item id="item:${escapeXmlAttr(c.id)}" kind="${escapeXmlAttr(c.kind)}" importance="${c.importance.toFixed(2)}" timestamp="${escapeXmlAttr(timestamp)}"${fromAttr}${pathAttr}>${text}${supersessionSuffix}</item>`;
-    case "segment":
-      return `<segment id="seg:${escapeXmlAttr(c.id)}" timestamp="${escapeXmlAttr(timestamp)}"${fromAttr}${pathAttr}>${text}</segment>`;
-    case "summary":
-      return `<summary id="sum:${escapeXmlAttr(c.id)}" timestamp="${escapeXmlAttr(timestamp)}"${fromAttr}${pathAttr}>${text}</summary>`;
-    default:
-      // media or unknown types — render as item
-      return `<item id="item:${escapeXmlAttr(c.id)}" kind="${escapeXmlAttr(c.kind)}" importance="${c.importance.toFixed(2)}" timestamp="${escapeXmlAttr(timestamp)}"${fromAttr}${pathAttr}>${text}${supersessionSuffix}</item>`;
-  }
-}
-
-function escapeXmlAttr(text: string): string {
-  return text
-    .replace(/&/g, "&amp;")
-    .replace(/"/g, "&quot;")
-    .replace(/</g, "&lt;")
-    .replace(/>/g, "&gt;");
-}

package/src/memory/search/mmr.ts

@@ -1,139 +0,0 @@
-import { generateSparseEmbedding } from "../embedding-backend.js";
-import type { SparseEmbedding } from "../embedding-types.js";
-import type { TieredCandidate } from "./tier-classifier.js";
-
-/**
- * Compute cosine similarity between two sparse vectors.
- * Returns 0 if either vector has zero magnitude.
- */
-function sparseCosine(a: SparseEmbedding, b: SparseEmbedding): number {
-  // Build index→value map for vector b
-  const bMap = new Map<number, number>();
-  for (let i = 0; i < b.indices.length; i++) {
-    bMap.set(b.indices[i]!, b.values[i]!);
-  }
-
-  // Compute dot product over shared indices
-  let dotProduct = 0;
-  for (let i = 0; i < a.indices.length; i++) {
-    const bVal = bMap.get(a.indices[i]!);
-    if (bVal !== undefined) {
-      dotProduct += a.values[i]! * bVal;
-    }
-  }
-
-  // Compute magnitudes
-  let magA = 0;
-  for (const v of a.values) magA += v * v;
-  magA = Math.sqrt(magA);
-
-  let magB = 0;
-  for (const v of b.values) magB += v * v;
-  magB = Math.sqrt(magB);
-
-  if (magA === 0 || magB === 0) return 0;
-  return dotProduct / (magA * magB);
-}
-
-/**
- * Apply Maximal Marginal Relevance (MMR) diversity ranking to candidates.
- *
- * Items are re-ranked using a greedy selection loop that progressively
- * penalizes candidates whose text is similar to already-selected ones.
- * Non-item candidates (segments, summaries, media) pass through unpenalized
- * since they represent different conversation windows.
- *
- * @param candidates - Scored candidates from upstream ranking
- * @param penalty - Float 0..1. 0 = no diversity pressure, 1 = maximum
- * @returns Re-ranked candidates with adjusted finalScores
- */
-export function applyMMR(
-  candidates: TieredCandidate[],
-  penalty: number,
-): TieredCandidate[] {
-  // Separate items from non-items
-  const items: { index: number; candidate: TieredCandidate }[] = [];
-  const nonItems: TieredCandidate[] = [];
-
-  for (let i = 0; i < candidates.length; i++) {
-    const c = candidates[i]!;
-    if (c.type === "item") {
-      items.push({ index: i, candidate: c });
-    } else {
-      nonItems.push(c);
-    }
-  }
-
-  // If no items or no penalty, pass through in original order
-  if (items.length === 0 || penalty === 0) {
-    return candidates;
-  }
-
-  // Pre-compute sparse embeddings only for items (not segments, summaries, media)
-  const embeddings = new Map<number, SparseEmbedding>();
-  for (const item of items) {
-    embeddings.set(item.index, generateSparseEmbedding(item.candidate.text));
-  }
-
-  // Greedy MMR selection loop
-  const selected: number[] = [];
-  const remaining = new Set<number>(items.map((_, i) => i));
-  const adjustedScores = new Map<number, number>();
-
-  // Select the item with the highest finalScore first
-  let bestIdx = -1;
-  let bestScore = -Infinity;
-  for (const idx of remaining) {
-    const score = items[idx]!.candidate.finalScore;
-    if (score > bestScore) {
-      bestScore = score;
-      bestIdx = idx;
-    }
-  }
-  selected.push(bestIdx);
-  remaining.delete(bestIdx);
-  adjustedScores.set(bestIdx, items[bestIdx]!.candidate.finalScore);
-
-  // Iteratively select remaining items
-  while (remaining.size > 0) {
-    let nextBestIdx = -1;
-    let nextBestScore = -Infinity;
-
-    for (const idx of remaining) {
-      const itemEmbIdx = items[idx]!.index;
-
-      // Compute max similarity to any already-selected item
-      let maxSim = 0;
-      for (const selIdx of selected) {
-        const selEmbIdx = items[selIdx]!.index;
-        const sim = sparseCosine(
-          embeddings.get(itemEmbIdx)!,
-          embeddings.get(selEmbIdx)!,
-        );
-        if (sim > maxSim) maxSim = sim;
-      }
-
-      const adjustedScore =
-        items[idx]!.candidate.finalScore * (1 - maxSim * penalty);
-      if (adjustedScore > nextBestScore) {
-        nextBestScore = adjustedScore;
-        nextBestIdx = idx;
-      }
-    }
-
-    selected.push(nextBestIdx);
-    remaining.delete(nextBestIdx);
-    adjustedScores.set(nextBestIdx, nextBestScore);
-  }
-
-  // Rebuild output: non-items first (original order), then items in selected order
-  const result: TieredCandidate[] = [...nonItems];
-  for (const idx of selected) {
-    result.push({
-      ...items[idx]!.candidate,
-      finalScore: adjustedScores.get(idx)!,
-    });
-  }
-
-  return result;
-}

package/src/memory/search/ranking.ts

@@ -1,15 +0,0 @@
-/**
- * Logarithmic recency decay (ACT-R inspired).
- *
- * Old formula `1/(1+ageDays)` decays far too aggressively:
- *   - 30 days -> 0.032, 1 year -> 0.003
- *
- * New formula `1/(1+log2(1+ageDays))` preserves long-term recall:
- *   - 1 day -> 0.50, 7 days -> 0.25, 30 days -> 0.17
- *   - 90 days -> 0.15, 1 year -> 0.12, 2 years -> 0.10
- */
-export function computeRecencyScore(createdAt: number): number {
-  const ageMs = Math.max(0, Date.now() - createdAt);
-  const ageDays = ageMs / (24 * 60 * 60 * 1000);
-  return 1 / (1 + Math.log2(1 + ageDays));
-}

package/src/memory/search/staleness.ts

@@ -1,40 +0,0 @@
-import type { StalenessLevel } from "./types.js";
-
-const BASE_LIFETIME_MS: Record<string, number> = {
-  identity: 180 * 86_400_000, // 6 months
-  preference: 90 * 86_400_000, // 3 months
-  constraint: 30 * 86_400_000, // 1 month
-  project: 14 * 86_400_000, // 2 weeks
-  decision: 14 * 86_400_000, // 2 weeks
-  event: 3 * 86_400_000, // 3 days
-  // Journals are experiential reflections and forward-looking notes — more
-  // durable than ephemeral events or decisions, but not as permanent as
-  // identity. 90 days mirrors "preference" lifetime.
-  journal: 90 * 86_400_000, // 3 months
-  capability: Infinity,
-};
-
-const DEFAULT_LIFETIME_MS = 30 * 86_400_000;
-
-export function computeStaleness(
-  item: {
-    kind: string;
-    firstSeenAt: number;
-    sourceConversationCount: number;
-  },
-  now: number,
-): { level: StalenessLevel; ratio: number } {
-  const baseLifetime = BASE_LIFETIME_MS[item.kind] ?? DEFAULT_LIFETIME_MS;
-  const reinforcement = Math.max(
-    1,
-    1 + 0.3 * (item.sourceConversationCount - 1),
-  );
-  const effectiveLifetime = baseLifetime * reinforcement;
-  const age = now - item.firstSeenAt;
-  const ratio = age / effectiveLifetime;
-
-  if (ratio < 0.5) return { level: "fresh", ratio };
-  if (ratio <= 1) return { level: "aging", ratio };
-  if (ratio <= 2) return { level: "stale", ratio };
-  return { level: "very_stale", ratio };
-}

package/src/memory/search/tier-classifier.ts

@@ -1,18 +0,0 @@
-import type { Candidate } from "./types.js";
-
-/** Backward-compatible alias — downstream files import this type. */
-export type TieredCandidate = Candidate & {
-  /** Human-readable label for the source conversation/summary (e.g. conversation title). */
-  sourceLabel?: string;
-};
-
-const MIN_SCORE_THRESHOLD = 0.2;
-
-/**
- * Filter candidates to those exceeding the minimum relevance threshold.
- * Replaces the former tier 1/tier 2 classification — all surviving candidates
- * are treated equally and ranked by score.
- */
-export function filterByMinScore(candidates: Candidate[]): TieredCandidate[] {
-  return candidates.filter((c) => c.finalScore > MIN_SCORE_THRESHOLD);
-}