@vellumai/assistant 0.8.3 → 0.8.5

package/src/runtime/routes/memory-v2-routes.ts

@@ -8,6 +8,8 @@ import { join } from "node:path";
 import { z } from "zod";
 
 import { loadConfig } from "../../config/loader.js";
+import type { AssistantConfig } from "../../config/types.js";
+import { getDb } from "../../memory/db-connection.js";
 import {
   enqueueMemoryJob,
   type MemoryJobType,
@@ -21,13 +23,23 @@ import {
   totalEdgeCount,
   validateEdgeTargets,
 } from "../../memory/v2/edge-index.js";
+import { runComparisonOverHistory } from "../../memory/v2/harness/compare.js";
+import type { Retriever } from "../../memory/v2/harness/retriever.js";
+import { createRouterRetriever } from "../../memory/v2/harness/router-retriever.js";
+import type { ComparisonReport } from "../../memory/v2/harness/runner.js";
+import { computeInjectionScores } from "../../memory/v2/injection-events.js";
+import { loadNowText } from "../../memory/v2/now-text.js";
+import { getPageIndex } from "../../memory/v2/page-index.js";
 import {
   getConceptsDir,
   listPages,
   readPage,
   renderPageContent,
 } from "../../memory/v2/page-store.js";
+import { ROUTER_PROMPT } from "../../memory/v2/prompts/router.js";
+import { type RouterSource, runRouter } from "../../memory/v2/router.js";
 import { seedV2SkillEntries } from "../../memory/v2/skill-store.js";
+import { createV3Retriever } from "../../memory/v3/retriever.js";
 import { getLogger } from "../../util/logger.js";
 import { getWorkspaceDir } from "../../util/platform.js";
 import { RouteError } from "./errors.js";
@@ -290,6 +302,367 @@ async function handleConceptFrequency({
   return getConceptFrequencySummary(workspaceDir, { conversationId, sinceMs });
 }
 
+// ── EMA scores ──────────────────────────────────────────────────────────
+
+const MemoryV2EmaScoresParams = z.object({}).strict();
+
+export interface MemoryV2EmaScoresEntry {
+  slug: string;
+  /** Time-decayed injection frequency; 0 when no events in the read window. */
+  score: number;
+  /** File mtime in epoch ms; 0 for synthetic entries (skills, CLI commands). */
+  modifiedAt: number;
+}
+
+export interface MemoryV2EmaScoresResult {
+  /** Every page index entry, sorted by score descending then slug ASCII. */
+  entries: MemoryV2EmaScoresEntry[];
+}
+
+async function handleEmaScores({
+  body = {},
+}: RouteHandlerArgs): Promise<MemoryV2EmaScoresResult> {
+  // Intentionally NOT gated on `memory.v2.enabled` — operators inspecting
+  // EMA data before flipping tier-2 routing on is a legitimate dry-run
+  // use case, mirroring `memory_v2_validate`.
+  MemoryV2EmaScoresParams.parse(body);
+
+  const pageIndex = await getPageIndex(getWorkspaceDir());
+  const slugs = pageIndex.entries.map((e) => e.slug);
+  const scores = computeInjectionScores(getDb(), slugs, Date.now());
+
+  const entries: MemoryV2EmaScoresEntry[] = pageIndex.entries.map((entry) => ({
+    slug: entry.slug,
+    score: scores.get(entry.slug) ?? 0,
+    modifiedAt: entry.modifiedAt,
+  }));
+  entries.sort((a, b) => {
+    if (a.score !== b.score) return b.score - a.score;
+    return a.slug < b.slug ? -1 : a.slug > b.slug ? 1 : 0;
+  });
+  return { entries };
+}
+
+// ── Simulate router (dry-run playground) ────────────────────────────────
+
+const SimulateRouterOverridesSchema = z
+  .object({
+    tier1_size: z.number().int().min(1).nullable().optional(),
+    tier2_size: z.number().int().min(1).nullable().optional(),
+    batch_size: z.number().int().min(1).nullable().optional(),
+  })
+  .strict();
+
+const RecentTurnPairSchema = z
+  .object({
+    assistantMessage: z.string(),
+    userMessage: z.string(),
+  })
+  .strict();
+
+const MemoryV2SimulateRouterParams = z
+  .object({
+    /**
+     * Recent (assistant, user) turn pairs to render inside `<last_turn>`,
+     * oldest first. Required; must contain at least one entry. The last
+     * entry's `userMessage` is the just-arrived turn the router is
+     * routing for (must be non-empty); earlier entries are conversation
+     * history. The oldest pair's `assistantMessage` may be empty for a
+     * first-turn scenario — the daemon skips that `[assistant]:` line
+     * the same way `runRouterBatch` does in prod.
+     */
+    recentTurnPairs: z
+      .array(RecentTurnPairSchema)
+      .min(1, "recentTurnPairs must contain at least one entry")
+      .refine(
+        (pairs) => pairs[pairs.length - 1].userMessage.length > 0,
+        "the last recentTurnPairs entry's userMessage must be non-empty",
+      ),
+    /**
+     * Verbatim `<now>` body. When omitted, the daemon loads the workspace's
+     * live NOW.md so callers that don't care about per-call now context get
+     * production-like behavior for free. Pass an explicit string (including
+     * the empty string) to override.
+     */
+    nowText: z.string().optional(),
+    configOverrides: SimulateRouterOverridesSchema.optional(),
+    profileOverride: z.string().min(1).optional(),
+    /**
+     * Inline router system-prompt override (simulator only). Empty /
+     * whitespace-only strings are normalized to "no override" so a
+     * cleared textarea behaves the same as never opening it. The 1 MiB
+     * cap mirrors the file-path size guard in `resolveRouterPrompt`.
+     */
+    routerPromptOverride: z.string().max(1_000_000).optional(),
+  })
+  .strict();
+
+export interface MemoryV2SimulateRouterEffectiveConfig {
+  tier1_size: number | null;
+  tier2_size: number | null;
+  batch_size: number | null;
+  max_page_ids: number;
+}
+
+export interface MemoryV2SimulateRouterResult {
+  /** Slugs the router would select, in model-returned order. */
+  selectedSlugs: string[];
+  /** Per-slug provenance: `"tier1"`, `"tier2"`, or `"tier3:<bucket>"`. */
+  sourceBySlug: Record<string, RouterSource>;
+  /** EMA scores for the selected slugs (0 when the slug has no events). */
+  scores: Record<string, number>;
+  /** `null` on success; otherwise one of the router failure reasons. */
+  failureReason: string | null;
+  /** The router config that actually ran (live merged with overrides). */
+  effectiveConfig: MemoryV2SimulateRouterEffectiveConfig;
+  /** The overrides the caller submitted, for display. */
+  overrides: {
+    tier1_size?: number | null;
+    tier2_size?: number | null;
+    batch_size?: number | null;
+  };
+  /** Page index size the router was given (post-tier-carve, all batches). */
+  totalCandidatePages: number;
+  /** The profile name passed as a per-call override, if any. */
+  profileOverride: string | null;
+  /** `true` when an inline `routerPromptOverride` was applied this call. */
+  routerPromptOverridden: boolean;
+}
+
+/**
+ * Build the config the router will see by overlaying override values on top
+ * of the live workspace config. Only the three new tier knobs are exposed —
+ * everything else (provider, prompts, weights) stays exactly as it would on
+ * a real turn. `undefined` means "inherit live"; `null` is a valid override
+ * value (meaning "disable this tier").
+ */
+function applySimulateOverrides(
+  live: AssistantConfig,
+  overrides: z.infer<typeof SimulateRouterOverridesSchema> | undefined,
+): AssistantConfig {
+  if (!overrides) return live;
+  const liveRouter = live.memory.v2.router;
+  const mergedRouter = {
+    ...liveRouter,
+    ...("tier1_size" in overrides && overrides.tier1_size !== undefined
+      ? { tier1_size: overrides.tier1_size }
+      : {}),
+    ...("tier2_size" in overrides && overrides.tier2_size !== undefined
+      ? { tier2_size: overrides.tier2_size }
+      : {}),
+    ...("batch_size" in overrides && overrides.batch_size !== undefined
+      ? { batch_size: overrides.batch_size }
+      : {}),
+  };
+  return {
+    ...live,
+    memory: {
+      ...live.memory,
+      v2: {
+        ...live.memory.v2,
+        router: mergedRouter,
+      },
+    },
+  };
+}
+
+export async function handleSimulateRouter({
+  body = {},
+}: RouteHandlerArgs): Promise<MemoryV2SimulateRouterResult> {
+  requireMemoryV2Enabled();
+  const {
+    recentTurnPairs,
+    nowText: rawNowText,
+    configOverrides,
+    profileOverride,
+    routerPromptOverride: rawRouterPromptOverride,
+  } = MemoryV2SimulateRouterParams.parse(body);
+
+  // Normalize whitespace-only strings to "no override" so the
+  // bundled/file prompt resolution behaves the same as a cleared editor.
+  const routerPromptOverride =
+    rawRouterPromptOverride !== undefined &&
+    rawRouterPromptOverride.trim().length > 0
+      ? rawRouterPromptOverride
+      : undefined;
+
+  const liveConfig = loadConfig();
+  const mergedConfig = applySimulateOverrides(liveConfig, configOverrides);
+  const effectiveRouter = mergedConfig.memory.v2.router;
+
+  // Validate the requested profile name against the configured profile
+  // catalog so the caller gets a structured 400 instead of a silent fall-
+  // through (the resolver tolerates missing override-profile references by
+  // design, but the playground wants the user to know they typo'd).
+  if (profileOverride !== undefined) {
+    const profiles = liveConfig.llm?.profiles ?? {};
+    if (!Object.prototype.hasOwnProperty.call(profiles, profileOverride)) {
+      const available = Object.keys(profiles).sort();
+      const hint =
+        available.length > 0
+          ? ` Available profiles: ${available.join(", ")}.`
+          : " No profiles defined in llm.profiles.";
+      throw new RouteError(
+        `Profile "${profileOverride}" is not defined in llm.profiles.${hint}`,
+        "MEMORY_V2_SIMULATE_INVALID_PROFILE",
+        400,
+      );
+    }
+  }
+
+  const workspaceDir = getWorkspaceDir();
+  // Caller can override `<now>` explicitly; otherwise fall back to the
+  // live workspace NOW.md so a UI that doesn't supply nowText still
+  // exercises a production-like context.
+  const nowText =
+    rawNowText !== undefined ? rawNowText : await loadNowText(workspaceDir);
+
+  const routerResult = await runRouter({
+    workspaceDir,
+    recentTurnPairs,
+    nowText,
+    priorEverInjected: [],
+    config: mergedConfig,
+    database: getDb(),
+    ...(profileOverride !== undefined
+      ? { overrideProfile: profileOverride }
+      : {}),
+    ...(routerPromptOverride !== undefined ? { routerPromptOverride } : {}),
+    // Always return the full union — the simulator's job is to surface
+    // what the router actually picked across all batches, not what
+    // injection.ts would have trimmed it to. The `max_page_ids` knob is
+    // still echoed in `effectiveConfig` so the UI can show the live cap
+    // as informational context.
+    disableUnionCap: true,
+  });
+
+  const pageIndex = await getPageIndex(workspaceDir);
+  const scores = computeInjectionScores(
+    getDb(),
+    routerResult.selectedSlugs,
+    Date.now(),
+  );
+
+  const sourceBySlug: Record<string, RouterSource> = {};
+  for (const [slug, source] of routerResult.sourceBySlug.entries()) {
+    sourceBySlug[slug] = source;
+  }
+
+  const scoresOut: Record<string, number> = {};
+  for (const slug of routerResult.selectedSlugs) {
+    scoresOut[slug] = scores.get(slug) ?? 0;
+  }
+
+  return {
+    selectedSlugs: routerResult.selectedSlugs,
+    sourceBySlug,
+    scores: scoresOut,
+    failureReason: routerResult.failureReason,
+    effectiveConfig: {
+      tier1_size: effectiveRouter.tier1_size,
+      tier2_size: effectiveRouter.tier2_size,
+      batch_size: effectiveRouter.batch_size,
+      max_page_ids: effectiveRouter.max_page_ids,
+    },
+    overrides: {
+      ...(configOverrides?.tier1_size !== undefined
+        ? { tier1_size: configOverrides.tier1_size }
+        : {}),
+      ...(configOverrides?.tier2_size !== undefined
+        ? { tier2_size: configOverrides.tier2_size }
+        : {}),
+      ...(configOverrides?.batch_size !== undefined
+        ? { batch_size: configOverrides.batch_size }
+        : {}),
+    },
+    totalCandidatePages: pageIndex.entries.length,
+    profileOverride: profileOverride ?? null,
+    routerPromptOverridden: routerPromptOverride !== undefined,
+  };
+}
+
+// ── Router prompt template (bundled default for the playground editor) ──
+
+export interface MemoryV2RouterPromptTemplateResult {
+  /** The bundled router prompt body, placeholders intact. */
+  template: string;
+}
+
+async function handleGetRouterPromptTemplate(): Promise<MemoryV2RouterPromptTemplateResult> {
+  requireMemoryV2Enabled();
+  return { template: ROUTER_PROMPT };
+}
+
+// ── Current `<now>` body (default value for the playground editor) ──────
+
+export interface MemoryV2NowTextResult {
+  /** The current rendered NOW.md body (autoloaded essentials/threads/recent). */
+  nowText: string;
+}
+
+async function handleGetNowText(): Promise<MemoryV2NowTextResult> {
+  requireMemoryV2Enabled();
+  const workspaceDir = getWorkspaceDir();
+  const nowText = await loadNowText(workspaceDir);
+  return { nowText };
+}
+
+// ── Compare retrievers over historical turns (read-only) ────────────────
+
+const MemoryV2CompareRetrieversParams = z
+  .object({
+    /**
+     * How many historical `mode='router'` turns to sample. Each scored turn
+     * re-runs the router (one LLM call), so keep this modest. Default 20.
+     */
+    limit: z.number().int().positive().optional(),
+    strategy: z.enum(["recent", "random"]).optional(),
+    conversationIds: z.array(z.string().min(1)).optional(),
+    ks: z.array(z.number().int().positive()).optional(),
+    includeNotInjected: z.boolean().optional(),
+  })
+  .strict();
+
+const DEFAULT_COMPARE_LIMIT = 20;
+const DEFAULT_COMPARE_KS = [5, 10, 25, 50];
+
+export async function handleCompareRetrievers({
+  body = {},
+  abortSignal,
+}: RouteHandlerArgs): Promise<ComparisonReport> {
+  requireMemoryV2Enabled();
+  const { limit, strategy, conversationIds, ks, includeNotInjected } =
+    MemoryV2CompareRetrieversParams.parse(body);
+
+  const config = loadConfig();
+  const workspaceDir = getWorkspaceDir();
+  const pageIndex = await getPageIndex(workspaceDir);
+  const db = getDb();
+
+  // The router is always comparand #1 (the harness self-test against its own
+  // logged ground truth). v3 joins as comparand #2 only when explicitly
+  // enabled, so the default compare surface is unchanged until v3 is switched
+  // on. v3 is offline-only here — the loop reads `db` but mutates nothing.
+  const retrievers: Retriever[] = [createRouterRetriever(db)];
+  if (config.memory.v3.enabled) {
+    retrievers.push(createV3Retriever(db));
+  }
+
+  return runComparisonOverHistory({
+    db,
+    workspaceDir,
+    config,
+    retrievers,
+    ks: ks ?? DEFAULT_COMPARE_KS,
+    limit: limit ?? DEFAULT_COMPARE_LIMIT,
+    pageExists: (slug) => pageIndex.bySlug.has(slug),
+    ...(strategy !== undefined ? { strategy } : {}),
+    ...(conversationIds !== undefined ? { conversationIds } : {}),
+    ...(includeNotInjected !== undefined ? { includeNotInjected } : {}),
+    ...(abortSignal !== undefined ? { signal: abortSignal } : {}),
+  });
+}
+
 // ── Route definitions ───────────────────────────────────────────────────
 
 export const ROUTES: RouteDefinition[] = [
@@ -359,4 +732,58 @@ export const ROUTES: RouteDefinition[] = [
     tags: ["memory"],
     requestBody: MemoryV2ConceptFrequencyParams,
   },
+  {
+    operationId: "memory_v2_ema_scores",
+    method: "POST",
+    endpoint: "memory/v2/ema-scores",
+    handler: handleEmaScores,
+    summary: "List every concept page with its injection-frequency EMA score",
+    description:
+      "Computes the time-decayed injection frequency (3-day half-life) for every entry in the current page index by reading memory_v2_injection_events. Returns entries sorted by score descending then slug ASCII, including zero-score pages so callers can decide whether to filter. Read-only; tier 2 of the v4 router uses the same computation to pick its top-M.",
+    tags: ["memory"],
+    requestBody: MemoryV2EmaScoresParams,
+  },
+  {
+    operationId: "memory_v2_simulate_router",
+    method: "POST",
+    endpoint: "memory/v2/simulate-router",
+    handler: handleSimulateRouter,
+    summary: "Dry-run the v4 router with config overrides (read-only)",
+    description:
+      "Runs the memory router against the live page index + EMA scores with optional tier_size / batch_size overrides, without recording an injection event or writing an activation log. Returns the slugs that would have been selected, per-slug tier provenance, EMA scores, and the effective router config so operators can validate knob changes before flipping them in workspace config.",
+    tags: ["memory"],
+    requestBody: MemoryV2SimulateRouterParams,
+  },
+  {
+    operationId: "memory_v2_compare_retrievers",
+    method: "POST",
+    endpoint: "memory/v2/compare-retrievers",
+    handler: handleCompareRetrievers,
+    summary:
+      "Compare retrievers against the router's logged selections (read-only)",
+    description:
+      "Runs one or more retrievers over a sample of historical turns (memory_v2_activation_logs, mode='router') and scores their selected pages against the logged selections as ground truth, reconstructing each turn's inputs from the messages table + current NOW. Read-only — writes nothing. Each scored turn re-runs the router (one LLM call), so keep `limit` modest. Today the only retriever is the router itself, so this is the harness self-test.",
+    tags: ["memory"],
+    requestBody: MemoryV2CompareRetrieversParams,
+  },
+  {
+    operationId: "memory_v2_router_prompt_template",
+    method: "GET",
+    endpoint: "memory/v2/router-prompt-template",
+    handler: handleGetRouterPromptTemplate,
+    summary: "Return the bundled router system-prompt template",
+    description:
+      "Returns the bundled `ROUTER_PROMPT` body with placeholders intact (`{{ASSISTANT_NAME}}`, `{{USER_NAME}}`, `{{PAGE_INDEX}}`). Used by the memory router playground's 'Load default' affordance so users have a known-good starting point when authoring an inline prompt override.",
+    tags: ["memory"],
+  },
+  {
+    operationId: "memory_v2_now_text",
+    method: "GET",
+    endpoint: "memory/v2/now-text",
+    handler: handleGetNowText,
+    summary: "Return the current rendered `<now>` body",
+    description:
+      "Returns the current NOW.md (autoloaded essentials/threads/recent). Used by the memory router playground to seed its `<now>` text area with a production-like default so callers can edit from a realistic baseline.",
+    tags: ["memory"],
+  },
 ];