@vellumai/assistant 0.8.4 → 0.8.5

package/src/runtime/routes/inference-profile-session-routes.ts

@@ -22,6 +22,7 @@ import type { RouteDefinition, RouteHandlerArgs } from "./types.js";
 
 async function handleOpenInferenceProfileSession({
   body = {},
+  headers,
 }: RouteHandlerArgs) {
   if (body.profile == null || typeof body.profile !== "string") {
     throw new BadRequestError("profile must be a non-empty string");
@@ -31,13 +32,18 @@ async function handleOpenInferenceProfileSession({
     profile: body.profile,
     ttlSeconds: body.ttlSeconds as number | null | undefined,
     sessionId: body.sessionId as string | undefined,
+    originClientId: headers?.["x-vellum-client-id"]?.trim() || undefined,
   });
 }
 
 async function handleCloseInferenceProfileSession({
   body = {},
+  headers,
 }: RouteHandlerArgs) {
-  return closeInferenceProfileSession(body.conversationId as string);
+  return closeInferenceProfileSession(
+    body.conversationId as string,
+    headers?.["x-vellum-client-id"]?.trim() || undefined,
+  );
 }
 
 function handleListInferenceProfileSessions({

package/src/runtime/routes/llm-call-sites-routes.ts

@@ -1,6 +1,9 @@
 import { resolveDefaultProfileKey } from "../../config/llm-resolver.js";
 import { loadConfig } from "../../config/loader.js";
-import { CALL_SITE_CATALOG, CALL_SITE_DOMAINS } from "../../config/schemas/call-site-catalog.js";
+import {
+  CALL_SITE_CATALOG,
+  CALL_SITE_DOMAINS,
+} from "../../config/schemas/call-site-catalog.js";
 import type { LLMCallSite } from "../../config/schemas/llm.js";
 import type { RouteDefinition } from "./types.js";
 
@@ -10,14 +13,28 @@ async function handleGetCallSites() {
     domains: CALL_SITE_DOMAINS,
     callSites: CALL_SITE_CATALOG.map((entry) => ({
       ...entry,
-      defaultProfile: resolveDefaultProfileKey(
-        entry.id as LLMCallSite,
-        llm,
-      ),
+      defaultProfile: resolveDefaultProfileKey(entry.id as LLMCallSite, llm),
     })),
   };
 }
 
+export interface LlmProfilesListResult {
+  /** Sorted list of profile names defined in `llm.profiles`. */
+  profiles: string[];
+  /** The workspace-wide active profile name, if one is set. */
+  activeProfile: string | null;
+}
+
+async function handleListProfiles(): Promise<LlmProfilesListResult> {
+  const { llm } = loadConfig();
+  const profiles = llm?.profiles ?? {};
+  return {
+    profiles: Object.keys(profiles).sort(),
+    activeProfile:
+      typeof llm?.activeProfile === "string" ? llm.activeProfile : null,
+  };
+}
+
 export const ROUTES: RouteDefinition[] = [
   {
     operationId: "llm_call_sites_list",
@@ -29,4 +46,14 @@ export const ROUTES: RouteDefinition[] = [
       "Returns the full catalog of LLM call sites with display names, descriptions, and domain groupings. Used by clients to render the per-call-site override settings UI.",
     tags: ["config"],
   },
+  {
+    operationId: "llm_profiles_list",
+    method: "GET",
+    endpoint: "config/llm/profiles",
+    handler: handleListProfiles,
+    summary: "List defined LLM profiles",
+    description:
+      "Returns the sorted list of profile names defined in `llm.profiles` plus the workspace-wide active profile. Used to populate per-call profile dropdowns (e.g. memory router playground) without requiring the caller to type profile names.",
+    tags: ["config"],
+  },
 ];

package/src/runtime/routes/memory-item-routes.ts

@@ -44,7 +44,10 @@ import type {
   MemoryType,
   NewNode,
 } from "../../memory/graph/types.js";
-import { enqueueMemoryJob } from "../../memory/jobs-store.js";
+import {
+  enqueueMemoryJob,
+  isMemoryEnabled,
+} from "../../memory/jobs-store.js";
 import { withQdrantBreaker } from "../../memory/qdrant-circuit-breaker.js";
 import { getQdrantClient } from "../../memory/qdrant-client.js";
 import { memoryGraphNodes } from "../../memory/schema.js";
@@ -525,7 +528,9 @@ async function handleCreateMemoryItem(body: Record<string, unknown>) {
   };
 
   const created = createNode(newNode);
-  enqueueMemoryJob("embed_graph_node", { nodeId: created.id });
+  if (isMemoryEnabled()) {
+    enqueueMemoryJob("embed_graph_node", { nodeId: created.id });
+  }
 
   return { item: nodeToPayload(created) };
 }
@@ -619,7 +624,7 @@ async function handleUpdateMemoryItem(
 
   updateNode(id, changes);
 
-  if (contentChanged) {
+  if (contentChanged && isMemoryEnabled()) {
     enqueueMemoryJob("embed_graph_node", { nodeId: id });
   }
 

package/src/runtime/routes/memory-v2-routes.ts

@@ -23,6 +23,10 @@ import {
   totalEdgeCount,
   validateEdgeTargets,
 } from "../../memory/v2/edge-index.js";
+import { runComparisonOverHistory } from "../../memory/v2/harness/compare.js";
+import type { Retriever } from "../../memory/v2/harness/retriever.js";
+import { createRouterRetriever } from "../../memory/v2/harness/router-retriever.js";
+import type { ComparisonReport } from "../../memory/v2/harness/runner.js";
 import { computeInjectionScores } from "../../memory/v2/injection-events.js";
 import { loadNowText } from "../../memory/v2/now-text.js";
 import { getPageIndex } from "../../memory/v2/page-index.js";
@@ -32,8 +36,10 @@ import {
   readPage,
   renderPageContent,
 } from "../../memory/v2/page-store.js";
+import { ROUTER_PROMPT } from "../../memory/v2/prompts/router.js";
 import { type RouterSource, runRouter } from "../../memory/v2/router.js";
 import { seedV2SkillEntries } from "../../memory/v2/skill-store.js";
+import { createV3Retriever } from "../../memory/v3/retriever.js";
 import { getLogger } from "../../util/logger.js";
 import { getWorkspaceDir } from "../../util/platform.js";
 import { RouteError } from "./errors.js";
@@ -347,10 +353,47 @@ const SimulateRouterOverridesSchema = z
   })
   .strict();
 
+const RecentTurnPairSchema = z
+  .object({
+    assistantMessage: z.string(),
+    userMessage: z.string(),
+  })
+  .strict();
+
 const MemoryV2SimulateRouterParams = z
   .object({
-    query: z.string().min(1, "query must be non-empty"),
+    /**
+     * Recent (assistant, user) turn pairs to render inside `<last_turn>`,
+     * oldest first. Required; must contain at least one entry. The last
+     * entry's `userMessage` is the just-arrived turn the router is
+     * routing for (must be non-empty); earlier entries are conversation
+     * history. The oldest pair's `assistantMessage` may be empty for a
+     * first-turn scenario — the daemon skips that `[assistant]:` line
+     * the same way `runRouterBatch` does in prod.
+     */
+    recentTurnPairs: z
+      .array(RecentTurnPairSchema)
+      .min(1, "recentTurnPairs must contain at least one entry")
+      .refine(
+        (pairs) => pairs[pairs.length - 1].userMessage.length > 0,
+        "the last recentTurnPairs entry's userMessage must be non-empty",
+      ),
+    /**
+     * Verbatim `<now>` body. When omitted, the daemon loads the workspace's
+     * live NOW.md so callers that don't care about per-call now context get
+     * production-like behavior for free. Pass an explicit string (including
+     * the empty string) to override.
+     */
+    nowText: z.string().optional(),
     configOverrides: SimulateRouterOverridesSchema.optional(),
+    profileOverride: z.string().min(1).optional(),
+    /**
+     * Inline router system-prompt override (simulator only). Empty /
+     * whitespace-only strings are normalized to "no override" so a
+     * cleared textarea behaves the same as never opening it. The 1 MiB
+     * cap mirrors the file-path size guard in `resolveRouterPrompt`.
+     */
+    routerPromptOverride: z.string().max(1_000_000).optional(),
   })
   .strict();
 
@@ -380,6 +423,10 @@ export interface MemoryV2SimulateRouterResult {
   };
   /** Page index size the router was given (post-tier-carve, all batches). */
   totalCandidatePages: number;
+  /** The profile name passed as a per-call override, if any. */
+  profileOverride: string | null;
+  /** `true` when an inline `routerPromptOverride` was applied this call. */
+  routerPromptOverridden: boolean;
 }
 
 /**
@@ -423,23 +470,70 @@ export async function handleSimulateRouter({
   body = {},
 }: RouteHandlerArgs): Promise<MemoryV2SimulateRouterResult> {
   requireMemoryV2Enabled();
-  const { query, configOverrides } = MemoryV2SimulateRouterParams.parse(body);
+  const {
+    recentTurnPairs,
+    nowText: rawNowText,
+    configOverrides,
+    profileOverride,
+    routerPromptOverride: rawRouterPromptOverride,
+  } = MemoryV2SimulateRouterParams.parse(body);
+
+  // Normalize whitespace-only strings to "no override" so the
+  // bundled/file prompt resolution behaves the same as a cleared editor.
+  const routerPromptOverride =
+    rawRouterPromptOverride !== undefined &&
+    rawRouterPromptOverride.trim().length > 0
+      ? rawRouterPromptOverride
+      : undefined;
 
   const liveConfig = loadConfig();
   const mergedConfig = applySimulateOverrides(liveConfig, configOverrides);
   const effectiveRouter = mergedConfig.memory.v2.router;
 
+  // Validate the requested profile name against the configured profile
+  // catalog so the caller gets a structured 400 instead of a silent fall-
+  // through (the resolver tolerates missing override-profile references by
+  // design, but the playground wants the user to know they typo'd).
+  if (profileOverride !== undefined) {
+    const profiles = liveConfig.llm?.profiles ?? {};
+    if (!Object.prototype.hasOwnProperty.call(profiles, profileOverride)) {
+      const available = Object.keys(profiles).sort();
+      const hint =
+        available.length > 0
+          ? ` Available profiles: ${available.join(", ")}.`
+          : " No profiles defined in llm.profiles.";
+      throw new RouteError(
+        `Profile "${profileOverride}" is not defined in llm.profiles.${hint}`,
+        "MEMORY_V2_SIMULATE_INVALID_PROFILE",
+        400,
+      );
+    }
+  }
+
   const workspaceDir = getWorkspaceDir();
-  const nowText = await loadNowText(workspaceDir);
+  // Caller can override `<now>` explicitly; otherwise fall back to the
+  // live workspace NOW.md so a UI that doesn't supply nowText still
+  // exercises a production-like context.
+  const nowText =
+    rawNowText !== undefined ? rawNowText : await loadNowText(workspaceDir);
 
   const routerResult = await runRouter({
     workspaceDir,
-    userMessage: query,
-    assistantMessage: "",
+    recentTurnPairs,
     nowText,
     priorEverInjected: [],
     config: mergedConfig,
     database: getDb(),
+    ...(profileOverride !== undefined
+      ? { overrideProfile: profileOverride }
+      : {}),
+    ...(routerPromptOverride !== undefined ? { routerPromptOverride } : {}),
+    // Always return the full union — the simulator's job is to surface
+    // what the router actually picked across all batches, not what
+    // injection.ts would have trimmed it to. The `max_page_ids` knob is
+    // still echoed in `effectiveConfig` so the UI can show the live cap
+    // as informational context.
+    disableUnionCap: true,
   });
 
   const pageIndex = await getPageIndex(workspaceDir);
@@ -482,9 +576,93 @@ export async function handleSimulateRouter({
         : {}),
     },
     totalCandidatePages: pageIndex.entries.length,
+    profileOverride: profileOverride ?? null,
+    routerPromptOverridden: routerPromptOverride !== undefined,
   };
 }
 
+// ── Router prompt template (bundled default for the playground editor) ──
+
+export interface MemoryV2RouterPromptTemplateResult {
+  /** The bundled router prompt body, placeholders intact. */
+  template: string;
+}
+
+async function handleGetRouterPromptTemplate(): Promise<MemoryV2RouterPromptTemplateResult> {
+  requireMemoryV2Enabled();
+  return { template: ROUTER_PROMPT };
+}
+
+// ── Current `<now>` body (default value for the playground editor) ──────
+
+export interface MemoryV2NowTextResult {
+  /** The current rendered NOW.md body (autoloaded essentials/threads/recent). */
+  nowText: string;
+}
+
+async function handleGetNowText(): Promise<MemoryV2NowTextResult> {
+  requireMemoryV2Enabled();
+  const workspaceDir = getWorkspaceDir();
+  const nowText = await loadNowText(workspaceDir);
+  return { nowText };
+}
+
+// ── Compare retrievers over historical turns (read-only) ────────────────
+
+const MemoryV2CompareRetrieversParams = z
+  .object({
+    /**
+     * How many historical `mode='router'` turns to sample. Each scored turn
+     * re-runs the router (one LLM call), so keep this modest. Default 20.
+     */
+    limit: z.number().int().positive().optional(),
+    strategy: z.enum(["recent", "random"]).optional(),
+    conversationIds: z.array(z.string().min(1)).optional(),
+    ks: z.array(z.number().int().positive()).optional(),
+    includeNotInjected: z.boolean().optional(),
+  })
+  .strict();
+
+const DEFAULT_COMPARE_LIMIT = 20;
+const DEFAULT_COMPARE_KS = [5, 10, 25, 50];
+
+export async function handleCompareRetrievers({
+  body = {},
+  abortSignal,
+}: RouteHandlerArgs): Promise<ComparisonReport> {
+  requireMemoryV2Enabled();
+  const { limit, strategy, conversationIds, ks, includeNotInjected } =
+    MemoryV2CompareRetrieversParams.parse(body);
+
+  const config = loadConfig();
+  const workspaceDir = getWorkspaceDir();
+  const pageIndex = await getPageIndex(workspaceDir);
+  const db = getDb();
+
+  // The router is always comparand #1 (the harness self-test against its own
+  // logged ground truth). v3 joins as comparand #2 only when explicitly
+  // enabled, so the default compare surface is unchanged until v3 is switched
+  // on. v3 is offline-only here — the loop reads `db` but mutates nothing.
+  const retrievers: Retriever[] = [createRouterRetriever(db)];
+  if (config.memory.v3.enabled) {
+    retrievers.push(createV3Retriever(db));
+  }
+
+  return runComparisonOverHistory({
+    db,
+    workspaceDir,
+    config,
+    retrievers,
+    ks: ks ?? DEFAULT_COMPARE_KS,
+    limit: limit ?? DEFAULT_COMPARE_LIMIT,
+    pageExists: (slug) => pageIndex.bySlug.has(slug),
+    ...(strategy !== undefined ? { strategy } : {}),
+    ...(conversationIds !== undefined ? { conversationIds } : {}),
+    ...(includeNotInjected !== undefined ? { includeNotInjected } : {}),
+    ...(abortSignal !== undefined ? { signal: abortSignal } : {}),
+  });
+}
+
 // ── Route definitions ───────────────────────────────────────────────────
 
 export const ROUTES: RouteDefinition[] = [
@@ -576,4 +754,36 @@ export const ROUTES: RouteDefinition[] = [
     tags: ["memory"],
     requestBody: MemoryV2SimulateRouterParams,
   },
+  {
+    operationId: "memory_v2_compare_retrievers",
+    method: "POST",
+    endpoint: "memory/v2/compare-retrievers",
+    handler: handleCompareRetrievers,
+    summary:
+      "Compare retrievers against the router's logged selections (read-only)",
+    description:
+      "Runs one or more retrievers over a sample of historical turns (memory_v2_activation_logs, mode='router') and scores their selected pages against the logged selections as ground truth, reconstructing each turn's inputs from the messages table + current NOW. Read-only — writes nothing. Each scored turn re-runs the router (one LLM call), so keep `limit` modest. Today the only retriever is the router itself, so this is the harness self-test.",
+    tags: ["memory"],
+    requestBody: MemoryV2CompareRetrieversParams,
+  },
+  {
+    operationId: "memory_v2_router_prompt_template",
+    method: "GET",
+    endpoint: "memory/v2/router-prompt-template",
+    handler: handleGetRouterPromptTemplate,
+    summary: "Return the bundled router system-prompt template",
+    description:
+      "Returns the bundled `ROUTER_PROMPT` body with placeholders intact (`{{ASSISTANT_NAME}}`, `{{USER_NAME}}`, `{{PAGE_INDEX}}`). Used by the memory router playground's 'Load default' affordance so users have a known-good starting point when authoring an inline prompt override.",
+    tags: ["memory"],
+  },
+  {
+    operationId: "memory_v2_now_text",
+    method: "GET",
+    endpoint: "memory/v2/now-text",
+    handler: handleGetNowText,
+    summary: "Return the current rendered `<now>` body",
+    description:
+      "Returns the current NOW.md (autoloaded essentials/threads/recent). Used by the memory router playground to seed its `<now>` text area with a production-like default so callers can edit from a realistic baseline.",
+    tags: ["memory"],
+  },
 ];