@vellumai/assistant 0.8.0 → 0.8.2

package/src/memory/llm-request-log-source-clickhouse.ts

@@ -0,0 +1,324 @@
+/**
+ * ClickHouse-backed LLM request log read source.
+ *
+ * Reads from the ClickHouse mirror (populated out-of-band by the
+ * `mirror-llm-logs-to-clickhouse` cron). Scoped to the running
+ * assistant's own `assistant_id` — never cross-assistant. URL and
+ * password are resolved lazily from the credential store
+ * (`clickhouse:url`, `clickhouse:password`); database/table/user come
+ * from workspace config.
+ *
+ * Known limitation: the mirror is INSERT-only. A row inserted locally
+ * with `message_id = NULL` and backfilled later will appear in
+ * ClickHouse with `message_id = ''` forever. Reads via this source for
+ * the most-recent ~minute of activity therefore have lower fidelity
+ * than the local source. Acceptable for the "internal use while we
+ * finetune prompts" use case; revisit when mirror updates are added.
+ */
+import type { LlmRequestLogsClickHouseConfig } from "../config/schemas/llm-request-logs.js";
+import { credentialKey } from "../security/credential-key.js";
+import { getSecureKeyAsync } from "../security/secure-keys.js";
+import { getLogger } from "../util/logger.js";
+import {
+  getAssistantMessageIdsInTurn,
+  getMessageById,
+  messageMetadataSchema,
+} from "./conversation-crud.js";
+import type { LlmRequestLogSource } from "./llm-request-log-source.js";
+import type { LogRow } from "./llm-request-log-store.js";
+
+const log = getLogger("clickhouse-llm-request-log-source");
+
+/**
+ * Read a credential and normalize `undefined` → `null`. The credential
+ * resolver factories on this class are typed `() => Promise<string | null>`;
+ * `getSecureKeyAsync` returns `Promise<string | undefined>`. Keep the
+ * coercion in one place so TypeScript stays happy without per-call casts.
+ */
+async function readCredentialOrNull(
+  service: string,
+  field: string,
+): Promise<string | null> {
+  const value = await getSecureKeyAsync(credentialKey(service, field));
+  return value ?? null;
+}
+
+/**
+ * Wire-format row returned by ClickHouse for our query columns. Note
+ * that `created_at` arrives as a string because Int64 is emitted as a
+ * quoted string under the default `output_format_json_quote_64bit_integers=1`
+ * setting; we coerce to `number` in `toLogRow`.
+ */
+interface ClickHouseRow {
+  id: string;
+  conversation_id: string;
+  message_id: string;
+  provider: string;
+  request_payload: string;
+  response_payload: string;
+  created_at: string;
+}
+
+/** Injectable fetch override for tests. Defaults to globalThis.fetch. */
+export type ClickHouseFetch = typeof fetch;
+
+/** Minimal subset of the SQLite message row the fork-source fallback needs. */
+export interface ClickHouseMessageRow {
+  metadata: string | null;
+}
+
+export interface ClickHouseLlmRequestLogSourceDeps {
+  /** Override the credential read for `clickhouse:url`. */
+  resolveUrl?: () => Promise<string | null>;
+  /** Override the credential read for `clickhouse:password`. */
+  resolvePassword?: () => Promise<string | null>;
+  /** Override the credential read for `vellum:platform_assistant_id`. */
+  resolveAssistantId?: () => Promise<string | null>;
+  /** Override the turn-id resolver (default: `getAssistantMessageIdsInTurn`). */
+  resolveTurnMessageIds?: (messageId: string) => string[];
+  /** Override the message lookup (default: `getMessageById`). */
+  resolveMessage?: (messageId: string) => ClickHouseMessageRow | null;
+  /** Override fetch for testing. */
+  fetchImpl?: ClickHouseFetch;
+}
+
+export class ClickHouseLlmRequestLogSource implements LlmRequestLogSource {
+  private cachedUrl: string | null = null;
+  private cachedPassword: string | null = null;
+  private cachedAssistantId: string | null = null;
+
+  private readonly resolveUrl: () => Promise<string | null>;
+  private readonly resolvePassword: () => Promise<string | null>;
+  private readonly resolveAssistantId: () => Promise<string | null>;
+  private readonly resolveTurnMessageIds: (messageId: string) => string[];
+  private readonly resolveMessage: (
+    messageId: string,
+  ) => ClickHouseMessageRow | null;
+  private readonly fetchImpl: ClickHouseFetch;
+
+  constructor(
+    private readonly config: LlmRequestLogsClickHouseConfig,
+    deps: ClickHouseLlmRequestLogSourceDeps = {},
+  ) {
+    this.resolveUrl =
+      deps.resolveUrl ?? (() => readCredentialOrNull("clickhouse", "url"));
+    this.resolvePassword =
+      deps.resolvePassword ??
+      (() => readCredentialOrNull("clickhouse", "password"));
+    this.resolveAssistantId =
+      deps.resolveAssistantId ??
+      (() => readCredentialOrNull("vellum", "platform_assistant_id"));
+    this.resolveTurnMessageIds =
+      deps.resolveTurnMessageIds ?? getAssistantMessageIdsInTurn;
+    this.resolveMessage = deps.resolveMessage ?? getMessageById;
+    this.fetchImpl = deps.fetchImpl ?? globalThis.fetch.bind(globalThis);
+  }
+
+  async getRequestLogById(logId: string): Promise<LogRow | null> {
+    const aid = await this.assistantId();
+    const sql = `SELECT
+        id,
+        conversation_id,
+        message_id,
+        provider,
+        request_payload,
+        response_payload,
+        toUnixTimestamp64Milli(created_at) AS created_at
+      FROM ${this.tableRef()}
+      WHERE assistant_id = {assistant_id:String}
+        AND id = {log_id:String}
+      ORDER BY created_at DESC
+      LIMIT 1
+      FORMAT JSONEachRow`;
+    const rows = await this.exec(sql, { assistant_id: aid, log_id: logId });
+    return rows[0] ? this.toLogRow(rows[0]) : null;
+  }
+
+  async getRequestLogsByMessageId(messageId: string): Promise<LogRow[]> {
+    const turnIds = this.resolveTurnMessageIds(messageId);
+    let rows = await this.selectByMessageIds(turnIds);
+
+    if (rows.length === 0) {
+      // Fork-source fallback. Mirror behavior of the local source: when no
+      // logs match the queried message's turn, see if it was forked from
+      // another and resolve that source's turn. The fork relationship lives
+      // in local SQLite (message.metadata.forkSourceMessageId), not CH.
+      const message = this.resolveMessage(messageId);
+      if (message?.metadata) {
+        try {
+          const parsed = messageMetadataSchema.safeParse(
+            JSON.parse(message.metadata),
+          );
+          const sourceMessageId =
+            parsed.success &&
+            typeof parsed.data.forkSourceMessageId === "string"
+              ? parsed.data.forkSourceMessageId
+              : null;
+          if (sourceMessageId && sourceMessageId !== messageId) {
+            const sourceTurnIds = this.resolveTurnMessageIds(sourceMessageId);
+            rows = await this.selectByMessageIds(sourceTurnIds);
+          }
+        } catch {
+          // metadata not JSON / schema mismatch — no fork fallback, return []
+        }
+      }
+    }
+
+    return rows.sort(
+      (a, b) => a.createdAt - b.createdAt || a.id.localeCompare(b.id),
+    );
+  }
+
+  private async selectByMessageIds(ids: string[]): Promise<LogRow[]> {
+    if (ids.length === 0) return [];
+    const aid = await this.assistantId();
+    // Bind each id as its own {id_N:String} placeholder. The IDs ultimately
+    // come from a caller-supplied path parameter — `getAssistantMessageIdsInTurn`
+    // passes the input straight through when the message lookup misses — so
+    // inline literal building (even with quote-doubling) is unsafe: ClickHouse
+    // honors `\'` as an escaped quote inside string literals, letting a
+    // backslash-suffixed id break out of the IN clause and bypass the
+    // `assistant_id` scope filter. Type-bound parameters carry value, not
+    // syntax, regardless of content.
+    const params: Record<string, string> = { assistant_id: aid };
+    const placeholders: string[] = [];
+    for (let i = 0; i < ids.length; i++) {
+      const key = `id_${i}`;
+      params[key] = ids[i]!;
+      placeholders.push(`{${key}:String}`);
+    }
+    const sql = `SELECT
+        id,
+        conversation_id,
+        message_id,
+        provider,
+        request_payload,
+        response_payload,
+        toUnixTimestamp64Milli(created_at) AS created_at
+      FROM ${this.tableRef()}
+      WHERE assistant_id = {assistant_id:String}
+        AND message_id IN (${placeholders.join(",")})
+      ORDER BY created_at ASC, id ASC
+      LIMIT 1 BY id
+      FORMAT JSONEachRow`;
+    const rows = await this.exec(sql, params);
+    return rows.map((r) => this.toLogRow(r));
+  }
+
+  private tableRef(): string {
+    // Database is set via the `database=` URL param in `exec`, so we only
+    // need to quote the table identifier here. Backtick-quote both to
+    // tolerate non-default names with special characters.
+    return `\`${this.config.table.replace(/`/g, "``")}\``;
+  }
+
+  private async exec(
+    sql: string,
+    params: Record<string, string>,
+  ): Promise<ClickHouseRow[]> {
+    const baseUrl = await this.url();
+    const password = await this.password();
+
+    let target: URL;
+    try {
+      target = new URL(baseUrl);
+    } catch (err) {
+      throw new Error(
+        `clickhouse:url is not a valid URL: ${err instanceof Error ? err.message : String(err)}`,
+      );
+    }
+    target.searchParams.set("database", this.config.database);
+    for (const [k, v] of Object.entries(params)) {
+      target.searchParams.set(`param_${k}`, v);
+    }
+
+    const auth =
+      "Basic " +
+      Buffer.from(`${this.config.user}:${password}`, "utf8").toString("base64");
+
+    const res = await this.fetchImpl(target.toString(), {
+      method: "POST",
+      headers: { Authorization: auth, "Content-Type": "text/plain" },
+      body: sql,
+    });
+
+    if (!res.ok) {
+      const body = await res.text().catch(() => "");
+      log.error(
+        { status: res.status, table: this.config.table, bodySnippet: body.slice(0, 200) },
+        "ClickHouse query failed",
+      );
+      throw new Error(
+        `ClickHouse query failed (HTTP ${res.status}): ${body.slice(0, 500)}`,
+      );
+    }
+
+    const text = await res.text();
+    if (text.trim().length === 0) return [];
+
+    const rows: ClickHouseRow[] = [];
+    for (const line of text.split("\n")) {
+      const trimmed = line.trim();
+      if (trimmed.length === 0) continue;
+      try {
+        rows.push(JSON.parse(trimmed) as ClickHouseRow);
+      } catch (err) {
+        throw new Error(
+          `Failed to parse ClickHouse JSONEachRow line: ${err instanceof Error ? err.message : String(err)}`,
+        );
+      }
+    }
+    return rows;
+  }
+
+  private toLogRow(row: ClickHouseRow): LogRow {
+    return {
+      id: row.id,
+      conversationId: row.conversation_id,
+      // The mirror writes empty-string for missing message_id/provider
+      // because the CH table columns have `DEFAULT ''` (Nullable adds
+      // overhead). Map empty back to null to match the local LogRow shape.
+      messageId: row.message_id === "" ? null : row.message_id,
+      provider: row.provider === "" ? null : row.provider,
+      requestPayload: row.request_payload,
+      responsePayload: row.response_payload,
+      createdAt: Number(row.created_at),
+    };
+  }
+
+  private async assistantId(): Promise<string> {
+    if (this.cachedAssistantId) return this.cachedAssistantId;
+    const val = await this.resolveAssistantId();
+    if (!val) {
+      throw new Error(
+        "vellum:platform_assistant_id credential is required when readSource=clickhouse",
+      );
+    }
+    this.cachedAssistantId = val;
+    return val;
+  }
+
+  private async url(): Promise<string> {
+    if (this.cachedUrl) return this.cachedUrl;
+    const val = await this.resolveUrl();
+    if (!val) {
+      throw new Error(
+        "clickhouse:url credential is required when readSource=clickhouse",
+      );
+    }
+    this.cachedUrl = val;
+    return val;
+  }
+
+  private async password(): Promise<string> {
+    if (this.cachedPassword) return this.cachedPassword;
+    const val = await this.resolvePassword();
+    if (!val) {
+      throw new Error(
+        "clickhouse:password credential is required when readSource=clickhouse",
+      );
+    }
+    this.cachedPassword = val;
+    return val;
+  }
+}

package/src/memory/llm-request-log-source-local.ts

@@ -0,0 +1,26 @@
+/**
+ * Default LLM request log read source: thin async wrapper around the
+ * existing synchronous `llm-request-log-store.ts` functions.
+ *
+ * Keeps the local read path identical to its pre-pluggable behavior —
+ * including turn resolution, orphan/unlinked recovery, opportunistic
+ * backfill, and fork-source fallback. The wrapper exists only to satisfy
+ * the `LlmRequestLogSource` interface (which is `Promise`-returning so
+ * implementations with real I/O can fit).
+ */
+import type { LlmRequestLogSource } from "./llm-request-log-source.js";
+import {
+  getRequestLogById,
+  getRequestLogsByMessageId,
+  type LogRow,
+} from "./llm-request-log-store.js";
+
+export class LocalLlmRequestLogSource implements LlmRequestLogSource {
+  async getRequestLogById(logId: string): Promise<LogRow | null> {
+    return getRequestLogById(logId);
+  }
+
+  async getRequestLogsByMessageId(messageId: string): Promise<LogRow[]> {
+    return getRequestLogsByMessageId(messageId);
+  }
+}

package/src/memory/llm-request-log-source.ts

@@ -0,0 +1,64 @@
+/**
+ * Pluggable read source for LLM request logs.
+ *
+ * The Inspector view at `GET /v1/messages/:id/llm-context` (and the
+ * single-log payload route) historically read directly from the local
+ * SQLite `llm_request_logs` table via `llm-request-log-store.ts`. The
+ * source-of-truth remains local, but the *read path* is now configurable
+ * via `llmRequestLogs.readSource` in workspace config.
+ *
+ * - `local` (default): wraps the existing store functions verbatim.
+ * - `clickhouse`: queries the ClickHouse mirror (longer retention, but
+ *   only sees rows the mirror cron has flushed). See
+ *   `llm-request-log-source-clickhouse.ts`.
+ *
+ * Implementations are cheap to instantiate, so there's no module-level
+ * cache — each call resolves config fresh and constructs a new instance.
+ * Config edits take effect on the next request without an invalidation hook.
+ */
+import { getConfig } from "../config/loader.js";
+import type { LogRow } from "./llm-request-log-store.js";
+
+export interface LlmRequestLogSource {
+  /** Fetch a single log row by its primary key. Returns null if not found. */
+  getRequestLogById(logId: string): Promise<LogRow | null>;
+
+  /**
+   * Fetch every LLM request log associated with the given message,
+   * including all assistant messages in the same agent turn. Implementations
+   * MAY additionally apply orphan/unlinked/fork-source recovery — the
+   * local implementation does, the ClickHouse mirror does not (it is
+   * INSERT-only against the source-of-truth).
+   */
+  getRequestLogsByMessageId(messageId: string): Promise<LogRow[]>;
+}
+
+/**
+ * Return the configured LLM request log source.
+ *
+ * The factory is async because both implementations are loaded via
+ * dynamic `import()` on first use. This is deliberate: it keeps the
+ * static module graph for `llm-request-log-source.ts` (and for
+ * everything that transitively imports it) free of
+ * `llm-request-log-store → conversation-crud → indexer → embedding-backend`,
+ * which would otherwise force test files that stub `embedding-backend.js`
+ * to also stub every export `indexer.ts` reaches for.
+ *
+ * Callers must `await` both the factory and the source methods.
+ */
+export async function getLlmRequestLogSource(): Promise<LlmRequestLogSource> {
+  const config = getConfig();
+  const cfg = config.llmRequestLogs ?? { readSource: "local" as const };
+
+  if (cfg.readSource === "clickhouse") {
+    const { ClickHouseLlmRequestLogSource } = await import(
+      "./llm-request-log-source-clickhouse.js"
+    );
+    return new ClickHouseLlmRequestLogSource(cfg.clickhouse);
+  }
+
+  const { LocalLlmRequestLogSource } = await import(
+    "./llm-request-log-source-local.js"
+  );
+  return new LocalLlmRequestLogSource();
+}

package/src/memory/llm-request-log-store.ts

@@ -10,7 +10,7 @@ import {
 import { getDb } from "./db-connection.js";
 import { llmRequestLogs, messages } from "./schema.js";
 
-type LogRow = {
+export type LogRow = {
   id: string;
   conversationId: string;
   messageId: string | null;

package/src/memory/llm-usage-store.ts

@@ -1,4 +1,4 @@
-import { and, asc, desc, eq, gt, or } from "drizzle-orm";
+import { and, asc, desc, eq, gt, or, sql } from "drizzle-orm";
 import { v4 as uuid } from "uuid";
 
 import type {
@@ -8,7 +8,7 @@ import type {
 } from "../usage/types.js";
 import { getDb } from "./db-connection.js";
 import { rawAll } from "./raw-query.js";
-import { llmUsageEvents } from "./schema.js";
+import { conversations, llmUsageEvents } from "./schema.js";
 import {
   bucketEventsByDay,
   bucketEventsByHour,
@@ -123,15 +123,96 @@ export function listUsageEvents(options?: { limit?: number }): UsageEvent[] {
   return rows.map(rowToUsageEvent);
 }
 
+/**
+ * Telemetry-flavoured `UsageEvent`: the persisted columns plus the two
+ * JOIN-computed conversation-level fields the reporter needs to emit
+ * for analytics (`avg turns per conversation`, `tokens on first turn`,
+ * foreground/background split on llm_usage rows themselves).
+ *
+ * Lives next to the query that produces it so the shape stays in lockstep
+ * with the SELECT; broader `UsageEvent` consumers stay untouched.
+ */
+export interface UnreportedUsageEvent extends UsageEvent {
+  /**
+   * Type of the parent conversation (`"standard"` / `"background"` /
+   * `"scheduled"`). Null when the LLM call has no `conversationId`
+   * (memory consolidation, background embedding work, etc.) and so no
+   * `conversations` row to join against.
+   */
+  conversationType: string | null;
+  /**
+   * 1-indexed position of the user turn this LLM call belongs to within
+   * the parent conversation, counting only real user turns (tool-result
+   * rows persisted with role="user" are excluded — same filter as the
+   * turn-event eligibility predicate). Computed as the count of user
+   * messages with `created_at <= this LLM call's created_at` in the
+   * parent conversation. Null when there's no parent conversation, or
+   * when the LLM call fired before any user turn (rare — covers seed
+   * agent starts).
+   */
+  turnIndex: number | null;
+}
+
 export function queryUnreportedUsageEvents(
   afterCreatedAt: number,
   afterId: string | undefined,
   limit: number,
-): UsageEvent[] {
+): UnreportedUsageEvent[] {
   const db = getDb();
+  // JOIN to `conversations` to attach `conversationType`. LEFT JOIN
+  // because `llm_usage_events.conversationId` is nullable — calls that
+  // aren't tied to a conversation (memory consolidation, etc.) still
+  // need to flush through telemetry.
+  //
+  // `turnIndex` is a correlated subquery counting real user turns in
+  // the same conversation up to and including this LLM call's
+  // `created_at`. The filter mirrors `queryUnreportedTurnEvents` so the
+  // two indexes stay aligned: an LLM call fired during processing of
+  // turn N reports `turn_index = N`, matching what the turn event
+  // stream emitted for the triggering user message.
   const rows = db
-    .select()
+    .select({
+      id: llmUsageEvents.id,
+      createdAt: llmUsageEvents.createdAt,
+      conversationId: llmUsageEvents.conversationId,
+      runId: llmUsageEvents.runId,
+      requestId: llmUsageEvents.requestId,
+      actor: llmUsageEvents.actor,
+      callSite: llmUsageEvents.callSite,
+      inferenceProfile: llmUsageEvents.inferenceProfile,
+      inferenceProfileSource: llmUsageEvents.inferenceProfileSource,
+      provider: llmUsageEvents.provider,
+      model: llmUsageEvents.model,
+      inputTokens: llmUsageEvents.inputTokens,
+      outputTokens: llmUsageEvents.outputTokens,
+      cacheCreationInputTokens: llmUsageEvents.cacheCreationInputTokens,
+      cacheReadInputTokens: llmUsageEvents.cacheReadInputTokens,
+      estimatedCostUsd: llmUsageEvents.estimatedCostUsd,
+      pricingStatus: llmUsageEvents.pricingStatus,
+      conversationType: conversations.conversationType,
+      // Null when conversationId is null (no parent conversation).
+      // Otherwise the count of eligible user turns up to and including
+      // this LLM call's createdAt. The COALESCE guard returns null
+      // (rather than 0) for the "no user turn yet" edge case so the
+      // analytics layer can distinguish "before-first-turn" LLM calls.
+      turnIndex: sql<number | null>`(
+        CASE WHEN ${llmUsageEvents.conversationId} IS NULL THEN NULL
+        ELSE (
+          SELECT COUNT(*) FROM messages AS m2
+          WHERE m2.conversation_id = ${llmUsageEvents.conversationId}
+            AND m2.role = 'user'
+            AND m2.content NOT LIKE '%"type":"tool\\_result"%' ESCAPE '\\'
+            AND m2.content NOT LIKE '%"type":"web\\_search\\_tool\\_result"%' ESCAPE '\\'
+            AND m2.created_at <= ${llmUsageEvents.createdAt}
+        )
+        END
+      )`.as("turn_index"),
+    })
     .from(llmUsageEvents)
+    .leftJoin(
+      conversations,
+      eq(llmUsageEvents.conversationId, conversations.id),
+    )
     .where(
       afterId
         ? or(
@@ -146,7 +227,15 @@ export function queryUnreportedUsageEvents(
     .orderBy(asc(llmUsageEvents.createdAt), asc(llmUsageEvents.id))
     .limit(limit)
     .all();
-  return rows.map(rowToUsageEvent);
+  return rows.map((row) => ({
+    ...rowToUsageEvent(row),
+    conversationType: row.conversationType,
+    // SQLite returns COUNT(*) as 0 when no rows match; the CASE in the
+    // subquery already collapses the no-conversation case to NULL.
+    // Convert the integer column to `number | null` for the typed
+    // return value.
+    turnIndex: row.turnIndex === null ? null : Number(row.turnIndex),
+  }));
 }
 
 // ---------------------------------------------------------------------------
@@ -252,6 +341,37 @@ interface GroupRow {
   event_count: number;
 }
 
+/**
+ * Return aggregate usage for a single conversation (e.g. a subagent).
+ */
+export function getConversationUsageTotals(conversationId: string): {
+  inputTokens: number;
+  outputTokens: number;
+  estimatedCost: number;
+} {
+  const rows = rawAll<{
+    total_input: number;
+    total_output: number;
+    total_cost: number | null;
+  }>(
+    /*sql*/ `
+    SELECT
+      COALESCE(SUM(input_tokens + COALESCE(cache_creation_input_tokens, 0) + COALESCE(cache_read_input_tokens, 0)), 0) AS total_input,
+      COALESCE(SUM(output_tokens), 0) AS total_output,
+      COALESCE(SUM(estimated_cost_usd), 0) AS total_cost
+    FROM llm_usage_events
+    WHERE conversation_id = ?1
+    `,
+    conversationId,
+  );
+  const row = rows[0];
+  return {
+    inputTokens: row.total_input,
+    outputTokens: row.total_output,
+    estimatedCost: row.total_cost ?? 0,
+  };
+}
+
 /**
  * Return aggregate totals for all usage events within the given time range.
  */

package/src/memory/memory-retrospective-constants.ts

@@ -0,0 +1,13 @@
+/**
+ * Sentinel value for the `source` column of memory-retrospective background
+ * conversations. Used both when creating them and when filtering them out of
+ * recursion / orphan-cleanup queries.
+ */
+export const MEMORY_RETROSPECTIVE_SOURCE = "memory-retrospective";
+
+/**
+ * Dedicated `group_id` value for memory-retrospective background
+ * conversations. Placed under `system:background` alongside auto-analysis,
+ * heartbeat, and filing conversations.
+ */
+export const MEMORY_RETROSPECTIVE_GROUP_ID = "system:background";