@vellumai/assistant 0.8.3 → 0.8.5

package/src/memory/schema/calls.ts

@@ -78,6 +78,7 @@ export const externalConversationBindings = sqliteTable(
       .references(() => conversations.id, { onDelete: "cascade" }),
     sourceChannel: text("source_channel").notNull(),
     externalChatId: text("external_chat_id").notNull(),
+    externalChatName: text("external_chat_name"),
     externalThreadId: text("external_thread_id"),
     externalUserId: text("external_user_id"),
     displayName: text("display_name"),

package/src/memory/schema/conversations.ts

@@ -21,6 +21,7 @@ export const conversations = sqliteTable(
       .notNull()
       .default(0),
     contextCompactedAt: integer("context_compacted_at"),
+    historyStrippedAt: integer("history_stripped_at"),
     slackContextCompactionWatermarkTs: text(
       "slack_context_compaction_watermark_ts",
     ),
@@ -41,6 +42,7 @@ export const conversations = sqliteTable(
     inferenceProfile: text("inference_profile"),
     inferenceProfileSessionId: text("inference_profile_session_id"),
     inferenceProfileExpiresAt: integer("inference_profile_expires_at"),
+    lastNotifiedInferenceProfile: text("last_notified_inference_profile"),
   },
   (table) => [
     index("idx_conversations_updated_at").on(table.updatedAt),
@@ -150,6 +152,7 @@ export const channelInboundEvents = sqliteTable("channel_inbound_events", {
   deliveryStatus: text("delivery_status").notNull().default("pending"),
   processingStatus: text("processing_status").notNull().default("pending"),
   processingAttempts: integer("processing_attempts").notNull().default(0),
+  deliveryAttempts: integer("delivery_attempts").notNull().default(0),
   lastProcessingError: text("last_processing_error"),
   retryAfter: integer("retry_after"),
   rawPayload: text("raw_payload"),

package/src/memory/schema/infrastructure.ts

@@ -138,6 +138,16 @@ export const llmRequestLogs = sqliteTable(
     responsePayload: text("response_payload").notNull(),
     createdAt: integer("created_at").notNull(),
     agentLoopExitReason: text("agent_loop_exit_reason"),
+    /**
+     * Logical call site that produced this row — e.g. `mainAgent`,
+     * `compactionAgent`. Stored as free-form text rather than enum-bound
+     * so a new call site can ship without a schema bump, but in practice
+     * callers pass values from `LLMCallSite` (`config/schemas/llm.ts`).
+     *
+     * Historical rows (pre-migration 264) stay NULL — "we don't know"
+     * rather than guessing `mainAgent`.
+     */
+    callSite: text("call_site"),
   },
   (table) => [
     index("idx_llm_request_logs_message_id").on(table.messageId),
@@ -217,6 +227,17 @@ export const llmUsageEvents = sqliteTable(
     outputTokens: integer("output_tokens").notNull(),
     cacheCreationInputTokens: integer("cache_creation_input_tokens"),
     cacheReadInputTokens: integer("cache_read_input_tokens"),
+    /**
+     * The provider's untouched `usage` block, serialized as JSON. Anthropic
+     * nests the TTL breakdown under `cache_creation.ephemeral_{5m,1h}_input_tokens`;
+     * OpenAI nests cached-read details under `prompt_tokens_details`; both are
+     * just preserved verbatim so admin charts and downstream consumers can
+     * extract whatever provider-specific detail they need without a schema
+     * change every time a provider exposes a new field. `null` when the
+     * provider didn't return a usage block or for rows persisted before
+     * migration 260.
+     */
+    rawUsage: text("raw_usage"),
     estimatedCostUsd: real("estimated_cost_usd"),
     pricingStatus: text("pricing_status").notNull(),
     llmCallCount: integer("llm_call_count"),
@@ -242,6 +263,7 @@ export const onboardingEvents = sqliteTable("onboarding_events", {
   tone: text("tone"),
   googleConnected: integer("google_connected", { mode: "boolean" }),
   googleScopesJson: text("google_scopes_json"),
+  priorAssistantsJson: text("prior_assistants_json"),
   abVariant: text("ab_variant"),
 });
 

package/src/memory/tool-usage-store.ts

@@ -2,6 +2,7 @@ import { count, desc, lt } from "drizzle-orm";
 import { v4 as uuid } from "uuid";
 
 import { getLogger } from "../util/logger.js";
+import { runAsyncSqlite } from "./db-async-query.js";
 import { getDb } from "./db-connection.js";
 import { toolInvocations } from "./schema.js";
 
@@ -47,25 +48,52 @@ export function getRecentInvocations(limit: number) {
 const log = getLogger("audit-log");
 
 /**
- * Delete tool invocation records older than the specified number of days.
- * Returns the number of deleted records. Does nothing if retentionDays is 0.
+ * Delete tool invocation records older than the specified number of
+ * days. Returns the number of deleted records. Does nothing if
+ * `retentionDays` is zero, negative, or non-finite.
+ *
+ * The DELETE runs through {@link runAsyncSqlite}: when the host has a
+ * `sqlite3` CLI it executes in a subprocess and the daemon's main
+ * event loop stays responsive. On hosts without the CLI the
+ * abstraction falls back to in-process blocking execution — the same
+ * behaviour the daemon had before this abstraction existed.
  */
-export function rotateToolInvocations(retentionDays: number): number {
-  if (retentionDays <= 0) return 0;
+export async function rotateToolInvocations(
+  retentionDays: number,
+): Promise<number> {
+  if (!Number.isFinite(retentionDays) || retentionDays <= 0) return 0;
 
-  const cutoff = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
+  // Math.floor guarantees a plain integer literal in the inlined SQL
+  // below; no decimal, no exponent, no surprise characters.
+  const cutoffMs = Math.floor(
+    Date.now() - retentionDays * 24 * 60 * 60 * 1000,
+  );
   const db = getDb();
 
-  // Count before delete (Drizzle's .run() returns void on bun-sqlite)
+  // Count before delete so we can return + log the affected row count
+  // (`runAsyncSqlite` does not surface SQLite's `changes()` value).
   const [countRow] = db
     .select({ value: count() })
     .from(toolInvocations)
-    .where(lt(toolInvocations.createdAt, cutoff))
+    .where(lt(toolInvocations.createdAt, cutoffMs))
     .all();
   const toDelete = countRow?.value ?? 0;
   if (toDelete === 0) return 0;
 
-  db.delete(toolInvocations).where(lt(toolInvocations.createdAt, cutoff)).run();
+  // `runAsyncSqlite` takes a raw SQL string — sqlite3 CLI subprocesses
+  // see SQL on stdin without a binding layer. `cutoffMs` is a plain
+  // integer (see Math.floor above), so inlining it here is safe.
+  const result = await runAsyncSqlite(
+    `DELETE FROM tool_invocations WHERE created_at < ${cutoffMs}`,
+  );
+  if (!result.ok) {
+    log.error(
+      { error: result.error, backend: result.backend, toDelete },
+      "tool_invocations purge failed",
+    );
+    return 0;
+  }
+
   log.info(
     `Rotated ${toDelete} audit log entries older than ${retentionDays} day(s)`,
   );

package/src/memory/v2/__tests__/consolidation-job.test.ts

@@ -102,6 +102,7 @@ mock.module("../../jobs-store.js", () => ({
     nextJobIdCounter += 1;
     return `job-${nextJobIdCounter}`;
   },
+  isMemoryEnabled: () => true,
 }));
 
 // ── Workspace pin ───────────────────────────────────────────────────

package/src/memory/v2/__tests__/harness-compare.test.ts

@@ -0,0 +1,186 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+mock.module("../../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, { get: () => () => {} }),
+}));
+
+mock.module("../../../config/loader.js", () => ({
+  getConfig: () => ({ memory: { enabled: false } }),
+}));
+
+import type { AssistantConfig } from "../../../config/types.js";
+import { getDb } from "../../db-connection.js";
+import { initializeDb } from "../../db-init.js";
+import type { MemoryV2ConceptRowRecord } from "../../memory-v2-activation-log-store.js";
+import {
+  conversations,
+  memoryV2ActivationLogs,
+  messages,
+} from "../../schema.js";
+import { runComparisonOverHistory } from "../harness/compare.js";
+import type { RetrievalOutput, Retriever } from "../harness/retriever.js";
+
+initializeDb();
+
+// loadNowText reads workspace files; a nonexistent dir yields "".
+const WORKSPACE = "/tmp/harness-compare-nonexistent-workspace";
+
+const ZERO_CONFIG = {
+  d: 0,
+  c_user: 0,
+  c_assistant: 0,
+  c_now: 0,
+  k: 0,
+  hops: 0,
+  top_k: 0,
+  epsilon: 0,
+};
+
+let seq = 0;
+
+function config(historicalPairs: number): AssistantConfig {
+  return {
+    memory: { v2: { router: { historical_pairs: historicalPairs } } },
+  } as unknown as AssistantConfig;
+}
+
+function ensureConversation(id: string): void {
+  getDb()
+    .insert(conversations)
+    .values({ id, createdAt: 0, updatedAt: 0 })
+    .onConflictDoNothing()
+    .run();
+}
+
+function insertMessage(
+  id: string,
+  conversationId: string,
+  role: string,
+  text: string,
+  createdAt: number,
+): void {
+  ensureConversation(conversationId);
+  getDb()
+    .insert(messages)
+    .values({
+      id,
+      conversationId,
+      role,
+      content: JSON.stringify([{ type: "text", text }]),
+      createdAt,
+    })
+    .run();
+}
+
+function makeConcept(
+  slug: string,
+  status: MemoryV2ConceptRowRecord["status"],
+): MemoryV2ConceptRowRecord {
+  return {
+    slug,
+    finalActivation: 0,
+    ownActivation: 0,
+    priorActivation: 0,
+    simUser: 0,
+    simAssistant: 0,
+    simNow: 0,
+    simUserRerankBoost: 0,
+    simAssistantRerankBoost: 0,
+    inRerankPool: false,
+    spreadContribution: 0,
+    source: "router",
+    status,
+  };
+}
+
+function insertRouterLog(
+  conversationId: string,
+  messageId: string,
+  turn: number,
+  concepts: MemoryV2ConceptRowRecord[],
+  createdAt: number,
+): void {
+  ensureConversation(conversationId);
+  getDb()
+    .insert(memoryV2ActivationLogs)
+    .values({
+      id: `log-${seq++}`,
+      conversationId,
+      messageId,
+      turn,
+      mode: "router",
+      conceptsJson: JSON.stringify(concepts),
+      skillsJson: "[]",
+      configJson: JSON.stringify(ZERO_CONFIG),
+      createdAt,
+    })
+    .run();
+}
+
+function stubRetriever(name: string, selected: string[]): Retriever {
+  return {
+    name,
+    retrieve: async (): Promise<RetrievalOutput> => ({
+      selectedSlugs: selected,
+      sourceBySlug: new Map(selected.map((s): [string, string] => [s, name])),
+    }),
+  };
+}
+
+function reset(): void {
+  const db = getDb();
+  db.delete(memoryV2ActivationLogs).run();
+  db.delete(messages).run();
+}
+
+/** Seed one router turn: user msg, assistant anchor, and the logged picks. */
+function seedTurn(groundTruth: string[]): void {
+  insertMessage("u1", "c1", "user", "hello", 10);
+  insertMessage("a1", "c1", "assistant", "hi", 20); // anchor for turn 1
+  insertRouterLog(
+    "c1",
+    "a1",
+    1,
+    groundTruth.map((slug) => makeConcept(slug, "injected")),
+    20,
+  );
+}
+
+describe("harness/compare runComparisonOverHistory", () => {
+  beforeEach(reset);
+
+  test("scores a stub retriever against the logged ground truth", async () => {
+    seedTurn(["p1", "p2"]);
+
+    const report = await runComparisonOverHistory({
+      db: getDb(),
+      workspaceDir: WORKSPACE,
+      config: config(1),
+      retrievers: [stubRetriever("stub", ["p1", "z"])],
+      ks: [5],
+    });
+
+    expect(report.turnsConsidered).toBe(1);
+    expect(report.turnsScored).toBe(1);
+    expect(report.turnsSkipped).toBe(0);
+    // hit p1, miss p2 → recall@5 = 1/2
+    expect(report.retrievers[0]?.aggregate.meanRecallAtK[5]).toBeCloseTo(0.5);
+  });
+
+  test("pageExists narrows the ground truth set", async () => {
+    seedTurn(["p1", "p2"]);
+
+    const report = await runComparisonOverHistory({
+      db: getDb(),
+      workspaceDir: WORKSPACE,
+      config: config(1),
+      retrievers: [stubRetriever("stub", ["p1", "z"])],
+      ks: [5],
+      pageExists: (slug) => slug === "p1", // p2 no longer exists
+    });
+
+    // ground truth is just {p1}; stub selected p1 → recall 1.0
+    expect(report.retrievers[0]?.aggregate.meanRecallAtK[5]).toBeCloseTo(1);
+  });
+});

package/src/memory/v2/__tests__/harness-metrics.test.ts

@@ -0,0 +1,74 @@
+import { describe, expect, test } from "bun:test";
+
+import { aggregate, evalTurn, recallAtK } from "../harness/metrics.js";
+import type { RetrievalOutput } from "../harness/retriever.js";
+
+function out(
+  selected: string[],
+  sourceBySlug: Record<string, string> = {},
+): RetrievalOutput {
+  return {
+    selectedSlugs: selected,
+    sourceBySlug: new Map(Object.entries(sourceBySlug)),
+  };
+}
+
+describe("harness/metrics recallAtK", () => {
+  test("fraction of ground truth recovered within k", () => {
+    const gt = new Set(["a", "b", "c", "d"]);
+    expect(recallAtK(["a", "b", "x"], gt, 10)).toBeCloseTo(0.5);
+    // only the top-2 selections count toward recall@2
+    expect(recallAtK(["a", "b", "c", "d"], gt, 2)).toBeCloseTo(0.5);
+    expect(recallAtK(["a", "b", "c", "d"], gt, 10)).toBeCloseTo(1);
+  });
+
+  test("empty ground truth is vacuously complete (recall 1)", () => {
+    expect(recallAtK([], new Set<string>(), 5)).toBe(1);
+  });
+});
+
+describe("harness/metrics evalTurn", () => {
+  test("hits / misses / extras and per-lane attribution", () => {
+    const e = evalTurn(
+      out(["a", "b", "z"], { a: "tree", b: "sparse", z: "dense" }),
+      ["a", "b", "c"],
+      [5],
+    );
+    expect(e.hits.sort()).toEqual(["a", "b"]);
+    expect(e.misses).toEqual(["c"]);
+    // selected but not in ground truth — reported as diff, not error
+    expect(e.extras).toEqual(["z"]);
+    expect(e.recallAtK[5]).toBeCloseTo(2 / 3);
+    expect(e.hitsByLane).toEqual({ tree: 1, sparse: 1 });
+  });
+
+  test("failureReason surfaces and recall is 0", () => {
+    const o: RetrievalOutput = {
+      selectedSlugs: [],
+      sourceBySlug: new Map(),
+      failureReason: "no_provider",
+    };
+    const e = evalTurn(o, ["a"], [5]);
+    expect(e.failureReason).toBe("no_provider");
+    expect(e.recallAtK[5]).toBe(0);
+  });
+});
+
+describe("harness/metrics aggregate", () => {
+  test("means recall@k and failure rate across turns", () => {
+    const ks = [5];
+    const t1 = evalTurn(out(["a"]), ["a"], ks); // recall 1
+    const t2 = evalTurn(out([]), ["a"], ks); // recall 0
+    const agg = aggregate([t1, t2], ks);
+    expect(agg.turns).toBe(2);
+    expect(agg.meanRecallAtK[5]).toBeCloseTo(0.5);
+    expect(agg.failureRate).toBe(0);
+  });
+
+  test("empty input is well-defined", () => {
+    const agg = aggregate([], [5]);
+    expect(agg.turns).toBe(0);
+    expect(agg.meanRecallAtK[5]).toBe(0);
+    expect(agg.failureRate).toBe(0);
+  });
+});

package/src/memory/v2/__tests__/harness-oracle.test.ts

@@ -0,0 +1,257 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+mock.module("../../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, { get: () => () => {} }),
+}));
+
+mock.module("../../../config/loader.js", () => ({
+  getConfig: () => ({ memory: { enabled: false } }),
+}));
+
+import { getDb } from "../../db-connection.js";
+import { initializeDb } from "../../db-init.js";
+import type { MemoryV2ConceptRowRecord } from "../../memory-v2-activation-log-store.js";
+import {
+  conversations,
+  memoryV2ActivationLogs,
+  messages,
+} from "../../schema.js";
+import { extractOracleTurns } from "../harness/oracle.js";
+
+initializeDb();
+
+const CONFIG_JSON = JSON.stringify({
+  d: 0,
+  c_user: 0,
+  c_assistant: 0,
+  c_now: 0,
+  k: 0,
+  hops: 0,
+  top_k: 0,
+  epsilon: 0,
+});
+
+let seq = 0;
+
+function ensureConversation(id: string): void {
+  getDb()
+    .insert(conversations)
+    .values({ id, createdAt: 0, updatedAt: 0 })
+    .onConflictDoNothing()
+    .run();
+}
+
+function makeConcept(
+  slug: string,
+  status: MemoryV2ConceptRowRecord["status"],
+): MemoryV2ConceptRowRecord {
+  return {
+    slug,
+    finalActivation: 0,
+    ownActivation: 0,
+    priorActivation: 0,
+    simUser: 0,
+    simAssistant: 0,
+    simNow: 0,
+    simUserRerankBoost: 0,
+    simAssistantRerankBoost: 0,
+    inRerankPool: false,
+    spreadContribution: 0,
+    source: "router",
+    status,
+  };
+}
+
+function insertLog(opts: {
+  conversationId: string;
+  messageId: string | null;
+  turn: number;
+  mode: string;
+  concepts: MemoryV2ConceptRowRecord[];
+  createdAt: number;
+}): void {
+  ensureConversation(opts.conversationId);
+  getDb()
+    .insert(memoryV2ActivationLogs)
+    .values({
+      id: `log-${seq++}`,
+      conversationId: opts.conversationId,
+      messageId: opts.messageId,
+      turn: opts.turn,
+      mode: opts.mode,
+      conceptsJson: JSON.stringify(opts.concepts),
+      skillsJson: "[]",
+      configJson: CONFIG_JSON,
+      createdAt: opts.createdAt,
+    })
+    .run();
+}
+
+function insertMessage(
+  id: string,
+  conversationId: string,
+  createdAt: number,
+): void {
+  ensureConversation(conversationId);
+  getDb()
+    .insert(messages)
+    .values({
+      id,
+      conversationId,
+      role: "assistant",
+      content: JSON.stringify([{ type: "text", text: "reply" }]),
+      createdAt,
+    })
+    .run();
+}
+
+function reset(): void {
+  const db = getDb();
+  db.delete(memoryV2ActivationLogs).run();
+  db.delete(messages).run();
+}
+
+describe("harness/oracle extractOracleTurns", () => {
+  beforeEach(reset);
+
+  test("returns only mode='router' rows with a resolvable messageId", () => {
+    insertMessage("m1", "c1", 100);
+    insertLog({
+      conversationId: "c1",
+      messageId: "m1",
+      turn: 1,
+      mode: "router",
+      concepts: [makeConcept("a", "injected")],
+      createdAt: 100,
+    });
+    // non-router row — ignored
+    insertMessage("m2", "c1", 200);
+    insertLog({
+      conversationId: "c1",
+      messageId: "m2",
+      turn: 2,
+      mode: "per-turn",
+      concepts: [makeConcept("b", "injected")],
+      createdAt: 200,
+    });
+    // router row, null messageId — skipped (no anchor)
+    insertLog({
+      conversationId: "c1",
+      messageId: null,
+      turn: 3,
+      mode: "router",
+      concepts: [makeConcept("c", "injected")],
+      createdAt: 300,
+    });
+    // router row, messageId does not resolve — skipped
+    insertLog({
+      conversationId: "c1",
+      messageId: "ghost",
+      turn: 4,
+      mode: "router",
+      concepts: [makeConcept("d", "injected")],
+      createdAt: 400,
+    });
+
+    const turns = extractOracleTurns(getDb(), { limit: 50 });
+    expect(turns.length).toBe(1);
+    expect(turns[0]?.turn).toBe(1);
+    expect(turns[0]?.groundTruthSlugs).toEqual(["a"]);
+    expect(turns[0]?.anchorMessageId).toBe("m1");
+    expect(turns[0]?.anchorCreatedAt).toBe(100);
+  });
+
+  test("ground truth keeps injected/in_context, drops the rest", () => {
+    insertMessage("m1", "c1", 100);
+    insertLog({
+      conversationId: "c1",
+      messageId: "m1",
+      turn: 1,
+      mode: "router",
+      createdAt: 100,
+      concepts: [
+        makeConcept("inj", "injected"),
+        makeConcept("ctx", "in_context"),
+        makeConcept("noinj", "not_injected"),
+        makeConcept("miss", "page_missing"),
+        makeConcept("bad", "corrupt"),
+      ],
+    });
+    const turns = extractOracleTurns(getDb());
+    expect(turns[0]?.groundTruthSlugs.sort()).toEqual(["ctx", "inj"]);
+  });
+
+  test("includeNotInjected adds not_injected to ground truth", () => {
+    insertMessage("m1", "c1", 100);
+    insertLog({
+      conversationId: "c1",
+      messageId: "m1",
+      turn: 1,
+      mode: "router",
+      createdAt: 100,
+      concepts: [
+        makeConcept("inj", "injected"),
+        makeConcept("noinj", "not_injected"),
+      ],
+    });
+    const turns = extractOracleTurns(getDb(), { includeNotInjected: true });
+    expect(turns[0]?.groundTruthSlugs.sort()).toEqual(["inj", "noinj"]);
+  });
+
+  test("pageExists predicate drops slugs whose page is gone", () => {
+    insertMessage("m1", "c1", 100);
+    insertLog({
+      conversationId: "c1",
+      messageId: "m1",
+      turn: 1,
+      mode: "router",
+      createdAt: 100,
+      concepts: [
+        makeConcept("present", "injected"),
+        makeConcept("gone", "injected"),
+      ],
+    });
+    const turns = extractOracleTurns(getDb(), {
+      pageExists: (slug) => slug === "present",
+    });
+    expect(turns[0]?.groundTruthSlugs).toEqual(["present"]);
+  });
+
+  test("skips rows whose ground truth is empty after filtering", () => {
+    insertMessage("m1", "c1", 100);
+    insertLog({
+      conversationId: "c1",
+      messageId: "m1",
+      turn: 1,
+      mode: "router",
+      createdAt: 100,
+      concepts: [makeConcept("noinj", "not_injected")],
+    });
+    expect(extractOracleTurns(getDb()).length).toBe(0);
+  });
+
+  test("conversationIds filter restricts the scan", () => {
+    insertMessage("m1", "c1", 100);
+    insertLog({
+      conversationId: "c1",
+      messageId: "m1",
+      turn: 1,
+      mode: "router",
+      concepts: [makeConcept("a", "injected")],
+      createdAt: 100,
+    });
+    insertMessage("m2", "c2", 200);
+    insertLog({
+      conversationId: "c2",
+      messageId: "m2",
+      turn: 1,
+      mode: "router",
+      concepts: [makeConcept("b", "injected")],
+      createdAt: 200,
+    });
+    const turns = extractOracleTurns(getDb(), { conversationIds: ["c2"] });
+    expect(turns.length).toBe(1);
+    expect(turns[0]?.conversationId).toBe("c2");
+  });
+});