@vellumai/assistant 0.8.0 → 0.8.2

package/src/memory/onboarding-events-store.ts

@@ -0,0 +1,106 @@
+import { and, asc, eq, gt, or } from "drizzle-orm";
+import { v4 as uuid } from "uuid";
+
+import { getConfig } from "../config/loader.js";
+import { getDb } from "./db-connection.js";
+import { onboardingEvents } from "./schema.js";
+
+export interface OnboardingEvent {
+  id: string;
+  createdAt: number;
+  screen: string;
+  toolsJson: string | null;
+  tasksJson: string | null;
+  tone: string | null;
+  googleConnected: boolean | null;
+  googleScopesJson: string | null;
+  abVariant: string | null;
+}
+
+export interface RecordOnboardingEventParams {
+  screen: string;
+  tools?: string[];
+  tasks?: string[];
+  tone?: string;
+  googleConnected?: boolean;
+  googleScopes?: string[];
+  abVariant?: string;
+}
+
+/**
+ * Record an onboarding event (pre-chat selections and Google connect status).
+ * Returns null when usage data collection is disabled.
+ */
+export function recordOnboardingEvent(
+  params: RecordOnboardingEventParams,
+): OnboardingEvent | null {
+  if (!getConfig().collectUsageData) return null;
+  const db = getDb();
+  const event: OnboardingEvent = {
+    id: uuid(),
+    createdAt: Date.now(),
+    screen: params.screen,
+    toolsJson: params.tools ? JSON.stringify(params.tools) : null,
+    tasksJson: params.tasks ? JSON.stringify(params.tasks) : null,
+    tone: params.tone ?? null,
+    googleConnected: params.googleConnected ?? null,
+    googleScopesJson: params.googleScopes
+      ? JSON.stringify(params.googleScopes)
+      : null,
+    abVariant: params.abVariant ?? null,
+  };
+  db.insert(onboardingEvents)
+    .values({
+      id: event.id,
+      createdAt: event.createdAt,
+      screen: event.screen,
+      toolsJson: event.toolsJson,
+      tasksJson: event.tasksJson,
+      tone: event.tone,
+      googleConnected: event.googleConnected,
+      googleScopesJson: event.googleScopesJson,
+      abVariant: event.abVariant,
+    })
+    .run();
+  return event;
+}
+
+/**
+ * Query onboarding events that haven't been reported to telemetry yet.
+ * Uses a compound cursor (createdAt + id) for reliable watermarking.
+ */
+export function queryUnreportedOnboardingEvents(
+  afterCreatedAt: number,
+  afterId: string | undefined,
+  limit: number,
+): OnboardingEvent[] {
+  const db = getDb();
+  const rows = db
+    .select({
+      id: onboardingEvents.id,
+      createdAt: onboardingEvents.createdAt,
+      screen: onboardingEvents.screen,
+      toolsJson: onboardingEvents.toolsJson,
+      tasksJson: onboardingEvents.tasksJson,
+      tone: onboardingEvents.tone,
+      googleConnected: onboardingEvents.googleConnected,
+      googleScopesJson: onboardingEvents.googleScopesJson,
+      abVariant: onboardingEvents.abVariant,
+    })
+    .from(onboardingEvents)
+    .where(
+      afterId
+        ? or(
+            gt(onboardingEvents.createdAt, afterCreatedAt),
+            and(
+              eq(onboardingEvents.createdAt, afterCreatedAt),
+              gt(onboardingEvents.id, afterId),
+            ),
+          )
+        : gt(onboardingEvents.createdAt, afterCreatedAt),
+    )
+    .orderBy(asc(onboardingEvents.createdAt), asc(onboardingEvents.id))
+    .limit(limit)
+    .all();
+  return rows;
+}

package/src/memory/published-pages-store.ts

@@ -36,6 +36,21 @@ export function getActivePublishedPageByAppId(
   return row ?? null;
 }
 
+export function insertPublishedPage(record: {
+  id: string;
+  deploymentId: string;
+  publicUrl: string;
+  pageTitle: string | null;
+  htmlHash: string;
+  publishedAt: number;
+  status: string;
+  appId: string | null;
+  projectSlug: string | null;
+}): void {
+  const db = getDb();
+  db.insert(publishedPages).values(record).run();
+}
+
 export function updatePublishedPage(
   id: string,
   updates: {
@@ -44,6 +59,7 @@ export function updatePublishedPage(
     htmlHash?: string;
     publishedAt?: number;
     appId?: string;
+    status?: string;
   },
 ): void {
   const db = getDb();

package/src/memory/schema/bookmarks.ts

@@ -0,0 +1,36 @@
+import {
+  index,
+  integer,
+  sqliteTable,
+  text,
+  uniqueIndex,
+} from "drizzle-orm/sqlite-core";
+
+import { conversations, messages } from "./conversations.js";
+
+/**
+ * User-saved bookmarks for individual messages. Surfaced in the macOS app
+ * Settings → Bookmarks tab and on the message hover overflow menu, behind the
+ * `bookmarks` client feature flag.
+ *
+ * Both foreign keys CASCADE so bookmarks disappear automatically when their
+ * parent message or conversation is deleted. A unique index on `message_id`
+ * keeps the create-bookmark path idempotent.
+ */
+export const messageBookmarks = sqliteTable(
+  "message_bookmarks",
+  {
+    id: text("id").primaryKey(),
+    messageId: text("message_id")
+      .notNull()
+      .references(() => messages.id, { onDelete: "cascade" }),
+    conversationId: text("conversation_id")
+      .notNull()
+      .references(() => conversations.id, { onDelete: "cascade" }),
+    createdAt: integer("created_at").notNull(),
+  },
+  (table) => [
+    uniqueIndex("message_bookmarks_message_id_uniq").on(table.messageId),
+    index("message_bookmarks_created_at_idx").on(table.createdAt),
+  ],
+);

package/src/memory/schema/calls.ts

@@ -78,6 +78,7 @@ export const externalConversationBindings = sqliteTable(
       .references(() => conversations.id, { onDelete: "cascade" }),
     sourceChannel: text("source_channel").notNull(),
     externalChatId: text("external_chat_id").notNull(),
+    externalThreadId: text("external_thread_id"),
     externalUserId: text("external_user_id"),
     displayName: text("display_name"),
     username: text("username"),

package/src/memory/schema/conversations.ts

@@ -39,6 +39,8 @@ export const conversations = sqliteTable(
     lastMessageAt: integer("last_message_at"),
     archivedAt: integer("archived_at"),
     inferenceProfile: text("inference_profile"),
+    inferenceProfileSessionId: text("inference_profile_session_id"),
+    inferenceProfileExpiresAt: integer("inference_profile_expires_at"),
   },
   (table) => [
     index("idx_conversations_updated_at").on(table.updatedAt),

package/src/memory/schema/index.ts

@@ -1,8 +1,10 @@
 export * from "./acp.js";
+export * from "./bookmarks.js";
 export * from "./calls.js";
 export * from "./contacts.js";
 export * from "./conversations.js";
 export * from "./guardian.js";
+export * from "./inference.js";
 export * from "./infrastructure.js";
 export * from "./memory-core.js";
 export * from "./memory-graph.js";

package/src/memory/schema/inference.ts

@@ -0,0 +1,27 @@
+import { index, integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
+
+/**
+ * Named provider connections.
+ *
+ * Each row is a named auth-config instance for a code-defined provider.
+ * Profiles in config.json reference connections by `name` via the
+ * `provider_connection` field.
+ *
+ * Created by migration 243.
+ */
+export const providerConnections = sqliteTable(
+  "provider_connections",
+  {
+    name: text("name").primaryKey(),
+    provider: text("provider").notNull(),
+    auth: text("auth").notNull(),
+    status: text("status").notNull().default("active"),
+    label: text("label"),
+    createdAt: integer("created_at").notNull(),
+    updatedAt: integer("updated_at").notNull(),
+  },
+  (table) => [index("idx_provider_connections_provider").on(table.provider)],
+);
+
+export type ProviderConnectionRow = typeof providerConnections.$inferSelect;
+export type NewProviderConnectionRow = typeof providerConnections.$inferInsert;

package/src/memory/schema/infrastructure.ts

@@ -232,6 +232,18 @@ export const lifecycleEvents = sqliteTable("lifecycle_events", {
   createdAt: integer("created_at").notNull(),
 });
 
+export const onboardingEvents = sqliteTable("onboarding_events", {
+  id: text("id").primaryKey(),
+  createdAt: integer("created_at").notNull(),
+  screen: text("screen").notNull(),
+  toolsJson: text("tools_json"),
+  tasksJson: text("tasks_json"),
+  tone: text("tone"),
+  googleConnected: integer("google_connected", { mode: "boolean" }),
+  googleScopesJson: text("google_scopes_json"),
+  abVariant: text("ab_variant"),
+});
+
 export const traceEvents = sqliteTable(
   "trace_events",
   {

package/src/memory/schema/memory-core.ts

@@ -100,6 +100,15 @@ export const memoryCheckpoints = sqliteTable("memory_checkpoints", {
   updatedAt: integer("updated_at").notNull(),
 });
 
+export const memoryRetrospectiveState = sqliteTable(
+  "memory_retrospective_state",
+  {
+    conversationId: text("conversation_id").primaryKey(),
+    lastProcessedMessageId: text("last_processed_message_id").notNull(),
+    lastRunAt: integer("last_run_at").notNull(),
+  },
+);
+
 export const conversationStarters = sqliteTable(
   "conversation_starters",
   {

package/src/memory/search/semantic.ts

@@ -9,6 +9,7 @@ import type {
 } from "../qdrant-client.js";
 import { getQdrantClient } from "../qdrant-client.js";
 import { memorySegments, memorySummaries } from "../schema.js";
+import { mapCosineToUnit } from "../validation.js";
 // ── Types (inlined from deleted types.ts) ──────────────────────────
 
 type CandidateType = "segment" | "item" | "summary" | "media";
@@ -235,7 +236,3 @@ function buildHybridFilter(
     must_not: mustNotConditions,
   };
 }
-
-function mapCosineToUnit(value: number): number {
-  return Math.max(0, Math.min(1, (value + 1) / 2));
-}

package/src/memory/turn-events-store.ts

@@ -1,16 +1,102 @@
 import { and, asc, eq, gt, or, sql } from "drizzle-orm";
 
 import { getDb } from "./db-connection.js";
-import { messages } from "./schema.js";
+import { conversations, messages } from "./schema.js";
 
 export interface TurnEvent {
   id: string;
   createdAt: number;
+  /**
+   * Parent conversation id. Lets downstream analytics group turns by
+   * conversation (e.g. avg turns per conversation).
+   */
+  conversationId: string;
+  /**
+   * Conversation type of the parent conversation. Used downstream to
+   * distinguish user-initiated turns (`"standard"`) from system-generated
+   * prompts in `"background"` / `"scheduled"` conversations so analytics
+   * (e.g. DAU) can exclude the latter.
+   */
+  conversationType: string;
+  /**
+   * 1-indexed position of this user turn within the parent conversation,
+   * counting only real user turns (tool-result rows persisted with
+   * role="user" are excluded — same filter as the eligibility predicate
+   * below). The first user turn in a conversation is `1`.
+   *
+   * Computed via correlated subquery on the same filtered set used for
+   * eligibility; this scales with batch size (≤ BATCH_SIZE turns per
+   * flush) and uses the `idx_messages_conversation_id` index for the
+   * partition lookup.
+   */
+  turnIndex: number;
+  /**
+   * Canonical `InterfaceId` enum value identifying the UI surface the user
+   * was interacting from at message-creation time (`"macos"`, `"ios"`,
+   * `"cli"`, `"web"`, `"chrome-extension"`, `"slack"`, `"telegram"`,
+   * `"whatsapp"`, `"email"`, `"phone"`). Sourced from
+   * `messages.metadata.userMessageInterface` (stamped on insert by every
+   * `persistUserMessage` path that flows through `TurnChannelContext`).
+   *
+   * Null when the metadata didn't carry the field — historical rows
+   * predating the threading, or system-initiated turns with no inbound
+   * client context. Downstream analytics should treat null as
+   * `"unknown"`.
+   */
+  interfaceId: string | null;
+  /**
+   * Canonical `ChannelId` enum value identifying the messaging fabric the
+   * user message arrived on (`"vellum"` for in-app messaging from
+   * macos/ios/web/cli; `"slack"`/`"telegram"`/`"whatsapp"`/`"email"`/
+   * `"phone"` for channel-based interfaces). Sourced from
+   * `messages.metadata.userMessageChannel`.
+   *
+   * The 7th `ChannelId` value (`"platform"`) is APNs-push outbound-only
+   * and should never appear on a user-message row.
+   */
+  channelId: string | null;
+  /**
+   * Flexible client metadata stashed under `messages.metadata.client` by
+   * the HTTP header middleware. Carries optional `browserFamily`,
+   * `browserVersion`, `os`, `interfaceVersion` (and is extensible without
+   * a migration since it lives inside the JSON column). Null when no
+   * client headers were attached.
+   *
+   * Returned as raw JSON text — the reporter parses + re-shapes for the
+   * wire format.
+   */
+  clientMetadata: string | null;
+}
+
+/**
+ * SQL fragment that excludes tool-result rows persisted with role="user".
+ * Kept as a single source of truth so the eligibility predicate and the
+ * correlated `turn_index` count stay in lockstep — otherwise the index
+ * can drift from the visible turn stream and break "first turn" /
+ * "turns per conversation" math.
+ *
+ * `<alias>` is interpolated as the SQL identifier for the table whose
+ * `content` column should be filtered (e.g. `messages` for the outer
+ * query, `m2` for the correlated subquery).
+ */
+function realUserTurnContentFilter(alias: string): ReturnType<typeof sql> {
+  return sql.raw(
+    `${alias}.content NOT LIKE '%"type":"tool\\_result"%' ESCAPE '\\' ` +
+      `AND ${alias}.content NOT LIKE '%"type":"web\\_search\\_tool\\_result"%' ESCAPE '\\'`,
+  );
 }
 
 /**
  * Query user messages (turns) that haven't been reported to telemetry yet.
  * Uses a compound cursor (createdAt + id) for reliable watermarking.
+ *
+ * Joins to `conversations` so each turn carries its `conversationType`.
+ * The inner join is safe because `messages.conversationId` has a
+ * not-null FK to `conversations.id` (cascade on delete): every message
+ * row has a matching conversation row.
+ *
+ * `turnIndex` is computed via a correlated subquery counting the real
+ * user turns in the same conversation up to and including this row.
  */
 export function queryUnreportedTurnEvents(
   afterCreatedAt: number,
@@ -19,8 +105,47 @@ export function queryUnreportedTurnEvents(
 ): TurnEvent[] {
   const db = getDb();
   const rows = db
-    .select({ id: messages.id, createdAt: messages.createdAt })
+    .select({
+      id: messages.id,
+      createdAt: messages.createdAt,
+      conversationId: messages.conversationId,
+      conversationType: conversations.conversationType,
+      // 1-indexed turn position within the parent conversation. Counts
+      // only real user turns (same filter applied to the outer query).
+      // `(created_at, id)` lex-comparison matches the watermark cursor
+      // ordering so ties on `created_at` are broken deterministically.
+      turnIndex: sql<number>`(
+        SELECT COUNT(*) FROM messages AS m2
+        WHERE m2.conversation_id = ${messages.conversationId}
+          AND m2.role = 'user'
+          AND ${realUserTurnContentFilter("m2")}
+          AND (m2.created_at < ${messages.createdAt}
+               OR (m2.created_at = ${messages.createdAt}
+                   AND m2.id <= ${messages.id}))
+      )`.as("turn_index"),
+      // Client attribution: extract from `messages.metadata` JSON.
+      // `userMessageInterface` and `userMessageChannel` are stamped on
+      // insert by every `persistUserMessage` path that flows through
+      // `TurnChannelContext`. `client` is the flexible namespace for
+      // browser/os/version metadata attached by HTTP header middleware.
+      // `json_extract` returns SQL NULL when the JSON path is absent —
+      // exactly the null semantics we want on the wire.
+      interfaceId: sql<
+        string | null
+      >`json_extract(${messages.metadata}, '$.userMessageInterface')`.as(
+        "interface_id",
+      ),
+      channelId: sql<
+        string | null
+      >`json_extract(${messages.metadata}, '$.userMessageChannel')`.as(
+        "channel_id",
+      ),
+      clientMetadata: sql<
+        string | null
+      >`json_extract(${messages.metadata}, '$.client')`.as("client_metadata"),
+    })
     .from(messages)
+    .innerJoin(conversations, eq(messages.conversationId, conversations.id))
     .where(
       and(
         eq(messages.role, "user"),

package/src/memory/v2/__tests__/__snapshots__/prompts-router.test.ts.snap

@@ -0,0 +1,27 @@
+// Bun Snapshot v1, https://bun.sh/docs/test/snapshots
+
+exports[`renderRouterPrompt — determinism & snapshot stability snapshot of fixed inputs 1`] = `
+"You are a background helper for Aria. Your job is to route memory pages for the next assistant turn between Aria and Alice.
+
+You will be shown the recent conversation, a \`<now>\` marker for the current time, an \`<already_injected_ids>\` block listing pages picked on the previous turn, and a \`# Concept Page Index\` listing every routable page on this workspace.
+
+Pick the concept pages whose contents would help Aria respond well on this turn. Lean toward inclusion when in doubt — missing a relevant page is a worse error than surfacing a few unused ones, because the assistant can ignore extras but can't summon context that wasn't loaded. Abstain (return an empty list) only when nothing in the index plausibly bears on the turn.
+
+Index format. Each line of the index has the shape:
+
+    [id] slug — summary (edges: a, b, c)
+
+\`id\` is a small integer used to refer to this page. \`edges\` are numeric IDs into the same list, pointing at related pages; you may follow them when one page strongly implies another.
+
+Already-injected pages. Pages whose IDs appear in \`<already_injected_ids>\` were picked on the previous turn. Do not pick them again unless Aria should re-anchor on that material — e.g., the topic genuinely returns after drifting away. Routine continuity does not require re-picking; the prior turn's pages are already in the assistant's working context.
+
+Time. Bias toward pages that match the current state implied by \`<now>\` and the active conversational threads (what is happening today, what was just decided, who is being discussed). Stale pages with no bearing on the live conversation should be skipped even if their summaries look superficially relevant.
+
+Emit your selection by calling \`select_pages_to_inject\` with the chosen \`page_ids\`. Return an empty array to abstain.
+
+# Concept Page Index
+
+[1] morning-routine — coffee, walk, journal (edges: 2)
+[2] journal-style — terse, dated, no fluff (edges: 1)
+[3] taxes-2025 — Q1 estimate due April 15 (edges: )"
+`;

package/src/memory/v2/__tests__/activation-store.test.ts

@@ -8,7 +8,7 @@ import { migrateActivationState } from "../../migrations/232-activation-state.js
 import * as schema from "../../schema.js";
 import {
   evictCompactedTurns,
-  fork,
+  forkActivationState,
   hydrate,
   save,
 } from "../activation-store.js";
@@ -113,12 +113,12 @@ describe("activation-store", () => {
     });
   });
 
-  describe("fork", () => {
+  describe("forkActivationState", () => {
     test("copies parent state to a new conversation id", async () => {
       const parentState = buildState();
       await save(db, "conv-parent", parentState);
 
-      await fork(db, "conv-parent", "conv-child");
+      forkActivationState(db, "conv-parent", "conv-child");
 
       const child = await hydrate(db, "conv-child");
       expect(child).toEqual(parentState);
@@ -129,7 +129,7 @@ describe("activation-store", () => {
     });
 
     test("is a no-op when the parent has no state", async () => {
-      await fork(db, "conv-parent-missing", "conv-child");
+      forkActivationState(db, "conv-parent-missing", "conv-child");
 
       expect(await hydrate(db, "conv-child")).toBeNull();
     });
@@ -139,7 +139,7 @@ describe("activation-store", () => {
       await save(db, "conv-parent", parentState);
       await save(db, "conv-child", buildState({ currentTurn: 99 }));
 
-      await fork(db, "conv-parent", "conv-child");
+      forkActivationState(db, "conv-parent", "conv-child");
 
       const child = await hydrate(db, "conv-child");
       expect(child?.currentTurn).toBe(7);

package/src/memory/v2/__tests__/activation.test.ts

@@ -52,9 +52,7 @@ mock.module("../../qdrant-client.js", () => ({
 
 const state = {
   embedCalls: [] as Array<{ inputs: unknown[] }>,
-  sparseCalls: [] as string[],
   embedReturn: [[0.1, 0.2, 0.3]] as number[][],
-  sparseReturn: { indices: [1, 2, 3], values: [0.5, 0.5, 0.5] },
   /**
    * Programmable Qdrant query response queues — one per channel. Each test
    * stages whatever ordered hits it needs and lets `simBatch` /
@@ -87,10 +85,6 @@ mock.module("../../embedding-backend.js", () => ({
       vectors: state.embedReturn,
     };
   },
-  generateSparseEmbedding: (text: string) => {
-    state.sparseCalls.push(text);
-    return state.sparseReturn;
-  },
 }));
 
 class MockQdrantClient {
@@ -176,9 +170,7 @@ const { _resetMemoryV2QdrantForTests } = await import("../qdrant.js");
 
 function resetState(): void {
   state.embedCalls.length = 0;
-  state.sparseCalls.length = 0;
   state.embedReturn = [[0.1, 0.2, 0.3]];
-  state.sparseReturn = { indices: [1, 2, 3], values: [0.5, 0.5, 0.5] };
   state.queryResponses.dense.length = 0;
   state.queryResponses.sparse.length = 0;
   state.queryCalls.length = 0;
@@ -470,7 +462,9 @@ describe("computeOwnActivation", () => {
         c_now: 0.2,
       }),
     });
-    // Expected: 0.3*0.6 + 0.3*0.5 + 0.2*0.4 + 0.2*0.2 = 0.18+0.15+0.08+0.04 = 0.45
+    // Positive cosines pass through unchanged (negatives clamp to 0).
+    // Expected: 0.3*0.6 + 0.3*0.5 + 0.2*0.4 + 0.2*0.2
+    //         = 0.18 + 0.15 + 0.08 + 0.04 = 0.45
     expect(out.activation.get("alice")).toBeCloseTo(0.45, 6);
   });
 
@@ -523,6 +517,7 @@ describe("computeOwnActivation", () => {
         c_now: 0.2,
       }),
     });
+    // Positive cosines pass through unchanged: 1.0 stays 1.0; 0 stays 0.
     // 0.3*0 + 0.3*1 + 0.2*0 + 0.2*0 = 0.3
     expect(out.activation.get("fresh")).toBeCloseTo(0.3, 6);
   });
@@ -573,7 +568,8 @@ describe("computeOwnActivation", () => {
     expect(breakdown).toBeDefined();
     // priorContribution is `d * prev`, not the weighted sim term.
     expect(breakdown?.priorContribution).toBeCloseTo(d * 0.6, 6);
-    // Raw sims, captured before c_user / c_assistant / c_now weighting.
+    // Fused sims (positive cosines pass through unchanged), captured before
+    // c_user / c_assistant / c_now weighting.
     expect(breakdown?.simUser).toBeCloseTo(0.5, 6);
     expect(breakdown?.simAssistant).toBeCloseTo(0.4, 6);
     expect(breakdown?.simNow).toBeCloseTo(0.2, 6);
@@ -801,8 +797,10 @@ describe("computeOwnActivation", () => {
   });
 
   test("rerank boost is additive on A_o and leaves raw simUser / simAssistant untouched", async () => {
-    stageHybridResponse([{ slug: "a", denseScore: 0.5 }]); // user
-    stageHybridResponse([{ slug: "a", denseScore: 0.4 }]); // assistant
+    // Positive cosines pass through unchanged; pick 0.5 / 0.4 directly so
+    // the round fused values used in the formula assertion below hold.
+    stageHybridResponse([{ slug: "a", denseScore: 0.5 }]); // user — fused 0.5
+    stageHybridResponse([{ slug: "a", denseScore: 0.4 }]); // assistant — fused 0.4
     stageHybridResponse([]); // now
     rerankState.scores = new Map([["a", 0.8]]);
 
@@ -1219,6 +1217,7 @@ describe("skills participate in the unified pipeline", () => {
     });
 
     // No prior state → priorContribution = 0.
+    // Positive cosines pass through unchanged.
     // 0.3*0.5 + 0.2*0.4 + 0.2*0.2 = 0.15 + 0.08 + 0.04 = 0.27
     expect(out.activation.get("skills/example-skill-a")).toBeCloseTo(0.27, 6);
     expect(out.breakdown.get("skills/example-skill-a")?.priorContribution).toBe(