@vellumai/assistant 0.11.2 → 0.11.3-staging.2

package/src/persistence/conversation-queries.ts

@@ -8,12 +8,18 @@ import {
 } from "../security/untrusted-content.js";
 import { getLogger } from "../util/logger.js";
 import { isLexicalBackfillComplete } from "./checkpoints.js";
+import { unseenAttentionStateConditions } from "./conversation-attention-store.js";
 import type { ConversationRow } from "./conversation-crud.js";
 import { parseConversation } from "./conversation-crud.js";
 import { ensureDisplayOrderMigration } from "./conversation-display-order-migration.js";
 import { ensureGroupMigration } from "./conversation-group-migration.js";
 import { searchMessageIdsLexical } from "./conversation-search-lexical.js";
-import type { ConversationType } from "./conversation-types.js";
+import {
+  type ConversationType,
+  NATIVE_ORIGIN_CHANNEL,
+  PINNED_GROUP_ID,
+  UNGROUPED_GROUP_ID,
+} from "./conversation-types.js";
 import { getDb } from "./db-connection.js";
 import { tokenize } from "./embeddings/sparse-tokenize.js";
 import {
@@ -21,7 +27,11 @@ import {
   resolveMessageContentBlocks,
 } from "./message-content-file.js";
 import { rawAll } from "./raw-query.js";
-import { conversations, messages } from "./schema/index.js";
+import {
+  conversationAssistantAttentionState,
+  conversations,
+  messages,
+} from "./schema/index.js";
 
 const log = getLogger("conversation-store");
 
@@ -169,6 +179,26 @@ function surfacedVisibilitySql(alias = "conversations"): string {
   );
 }
 
+/**
+ * Raw SQL predicate for "not an automated background/scheduled row".
+ *
+ * A background or scheduled row counts as foreground only once it has been
+ * explicitly surfaced. {@link standardListingVisibilitySql} already admits
+ * such rows through its surfaced and custom-group arms, so a caller that
+ * must distinguish "visible in the listing" from "user-facing foreground"
+ * (unread counting) applies this on top.
+ *
+ * The SQL twin of `isBackgroundConversation` in the web client's
+ * `utils/conversation-predicates.ts`.
+ */
+function notBackgroundVisibilitySql(alias = "conversations"): string {
+  return (
+    `NOT ((${alias}.conversation_type IN ('background', 'scheduled')` +
+    ` OR COALESCE(${alias}.group_id, 'system:all') IN ('system:background', 'system:scheduled'))` +
+    ` AND ${alias}.surfaced_at IS NULL)`
+  );
+}
+
 /**
  * SQL predicate selecting which bucket {@link listConversations} and
  * {@link countConversations} return, keyed by the canonical
@@ -211,34 +241,133 @@ function conversationTypeClause(type: ConversationType) {
   }
 }
 
+/**
+ * The predicates that select which conversations a listing covers, shared by
+ * {@link listConversations} and {@link countConversations} so a list and its
+ * count can never disagree about what they are describing.
+ *
+ * Named rather than positional on purpose: `originChannel` and `groupId` are
+ * both optional strings, and as adjacent positional arguments they were a
+ * transposition waiting to happen.
+ */
+export interface ConversationListFilter {
+  conversationType?: ConversationType;
+  archiveStatus?: ArchiveStatusFilter;
+  originChannel?: string;
+  /**
+   * Restrict to one group. {@link UNGROUPED_GROUP_ID} selects rows in no
+   * group (the sidebar's flat list), {@link PINNED_GROUP_ID} the Pinned
+   * section, and a UUID one custom group. Omit to span every group.
+   */
+  groupId?: string;
+}
+
+export interface ConversationListQuery extends ConversationListFilter {
+  limit?: number;
+  offset?: number;
+}
+
+/**
+ * SQL predicate for {@link ConversationListFilter.groupId}.
+ *
+ * The ungrouped case has to match NULL as well as the literal sentinel:
+ * `group_id` is only written when a conversation is filed somewhere, so the
+ * overwhelming majority of rows carry NULL rather than `'system:all'`.
+ *
+ * `group_id` is referenced as raw SQL because it is not a Drizzle column:
+ * it, `display_order`, and `is_pinned` are added by the lazy
+ * `ensureDisplayOrderMigration` / `ensureGroupMigration` steps rather than
+ * declared in `schema/conversations.ts`.
+ */
+function groupIdClause(groupId: string) {
+  if (groupId === UNGROUPED_GROUP_ID) {
+    // "Ungrouped" means every row the sidebar's flat list shows: not pinned
+    // and not filed in a custom group. It deliberately admits the system
+    // buckets, because a surfaced background or scheduled conversation keeps
+    // its `system:background` / `system:scheduled` group id (surfacing writes
+    // only `surfaced_at`) while the standard listing renders it in Recents.
+    // Whether such a row is actually visible stays with
+    // `conversationTypeClause`, which admits the surfaced ones and excludes
+    // the rest; matching on group id alone here would drop them.
+    return sql`(group_id IS NULL OR (group_id LIKE 'system:%' AND group_id != ${PINNED_GROUP_ID}))`;
+  }
+  // `group_id` is single-valued and authoritative, so a row belongs to
+  // exactly one section by construction. `is_pinned` is a derived duplicate
+  // that reads do not consult: `ensureGroupMigration` backfills
+  // `group_id = 'system:pinned'` for every legacy `is_pinned` row before any
+  // query runs, and pinning writes both together thereafter.
+  return sql`group_id = ${groupId}`;
+}
+
+/**
+ * SQL predicate for {@link ConversationListFilter.originChannel}.
+ *
+ * `'vellum'` additionally matches NULL. `origin_channel` is left NULL at
+ * insert precisely so an inbound message can still claim the conversation for
+ * its channel (`setConversationOriginChannelIfUnset` is guarded on `isNull`),
+ * and migration 288 settles whatever was never claimed to `'vellum'` at daemon
+ * startup. NULL is therefore "not yet attributed", and it is what the
+ * overwhelming majority of rows carry between one boot and the next.
+ *
+ * A strict equality would put those rows in no section at all: not native, not
+ * any channel. Reading NULL as native is the self-correcting error of the two,
+ * since the only rows it can misplace are ones whose inbound message has not
+ * arrived yet, and that message moves them the moment it does.
+ *
+ * Every other channel matches exactly. A row is claimed for a channel only by
+ * that channel, so there is no ambiguity to be tolerant about.
+ */
+function originChannelClause(originChannel: string) {
+  if (originChannel === NATIVE_ORIGIN_CHANNEL) {
+    return sql`(origin_channel IS NULL OR origin_channel = ${NATIVE_ORIGIN_CHANNEL})`;
+  }
+  return eq(conversations.originChannel, originChannel);
+}
+
+function conversationListWhere(filter: ConversationListFilter) {
+  const {
+    conversationType = "standard",
+    archiveStatus = "active",
+    originChannel,
+    groupId,
+  } = filter;
+  return and(
+    conversationTypeClause(conversationType),
+    archiveStatusClause(archiveStatus) ?? undefined,
+    originChannel ? originChannelClause(originChannel) : undefined,
+    groupId ? groupIdClause(groupId) : undefined,
+  );
+}
+
 export function listConversations(
-  limit?: number,
-  conversationType: ConversationType = "standard",
-  offset = 0,
-  archiveStatus: ArchiveStatusFilter = "active",
-  originChannel?: string,
+  query: ConversationListQuery = {},
 ): ConversationRow[] {
   ensureDisplayOrderMigration();
   ensureGroupMigration();
   const db = getDb();
-  const typeCond = conversationTypeClause(conversationType);
-  const archiveCond = archiveStatusClause(archiveStatus);
-  const channelCond = originChannel
-    ? eq(conversations.originChannel, originChannel)
-    : undefined;
-  const where = and(typeCond, archiveCond ?? undefined, channelCond);
-  const query = db
+  const { limit, offset = 0, ...filter } = query;
+  const recency = desc(
+    sql`COALESCE(${conversations.lastMessageAt}, ${conversations.updatedAt})`,
+  );
+  // `id` closes the ordering so the sort is a total order. Rows tied on every
+  // preceding key would otherwise have no defined relative order, which
+  // becomes duplicated and skipped rows once a cursor pages across a tie
+  // boundary.
+  //
+  // Deliberately untested here: SQLite returns a stable arrangement for
+  // identical queries over identical data, so no assertion at this layer can
+  // distinguish a total order from a lucky one. The guarantee becomes
+  // observable, and gets its test, with the keyset pagination work.
+  const tiebreak = desc(conversations.id);
+  return db
     .select()
     .from(conversations)
-    .where(where)
-    .orderBy(
-      desc(
-        sql`COALESCE(${conversations.lastMessageAt}, ${conversations.updatedAt})`,
-      ),
-    )
+    .where(conversationListWhere(filter))
+    .orderBy(recency, tiebreak)
     .limit(limit ?? 100)
-    .offset(offset);
-  return query.all().map(parseConversation);
+    .offset(offset)
+    .all()
+    .map(parseConversation);
 }
 
 /**
@@ -360,7 +489,6 @@ export function listPinnedConversations(
       ),
     )
     .orderBy(
-      sql`COALESCE(display_order, 999999) ASC`,
       desc(
         sql`COALESCE(${conversations.lastMessageAt}, ${conversations.updatedAt})`,
       ),
@@ -422,23 +550,69 @@ export function listConversationsByTitlePrefix(
   }));
 }
 
+/**
+ * How many conversations {@link listConversations} would return for the same
+ * filter, ignoring `limit` / `offset`. Both read through
+ * {@link conversationListWhere}, so a page and its total always describe the
+ * same set.
+ */
 export function countConversations(
-  conversationType: ConversationType = "standard",
-  archiveStatus: ArchiveStatusFilter = "active",
-  originChannel?: string,
+  filter: ConversationListFilter = {},
 ): number {
   ensureGroupMigration();
   const db = getDb();
-  const typeCond = conversationTypeClause(conversationType);
-  const archiveCond = archiveStatusClause(archiveStatus);
-  const channelCond = originChannel
-    ? eq(conversations.originChannel, originChannel)
-    : undefined;
-  const where = and(typeCond, archiveCond ?? undefined, channelCond);
   const [{ total }] = db
     .select({ total: count() })
     .from(conversations)
-    .where(where)
+    .where(conversationListWhere(filter))
+    .all();
+  return total;
+}
+
+/**
+ * Count of foreground conversations whose latest assistant message is unseen.
+ *
+ * The server-side twin of the web client's `contributesToUnreadCount`
+ * predicate (`clients/web/src/utils/conversation-predicates.ts`), so the
+ * count returned by `GET /v1/conversations/unread-count` always matches what
+ * the sidebar's unread indicators would sum to over the full list.
+ *
+ * **Two definitions of one rule.** Changing what counts here means changing
+ * the client predicate too. Both sides assert the same named scenario matrix
+ * ("unread-count contract") so a one-sided change is visible in review: see
+ * `__tests__/conversation-list-routes.test.ts` here and
+ * `conversation-predicates.test.ts` there. The duplication exists only to
+ * keep a fallback for assistants without this route, and goes away with it.
+ *
+ * The rules:
+ *
+ * - visible in the standard (Recents) listing ({@link conversationTypeClause}),
+ * - not archived,
+ * - not an unsurfaced background/scheduled row
+ *   ({@link notBackgroundVisibilitySql}), which excludes automated rows the
+ *   listing admits through its custom-group arm,
+ * - unseen per the attention projection
+ *   ({@link unseenAttentionStateConditions}); conversations without a
+ *   projection row read as seen, hence the inner join.
+ */
+export function countUnreadConversations(): number {
+  ensureGroupMigration();
+  const db = getDb();
+  const [{ total }] = db
+    .select({ total: count() })
+    .from(conversations)
+    .innerJoin(
+      conversationAssistantAttentionState,
+      eq(conversationAssistantAttentionState.conversationId, conversations.id),
+    )
+    .where(
+      and(
+        conversationTypeClause("standard"),
+        isNull(conversations.archivedAt),
+        sql.raw(notBackgroundVisibilitySql()),
+        ...unseenAttentionStateConditions(),
+      ),
+    )
     .all();
   return total;
 }

package/src/persistence/conversation-types.ts

@@ -7,7 +7,32 @@
 // `resolveConversationKind` classifier, and the pure predicates over a
 // persisted message's `metadata` record.
 
-import { parseChannelId } from "../channels/types.js";
+import {
+  type ChannelId,
+  parseChannelId,
+  parseClientOs,
+} from "../channels/types.js";
+
+/**
+ * Where a conversation came from, stated by whoever creates it.
+ *
+ * `conversations.origin_channel` records this. It is the caller's job to
+ * supply it, because the caller is the only party that knows: an inbound
+ * Slack message creates a conversation *because* it is Slack, and nothing
+ * downstream can recover that fact once the moment has passed.
+ *
+ * A bare {@link ChannelId} covers the cases where the origin is known
+ * outright, including `"vellum"` for a conversation started in the app.
+ * `inheritFrom` covers the derived ones, where the conversation belongs to
+ * whatever surface its parent belongs to: forks, subagent spawns, and
+ * scheduled wakes.
+ *
+ * Deliberately closed, and deliberately not optional. An optional string
+ * would let a caller decline to answer, which is how the column came to
+ * carry a third "not yet attributed" state that different readers resolve
+ * in contradictory directions. See JARVIS-1466.
+ */
+export type ConversationOrigin = ChannelId | { readonly inheritFrom: string };
 
 /**
  * Canonical `conversations.source` string for background memory v2
@@ -24,6 +49,15 @@ export type ConversationCreateType = "standard" | "background" | "scheduled";
 /** Read-side alias of {@link ConversationCreateType}. */
 export type ConversationType = ConversationCreateType;
 
+/**
+ * Conversation types minted outside the schedule pipeline. Scheduled rows are
+ * owned by it and created through `createConversation`.
+ */
+export type NonScheduledConversationType = Exclude<
+  ConversationCreateType,
+  "scheduled"
+>;
+
 /**
  * Conversation types created by background machinery (heartbeat runs,
  * scheduled runs, retrospective forks) rather than by a person. Exported as a
@@ -128,6 +162,29 @@ export function isEchoSuppressedUserMessage(
   );
 }
 
+/**
+ * True when a queued message has no client-visible queued row, so every event
+ * and snapshot that describes the queue to clients must skip it.
+ *
+ * Same set as {@link isEchoSuppressedUserMessage}: a row that never renders as
+ * a user bubble must not render as a queued one either. A subagent's
+ * completion notification injected into a busy parent (`subagent/notify.ts`)
+ * is the common case — the user sees that run through its inline card, and a
+ * `[Subagent "…" — …]` row in the queue drawer is daemon scaffolding leaking
+ * into the transcript's place.
+ *
+ * Named separately from the echo predicate for the queue sites that must
+ * agree: the `message_queued` ack and its visible-position count, the
+ * corrective `message_requeued` position, the terminal
+ * `message_queued_deleted` on user delete and on abort discard, and the
+ * list-messages queued snapshot a cold reload reads.
+ */
+export function isSuppressedQueuedMessage(
+  metadata: Record<string, unknown> | undefined,
+): boolean {
+  return isEchoSuppressedUserMessage(metadata);
+}
+
 /**
  * True when a role-`"user"` row was persisted by the voice bridge for a live
  * phone call or in-app voice session (every row `startVoiceTurn` persists, see
@@ -148,6 +205,34 @@ export function isVoiceSessionUserMessage(
   return metadata?.voiceSessionTurn === true;
 }
 
+/**
+ * True when the row that opened the turn was sent from the macOS app, on that
+ * row's own evidence.
+ *
+ * Two markers, both required. The `client` bag's `os` entry is the only
+ * per-platform attribution on a message: `userMessageInterface` is `"web"` for
+ * the macOS app, the iOS app, and a desktop browser alike. `clientOsFromRequest`
+ * says that `os` was reported by this row's request or transport rather than
+ * inherited from the conversation's live client state, which names the surface
+ * of an earlier turn: a button tapped on the phone against a conversation last
+ * sent to from the Mac persists `os: "macos"` with no marker.
+ *
+ * Origin a row did not report itself is origin unknown. Callers gate
+ * suppression on this, so unknown has to read as not-macOS.
+ */
+export function isMacOriginatedUserMessage(
+  metadata: Record<string, unknown> | undefined,
+): boolean {
+  if (metadata?.clientOsFromRequest !== true) {
+    return false;
+  }
+  const client = metadata.client;
+  if (typeof client !== "object" || client === null) {
+    return false;
+  }
+  return parseClientOs((client as Record<string, unknown>).os) === "macos";
+}
+
 /**
  * True when the row that opened the turn arrived over an external messaging
  * surface (Slack, Telegram, WhatsApp, email, a phone call) rather than the
@@ -220,3 +305,24 @@ export function isReplyPushIneligibleUserMessage(
     (!options?.replyDeliveredInAppOnly && isReplyDeliveredOffApp(metadata))
   );
 }
+
+/**
+ * The reserved `group_id` values the sidebar's system sections use.
+ *
+ * `group_id` is nullable, and a conversation that has never been filed
+ * carries NULL rather than {@link UNGROUPED_GROUP_ID}, so predicates over the
+ * column read it through `COALESCE(group_id, 'system:all')`.
+ */
+export const UNGROUPED_GROUP_ID = "system:all";
+export const PINNED_GROUP_ID = "system:pinned";
+
+/**
+ * The `origin_channel` value for a conversation started in Vellum itself
+ * rather than arriving from an external channel.
+ *
+ * Named because reads have to treat it as "vellum or not yet attributed":
+ * `origin_channel` is deliberately NULL at insert so an inbound message can
+ * claim the conversation for its own channel, and migration 288 settles the
+ * unclaimed ones to this value at daemon startup.
+ */
+export const NATIVE_ORIGIN_CHANNEL = "vellum";

package/src/persistence/jobs-store.ts

@@ -794,10 +794,19 @@ export function claimMemoryJobs(
 
 export function completeMemoryJob(id: string): void {
   const db = memoryDb();
+  // Guarded on `running` so a late return from an abandoned execution cannot
+  // overwrite a terminal state another writer already recorded (the stalled-job
+  // watchdog's `failed`, or a successor worker's re-claimed row).
   db.update(memoryJobs)
     .set({ status: "completed", updatedAt: Date.now(), lastError: null })
-    .where(eq(memoryJobs.id, id))
+    .where(and(eq(memoryJobs.id, id), eq(memoryJobs.status, "running")))
     .run();
+  if (rawMemoryChanges() === 0) {
+    log.warn(
+      { jobId: id },
+      "completeMemoryJob: row was not running; late completion suppressed",
+    );
+  }
 }
 
 /** Max times a job can be deferred before it is marked as failed. */
@@ -820,7 +829,17 @@ const DEFER_MAX_DELAY_MS = 5 * 60 * 1000;
  * Returns `'deferred'` if the job was put back, or `'failed'` if max deferrals
  * were exceeded and the job was marked as failed.
  */
-export function deferMemoryJob(id: string): "deferred" | "failed" {
+export function deferMemoryJob(
+  id: string,
+  options?: {
+    /**
+     * Terminal `last_error` recorded when the deferral budget is exhausted.
+     * Defaults to the backend-unavailable wording used by the worker's
+     * infrastructure deferrals.
+     */
+    exhaustedMessage?: string;
+  },
+): "deferred" | "failed" {
   const db = memoryDb();
   const row = db.select().from(memoryJobs).where(eq(memoryJobs.id, id)).get();
   if (!row) {
@@ -840,9 +859,11 @@ export function deferMemoryJob(id: string): "deferred" | "failed" {
         status: "failed",
         deferrals,
         updatedAt: now,
-        lastError: `Backend unavailable after ${deferrals} deferrals`,
+        lastError:
+          options?.exhaustedMessage ??
+          `Backend unavailable after ${deferrals} deferrals`,
       })
-      .where(eq(memoryJobs.id, id))
+      .where(and(eq(memoryJobs.id, id), eq(memoryJobs.status, "running")))
       .run();
     return "failed";
   }
@@ -868,7 +889,7 @@ export function deferMemoryJob(id: string): "deferred" | "failed" {
       runAfter: now + delay,
       updatedAt: now,
     })
-    .where(eq(memoryJobs.id, id))
+    .where(and(eq(memoryJobs.id, id), eq(memoryJobs.status, "running")))
     .run();
   return "deferred";
 }
@@ -886,7 +907,7 @@ export function rescheduleMemoryJob(id: string, delayMs: number): void {
   memoryDb()
     .update(memoryJobs)
     .set({ status: "pending", runAfter: now + delayMs, updatedAt: now })
-    .where(eq(memoryJobs.id, id))
+    .where(and(eq(memoryJobs.id, id), eq(memoryJobs.status, "running")))
     .run();
 }
 
@@ -904,6 +925,9 @@ export function failMemoryJob(
   }
   const attempts = row.attempts + 1;
   const now = Date.now();
+  // Both transitions are guarded on `running` for the same reason as
+  // `completeMemoryJob`: a late writer must not overwrite a terminal state
+  // recorded by the stalled-job watchdog or a successor worker.
   if (attempts >= maxAttempts) {
     db.update(memoryJobs)
       .set({
@@ -912,7 +936,7 @@ export function failMemoryJob(
         updatedAt: now,
         lastError: truncate(error, 2000, ""),
       })
-      .where(eq(memoryJobs.id, id))
+      .where(and(eq(memoryJobs.id, id), eq(memoryJobs.status, "running")))
       .run();
     return;
   }
@@ -924,7 +948,7 @@ export function failMemoryJob(
       updatedAt: now,
       lastError: truncate(error, 2000, ""),
     })
-    .where(eq(memoryJobs.id, id))
+    .where(and(eq(memoryJobs.id, id), eq(memoryJobs.status, "running")))
     .run();
 }
 

package/src/persistence/llm-usage-store.ts

@@ -258,6 +258,21 @@ export interface UnreportedUsageEvent extends UsageEvent {
    * Null when there's no parent conversation.
    */
   parentTurnIndex: number | null;
+  /**
+   * Role of the subagent that owns this LLM call's conversation, from
+   * `SUBAGENT_ROLE_REGISTRY`. Null for every LLM call that did not run inside
+   * a subagent conversation, and for subagent conversations created before
+   * migration 362 whose `subagents` row had already been disposed.
+   */
+  subagentRole: string | null;
+  /**
+   * How that subagent was spawned, one of the modes described on
+   * `SubagentSpawnMode` in `subagent/types.ts`. Orthogonal to `subagentRole`:
+   * the role selects the child's capabilities, the spawn mode selects its
+   * context inheritance and lifecycle. Null under the same conditions as
+   * `subagentRole`.
+   */
+  subagentSpawnMode: string | null;
 }
 
 export function queryUnreportedUsageEvents(
@@ -357,6 +372,11 @@ export function queryUnreportedUsageEvents(
       // the cutoff (child creation for subagent spawns, fork boundary
       // message for retrospective forks). Same eligibility filter as
       // `turnIndex` above.
+      // Stamped on the conversation row at spawn (migration 362) rather than
+      // joined from `subagents`, whose rows are deleted on dispose while this
+      // watermark query can trail far behind after an ingest outage.
+      subagentRole: conversations.subagentRole,
+      subagentSpawnMode: conversations.subagentSpawnMode,
       parentTurnIndex: sql<number | null>`(
         CASE WHEN ${parentIdSql} IS NULL THEN NULL
         ELSE (
@@ -399,6 +419,8 @@ export function queryUnreportedUsageEvents(
     parentConversationId: row.parentConversationId,
     parentTurnIndex:
       row.parentTurnIndex === null ? null : Number(row.parentTurnIndex),
+    subagentRole: row.subagentRole,
+    subagentSpawnMode: row.subagentSpawnMode,
   }));
 }