@vellumai/assistant 0.11.3 → 0.11.4-dev.202608190019.b94dbf2

package/src/persistence/conversation-queries.ts

@@ -1,4 +1,14 @@
-import { and, count, desc, eq, inArray, isNull, lt, sql } from "drizzle-orm";
+import {
+  and,
+  count,
+  desc,
+  eq,
+  getTableColumns,
+  isNull,
+  lt,
+  sql,
+} from "drizzle-orm";
+import type { SQLiteSelect } from "drizzle-orm/sqlite-core";
 
 import {
   parseExternalContentEnvelope,
@@ -26,11 +36,14 @@ import {
   parseContentRef,
   resolveMessageContentBlocks,
 } from "./message-content-file.js";
+import {
+  countMessagesByRoleForConversations,
+  latestUserMessageRawContent,
+} from "./message-reads.js";
 import { rawAll } from "./raw-query.js";
 import {
   conversationAssistantAttentionState,
   conversations,
-  messages,
 } from "./schema/index.js";
 
 const log = getLogger("conversation-store");
@@ -186,7 +199,8 @@ function surfacedVisibilitySql(alias = "conversations"): string {
  * explicitly surfaced. {@link standardListingVisibilitySql} already admits
  * such rows through its surfaced and custom-group arms, so a caller that
  * must distinguish "visible in the listing" from "user-facing foreground"
- * (unread counting) applies this on top.
+ * (unread counting, the section index, a `foregroundOnly` list read) applies
+ * this on top.
  *
  * The SQL twin of `isBackgroundConversation` in the web client's
  * `utils/conversation-predicates.ts`.
@@ -260,6 +274,25 @@ export interface ConversationListFilter {
    * section, and a UUID one custom group. Omit to span every group.
    */
   groupId?: string;
+  /**
+   * Restrict to conversations whose latest assistant message the user has
+   * not seen: the same "unseen" predicate the unread count and the section
+   * index apply ({@link unseenAttentionStateConditions}), so every reader
+   * of "needs attention" agrees. Only `true` narrows; omit to span every
+   * conversation. A client that keeps no complete list can ask for exactly
+   * the rows its attention surfaces need instead of scanning for them.
+   */
+  needsAttention?: true;
+  /**
+   * Restrict to user-facing foreground conversations: drop the automated
+   * background/scheduled rows the standard listing admits through its
+   * custom-group arm ({@link notBackgroundVisibilitySql}, the same predicate
+   * the unread count and the section index apply on top of visibility).
+   * Only `true` narrows; omit to keep every visible row. A client that
+   * needs the newest conversation a user can open can ask for it in one
+   * row instead of paging past runs it would skip.
+   */
+  foregroundOnly?: true;
 }
 
 export interface ConversationListQuery extends ConversationListFilter {
@@ -324,21 +357,57 @@ function originChannelClause(originChannel: string) {
   return eq(conversations.originChannel, originChannel);
 }
 
+/**
+ * The join between a conversation and its attention projection, one row per
+ * conversation in `conversation_assistant_attention_state`. Every reader of
+ * "unseen" joins on this: the list and count filters, the unread count, and
+ * the section index.
+ */
+const attentionJoin = eq(
+  conversationAssistantAttentionState.conversationId,
+  conversations.id,
+);
+
 function conversationListWhere(filter: ConversationListFilter) {
   const {
     conversationType = "standard",
     archiveStatus = "active",
     originChannel,
     groupId,
+    needsAttention,
+    foregroundOnly,
   } = filter;
   return and(
     conversationTypeClause(conversationType),
     archiveStatusClause(archiveStatus) ?? undefined,
     originChannel ? originChannelClause(originChannel) : undefined,
     groupId ? groupIdClause(groupId) : undefined,
+    ...(needsAttention ? unseenAttentionStateConditions() : []),
+    foregroundOnly ? sql.raw(notBackgroundVisibilitySql()) : undefined,
   );
 }
 
+/**
+ * The joins {@link conversationListWhere}'s predicates need, applied to a
+ * list or count query over `conversations`. The attention predicate lives on
+ * `conversation_assistant_attention_state`, so `needsAttention` joins it.
+ * Inner, not left: a conversation with no
+ * attention row has no unseen message by definition. Joined only when the
+ * filter asks, so a query without the filter runs without the join and
+ * lists every conversation the predicates admit. One place for both
+ * {@link listConversations} and {@link countConversations}, so a page and
+ * its total always describe the same set.
+ */
+function withListFilterJoins<T extends SQLiteSelect>(
+  query: T,
+  filter: ConversationListFilter,
+) {
+  if (!filter.needsAttention) {
+    return query;
+  }
+  return query.innerJoin(conversationAssistantAttentionState, attentionJoin);
+}
+
 export function listConversations(
   query: ConversationListQuery = {},
 ): ConversationRow[] {
@@ -359,9 +428,13 @@ export function listConversations(
   // distinguish a total order from a lucky one. The guarantee becomes
   // observable, and gets its test, with the keyset pagination work.
   const tiebreak = desc(conversations.id);
-  return db
-    .select()
-    .from(conversations)
+  // Selecting the conversation columns explicitly keeps the row shape
+  // `parseConversation` maps identical whether or not the attention join is
+  // present.
+  return withListFilterJoins(
+    db.select(getTableColumns(conversations)).from(conversations).$dynamic(),
+    filter,
+  )
     .where(conversationListWhere(filter))
     .orderBy(recency, tiebreak)
     .limit(limit ?? 100)
@@ -438,31 +511,7 @@ export function getMessageRoleStatsByConversation(
   conversationIds: string[],
   role: string = "assistant",
 ): Map<string, { count: number; lastAt: number }> {
-  if (conversationIds.length === 0) {
-    return new Map();
-  }
-  const db = getDb();
-  const rows = db
-    .select({
-      conversationId: messages.conversationId,
-      count: sql<number>`COUNT(*)`.as("count"),
-      lastAt: sql<number>`MAX(${messages.createdAt})`.as("last_at"),
-    })
-    .from(messages)
-    .where(
-      and(
-        inArray(messages.conversationId, conversationIds),
-        eq(messages.role, role),
-      ),
-    )
-    .groupBy(messages.conversationId)
-    .all();
-  return new Map(
-    rows.map((r) => [
-      r.conversationId,
-      { count: Number(r.count), lastAt: Number(r.lastAt) },
-    ]),
-  );
+  return countMessagesByRoleForConversations(conversationIds, role);
 }
 
 export function listPinnedConversations(
@@ -535,6 +584,8 @@ export function listConversationsByTitlePrefix(
   const rows = rawAll<Row>(
     "conversation:listByTitlePrefix",
     `SELECT c.id, c.title,
+            -- Any-state count (streaming rows included): a list-surface size
+            -- hint where a transient +1 during a live turn is immaterial.
             (SELECT COUNT(*) FROM messages WHERE conversation_id = c.id) AS message_count,
             c.created_at
      FROM conversations c
@@ -561,9 +612,10 @@ export function countConversations(
 ): number {
   ensureGroupMigration();
   const db = getDb();
-  const [{ total }] = db
-    .select({ total: count() })
-    .from(conversations)
+  const [{ total }] = withListFilterJoins(
+    db.select({ total: count() }).from(conversations).$dynamic(),
+    filter,
+  )
     .where(conversationListWhere(filter))
     .all();
   return total;
@@ -601,10 +653,7 @@ export function countUnreadConversations(): number {
   const [{ total }] = db
     .select({ total: count() })
     .from(conversations)
-    .innerJoin(
-      conversationAssistantAttentionState,
-      eq(conversationAssistantAttentionState.conversationId, conversations.id),
-    )
+    .innerJoin(conversationAssistantAttentionState, attentionJoin)
     .where(
       and(
         conversationTypeClause("standard"),
@@ -617,32 +666,109 @@ export function countUnreadConversations(): number {
   return total;
 }
 
+/** Totals for one sidebar section: every member, and the unread among them. */
+export interface ConversationSectionCount {
+  total: number;
+  unread: number;
+}
+
 /**
- * Check whether the last user message in a conversation is a tool_result-only
- * message (i.e., not a real user-typed message). This is used by undo() to
- * determine if additional exchanges need to be deleted from the DB.
+ * Row counts for every non-empty sidebar section, with no rows fetched.
+ *
+ * Two axes, disjoint by construction because `group_id` is single-valued:
+ *
+ * - `groups`: rows filed somewhere, bucketed by `group_id`. Contains
+ *   `'system:pinned'` and custom-group ids; the background/scheduled system
+ *   buckets are excluded because no section renders them.
+ * - `channels`: rows filed nowhere, bucketed by effective origin channel,
+ *   where NULL reads as {@link NATIVE_ORIGIN_CHANNEL} exactly as
+ *   {@link originChannelClause} reads it. The native bucket is the Chats
+ *   section; each other bucket is that channel's section.
+ *
+ * Only buckets with at least one row appear: GROUP BY cannot emit an empty
+ * bucket, and an empty section renders no card.
+ *
+ * `total` follows the standard-listing visibility
+ * ({@link conversationTypeClause}), active rows only, so it counts exactly
+ * the rows `GET /v1/conversations` would return for that section's filter.
+ * `unread` applies the same rules as {@link countUnreadConversations} on top
+ * (not an unsurfaced background/scheduled row, unseen per the attention
+ * projection), composed from the same predicates rather than restated, so
+ * the per-section numbers always sum against the global count.
  */
-export function isLastUserMessageToolResult(conversationId: string): boolean {
+export interface ConversationSectionCounts {
+  groups: Array<{ groupId: string } & ConversationSectionCount>;
+  channels: Array<{ channel: string } & ConversationSectionCount>;
+}
+
+export function countConversationSections(): ConversationSectionCounts {
+  ensureGroupMigration();
   const db = getDb();
-  const lastUserMsg = db
-    .select({ content: messages.content })
-    .from(messages)
+
+  /* `countUnreadConversations` gets the same effect with an INNER JOIN and
+     the unseen conditions in its WHERE; here every row must be counted and
+     only unread rows summed, so the join is LEFT and the conditions move
+     into the CASE. A row with no attention projection has NULL columns,
+     fails the conditions, and sums 0: identical membership. */
+  const unread = sql<number>`SUM(CASE WHEN ${and(
+    sql.raw(notBackgroundVisibilitySql()),
+    ...unseenAttentionStateConditions(),
+  )} THEN 1 ELSE 0 END)`;
+
+  const groups = db
+    .select({
+      groupId: sql<string>`group_id`,
+      total: count(),
+      unread,
+    })
+    .from(conversations)
+    .leftJoin(conversationAssistantAttentionState, attentionJoin)
     .where(
       and(
-        eq(messages.conversationId, conversationId),
-        eq(messages.role, "user"),
+        conversationTypeClause("standard"),
+        isNull(conversations.archivedAt),
+        sql`(group_id = ${PINNED_GROUP_ID} OR (group_id IS NOT NULL AND group_id NOT LIKE 'system:%'))`,
       ),
     )
-    .orderBy(sql`rowid DESC`)
-    .limit(1)
-    .get();
+    .groupBy(sql`group_id`)
+    .all();
+
+  const effectiveChannel = sql<string>`COALESCE(${conversations.originChannel}, ${NATIVE_ORIGIN_CHANNEL})`;
+  const channels = db
+    .select({
+      channel: effectiveChannel,
+      total: count(),
+      unread,
+    })
+    .from(conversations)
+    .leftJoin(conversationAssistantAttentionState, attentionJoin)
+    .where(
+      and(
+        conversationTypeClause("standard"),
+        isNull(conversations.archivedAt),
+        groupIdClause(UNGROUPED_GROUP_ID),
+      ),
+    )
+    .groupBy(effectiveChannel)
+    .all();
+
+  return { groups, channels };
+}
+
+/**
+ * Check whether the last user message in a conversation is a tool_result-only
+ * message (i.e., not a real user-typed message). This is used by undo() to
+ * determine if additional exchanges need to be deleted from the DB.
+ */
+export function isLastUserMessageToolResult(conversationId: string): boolean {
+  const lastUserContent = latestUserMessageRawContent(conversationId);
 
-  if (!lastUserMsg) {
+  if (lastUserContent === null) {
     return false;
   }
 
   try {
-    const parsed = JSON.parse(lastUserMsg.content);
+    const parsed = JSON.parse(lastUserContent);
     if (
       Array.isArray(parsed) &&
       parsed.length > 0 &&
@@ -816,6 +942,10 @@ export async function searchConversations(
       if (!opts?.includeArchived) {
         whereClauses.push(`c.archived_at IS NULL`);
       }
+      // No completeness predicate: candidate ids come from the lexical
+      // index, which filters finalized = 1 at index time, so an unfinalized
+      // row cannot be a candidate. Do not copy this shape for id sets with
+      // different provenance.
       const visibleRows = rawAll<CandidateRow>(
         "conversation:searchConversations:visibleCandidates",
         `

package/src/persistence/delivery-crud.ts

@@ -5,11 +5,11 @@
  * finding messages by source identifiers, and managing raw payload storage.
  */
 
-import { and, desc, eq, isNotNull, ne, or, sql } from "drizzle-orm";
+import { and, desc, eq, isNotNull, like, ne, or, sql } from "drizzle-orm";
 import { v4 as uuid } from "uuid";
 
+import type { ChannelId } from "../channels/types.js";
 import { readSlackMetadataFromMessageMetadata } from "../messaging/providers/slack/message-metadata.js";
-import { DAEMON_INTERNAL_ASSISTANT_ID } from "../runtime/assistant-scope.js";
 import type { SlackInboundMessageMetadata } from "../runtime/http-types.js";
 import { parseJsonSafe } from "../util/json.js";
 import { isPlainObject } from "../util/object.js";
@@ -19,8 +19,14 @@ import {
   getOrCreateConversation,
   setConversationKeyIfAbsent,
 } from "./conversation-key-store.js";
+import type { NonScheduledConversationType } from "./conversation-types.js";
 import { getDb } from "./db-connection.js";
-import { channelInboundEvents, conversations } from "./schema.js";
+import {
+  channelInboundEvents,
+  conversationKeys,
+  conversations,
+  messages,
+} from "./schema.js";
 
 export interface InboundResult {
   accepted: boolean;
@@ -31,13 +37,27 @@ export interface InboundResult {
 
 export interface RecordInboundOptions {
   sourceMessageId?: string;
-  assistantId?: string;
   sourceThreadId?: string;
+  /**
+   * Record the event against this conversation instead of resolving one from
+   * the address. For events that belong to a message rather than to a chat: a
+   * reaction lives in the conversation of the message it was attached to, and
+   * resolving from its own address would mint a second one.
+   */
+  conversationId?: string;
 }
 
 const SLACK_LEGACY_THREAD_EVIDENCE_BATCH_SIZE = 50;
 const SLACK_LEGACY_THREAD_EVIDENCE_MAX_SCAN = 500;
 
+/**
+ * Rows examined when locating a Slack message the assistant itself posted.
+ * Bounded on purpose: the scan runs on the inbound path while the gateway
+ * waits for its ack, and reactions land on recent messages, so a cap costs
+ * almost no recall and keeps the cost flat as the database grows.
+ */
+const SLACK_OUTBOUND_TS_MAX_SCAN = 400;
+
 /**
  * Channels where an inbound thread id scopes the conversation: a Slack thread
  * or a Telegram private-chat topic each maps to its own conversation. A
@@ -45,30 +65,25 @@ const SLACK_LEGACY_THREAD_EVIDENCE_MAX_SCAN = 500;
  */
 const THREAD_SCOPED_CHANNELS = new Set(["slack", "telegram"]);
 
-function buildScopedConversationKeyForAssistant(
-  assistantId: string,
+/**
+ * Scope prefix on every conversation key.
+ *
+ * Part of the stored key format, so it is fixed: rows are written under
+ * `asst:self:` and stop resolving if it changes. The daemon is single-tenant
+ * and scopes all its storage to `self` (`DAEMON_INTERNAL_ASSISTANT_ID`).
+ */
+const CONVERSATION_KEY_SCOPE = "asst:self";
+
+export function buildScopedConversationKey(
   sourceChannel: string,
   externalChatId: string,
   sourceThreadId?: string | null,
 ): string {
   const threadId = sourceThreadId?.trim();
   if (THREAD_SCOPED_CHANNELS.has(sourceChannel) && threadId) {
-    return `asst:${assistantId}:${sourceChannel}:${externalChatId}:thread:${threadId}`;
+    return `${CONVERSATION_KEY_SCOPE}:${sourceChannel}:${externalChatId}:thread:${threadId}`;
   }
-  return `asst:${assistantId}:${sourceChannel}:${externalChatId}`;
-}
-
-export function buildScopedConversationKey(
-  sourceChannel: string,
-  externalChatId: string,
-  sourceThreadId?: string | null,
-): string {
-  return buildScopedConversationKeyForAssistant(
-    DAEMON_INTERNAL_ASSISTANT_ID,
-    sourceChannel,
-    externalChatId,
-    sourceThreadId,
-  );
+  return `${CONVERSATION_KEY_SCOPE}:${sourceChannel}:${externalChatId}`;
 }
 
 function legacySlackConversationHasThreadEvidence(
@@ -135,14 +150,24 @@ function legacySlackConversationHasThreadEvidence(
   return false;
 }
 
-function resolveInboundConversation(
-  assistantId: string,
+/**
+ * Applied only where this call is what creates the conversation. An address
+ * that already resolves is an existing conversation, and neither its type nor
+ * where it came from is the current message's to restate.
+ */
+export interface ResolveInboundConversationOptions {
+  conversationType?: NonScheduledConversationType;
+  /** Channel this conversation originates on, for first-message attribution. */
+  origin?: ChannelId;
+}
+
+export function resolveInboundConversation(
   sourceChannel: string,
   externalChatId: string,
   sourceThreadId?: string | null,
+  opts?: ResolveInboundConversationOptions,
 ): { conversationId: string } {
-  const threadedKey = buildScopedConversationKeyForAssistant(
-    assistantId,
+  const threadedKey = buildScopedConversationKey(
     sourceChannel,
     externalChatId,
     sourceThreadId,
@@ -155,7 +180,7 @@ function resolveInboundConversation(
   // other thread-scoped channel (Telegram topics) has no flat-key aliasing —
   // a thread id always resolves the threaded key directly.
   if (sourceChannel !== "slack" || !threadId) {
-    return getOrCreateConversation(threadedKey);
+    return getOrCreateConversation(threadedKey, opts);
   }
 
   const threadedMapping = getConversationByKey(threadedKey);
@@ -163,8 +188,7 @@ function resolveInboundConversation(
     return { conversationId: threadedMapping.conversationId };
   }
 
-  const legacyKey = buildScopedConversationKeyForAssistant(
-    assistantId,
+  const legacyKey = buildScopedConversationKey(
     sourceChannel,
     externalChatId,
     null,
@@ -185,7 +209,7 @@ function resolveInboundConversation(
     }
   }
 
-  return getOrCreateConversation(threadedKey);
+  return getOrCreateConversation(threadedKey, opts);
 }
 
 /**
@@ -236,7 +260,97 @@ export function findInboundConversationId(
 /**
  * Record an inbound channel event. Returns `duplicate: true` if this
  * exact (channel, chat, message) combination was already seen.
+ *
+ * The dedup half of this is also implemented gateway-side, on the same
+ * triple, in `gateway/src/db/inbound-dedup-store.ts`. Deliberately, not by
+ * accident: the gateway claims a delivery before handing it over, so a
+ * vendor's retry costs no crossing, and this stays as the record that binds
+ * the conversation and tracks the reply. Both must key on the same three
+ * fields for either to mean anything.
  */
+/**
+ * The inbound event already recorded for this address, if any. Read-only twin
+ * of the dedup check inside {@link recordInbound}, for callers that must know
+ * an event is a redelivery before doing work that recording would otherwise
+ * gate (routing a guardian decision, for one).
+ */
+export function findInboundEvent(
+  sourceChannel: string,
+  externalChatId: string,
+  externalMessageId: string,
+): { eventId: string; conversationId: string } | null {
+  const row = getDb()
+    .select({
+      id: channelInboundEvents.id,
+      conversationId: channelInboundEvents.conversationId,
+    })
+    .from(channelInboundEvents)
+    .where(
+      and(
+        eq(channelInboundEvents.sourceChannel, sourceChannel),
+        eq(channelInboundEvents.externalChatId, externalChatId),
+        eq(channelInboundEvents.externalMessageId, externalMessageId),
+      ),
+    )
+    .get();
+  return row ? { eventId: row.id, conversationId: row.conversationId } : null;
+}
+
+/**
+ * The conversation holding the Slack message with this `ts`, found by reading
+ * the `slackMeta` the assistant's own posts carry.
+ *
+ * `findMessageBySourceId` covers every message that arrived as an inbound
+ * event. It cannot see what the assistant posted, because an outbound reply
+ * opens no inbound event, so a reaction on the assistant's own message needs
+ * this. The search is confined to conversations already bound to the same
+ * Slack channel and to the most recent {@link SLACK_OUTBOUND_TS_MAX_SCAN}
+ * rows among them; beyond that it gives up and the caller drops the
+ * annotation, which is the same outcome as never finding it at all.
+ */
+export function findSlackConversationByMessageTs(
+  externalChatId: string,
+  channelTs: string,
+): string | null {
+  const db = getDb();
+  const keyPrefix = `${CONVERSATION_KEY_SCOPE}:slack:${externalChatId}`;
+  const rows = db
+    .select({
+      conversationId: messages.conversationId,
+      metadata: messages.metadata,
+    })
+    .from(messages)
+    .innerJoin(
+      conversationKeys,
+      eq(conversationKeys.conversationId, messages.conversationId),
+    )
+    .where(
+      and(
+        or(
+          eq(conversationKeys.conversationKey, keyPrefix),
+          like(conversationKeys.conversationKey, `${keyPrefix}:thread:%`),
+        ),
+        like(messages.metadata, '%"slackMeta"%'),
+      ),
+    )
+    .orderBy(desc(messages.createdAt))
+    .limit(SLACK_OUTBOUND_TS_MAX_SCAN)
+    .all();
+
+  for (const row of rows) {
+    const slackMeta = readSlackMetadataFromMessageMetadata(row.metadata, {
+      allowFlatLegacy: true,
+    });
+    if (
+      slackMeta?.channelId === externalChatId &&
+      slackMeta.channelTs === channelTs
+    ) {
+      return row.conversationId;
+    }
+  }
+  return null;
+}
+
 export function recordInbound(
   sourceChannel: string,
   externalChatId: string,
@@ -269,13 +383,13 @@ export function recordInbound(
     };
   }
 
-  const assistantId = options?.assistantId ?? DAEMON_INTERNAL_ASSISTANT_ID;
-  const mapping = resolveInboundConversation(
-    assistantId,
-    sourceChannel,
-    externalChatId,
-    options?.sourceThreadId,
-  );
+  const mapping = options?.conversationId
+    ? { conversationId: options.conversationId }
+    : resolveInboundConversation(
+        sourceChannel,
+        externalChatId,
+        options?.sourceThreadId,
+      );
   const now = Date.now();
   const eventId = uuid();