@vellumai/assistant 0.12.2-dev.202609181713.fb1ff61 → 0.12.2-dev.202609181913.1108ac3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/ARCHITECTURE.md +31 -22
  2. package/docs/mode-sessions.md +183 -0
  3. package/openapi.yaml +255 -0
  4. package/package.json +1 -1
  5. package/src/__tests__/annotate-activity-metadata.test.ts +2 -0
  6. package/src/__tests__/annotate-risk-options.test.ts +2 -0
  7. package/src/__tests__/answered-question-persistence.test.ts +2 -0
  8. package/src/__tests__/approval-routes-http.test.ts +3 -0
  9. package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
  10. package/src/__tests__/assistant-stream-state.test.ts +22 -0
  11. package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
  12. package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
  13. package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
  14. package/src/__tests__/conversation-agent-loop.test.ts +426 -14
  15. package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
  16. package/src/__tests__/conversation-evictor.test.ts +62 -37
  17. package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
  18. package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
  19. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
  20. package/src/__tests__/conversation-retry-route.test.ts +22 -0
  21. package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
  22. package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
  23. package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
  24. package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
  25. package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
  26. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
  27. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
  28. package/src/__tests__/conversation-undo.test.ts +61 -0
  29. package/src/__tests__/cu-unified-flow.test.ts +82 -1
  30. package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
  31. package/src/__tests__/events-tail-route.test.ts +25 -0
  32. package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
  33. package/src/__tests__/guardian-action-sweep.test.ts +16 -0
  34. package/src/__tests__/heartbeat-service.test.ts +2 -59
  35. package/src/__tests__/helpers/mock-conversation.ts +21 -0
  36. package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
  37. package/src/__tests__/host-cu-proxy.test.ts +46 -2
  38. package/src/__tests__/http-user-message-parity.test.ts +2 -0
  39. package/src/__tests__/mode-session-history.test.ts +245 -0
  40. package/src/__tests__/mode-session-recovery.test.ts +222 -0
  41. package/src/__tests__/mode-session-wire.test.ts +141 -0
  42. package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
  43. package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
  44. package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
  45. package/src/__tests__/secret-ingress-http.test.ts +2 -0
  46. package/src/__tests__/send-endpoint-busy.test.ts +163 -1
  47. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
  48. package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
  49. package/src/__tests__/tool-result-metadata-plumbing.test.ts +2 -0
  50. package/src/__tests__/tool-start-timestamp.test.ts +2 -0
  51. package/src/api/events/assistant-turn-start.ts +3 -0
  52. package/src/api/events/generation-handoff.ts +2 -0
  53. package/src/api/events/message-complete.ts +2 -0
  54. package/src/api/events/notification-conversation-created.ts +6 -4
  55. package/src/api/events/notification-intent.ts +4 -2
  56. package/src/api/events/tool-result.ts +2 -0
  57. package/src/api/events/tool-use-start.ts +3 -0
  58. package/src/api/events/user-message-echo.ts +3 -0
  59. package/src/api/index.ts +18 -6
  60. package/src/api/mode-session.test.ts +131 -0
  61. package/src/api/mode-session.ts +148 -0
  62. package/src/api/responses/conversation-message.ts +6 -0
  63. package/src/browser/__tests__/operations.test.ts +29 -0
  64. package/src/browser/operations.ts +29 -0
  65. package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
  66. package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
  67. package/src/calls/__tests__/voice-session-bridge.test.ts +288 -234
  68. package/src/calls/call-controller.ts +19 -0
  69. package/src/calls/guardian-action-sweep.ts +20 -5
  70. package/src/calls/voice-escalation-judge.ts +254 -0
  71. package/src/calls/voice-escalation-target.ts +0 -1
  72. package/src/calls/voice-leg-coordinator.ts +121 -3
  73. package/src/calls/voice-session-bridge.ts +164 -118
  74. package/src/calls/voice-triage-escalate.ts +7 -0
  75. package/src/config/call-site-defaults.ts +5 -0
  76. package/src/config/feature-flag-registry.json +8 -0
  77. package/src/config/schemas/call-site-catalog.ts +7 -0
  78. package/src/config/schemas/channels.ts +0 -18
  79. package/src/config/schemas/llm.ts +1 -0
  80. package/src/config/session-groups-gate.ts +21 -0
  81. package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
  82. package/src/conversations/message-consolidation.ts +102 -5
  83. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
  84. package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
  85. package/src/daemon/browser-mode-session.test.ts +402 -0
  86. package/src/daemon/browser-mode-session.ts +186 -0
  87. package/src/daemon/computer-use-mode-session.test.ts +210 -0
  88. package/src/daemon/computer-use-mode-session.ts +105 -0
  89. package/src/daemon/conversation-agent-loop-handlers.ts +52 -4
  90. package/src/daemon/conversation-agent-loop.ts +170 -2
  91. package/src/daemon/conversation-evictor.ts +5 -24
  92. package/src/daemon/conversation-history.ts +10 -0
  93. package/src/daemon/conversation-messaging.ts +33 -0
  94. package/src/daemon/conversation-mode-session.test.ts +1667 -0
  95. package/src/daemon/conversation-mode-session.ts +1008 -0
  96. package/src/daemon/conversation-process.ts +5 -0
  97. package/src/daemon/conversation-store.ts +4 -4
  98. package/src/daemon/conversation-surfaces.ts +86 -4
  99. package/src/daemon/conversation.ts +50 -1
  100. package/src/daemon/host-cu-proxy.ts +21 -3
  101. package/src/daemon/lifecycle.ts +25 -8
  102. package/src/daemon/mode-session-startup-recovery.ts +23 -0
  103. package/src/daemon/mode-session-tracking.ts +54 -0
  104. package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
  105. package/src/heartbeat/heartbeat-service.ts +5 -33
  106. package/src/ipc/__tests__/browser-ipc.test.ts +210 -0
  107. package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
  108. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
  109. package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
  110. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +52 -1
  111. package/src/live-voice/__tests__/live-voice-vad.test.ts +89 -2
  112. package/src/live-voice/camera-mode-session.test.ts +265 -0
  113. package/src/live-voice/camera-mode-session.ts +277 -0
  114. package/src/live-voice/live-voice-manager.ts +26 -1
  115. package/src/live-voice/live-voice-photo.ts +88 -7
  116. package/src/live-voice/live-voice-session.ts +409 -94
  117. package/src/live-voice/protocol.ts +161 -2
  118. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
  119. package/src/messaging/providers/telegram-bot/send.test.ts +207 -1
  120. package/src/messaging/providers/whatsapp/send.test.ts +169 -4
  121. package/src/notifications/README.md +24 -8
  122. package/src/notifications/adapters/macos.ts +29 -38
  123. package/src/notifications/emit-signal.ts +17 -10
  124. package/src/persistence/conversation-crud.ts +27 -0
  125. package/src/persistence/conversation-mode-sessions.test.ts +424 -0
  126. package/src/persistence/conversation-mode-sessions.ts +460 -0
  127. package/src/persistence/message-metadata.ts +25 -0
  128. package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
  129. package/src/persistence/schema/conversations.ts +67 -0
  130. package/src/persistence/steps.ts +15 -0
  131. package/src/runtime/assistant-event-hub.ts +25 -1
  132. package/src/runtime/assistant-event-publish-options.ts +7 -0
  133. package/src/runtime/assistant-stream-state.ts +12 -0
  134. package/src/runtime/background-job-runner.ts +5 -5
  135. package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
  136. package/src/runtime/pending-interactions.ts +29 -0
  137. package/src/runtime/routes/browser-context.ts +31 -11
  138. package/src/runtime/routes/browser-routes.ts +43 -4
  139. package/src/runtime/routes/conversation-management-routes.ts +4 -1
  140. package/src/runtime/routes/conversation-routes.ts +112 -26
  141. package/src/runtime/routes/events-routes.ts +59 -47
  142. package/src/api/events/heartbeat-alert.ts +0 -20
package/ARCHITECTURE.md CHANGED
@@ -96,7 +96,7 @@ All HTTP API requests use a single `Authorization: Bearer <jwt>` header for auth
96
96
 
97
97
  **Credential storage:** Only hashed tokens are persisted. Access token hashes go in `credential_records`; refresh token hashes in `refresh_token_records`. Raw tokens are returned once and never stored server-side.
98
98
 
99
- **Notification scoping:** Guardian-sensitive notifications are annotated with `targetGuardianPrincipalId` for identity-scoped delivery.
99
+ **Notification scoping:** Guardian-sensitive notifications (`notification_intent`, `notification_conversation_created`) are published with `targetActorPrincipalId`, so the event hub delivers them, live, on reconnect replay, and from the `events/tail` recovery route, only to SSE connections whose verified `actorPrincipalId` is the guardian's. Connections without a principal never receive them. The payload's `targetGuardianPrincipalId` records the scoping.
100
100
 
101
101
  **Key source files:**
102
102
 
@@ -289,32 +289,35 @@ The WhatsApp channel enables inbound and outbound messaging via the Meta WhatsAp
289
289
 
290
290
  **Ingress** (`GET /webhooks/whatsapp` — verification, `POST /webhooks/whatsapp` — messages):
291
291
 
292
- 1. **Webhook verification**: Meta sends a `GET` with `hub.mode=subscribe`, `hub.verify_token`, and `hub.challenge`. The gateway compares `hub.verify_token` against `WHATSAPP_WEBHOOK_VERIFY_TOKEN` and echoes `hub.challenge` as plain text.
293
- 2. On `POST`, the gateway verifies the `X-Hub-Signature-256` header (HMAC-SHA256 of the raw request body using `WHATSAPP_APP_SECRET`) when the app secret is configured. Fail-closed: requests are rejected when the secret is set but the signature fails.
292
+ 1. **Webhook verification**: Meta sends a `GET` with `hub.mode=subscribe`, `hub.verify_token`, and `hub.challenge`. The gateway compares `hub.verify_token` against the `webhook_verify_token` credential and echoes `hub.challenge` as plain text.
293
+ 2. On `POST`, the gateway verifies the `X-Hub-Signature-256` header (HMAC-SHA256 of the raw request body using the `app_secret` credential). Fail-closed: a request is rejected with 500 when no app secret is configured, and with 403 when the signature does not match.
294
294
  3. **Normalization**: Text and media messages (image, audio, video, document, sticker) from `messages` change fields are forwarded. Delivery receipts, read receipts, and unsupported message types (contacts, location) are silently acknowledged with `{ ok: true }`. Media attachments are downloaded from the WhatsApp Cloud API, uploaded to the runtime attachment store, and their IDs are passed alongside the message content.
295
295
  4. **`/new` command**: When the message body is `/new` (case-insensitive), the gateway resolves routing, resets the conversation, and sends a confirmation message without forwarding to the runtime.
296
296
  5. The payload is normalized into a `GatewayInboundEvent` with `sourceChannel: "whatsapp"` and `conversationExternalId` set to the sender's WhatsApp phone number (E.164).
297
297
  6. WhatsApp message IDs are deduplicated via `StringDedupCache` (24-hour TTL).
298
298
  7. The gateway marks each inbound message as read (best-effort, fire-and-forget).
299
- 8. The event is forwarded to the runtime via `POST /channels/inbound` with WhatsApp-specific transport hints and a `replyCallbackUrl` pointing to `/deliver/whatsapp`.
299
+ 8. The event is forwarded to the runtime via `POST /channels/inbound` with WhatsApp-specific transport hints and a `replyCallbackUrl` of `<gatewayInternalBaseUrl>/deliver/whatsapp`. The gateway serves no route at that path: the daemon uses the URL only to address the reply (see Egress).
300
300
 
301
- **Egress** (`POST /deliver/whatsapp`):
301
+ **Egress** (daemon WhatsApp transport, `src/messaging/providers/whatsapp/`):
302
302
 
303
- 1. The runtime calls the gateway's `/deliver/whatsapp` endpoint with `{ to, text }` or `{ chatId, text }` (alias).
304
- 2. The gateway authenticates the request via bearer token (same fail-closed model as other deliver endpoints).
305
- 3. The gateway sends the message via the WhatsApp Cloud API `/{phoneNumberId}/messages` endpoint using the configured access token.
306
- 4. Text is split at 4096 characters if needed.
303
+ 1. Replies, approval prompts, and proactive sends go out through the daemon's `whatsappTransport` (`transport.ts`), which calls the Meta Cloud API itself. `isDirectDelivery()` in `src/messaging/providers/index.ts` resolves the `/deliver/whatsapp` callback URL to this transport through `channelForCallback()` (`src/messaging/providers/callback-routing.ts`), so the daemon never posts back to the gateway or dials the URL's host and port. The exception is on the gateway side: its replies to invite and verification codes it intercepts at ingress (`deliverVerificationReply` in `gateway/src/verification/reply-delivery.ts`) still POST to the callback URL, which no gateway route serves, so those replies are not delivered.
304
+ 2. `api.ts` reads `phone_number_id` and `access_token` from the secure store and sends through the Cloud API `/{phoneNumberId}/messages` endpoint.
305
+ 3. `send.ts` splits text into 4096-character chunks, preferring newline and then whitespace boundaries.
306
+ 4. An approval prompt renders as an interactive message with up to three reply buttons whose ids follow the shared `apr:<requestId>:<action>` convention. When the cap forces a cut, the reject and block actions are kept. The gateway normalizes the guardian's `button_reply` back into callback data on ingress.
307
+ 5. Attachments up to 25 MB are uploaded to the Cloud API media endpoint and sent as image, video, or document messages. Any that fail are listed in a follow-up text notice.
307
308
 
308
- **Required credentials**:
309
+ The gateway sends only its own notices to WhatsApp (the `/new` confirmation and routing-rejection notices), through `gateway/src/whatsapp/send.ts`.
309
310
 
310
- - `WHATSAPP_PHONE_NUMBER_ID` — the numeric WhatsApp Business phone number ID from Meta
311
- - `WHATSAPP_ACCESS_TOKEN` — System User or temporary access token
312
- - `WHATSAPP_APP_SECRET` — App secret for webhook signature verification
313
- - `WHATSAPP_WEBHOOK_VERIFY_TOKEN` — Token for the Meta webhook subscription handshake
311
+ **Required credentials** (credential vault, `whatsapp` service):
314
312
 
315
- These can be set via environment variables or stored in the credential vault (CES / encrypted store) under the `whatsapp` service prefix.
313
+ - `phone_number_id`: the numeric WhatsApp Business phone number ID from Meta
314
+ - `access_token`: System User or temporary access token
315
+ - `app_secret`: app secret for webhook signature verification
316
+ - `webhook_verify_token`: token for the Meta webhook subscription handshake
316
317
 
317
- **Limitations (v1)**: Rich approval UI (inline buttons) is not supported. Contacts and location message types are acknowledged but not forwarded.
318
+ The daemon reads `phone_number_id` and `access_token` to send. The gateway reads all four: the last two for ingress, and the first two for its own notices.
319
+
320
+ **Limitations**: Contacts and location message types are acknowledged but not forwarded.
318
321
 
319
322
  **Channel Readiness**: The channel readiness HTTP endpoints (`GET /v1/channels/readiness`, `POST /v1/channels/readiness/refresh`) backed by `ChannelReadinessService` in `src/runtime/channel-readiness-service.ts` provide a unified readiness subsystem for all channels. Each channel registers a `ChannelProbe` that runs synchronous local checks (credential presence, ingress config) and optional async remote checks with a 5-minute TTL cache. Built-in probes: Telegram (bot token, webhook secret, ingress). The GET endpoint returns cached snapshots; the refresh endpoint invalidates the cache first. Unknown channels return `unsupported_channel`. Route handlers live in `src/runtime/routes/channel-readiness-routes.ts`.
320
323
 
@@ -1993,7 +1996,7 @@ An SSE push event surfaces new conversations in the macOS client sidebar:
1993
1996
 
1994
1997
  - **`notification_conversation_created`** — Emitted by `broadcaster.ts` when a notification delivery **creates** a new vellum conversation (strategy `start_new_conversation`, `createdNewConversation: true`). **Not** emitted when a conversation is reused. Payload: `{ conversationId, title, sourceEventName }`.
1995
1998
 
1996
- The event follows this pattern: the daemon creates a server-side conversation, persists an initial message, and broadcasts the SSE event so the macOS `ConversationManager` can create a visible conversation in the sidebar.
1999
+ The daemon persists the conversation and its initial message before broadcasting, so a client that receives the event can fetch the conversation immediately. A guardian-sensitive conversation is announced only to the guardian's own connections.
1997
2000
 
1998
2001
  ### Conversation Routing Decision Flow
1999
2002
 
@@ -2031,12 +2034,15 @@ Reminders carry optional `routingIntent` (`single_channel` | `multi_channel` | `
2031
2034
 
2032
2035
  ### Channel Delivery
2033
2036
 
2034
- Notifications are delivered to three channel types:
2037
+ Notifications are delivered to five channel types. The daemon sends to external channels itself, calling each provider's API through the send modules in `src/messaging/providers/<channel>/`; nothing goes through the gateway.
2035
2038
 
2036
2039
  - **Vellum (always connected)**: SSE via the daemon's broadcast mechanism. The `VellumAdapter` emits a `notification_intent` message with rendered copy and optional `deepLinkMetadata` (includes `conversationId` for conversation navigation and `messageId` for message-level scroll anchoring).
2037
- - **Telegram (when guardian binding exists)**: HTTP POST to the gateway's `/deliver/telegram` endpoint. Requires an active guardian binding for the assistant.
2040
+ - **Platform (always connected)**: the `PlatformPushAdapter` posts the notification to the platform's `/v1/assistants/{id}/push/dispatch/` endpoint for native mobile push. Without platform credentials the delivery is recorded as failed.
2041
+ - **Telegram (when the guardian has a chat ID)**: the `TelegramAdapter` calls the Telegram Bot API through `telegram-bot/send.ts`. Approval cards carry inline keyboard buttons and fall back to plain text with typed-reply instructions if the rich send fails.
2042
+ - **Slack (when the guardian's chat is a DM)**: the `SlackAdapter` posts through `slack/send.ts`. Only `D`-prefixed DM channels qualify, so a binding made from a shared channel never receives notifications.
2043
+ - **Discord (when the guardian binding names a user)**: the `DiscordAdapter` opens the guardian's DM through the Discord REST API and sends through `discord/send.ts`.
2038
2044
 
2039
- Connected channels are resolved at signal emission time: vellum is always included, and binding-based channels (Telegram) are included only when an active guardian binding exists for the assistant.
2045
+ Connected channels are resolved at signal emission time by `getConnectedChannels()` in `emit-signal.ts`, from the guardian delivery list (`getGuardianDelivery()`): vellum and platform are always included, and Telegram, Slack, and Discord are included only when that list gives the guardian a deliverable endpoint on the channel.
2040
2046
 
2041
2047
  **Key modules:**
2042
2048
 
@@ -2050,8 +2056,11 @@ Connected channels are resolved at signal emission time: vellum is always includ
2050
2056
  | `assistant/src/notifications/conversation-pairing.ts` | Materializes conversation + message per delivery; executes conversation reuse decisions |
2051
2057
  | `assistant/src/notifications/conversation-candidates.ts` | Builds per-channel candidate set of recent conversations for the decision engine |
2052
2058
  | `assistant/src/notifications/adapters/macos.ts` | Vellum adapter — broadcasts `notification_intent` via SSE with deep-link metadata |
2053
- | `assistant/src/notifications/adapters/telegram.ts` | Telegram adapter — POSTs to gateway `/deliver/telegram` |
2054
- | `assistant/src/notifications/destination-resolver.ts` | Resolves per-channel endpoints (vellum SSE, Telegram chat ID from guardian binding) |
2059
+ | `assistant/src/notifications/adapters/platform.ts` | Platform adapter: posts to the platform push-dispatch endpoint for native mobile push |
2060
+ | `assistant/src/notifications/adapters/telegram.ts` | Telegram adapter: calls the Telegram Bot API through `messaging/providers/telegram-bot/send.ts` |
2061
+ | `assistant/src/notifications/adapters/slack.ts` | Slack adapter: posts to the guardian's DM through `messaging/providers/slack/send.ts` |
2062
+ | `assistant/src/notifications/adapters/discord.ts` | Discord adapter: sends to the guardian's DM through `messaging/providers/discord/send.ts` |
2063
+ | `assistant/src/notifications/destination-resolver.ts` | Resolves per-channel endpoints from the guardian delivery list (Telegram and Slack chat ID, Discord user ID) |
2055
2064
  | `assistant/src/notifications/copy-composer.ts` | Template-based fallback copy when LLM copy is unavailable |
2056
2065
  | `assistant/src/notifications/preference-extractor.ts` | Detects preference statements in conversation messages |
2057
2066
  | `assistant/src/notifications/preferences-store.ts` | CRUD for user notification preferences |
@@ -0,0 +1,183 @@
1
+ # Transcript mode sessions
2
+
3
+ Mode sessions are compact, conversation-owned lifecycle records for grouping
4
+ computer-use, browser, Live vision, and reserved Ambient transcript activity.
5
+ They connect runtime producers to immutable message membership while keeping
6
+ model input and stored message content unchanged.
7
+
8
+ ## API contract
9
+
10
+ `src/api/mode-session.ts` owns both shared shapes:
11
+
12
+ - `ModeSession` is immutable transcript membership: an opaque `id` and one of
13
+ `computer_use`, `browser`, `live_vision`, or `ambient`.
14
+ - `ModeSessionSummary` is the durable lifecycle record. It carries the owning
15
+ conversation, mode, timing bounds, boundary message IDs, terminal reason,
16
+ and a positive monotonic revision.
17
+
18
+ Only three statuses are stored: `active`, `completed`, and `interrupted`.
19
+ Waiting and draining are runtime presentation facts and do not add database
20
+ states. Duration is derived from recorded time bounds instead of stored as a
21
+ separate value. Active records have no terminal fields, completed records have
22
+ a confirmed end, and interrupted records may leave the end unknown. A known end
23
+ never precedes the last confirmed activity.
24
+
25
+ ## Persistence
26
+
27
+ Migration 381 creates `conversation_mode_sessions`. Each row belongs to one
28
+ conversation through an `ON DELETE CASCADE` foreign key. The
29
+ `(conversation_id, status)` index supports conversation-scoped active reads,
30
+ while the primary key supports bounded ID batches for transcript pages.
31
+
32
+ `conversation-mode-sessions.ts` is the only lifecycle writer. Begin creates
33
+ revision 1. Activity, boundary, revision-only, and terminal updates require the
34
+ caller's expected revision and run in an immediate transaction. Each accepted
35
+ write increments the revision. A stale revision, missing record, or terminal
36
+ record is returned as an expected write outcome. Terminal rows cannot return
37
+ to active or change to a different terminal result. Repeating the same terminal
38
+ outcome is an accepted no-op and does not advance the revision.
39
+
40
+ The first included boundary can precede source activation when a later action
41
+ claims assistant content already persisted in the same logical turn. Last
42
+ activity never moves backward. An interruption may have an unknown end, so
43
+ `endedAt` remains nullable and `lastActivityAt` remains the last confirmed
44
+ time.
45
+
46
+ ## Startup recovery
47
+
48
+ After database migrations succeed, the assistant marks every leftover active
49
+ record interrupted before database readiness is published. Recovery increments
50
+ the revision, preserves activity and message boundaries, records
51
+ `assistant_restarted`, and leaves `endedAt` null. It runs on every boot rather
52
+ than as a checkpointed migration.
53
+
54
+ If this auxiliary recovery fails, ordinary database readiness succeeds and
55
+ tracking stays unavailable for that boot. History omits unrecovered active
56
+ descriptors while preserving terminal records and message stamps. A subsequent
57
+ boot retries through the same startup path. Required migration failures still
58
+ block database readiness.
59
+
60
+ Runtime source generations and structural question associations are owned by
61
+ the producer and message-membership integration. Those process-local facts are
62
+ not reconstructed after restart. Resumed mode work uses a fresh session ID and
63
+ callbacks from retired source generations are rejected.
64
+
65
+ New producer admissions are controlled by the default-off `session-groups`
66
+ assistant feature flag. Browser and computer-use actions still execute when the
67
+ flag is off, but they do not activate or claim a mode session. Existing owners
68
+ and terminal cleanup continue to drain so disabling the flag cannot strand
69
+ accepted work. Live voice snapshots the flag when its socket session is
70
+ created: disabled sessions retain the existing camera transport without
71
+ advertising or preparing tracked sight-session epochs.
72
+
73
+ The same flag controls activity-label wording at the shared rendering
74
+ boundary. Disabled views retain their existing labels and phase grouping.
75
+
76
+ The migration, startup recovery, persisted message stamps, and history reads
77
+ remain available while the flag is off. This keeps records made by an enabled
78
+ cohort compatible with rollback and later re-enablement.
79
+
80
+ ## Runtime ownership
81
+
82
+ Each live `Conversation` owns one `ConversationModeSessionCoordinator`.
83
+ Computer-use, browser, and camera producers activate a source with a stable
84
+ source identity and guarded generation. The first accepted mode action claims
85
+ the logical turn. The coordinator backfills rows already persisted for that
86
+ turn, then stamps later user, assistant, and tool-result rows as they are
87
+ inserted. A turn keeps its first owner when another mode acts inside it.
88
+
89
+ Source eligibility and captured turn ownership end separately. Ending or
90
+ resetting a source prevents future claims immediately, but a captured turn
91
+ keeps its owner until its final output is persisted. Ordinary computer-use and
92
+ browser sources finish when the turn settles. Camera sources use a source
93
+ lifetime plus a synthetic run turn, so accepted frame writes and captured
94
+ voice turns form a settlement barrier before terminal persistence. A camera
95
+ epoch snapshots ownership before asynchronous frame persistence; stale epochs
96
+ cannot join a later run.
97
+
98
+ Tracking failures are logged and isolated from authorized actions and accepted
99
+ content persistence. They never repeat the underlying action. A refused camera
100
+ start clears client negotiation and falls back to untracked frames without
101
+ revoking consent or ending voice. Individual stale-frame rejection does not
102
+ use this fallback.
103
+
104
+ Server admission is the camera boundary: an admitted frame can finish saving
105
+ after stop, but a device-side capture delivered after its run ends is rejected
106
+ and reclaimed. Reconnect pauses capture delivery and announces a fresh run
107
+ after replacement readiness. In-flight uploads and native samples retain their
108
+ captured ownership and cannot join the replacement run.
109
+
110
+ The coordinator stores only `active`, `completed`, or `interrupted`. Waiting
111
+ and finishing are revisioned runtime hints. Terminal dispositions remain
112
+ pending in memory until all captured turns release, then one idempotent
113
+ compare-and-swap update records the terminal outcome. A successful turn with
114
+ no explicit producer end also finalizes, allowing later work from the same
115
+ resource generation to mint a new session.
116
+
117
+ Regular reload, TTL, LRU, and memory-pressure eviction retain a conversation
118
+ while it has an eligible mode source, captured owner, structural association,
119
+ or unsettled terminal disposition. This preserves accepted camera output and
120
+ exact structural continuation in the coordinator that admitted them. A long
121
+ structural wait therefore remains resident until it is consumed, invalidated,
122
+ or its source retires. Explicit conversation deletion and process shutdown
123
+ keep their existing teardown semantics; startup recovery interrupts any
124
+ durable active record left by process exit.
125
+
126
+ Invalidating the final structural association retires a turn-lifetime source
127
+ and lets accepted output settle. Successful abandoned waits end at their last
128
+ confirmed activity with reason `structural_wait_abandoned`; cleanup does not
129
+ extend their duration. Source-lifetime camera runs remain eligible. A retired
130
+ disposition that cannot persist its terminal update releases residency and
131
+ invalidates history without claiming completion. An active record without live
132
+ ownership has no descriptor and renders as unavailable.
133
+
134
+ ## Structural continuation
135
+
136
+ Structural waits associate an exact response identifier with the owned turn.
137
+ Ordinary `ask_question` awaits its resolver inside that turn, so its answer
138
+ resumes the existing owner. A blocking UI surface can accept a new turn: its
139
+ matching response consumes the association before surface cleanup and retains
140
+ the session ID. Rejected, stale, unrelated, or cross-conversation responses
141
+ cannot inherit ownership. Dismissal invalidates the association. Ordinary prose
142
+ and an idle reusable resource do not continue a session; later mode work starts
143
+ a fresh ID and duration.
144
+
145
+ Structural associations intentionally do not survive restart. Startup recovery
146
+ terminalizes the old record, and a late response remains part of the existing
147
+ interaction flow without reopening that record.
148
+
149
+ ## Transcript and synchronization
150
+
151
+ `modeSession` is optional message metadata and is stripped from caller-supplied
152
+ metadata. Membership is added only after a row insert succeeds. Text and
153
+ thinking deltas carry no repeated ownership field; row-boundary events carry
154
+ the current owner, and the normal conversation-messages invalidation refetches
155
+ retrospective stamps.
156
+
157
+ History returns one batched `modeSessions` descriptor list beside messages.
158
+ The query resolves records for page members and accepts bounded explicit IDs
159
+ for groups already loaded outside the newest page. Consolidation never crosses
160
+ a stamped/unstamped or different-owner boundary. Same-owner folds retain row
161
+ aliases and `modeSessionActivity.firstAt`/`lastAt`, so donor timestamps survive
162
+ history compaction.
163
+
164
+ Message deletion and retry paths recompute replaceable first/last display
165
+ boundaries for every affected conversation session. They preserve status,
166
+ terminal time, reason, and confirmed activity, and increment the revision.
167
+ Deleting every owned row leaves the lifecycle record available for audit but
168
+ produces no transcript container.
169
+
170
+ All membership, lifecycle, boundary, and runtime-hint changes reuse the
171
+ conversation-messages `sync_changed` tag. Clients merge descriptors by
172
+ revision in the history query cache and refetch through the existing history
173
+ reconciliation path. Explicit stale revisions cannot replace newer cached
174
+ descriptors; an omitted descriptor remains unavailable. There is no separate
175
+ descriptor store, session event stream, or polling endpoint.
176
+
177
+ The transcript keeps lightweight segment identities for stable disclosure keys,
178
+ not duplicate message bodies. Closed children unmount, and eligible mounted
179
+ headers share one coarse clock that pauses while the app is hidden. No header
180
+ visibility observer is required. The clock reads initial desktop visibility
181
+ from the existing attention state on its coarse tick, without publishing a
182
+ shared lifecycle edge. With grouping disabled, the load-older guard uses the
183
+ ordinary flat-transcript behavior and skips grouped DOM scans.
package/openapi.yaml CHANGED
@@ -20944,6 +20944,12 @@ paths:
20944
20944
  schema:
20945
20945
  type: integer
20946
20946
  description: Maximum number of messages to return.
20947
+ - name: modeSessionIds
20948
+ in: query
20949
+ required: false
20950
+ schema:
20951
+ type: string
20952
+ description: Comma-separated mode-session IDs to include with summaries for loaded active groups outside this page.
20947
20953
  responses:
20948
20954
  "200":
20949
20955
  description: Successful response
@@ -20959,6 +20965,38 @@ paths:
20959
20965
  properties:
20960
20966
  id:
20961
20967
  type: string
20968
+ modeSession:
20969
+ type: object
20970
+ properties:
20971
+ mode:
20972
+ type: string
20973
+ enum:
20974
+ - computer_use
20975
+ - browser
20976
+ - live_vision
20977
+ - ambient
20978
+ id:
20979
+ type: string
20980
+ minLength: 1
20981
+ required:
20982
+ - mode
20983
+ - id
20984
+ additionalProperties: false
20985
+ modeSessionActivity:
20986
+ type: object
20987
+ properties:
20988
+ firstAt:
20989
+ type: integer
20990
+ minimum: 0
20991
+ maximum: 8640000000000000
20992
+ lastAt:
20993
+ type: integer
20994
+ minimum: 0
20995
+ maximum: 8640000000000000
20996
+ required:
20997
+ - firstAt
20998
+ - lastAt
20999
+ additionalProperties: false
20962
21000
  mergedMessageIds:
20963
21001
  type: array
20964
21002
  items:
@@ -22277,6 +22315,223 @@ paths:
22277
22315
  locally but the server reports `processing: false`, the turn has ended (or died) and the UI should
22278
22316
  stop waiting rather than spin indefinitely. Absent on older daemons that predate this field."
22279
22317
  type: boolean
22318
+ modeSessions:
22319
+ description: Conversation-owned mode-session summaries referenced by this page or explicitly requested by ID.
22320
+ type: array
22321
+ items:
22322
+ type: object
22323
+ properties:
22324
+ summary:
22325
+ oneOf:
22326
+ - type: object
22327
+ properties:
22328
+ id:
22329
+ type: string
22330
+ minLength: 1
22331
+ conversationId:
22332
+ type: string
22333
+ minLength: 1
22334
+ mode:
22335
+ type: string
22336
+ enum:
22337
+ - computer_use
22338
+ - browser
22339
+ - live_vision
22340
+ - ambient
22341
+ sourceStartedAt:
22342
+ type: integer
22343
+ minimum: 0
22344
+ maximum: 9007199254740991
22345
+ firstIncludedAt:
22346
+ anyOf:
22347
+ - type: integer
22348
+ minimum: 0
22349
+ maximum: 9007199254740991
22350
+ - type: "null"
22351
+ firstIncludedMessageId:
22352
+ anyOf:
22353
+ - type: string
22354
+ minLength: 1
22355
+ - type: "null"
22356
+ lastActivityAt:
22357
+ type: integer
22358
+ minimum: 0
22359
+ maximum: 9007199254740991
22360
+ lastOwnedMessageId:
22361
+ anyOf:
22362
+ - type: string
22363
+ minLength: 1
22364
+ - type: "null"
22365
+ revision:
22366
+ type: integer
22367
+ exclusiveMinimum: 0
22368
+ maximum: 9007199254740991
22369
+ status:
22370
+ type: string
22371
+ const: active
22372
+ endedAt:
22373
+ type: "null"
22374
+ endReason:
22375
+ type: "null"
22376
+ required:
22377
+ - id
22378
+ - conversationId
22379
+ - mode
22380
+ - sourceStartedAt
22381
+ - firstIncludedAt
22382
+ - firstIncludedMessageId
22383
+ - lastActivityAt
22384
+ - lastOwnedMessageId
22385
+ - revision
22386
+ - status
22387
+ - endedAt
22388
+ - endReason
22389
+ additionalProperties: false
22390
+ - type: object
22391
+ properties:
22392
+ id:
22393
+ type: string
22394
+ minLength: 1
22395
+ conversationId:
22396
+ type: string
22397
+ minLength: 1
22398
+ mode:
22399
+ type: string
22400
+ enum:
22401
+ - computer_use
22402
+ - browser
22403
+ - live_vision
22404
+ - ambient
22405
+ sourceStartedAt:
22406
+ type: integer
22407
+ minimum: 0
22408
+ maximum: 9007199254740991
22409
+ firstIncludedAt:
22410
+ anyOf:
22411
+ - type: integer
22412
+ minimum: 0
22413
+ maximum: 9007199254740991
22414
+ - type: "null"
22415
+ firstIncludedMessageId:
22416
+ anyOf:
22417
+ - type: string
22418
+ minLength: 1
22419
+ - type: "null"
22420
+ lastActivityAt:
22421
+ type: integer
22422
+ minimum: 0
22423
+ maximum: 9007199254740991
22424
+ lastOwnedMessageId:
22425
+ anyOf:
22426
+ - type: string
22427
+ minLength: 1
22428
+ - type: "null"
22429
+ revision:
22430
+ type: integer
22431
+ exclusiveMinimum: 0
22432
+ maximum: 9007199254740991
22433
+ status:
22434
+ type: string
22435
+ const: completed
22436
+ endedAt:
22437
+ type: integer
22438
+ minimum: 0
22439
+ maximum: 9007199254740991
22440
+ endReason:
22441
+ type: string
22442
+ minLength: 1
22443
+ required:
22444
+ - id
22445
+ - conversationId
22446
+ - mode
22447
+ - sourceStartedAt
22448
+ - firstIncludedAt
22449
+ - firstIncludedMessageId
22450
+ - lastActivityAt
22451
+ - lastOwnedMessageId
22452
+ - revision
22453
+ - status
22454
+ - endedAt
22455
+ - endReason
22456
+ additionalProperties: false
22457
+ - type: object
22458
+ properties:
22459
+ id:
22460
+ type: string
22461
+ minLength: 1
22462
+ conversationId:
22463
+ type: string
22464
+ minLength: 1
22465
+ mode:
22466
+ type: string
22467
+ enum:
22468
+ - computer_use
22469
+ - browser
22470
+ - live_vision
22471
+ - ambient
22472
+ sourceStartedAt:
22473
+ type: integer
22474
+ minimum: 0
22475
+ maximum: 9007199254740991
22476
+ firstIncludedAt:
22477
+ anyOf:
22478
+ - type: integer
22479
+ minimum: 0
22480
+ maximum: 9007199254740991
22481
+ - type: "null"
22482
+ firstIncludedMessageId:
22483
+ anyOf:
22484
+ - type: string
22485
+ minLength: 1
22486
+ - type: "null"
22487
+ lastActivityAt:
22488
+ type: integer
22489
+ minimum: 0
22490
+ maximum: 9007199254740991
22491
+ lastOwnedMessageId:
22492
+ anyOf:
22493
+ - type: string
22494
+ minLength: 1
22495
+ - type: "null"
22496
+ revision:
22497
+ type: integer
22498
+ exclusiveMinimum: 0
22499
+ maximum: 9007199254740991
22500
+ status:
22501
+ type: string
22502
+ const: interrupted
22503
+ endedAt:
22504
+ anyOf:
22505
+ - type: integer
22506
+ minimum: 0
22507
+ maximum: 9007199254740991
22508
+ - type: "null"
22509
+ endReason:
22510
+ type: string
22511
+ minLength: 1
22512
+ required:
22513
+ - id
22514
+ - conversationId
22515
+ - mode
22516
+ - sourceStartedAt
22517
+ - firstIncludedAt
22518
+ - firstIncludedMessageId
22519
+ - lastActivityAt
22520
+ - lastOwnedMessageId
22521
+ - revision
22522
+ - status
22523
+ - endedAt
22524
+ - endReason
22525
+ additionalProperties: false
22526
+ type: object
22527
+ runtimeState:
22528
+ type: string
22529
+ enum:
22530
+ - waiting
22531
+ - finishing
22532
+ required:
22533
+ - summary
22534
+ additionalProperties: false
22280
22535
  required:
22281
22536
  - messages
22282
22537
  additionalProperties: false
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vellumai/assistant",
3
- "version": "0.12.2-dev.202609181713.fb1ff61",
3
+ "version": "0.12.2-dev.202609181913.1108ac3",
4
4
  "license": "MIT",
5
5
  "type": "module",
6
6
  "exports": {
@@ -53,6 +53,7 @@ import {
53
53
  handleToolResult,
54
54
  } from "../daemon/conversation-agent-loop-handlers.js";
55
55
  import type { ToolActivityMetadata } from "../daemon/message-types/web-activity.js";
56
+ import { mockUnownedModeSessions } from "./helpers/mock-conversation.js";
56
57
 
57
58
  // ── Helpers ───────────────────────────────────────────────────────────────────
58
59
 
@@ -65,6 +66,7 @@ function makeDeps(): EventHandlerDeps {
65
66
  emitActivityState: () => {},
66
67
  markWorkspaceTopLevelDirty: () => {},
67
68
  currentTurnSurfaces: [],
69
+ modeSessions: mockUnownedModeSessions(),
68
70
  } as unknown as EventHandlerDeps["ctx"],
69
71
  onEvent: () => {},
70
72
  reqId: "test-req",
@@ -47,6 +47,7 @@ import {
47
47
  createEventHandlerState,
48
48
  handleToolResult,
49
49
  } from "../daemon/conversation-agent-loop-handlers.js";
50
+ import { mockUnownedModeSessions } from "./helpers/mock-conversation.js";
50
51
 
51
52
  // ── Helpers ───────────────────────────────────────────────────────────────────
52
53
 
@@ -59,6 +60,7 @@ function makeDeps(): EventHandlerDeps {
59
60
  emitActivityState: () => {},
60
61
  markWorkspaceTopLevelDirty: () => {},
61
62
  currentTurnSurfaces: [],
63
+ modeSessions: mockUnownedModeSessions(),
62
64
  } as unknown as EventHandlerDeps["ctx"],
63
65
  onEvent: () => {},
64
66
  reqId: "test-req",
@@ -46,6 +46,7 @@ import {
46
46
  createEventHandlerState,
47
47
  handleToolResult,
48
48
  } from "../daemon/conversation-agent-loop-handlers.js";
49
+ import { mockUnownedModeSessions } from "./helpers/mock-conversation.js";
49
50
 
50
51
  // ── Helpers ───────────────────────────────────────────────────────────────────
51
52
 
@@ -60,6 +61,7 @@ function makeDeps(): EventHandlerDeps {
60
61
  emitActivityState: () => {},
61
62
  markWorkspaceTopLevelDirty: () => {},
62
63
  currentTurnSurfaces: [],
64
+ modeSessions: mockUnownedModeSessions(),
63
65
  } as unknown as EventHandlerDeps["ctx"],
64
66
  onEvent: (event: unknown) => {
65
67
  emitted.push(event as Record<string, unknown>);