@vellumai/assistant 0.11.5 → 0.11.6-staging.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. package/AGENTS.md +5 -1
  2. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +118 -0
  3. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/ingress.ts +103 -0
  4. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +118 -0
  5. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/ingress.ts +103 -0
  6. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +70 -0
  7. package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +16 -1
  8. package/node_modules/@vellumai/gateway-client/src/index.ts +6 -2
  9. package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +121 -61
  10. package/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +118 -0
  11. package/node_modules/@vellumai/service-contracts/src/ingress.ts +103 -0
  12. package/openapi.yaml +421 -15
  13. package/package.json +1 -1
  14. package/scripts/sync-web-search-catalog.ts +6 -0
  15. package/src/__tests__/app-pin-store.test.ts +149 -0
  16. package/src/__tests__/channel-availability-routes.test.ts +23 -1
  17. package/src/__tests__/channel-readiness-discord.test.ts +231 -0
  18. package/src/__tests__/channel-readiness-service.test.ts +126 -0
  19. package/src/__tests__/channel-readiness-slack-remote.test.ts +141 -0
  20. package/src/__tests__/channel-reply-delivery.test.ts +4 -4
  21. package/src/__tests__/client-os-metadata-persistence.test.ts +23 -10
  22. package/src/__tests__/conversation-delete-watch-timeline.test.ts +231 -0
  23. package/src/__tests__/conversation-error.test.ts +17 -0
  24. package/src/__tests__/conversation-seed-composer.test.ts +8 -0
  25. package/src/__tests__/conversation-slash-commands.test.ts +8 -0
  26. package/src/__tests__/disk-pressure-policy.test.ts +6 -0
  27. package/src/__tests__/gemini-provider.test.ts +138 -0
  28. package/src/__tests__/history-repair.test.ts +105 -3
  29. package/src/__tests__/identity-routes.test.ts +1 -0
  30. package/src/__tests__/llm-catalog-parity.test.ts +45 -0
  31. package/src/__tests__/migration-import-from-path.test.ts +349 -0
  32. package/src/__tests__/notification-telegram-adapter.test.ts +102 -0
  33. package/src/__tests__/oauth-commands-routes.test.ts +89 -0
  34. package/src/__tests__/oauth-provider-profiles.test.ts +7 -6
  35. package/src/__tests__/openai-provider.test.ts +18 -0
  36. package/src/__tests__/openai-responses-provider.test.ts +18 -0
  37. package/src/__tests__/platform-callback-registration.test.ts +184 -0
  38. package/src/__tests__/plugin-api-store-credential.test.ts +71 -3
  39. package/src/__tests__/pricing.test.ts +2 -2
  40. package/src/__tests__/public-ingress-urls.test.ts +36 -0
  41. package/src/__tests__/resolve-trust-class.test.ts +0 -48
  42. package/src/__tests__/sanitize-config-for-transfer.test.ts +28 -0
  43. package/src/__tests__/secret-routes-platform-proxy.test.ts +49 -0
  44. package/src/__tests__/settings-routes.test.ts +85 -3
  45. package/src/__tests__/web-search-catalog-parity.test.ts +8 -0
  46. package/src/agent/history-repair/history-repair.ts +45 -14
  47. package/src/agent/loop.ts +4 -1
  48. package/src/api/constants/profile-config-validation.ts +60 -0
  49. package/src/api/events/tool-result.ts +6 -1
  50. package/src/api/events/watch-retro-completed.ts +52 -0
  51. package/src/api/index.ts +11 -0
  52. package/src/apps/app-pin-reconciler.ts +92 -0
  53. package/src/apps/app-pin-store.ts +125 -0
  54. package/src/channels/gateway-channel-socket-health.ts +32 -0
  55. package/src/channels/gateway-discord-admission.ts +32 -0
  56. package/src/channels/types.ts +20 -0
  57. package/src/cli/commands/__tests__/conversations-slack.test.ts +1 -1
  58. package/src/cli/commands/__tests__/inference-profiles.test.ts +16 -4
  59. package/src/cli/commands/__tests__/inference-providers.test.ts +67 -2
  60. package/src/cli/commands/channels/__tests__/channels.test.ts +85 -0
  61. package/src/cli/commands/channels/index.ts +45 -31
  62. package/src/cli/commands/inference-profiles.ts +56 -3
  63. package/src/cli/commands/inference-providers.ts +28 -2
  64. package/src/cli/commands/oauth/index.help.ts +7 -1
  65. package/src/cli/commands/oauth/request.test.ts +290 -0
  66. package/src/cli/commands/oauth/request.ts +57 -41
  67. package/src/cli/lib/bundled-marketplace.json +14 -1
  68. package/src/cli/lib/open-browser.test.ts +67 -0
  69. package/src/cli/lib/open-browser.ts +24 -5
  70. package/src/config/__tests__/profile-materialization.test.ts +26 -0
  71. package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +6 -0
  72. package/src/config/bundled-skills/schedule/SKILL.md +1 -1
  73. package/src/config/feature-flag-registry.json +17 -1
  74. package/src/config/profile-materialization.ts +29 -0
  75. package/src/config/sanitize-for-transfer.ts +16 -0
  76. package/src/config/schemas/llm.ts +7 -0
  77. package/src/config/schemas/services.ts +6 -0
  78. package/src/context/outbound-sanitize.ts +6 -0
  79. package/src/daemon/__tests__/lifecycle-watch-timeline-sweep.test.ts +98 -0
  80. package/src/daemon/conversation-error.ts +24 -2
  81. package/src/daemon/conversation-slash.ts +6 -15
  82. package/src/daemon/daemon-control.ts +1 -0
  83. package/src/daemon/disk-pressure-policy.ts +7 -1
  84. package/src/daemon/handlers/__tests__/config-ingress-tunnel-records.test.ts +208 -0
  85. package/src/daemon/handlers/config-ingress.ts +115 -5
  86. package/src/daemon/lifecycle.ts +26 -0
  87. package/src/daemon/message-types/web-activity.ts +3 -2
  88. package/src/daemon/trust-context.ts +0 -37
  89. package/src/inbound/__tests__/tunnel-probe.test.ts +448 -0
  90. package/src/inbound/platform-callback-registration.ts +28 -2
  91. package/src/inbound/public-ingress-urls.ts +12 -0
  92. package/src/inbound/tunnel-probe.ts +261 -0
  93. package/src/live-voice/__tests__/live-voice-connection.test.ts +25 -0
  94. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +8 -2
  95. package/src/live-voice/__tests__/live-voice-session-manager.test.ts +212 -10
  96. package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +5 -2
  97. package/src/live-voice/live-voice-connection.ts +46 -8
  98. package/src/live-voice/live-voice-manager.ts +25 -0
  99. package/src/live-voice/live-voice-session-manager.ts +318 -2
  100. package/src/live-voice/live-voice-session.ts +52 -2
  101. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +126 -68
  102. package/src/messaging/providers/channel-transport.ts +64 -47
  103. package/src/messaging/providers/discord/send.test.ts +46 -1
  104. package/src/messaging/providers/discord/send.ts +51 -0
  105. package/src/messaging/providers/discord/transport.ts +26 -3
  106. package/src/messaging/providers/index.ts +22 -47
  107. package/src/messaging/providers/slack/send.test.ts +83 -26
  108. package/src/messaging/providers/slack/send.ts +120 -51
  109. package/src/messaging/providers/slack/stream-tasks.test.ts +26 -0
  110. package/src/messaging/providers/slack/stream-tasks.ts +39 -0
  111. package/src/messaging/providers/slack/transport.ts +24 -22
  112. package/src/messaging/providers/telegram-bot/send.test.ts +109 -12
  113. package/src/messaging/providers/telegram-bot/send.ts +43 -0
  114. package/src/messaging/providers/telegram-bot/transport.ts +25 -8
  115. package/src/notifications/__tests__/assistant-reply-producer.test.ts +30 -7
  116. package/src/notifications/adapters/telegram.ts +48 -1
  117. package/src/notifications/assistant-reply-producer.ts +7 -7
  118. package/src/notifications/conversation-seed-composer.ts +7 -2
  119. package/src/oauth/byo-connection.test.ts +63 -0
  120. package/src/oauth/byo-connection.ts +16 -15
  121. package/src/oauth/connection.test.ts +111 -0
  122. package/src/oauth/connection.ts +142 -1
  123. package/src/oauth/platform-connection.test.ts +34 -0
  124. package/src/oauth/platform-connection.ts +28 -5
  125. package/src/oauth/seed-providers.ts +15 -1
  126. package/src/permissions/types.ts +3 -1
  127. package/src/persistence/conversation-crud.ts +46 -0
  128. package/src/persistence/conversation-types.ts +11 -9
  129. package/src/persistence/db-async-query.ts +2 -1
  130. package/src/persistence/db-maintenance.ts +15 -0
  131. package/src/persistence/embeddings/qdrant-manager.ts +1 -0
  132. package/src/persistence/migrations/367-create-watch-timeline-entries.ts +46 -0
  133. package/src/persistence/migrations/368-watch-timeline-screenshot-blob.ts +33 -0
  134. package/src/persistence/migrations/369-create-app-pins.ts +37 -0
  135. package/src/persistence/migrations/__tests__/367-create-watch-timeline-entries.test.ts +98 -0
  136. package/src/persistence/migrations/__tests__/368-watch-timeline-screenshot-blob.test.ts +98 -0
  137. package/src/persistence/schema/index.ts +1 -0
  138. package/src/persistence/schema/infrastructure.ts +17 -0
  139. package/src/persistence/schema/watch.ts +29 -0
  140. package/src/persistence/steps.ts +6 -0
  141. package/src/plugins/mtime-cache.ts +11 -0
  142. package/src/providers/__tests__/retry-network-error.test.ts +84 -0
  143. package/src/providers/connection-resolution.ts +23 -1
  144. package/src/providers/content-blocks.ts +9 -0
  145. package/src/providers/fetch-provider-catalog.ts +19 -0
  146. package/src/providers/gemini/client.ts +13 -5
  147. package/src/providers/inference/__tests__/endpoint-probe.test.ts +92 -0
  148. package/src/providers/inference/__tests__/profile-config-validation.test.ts +39 -0
  149. package/src/providers/inference/__tests__/profile-probe-classify.test.ts +66 -0
  150. package/src/providers/inference/adapter-factory.ts +0 -9
  151. package/src/providers/inference/credential-rotation.ts +61 -0
  152. package/src/providers/inference/endpoint-probe.ts +115 -0
  153. package/src/providers/inference/profile-probe.ts +256 -0
  154. package/src/providers/model-catalog.ts +170 -125
  155. package/src/providers/openai/__tests__/api-error-normalization.test.ts +17 -1
  156. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +42 -60
  157. package/src/providers/openai/__tests__/connection-error-wrap.test.ts +44 -0
  158. package/src/providers/openai/__tests__/orphan-tool-result-guard.test.ts +34 -2
  159. package/src/providers/openai/api-error-normalization.ts +16 -2
  160. package/src/providers/openai/chat-completions-provider.ts +75 -29
  161. package/src/providers/openai/responses-provider.ts +5 -2
  162. package/src/providers/openrouter/client.ts +0 -1
  163. package/src/providers/provider-send-message.ts +11 -0
  164. package/src/providers/retry.ts +6 -0
  165. package/src/providers/search-provider-catalog.ts +20 -0
  166. package/src/providers/vercel-ai-gateway/client.ts +0 -1
  167. package/src/runtime/AGENTS.md +1 -0
  168. package/src/runtime/__tests__/desktop-presence.test.ts +27 -4
  169. package/src/runtime/__tests__/host-observe.test.ts +302 -0
  170. package/src/runtime/channel-readiness-service.ts +214 -14
  171. package/src/runtime/channel-readiness-types.ts +49 -2
  172. package/src/runtime/channel-reply-delivery.ts +2 -2
  173. package/src/runtime/desktop-presence.ts +24 -21
  174. package/src/runtime/host-observe.ts +246 -0
  175. package/src/runtime/http-server.ts +181 -1
  176. package/src/runtime/migrations/__tests__/staged-import-path.test.ts +104 -0
  177. package/src/runtime/migrations/staged-import-path.ts +116 -0
  178. package/src/runtime/routes/__tests__/app-pin-routes.test.ts +383 -0
  179. package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +80 -0
  180. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +118 -0
  181. package/src/runtime/routes/__tests__/inference-provider-connection-routes.test.ts +20 -0
  182. package/src/runtime/routes/__tests__/ingress-status-routes.test.ts +508 -0
  183. package/src/runtime/routes/__tests__/plugins-routes.test.ts +35 -56
  184. package/src/runtime/routes/__tests__/watch-routes-guardian-cache.test.ts +139 -0
  185. package/src/runtime/routes/__tests__/watch-routes.test.ts +598 -0
  186. package/src/runtime/routes/app-management-routes.ts +140 -29
  187. package/src/runtime/routes/channel-availability-routes.ts +1 -0
  188. package/src/runtime/routes/channel-readiness-routes.ts +14 -2
  189. package/src/runtime/routes/conversation-query-routes.ts +10 -0
  190. package/src/runtime/routes/guardian-approval-interception.ts +24 -33
  191. package/src/runtime/routes/host-cu-routes.ts +18 -0
  192. package/src/runtime/routes/identity-routes.ts +2 -0
  193. package/src/runtime/routes/inbound-message-handler.ts +10 -7
  194. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +166 -308
  195. package/src/runtime/routes/inbound-stages/background-dispatch.ts +158 -335
  196. package/src/runtime/routes/index.ts +2 -0
  197. package/src/runtime/routes/inference-profiles-routes.ts +232 -31
  198. package/src/runtime/routes/inference-provider-connection-routes.ts +24 -4
  199. package/src/runtime/routes/ingress-status-routes.ts +180 -0
  200. package/src/runtime/routes/live-voice-routes.test.ts +40 -1
  201. package/src/runtime/routes/live-voice-routes.ts +34 -0
  202. package/src/runtime/routes/migration-routes.ts +218 -10
  203. package/src/runtime/routes/oauth-commands-routes.ts +23 -16
  204. package/src/runtime/routes/plugins-routes.ts +12 -28
  205. package/src/runtime/routes/question-routes.ts +6 -0
  206. package/src/runtime/routes/secret-routes.ts +7 -27
  207. package/src/runtime/routes/settings-routes.ts +9 -6
  208. package/src/runtime/routes/watch-routes.ts +807 -0
  209. package/src/runtime/slack-reply-session.test.ts +230 -121
  210. package/src/runtime/slack-reply-session.ts +113 -81
  211. package/src/runtime/{slack-task-progress.test.ts → task-progress.test.ts} +1 -28
  212. package/src/runtime/{slack-task-progress.ts → task-progress.ts} +30 -51
  213. package/src/security/__tests__/untrusted-content.test.ts +42 -0
  214. package/src/security/untrusted-content.ts +28 -9
  215. package/src/telemetry/__tests__/live-voice-funnel.test.ts +108 -0
  216. package/src/telemetry/live-voice-funnel.ts +75 -8
  217. package/src/tools/credentials/store.ts +18 -6
  218. package/src/tools/network/__tests__/firecrawl-compat.test.ts +77 -0
  219. package/src/tools/network/__tests__/web-fetch-fastcrw.test.ts +169 -0
  220. package/src/tools/network/__tests__/web-search.test.ts +97 -2
  221. package/src/tools/network/firecrawl-compat.ts +90 -0
  222. package/src/tools/network/web-fetch.ts +142 -62
  223. package/src/tools/network/web-search.ts +141 -55
  224. package/src/tools/types.ts +2 -1
  225. package/src/util/oauth-request-body.test.ts +74 -0
  226. package/src/util/oauth-request-body.ts +60 -0
  227. package/src/util/worker-process.ts +1 -0
  228. package/src/watch/__tests__/watch-retro.test.ts +665 -0
  229. package/src/watch/__tests__/watch-session-manager.test.ts +566 -0
  230. package/src/watch/__tests__/watch-timeline.test.ts +670 -0
  231. package/src/watch/watch-retro.ts +480 -0
  232. package/src/watch/watch-session-manager.ts +575 -0
  233. package/src/watch/watch-timeline.ts +848 -0
@@ -4,8 +4,46 @@ import type { ChannelId } from "../channels/types.js";
4
4
 
5
5
  export type { ChannelId };
6
6
 
7
- /** Setup progress for a channel: not_configured → incomplete → ready. */
8
- export type SetupStatus = "not_configured" | "incomplete" | "ready";
7
+ /**
8
+ * Setup progress for a channel: not_configured → incomplete → ready.
9
+ *
10
+ * Progress only. A channel that is fully configured is `ready` here even when
11
+ * it is not currently working, because "have I finished setting this up" and
12
+ * "is it working right now" are different questions with different answers and
13
+ * different remedies. Operational state is {@link ChannelHealth}.
14
+ */
15
+ export const SETUP_STATUSES = [
16
+ "not_configured",
17
+ "incomplete",
18
+ "ready",
19
+ ] as const;
20
+ export type SetupStatus = (typeof SETUP_STATUSES)[number];
21
+
22
+ /** What a check establishes. */
23
+ export const CHECK_KINDS = [
24
+ /** Whether the channel is configured: credentials, ingress, policy. */
25
+ "configuration",
26
+ /** Whether the channel is currently working: is anything arriving. */
27
+ "operational",
28
+ ] as const;
29
+ export type CheckKind = (typeof CHECK_KINDS)[number];
30
+
31
+ /**
32
+ * Whether a configured channel is currently working.
33
+ *
34
+ * Absent when a channel measures no operational checks at all, which is not
35
+ * the same as failing to establish them: nothing was asked, so nothing is
36
+ * claimed either way, and readiness is decided on configuration alone.
37
+ */
38
+ export const CHANNEL_HEALTHS = [
39
+ /** Every operational check confirmed the channel is working. */
40
+ "ok",
41
+ /** An operational check found a fault. */
42
+ "failing",
43
+ /** Operational checks ran but established nothing either way. */
44
+ "unknown",
45
+ ] as const;
46
+ export type ChannelHealth = (typeof CHANNEL_HEALTHS)[number];
9
47
 
10
48
  /** Result of a single readiness check (local or remote). */
11
49
  export interface ReadinessCheckResult {
@@ -30,13 +68,22 @@ export interface ReadinessCheckResult {
30
68
  * require this to be absent or false.
31
69
  */
32
70
  indeterminate?: boolean;
71
+ /**
72
+ * Which question this check answers. Defaults to `configuration`, so a
73
+ * check that does not say is treated as setup evidence rather than as a
74
+ * claim about whether the channel is working.
75
+ */
76
+ kind?: CheckKind;
33
77
  }
34
78
 
35
79
  /** Point-in-time snapshot of a channel's readiness state. */
36
80
  export interface ChannelReadinessSnapshot {
37
81
  channel: ChannelId;
82
+ /** Configured and confirmed working: `setupStatus === "ready"` and health is not in doubt. */
38
83
  ready: boolean;
39
84
  setupStatus: SetupStatus;
85
+ /** Absent when the channel measures no operational checks. */
86
+ health?: ChannelHealth;
40
87
  checkedAt: number;
41
88
  stale: boolean;
42
89
  reasons: Array<{ code: string; text: string }>;
@@ -217,7 +217,7 @@ export async function deliverRenderedReplyViaCallback(
217
217
  chatId,
218
218
  messageId: editTarget,
219
219
  text: segmentText,
220
- useBlocks: true,
220
+ renderRichly: true,
221
221
  });
222
222
  // An edit replaces the text of one message. Attachments are always new
223
223
  // messages, so they still have to be posted alongside it.
@@ -245,7 +245,7 @@ export async function deliverRenderedReplyViaCallback(
245
245
  result = await deliverChannelReply(callbackUrl, {
246
246
  chatId,
247
247
  text: segmentText,
248
- useBlocks: true,
248
+ renderRichly: true,
249
249
  attachments: segmentAttachments,
250
250
  assistantId,
251
251
  audience,
@@ -2,12 +2,12 @@
2
2
  * Desktop presence policy.
3
3
  *
4
4
  * Answers exactly one question for notification producers: is the user
5
- * demonstrably at their Mac right now? Producers use the answer to skip a
5
+ * demonstrably at their desktop right now? Producers use the answer to skip a
6
6
  * push that the user would see on screen anyway.
7
7
  *
8
8
  * Fails open by design. Presence is in-memory, best-effort, and reported by
9
9
  * a client that can drop off at any time, so anything short of a fresh
10
- * `active` report from a `macos` client answers `false` and the push goes
10
+ * `active` report from a desktop client answers `false` and the push goes
11
11
  * out. A missed notification is strictly worse than a redundant one.
12
12
  */
13
13
 
@@ -15,16 +15,17 @@ import { getLogger } from "../util/logger.js";
15
15
  import { assistantEventHub } from "./assistant-event-hub.js";
16
16
 
17
17
  const log = getLogger("desktop-presence");
18
+ const DESKTOP_INTERFACES = ["macos", "windows"] as const;
18
19
 
19
20
  /**
20
- * Bounds how long suppression can outlive the Mac going to sleep or dropping
21
+ * Bounds how long suppression can outlive the desktop sleeping or dropping
21
22
  * off, while still tolerating two dropped reports at the 30s report interval.
22
23
  */
23
24
  export const PRESENCE_STALE_AFTER_MS = 90_000;
24
25
 
25
26
  export interface DesktopAttendanceOptions {
26
27
  /**
27
- * Only count macOS clients whose verified actor principal matches this id.
28
+ * Only count desktop clients whose verified actor principal matches this id.
28
29
  * A client that connected without a principal (legacy or service token)
29
30
  * never matches a supplied id.
30
31
  */
@@ -34,13 +35,13 @@ export interface DesktopAttendanceOptions {
34
35
  }
35
36
 
36
37
  /**
37
- * Whether some macOS client has reported `active` recently enough to trust.
38
- * Stale, absent, non-macOS, and error reads all answer `false`.
38
+ * Whether some desktop client has reported `active` recently enough to trust.
39
+ * Stale, absent, non-desktop, and error reads all answer `false`.
39
40
  *
40
41
  * Callers whose notification targets one recipient (guardian-scoped or
41
42
  * otherwise per-recipient pushes) must pass that recipient's
42
- * `actorPrincipalId`, or another user's attended Mac suppresses the push.
43
- * Omitting it treats any attended macOS client as attendance, which only
43
+ * `actorPrincipalId`, or another user's attended desktop suppresses the push.
44
+ * Omitting it treats any attended desktop client as attendance, which only
44
45
  * suits notifications with no single recipient.
45
46
  */
46
47
  export function isDesktopAttended(
@@ -48,19 +49,21 @@ export function isDesktopAttended(
48
49
  ): boolean {
49
50
  const { actorPrincipalId, now = new Date() } = options;
50
51
  try {
51
- return assistantEventHub.listClientsByInterface("macos").some((client) => {
52
- if (
53
- actorPrincipalId !== undefined &&
54
- client.actorPrincipalId !== actorPrincipalId
55
- ) {
56
- return false;
57
- }
58
- return (
59
- client.presence?.state === "active" &&
60
- now.getTime() - client.presence.reportedAt.getTime() <=
61
- PRESENCE_STALE_AFTER_MS
62
- );
63
- });
52
+ return DESKTOP_INTERFACES.some((interfaceId) =>
53
+ assistantEventHub.listClientsByInterface(interfaceId).some((client) => {
54
+ if (
55
+ actorPrincipalId !== undefined &&
56
+ client.actorPrincipalId !== actorPrincipalId
57
+ ) {
58
+ return false;
59
+ }
60
+ return (
61
+ client.presence?.state === "active" &&
62
+ now.getTime() - client.presence.reportedAt.getTime() <=
63
+ PRESENCE_STALE_AFTER_MS
64
+ );
65
+ }),
66
+ );
64
67
  } catch (err) {
65
68
  // Returning false sends the push, which is the safe direction.
66
69
  log.warn({ err }, "desktop presence read failed; treating as unattended");
@@ -0,0 +1,246 @@
1
+ /**
2
+ * Screen observation raised by the daemon outside any conversation or tool
3
+ * call.
4
+ *
5
+ * `computer_use_observe` normally reaches the desktop client through
6
+ * `HostCuProxy`, which only exists inside an agent turn. This helper drives the
7
+ * `host_cu` wire protocol directly for callers that have no turn to hang the
8
+ * request off: it mints a requestId, registers a pending interaction,
9
+ * broadcasts a `host_cu_request` envelope to a single client, and awaits that
10
+ * client's POST to `/v1/host-cu-result`.
11
+ *
12
+ * Every request is bound to the actor principal that initiated it, the same
13
+ * binding `HostBashProxy.request()` applies. The target client is either
14
+ * auto-resolved among that actor's own `host_cu` clients
15
+ * ({@link pickSameUserAutoResolve}) or, when named explicitly, checked against
16
+ * the actor before dispatch ({@link enforceSameActorOrErrorResult}), so a
17
+ * caller can only capture the accessibility tree and screenshot of a machine
18
+ * its own authenticated user is signed in on.
19
+ *
20
+ * Every failure path resolves to `{ ok: false, reason }` rather than throwing:
21
+ * a caller observing in the background must degrade, never crash.
22
+ */
23
+
24
+ import { randomUUID } from "node:crypto";
25
+
26
+ import { getLogger } from "../util/logger.js";
27
+ import { assistantEventHub, broadcastMessage } from "./assistant-event-hub.js";
28
+ import {
29
+ ambiguousSameUserError,
30
+ enforceSameActorOrErrorResult,
31
+ pickSameUserAutoResolve,
32
+ } from "./auth/same-actor.js";
33
+ import * as pendingInteractions from "./pending-interactions.js";
34
+
35
+ const log = getLogger("host-observe");
36
+
37
+ const DEFAULT_OBSERVE_TIMEOUT_MS = 30_000;
38
+ const MAX_OBSERVE_TIMEOUT_MS = 120_000;
39
+
40
+ const NO_CLIENT_REASON =
41
+ "No connected client supports screen observation for this user. Make sure the desktop app is running and signed in.";
42
+
43
+ /**
44
+ * Observation fields returned by the host CU executor. Mirrors the optional
45
+ * fields of `CU_RESULT_SCHEMA` (see
46
+ * `packages/electron-desktop/src/host-proxy/cu-executor.ts`) so the executor's
47
+ * result deserializes as-is. `executionResult` and `secondaryWindows` are
48
+ * omitted: an observe-only request executes no action, so they carry nothing
49
+ * for the caller.
50
+ */
51
+ export interface HostObservationFields {
52
+ axTree?: string;
53
+ axDiff?: string;
54
+ screenshot?: string;
55
+ screenshotWidthPx?: number;
56
+ screenshotHeightPx?: number;
57
+ screenWidthPt?: number;
58
+ screenHeightPt?: number;
59
+ executionError?: string;
60
+ }
61
+
62
+ /** A successful observation, or a structured failure. Never throws. */
63
+ export type HostObservation =
64
+ | ({ ok: true } & HostObservationFields)
65
+ | { ok: false; reason: string; timedOut?: boolean };
66
+
67
+ export interface ObserveHostScreenOptions {
68
+ /**
69
+ * Principal id of the actor on whose behalf the observation is taken.
70
+ * Required, and the only identity the target client is matched against.
71
+ * `undefined` (no authenticated actor) fails closed: it selects no client
72
+ * and rejects any explicitly named one.
73
+ */
74
+ sourceActorPrincipalId: string | undefined;
75
+ /** How long to wait for the client. Defaults to 30s, capped at 120s. */
76
+ timeoutMs?: number;
77
+ /** Keep the base64 screenshot in the result. Defaults to true. */
78
+ includeScreenshot?: boolean;
79
+ /**
80
+ * Target a specific client. Must belong to `sourceActorPrincipalId`.
81
+ * Defaults to the actor's own `host_cu` client when exactly one is
82
+ * connected.
83
+ */
84
+ clientId?: string;
85
+ }
86
+
87
+ /**
88
+ * Failure raised by this module's own lifecycle paths (timeout, send failure),
89
+ * as opposed to an observation delivered by the client. Distinguished from
90
+ * {@link HostObservationFields} by its `failureReason` key.
91
+ */
92
+ interface LocalFailure {
93
+ failureReason: string;
94
+ timedOut: boolean;
95
+ }
96
+
97
+ /**
98
+ * Resolve the `host_cu` client to observe, bound to the initiating actor.
99
+ * Returns the client id, or the failure to hand back to the caller.
100
+ */
101
+ function resolveObserveTarget(
102
+ sourceActorPrincipalId: string | undefined,
103
+ clientId: string | undefined,
104
+ ): { clientId: string } | { reason: string } {
105
+ let resolvedClientId: string;
106
+ if (clientId) {
107
+ const target = assistantEventHub.getClientById(clientId);
108
+ if (!target?.capabilities.includes("host_cu")) {
109
+ return {
110
+ reason: `Client "${clientId}" is not connected or does not support host_cu.`,
111
+ };
112
+ }
113
+ resolvedClientId = clientId;
114
+ } else {
115
+ // Auto-resolve to the unique same-user client. Refusing the ambiguous case
116
+ // keeps one observation from fanning out across every machine the user has
117
+ // connected.
118
+ const resolved = pickSameUserAutoResolve({
119
+ hub: assistantEventHub,
120
+ capability: "host_cu",
121
+ sourceActorPrincipalId,
122
+ });
123
+ if (resolved.kind === "ambiguous") {
124
+ return { reason: ambiguousSameUserError("host_cu").content };
125
+ }
126
+ if (resolved.kind === "none") {
127
+ return { reason: NO_CLIENT_REASON };
128
+ }
129
+ resolvedClientId = resolved.clientId;
130
+ }
131
+
132
+ // Fail closed before registration and before broadcast, so no caller reaches
133
+ // a client whose authenticated user is not its own.
134
+ const rejection = enforceSameActorOrErrorResult({
135
+ hub: assistantEventHub,
136
+ sourceActorPrincipalId,
137
+ targetClientId: resolvedClientId,
138
+ op: "host_cu",
139
+ });
140
+ if (rejection) {
141
+ return { reason: rejection.content };
142
+ }
143
+ return { clientId: resolvedClientId };
144
+ }
145
+
146
+ /**
147
+ * Ask the initiating actor's `host_cu`-capable client for the current screen
148
+ * state.
149
+ */
150
+ export async function observeHostScreen(
151
+ options: ObserveHostScreenOptions,
152
+ ): Promise<HostObservation> {
153
+ const { sourceActorPrincipalId, includeScreenshot = true } = options;
154
+ const timeoutMs = Math.min(
155
+ options.timeoutMs ?? DEFAULT_OBSERVE_TIMEOUT_MS,
156
+ MAX_OBSERVE_TIMEOUT_MS,
157
+ );
158
+
159
+ const target = resolveObserveTarget(sourceActorPrincipalId, options.clientId);
160
+ if ("reason" in target) {
161
+ return { ok: false, reason: target.reason };
162
+ }
163
+ const targetClientId = target.clientId;
164
+
165
+ const requestId = randomUUID();
166
+
167
+ // `/v1/host-cu-result` resolves this promise with the client's raw
168
+ // observation fields; the lifecycle paths below resolve it with a failure.
169
+ const payload = await new Promise<HostObservationFields | LocalFailure>(
170
+ (resolvePromise) => {
171
+ const timer = setTimeout(() => {
172
+ // Resolve the tracker first so a late client POST is tolerated as a
173
+ // no-op instead of double-resolving this promise.
174
+ if (!pendingInteractions.resolve(requestId, "cancelled")) {
175
+ return;
176
+ }
177
+ broadcastMessage(
178
+ { type: "host_cu_cancel", requestId, conversationId: "" },
179
+ undefined,
180
+ { targetClientId },
181
+ );
182
+ resolvePromise({
183
+ failureReason: `Timed out after ${timeoutMs}ms waiting for the desktop client. It may be busy, outdated, or disconnected.`,
184
+ timedOut: true,
185
+ });
186
+ }, timeoutMs);
187
+
188
+ // The target's actor principal is present by construction: the same-actor
189
+ // check above rejects a client that registered without one.
190
+ const targetActorPrincipalId =
191
+ assistantEventHub.getActorPrincipalIdForClient(targetClientId);
192
+ pendingInteractions.register(requestId, {
193
+ kind: "host_cu",
194
+ rpcResolve: resolvePromise as (value: unknown) => void,
195
+ timer,
196
+ targetClientId,
197
+ ...(targetActorPrincipalId ? { targetActorPrincipalId } : {}),
198
+ });
199
+
200
+ try {
201
+ broadcastMessage(
202
+ {
203
+ type: "host_cu_request",
204
+ requestId,
205
+ // Conversation-agnostic. The native helper keys its per-session
206
+ // state off this id, and an empty string is already what the
207
+ // desktop executor sends when no conversation is attached.
208
+ conversationId: "",
209
+ toolName: "computer_use_observe",
210
+ input: {},
211
+ stepNumber: 1,
212
+ },
213
+ undefined,
214
+ { targetClientId },
215
+ );
216
+ } catch (err) {
217
+ pendingInteractions.resolve(requestId, "cancelled");
218
+ log.warn({ requestId, err }, "Screen observation broadcast failed");
219
+ resolvePromise({
220
+ failureReason: `Failed to reach the desktop client: ${String(err)}`,
221
+ timedOut: false,
222
+ });
223
+ }
224
+ },
225
+ );
226
+
227
+ if ("failureReason" in payload) {
228
+ return {
229
+ ok: false,
230
+ reason: payload.failureReason,
231
+ ...(payload.timedOut ? { timedOut: true } : {}),
232
+ };
233
+ }
234
+
235
+ const { executionError, ...observation } = payload;
236
+ if (executionError) {
237
+ return { ok: false, reason: executionError };
238
+ }
239
+
240
+ if (!includeScreenshot) {
241
+ delete observation.screenshot;
242
+ delete observation.screenshotWidthPx;
243
+ delete observation.screenshotHeightPx;
244
+ }
245
+ return { ok: true, ...observation };
246
+ }
@@ -7,6 +7,10 @@
7
7
 
8
8
  import type { ServerWebSocket } from "bun";
9
9
 
10
+ import {
11
+ startAppPinReconcileSweep,
12
+ stopAppPinReconcileSweep,
13
+ } from "../apps/app-pin-reconciler.js";
10
14
  import {
11
15
  activeMediaStreamSessions,
12
16
  MediaStreamCallSession,
@@ -93,6 +97,12 @@ import {
93
97
  startInferenceProfileSessionReaper,
94
98
  stopInferenceProfileSessionReaper,
95
99
  } from "./routes/inference-profile-session-reaper.js";
100
+ import {
101
+ activeWatchStreamSessions,
102
+ closeWatchIngress,
103
+ drainWatchRetros,
104
+ WatchStreamSession,
105
+ } from "./routes/watch-routes.js";
96
106
 
97
107
  // Re-export for consumers
98
108
  export { isPrivateAddress } from "./middleware/auth.js";
@@ -166,6 +176,25 @@ interface LiveVoiceWebSocketData {
166
176
  wsType: "live-voice";
167
177
  }
168
178
 
179
+ /**
180
+ * WebSocket data attached to `/v1/watch/stream` connections. The `wsType`
181
+ * discriminator routes frames to the watch session orchestrator instead of
182
+ * the other WebSocket handlers.
183
+ */
184
+ interface WatchStreamWebSocketData {
185
+ wsType: "watch-stream";
186
+ mimeType: string;
187
+ sampleRate?: number;
188
+ /** Conversation the session's timeline is keyed on, when the client names one. */
189
+ conversationId?: string;
190
+ /** Desktop client to observe, when the actor has more than one connected. */
191
+ clientId?: string;
192
+ /** The session ID for tracking in the active sessions registry. */
193
+ sessionId: string;
194
+ /** Bound at open time so the close handler tears down the exact session. */
195
+ session?: WatchStreamSession;
196
+ }
197
+
169
198
  export class RuntimeHttpServer {
170
199
  private server: ReturnType<typeof Bun.serve> | null = null;
171
200
  private port: number;
@@ -204,7 +233,8 @@ export class RuntimeHttpServer {
204
233
  type AllWebSocketData =
205
234
  | MediaStreamWebSocketData
206
235
  | SttStreamWebSocketData
207
- | LiveVoiceWebSocketData;
236
+ | LiveVoiceWebSocketData
237
+ | WatchStreamWebSocketData;
208
238
  this.server = Bun.serve<AllWebSocketData>({
209
239
  port: this.port,
210
240
  hostname: this.hostname,
@@ -310,11 +340,41 @@ export class RuntimeHttpServer {
310
340
  send: (frame) => {
311
341
  ws.send(JSON.stringify(frame));
312
342
  },
343
+ // Lets the daemon hang up on a client that stopped answering.
344
+ // A normal close (not a retryable one) so the client ends the
345
+ // call rather than reconnecting into a session that is gone.
346
+ close: () => {
347
+ ws.close(1000, "Live voice session released");
348
+ },
313
349
  }),
314
350
  );
315
351
  log.info("Live voice WebSocket opened");
316
352
  return;
317
353
  }
354
+ if (data.wsType === "watch-stream") {
355
+ const watchData = data;
356
+ log.info(
357
+ {
358
+ sessionId: watchData.sessionId,
359
+ mimeType: watchData.mimeType,
360
+ },
361
+ "Watch stream WebSocket opened",
362
+ );
363
+ const session = new WatchStreamSession(ws, {
364
+ mimeType: watchData.mimeType,
365
+ ...(watchData.sampleRate !== undefined
366
+ ? { sampleRate: watchData.sampleRate }
367
+ : {}),
368
+ ...(watchData.conversationId
369
+ ? { conversationId: watchData.conversationId }
370
+ : {}),
371
+ ...(watchData.clientId ? { clientId: watchData.clientId } : {}),
372
+ });
373
+ watchData.session = session;
374
+ activeWatchStreamSessions.set(watchData.sessionId, session);
375
+ void session.start();
376
+ return;
377
+ }
318
378
  log.warn("WebSocket opened with unknown data type — closing");
319
379
  ws.close(1008, "Unknown WebSocket type");
320
380
  },
@@ -353,6 +413,18 @@ export class RuntimeHttpServer {
353
413
  void connection?.handleMessage(message);
354
414
  return;
355
415
  }
416
+ if (data.wsType === "watch-stream") {
417
+ const session = data.session;
418
+ if (!session) {
419
+ return;
420
+ }
421
+ if (typeof message === "string") {
422
+ session.handleMessage(message);
423
+ } else {
424
+ session.handleBinaryAudio(message);
425
+ }
426
+ return;
427
+ }
356
428
  log.warn("WebSocket message on unknown data type — closing");
357
429
  ws.close(1008, "Unknown WebSocket type");
358
430
  },
@@ -423,6 +495,29 @@ export class RuntimeHttpServer {
423
495
  connection?.release();
424
496
  return;
425
497
  }
498
+ if (data.wsType === "watch-stream") {
499
+ const watchData = data;
500
+ log.info(
501
+ {
502
+ sessionId: watchData.sessionId,
503
+ code,
504
+ reason: reason?.toString(),
505
+ },
506
+ "Watch stream WebSocket closed",
507
+ );
508
+ const session = watchData.session;
509
+ if (session) {
510
+ session.handleClose(code, reason?.toString());
511
+ // Only drop our own session from the registry, since a
512
+ // reconnect may have already replaced it under a new id.
513
+ if (
514
+ activeWatchStreamSessions.get(watchData.sessionId) === session
515
+ ) {
516
+ activeWatchStreamSessions.delete(watchData.sessionId);
517
+ }
518
+ }
519
+ return;
520
+ }
426
521
  log.warn(
427
522
  { code, reason: reason?.toString() },
428
523
  "WebSocket with unknown data type closed",
@@ -523,10 +618,16 @@ export class RuntimeHttpServer {
523
618
  // migrations are unready.
524
619
  startPluginScheduleReconcileSweep();
525
620
  log.info("Plugin schedule reconcile sweep started");
621
+
622
+ // Same backstop for sidebar pins: a disabled plugin's app stops existing,
623
+ // and its id returns intact when the plugin is re-enabled.
624
+ startAppPinReconcileSweep();
625
+ log.info("App pin reconcile sweep started");
526
626
  }
527
627
 
528
628
  async stop(): Promise<void> {
529
629
  stopGuardianExpirySweep();
630
+ stopAppPinReconcileSweep();
530
631
  stopInferenceProfileSessionReaper();
531
632
  stopTelegramWebhookHealthSweep();
532
633
  stopPluginScheduleReconcileSweep();
@@ -543,6 +644,20 @@ export class RuntimeHttpServer {
543
644
  activeSttStreamSessions.delete(sessionId);
544
645
  }
545
646
 
647
+ // Watch shuts down in order: refuse new sessions, tear down the open ones,
648
+ // then wait on the retrospectives they left running. Refusing first is what
649
+ // makes the wait meaningful, because the Bun server below keeps accepting
650
+ // connections until the very end and a session opened during the wait would
651
+ // register a retrospective after it had already taken its snapshot.
652
+ closeWatchIngress();
653
+ for (const [sessionId, session] of activeWatchStreamSessions) {
654
+ session.destroy();
655
+ activeWatchStreamSessions.delete(sessionId);
656
+ }
657
+ // Destroying a session starts no retrospective, so this waits only on one
658
+ // that a socket closing just before shutdown had already under way.
659
+ await drainWatchRetros();
660
+
546
661
  const liveVoiceManager = getLiveVoiceSessionManager();
547
662
  const liveVoiceSessionId = liveVoiceManager.activeSessionId;
548
663
  if (liveVoiceSessionId) {
@@ -636,6 +751,16 @@ export class RuntimeHttpServer {
636
751
  return this.handleLiveVoiceUpgrade(req, server);
637
752
  }
638
753
 
754
+ // WebSocket upgrade for watch narration capture, under the same
755
+ // private-network restrictions and gateway-service token verification as
756
+ // STT streaming.
757
+ if (
758
+ path === "/v1/watch/stream" &&
759
+ req.headers.get("upgrade")?.toLowerCase() === "websocket"
760
+ ) {
761
+ return this.handleWatchStreamUpgrade(req, server);
762
+ }
763
+
639
764
  // Twilio webhook endpoints — before auth check because Twilio
640
765
  // webhook POSTs don't include bearer tokens.
641
766
  const twilioResponse = await this.handleTwilioWebhook(req, path);
@@ -923,6 +1048,61 @@ export class RuntimeHttpServer {
923
1048
  return undefined!;
924
1049
  }
925
1050
 
1051
+ /**
1052
+ * Handle WebSocket upgrade for `/v1/watch/stream`.
1053
+ *
1054
+ * Gated exactly as `/v1/stt/stream` is: private network peers and origins
1055
+ * only, then a gateway service token. The gateway owns downstream client
1056
+ * auth and dials this upstream on the client's behalf.
1057
+ */
1058
+ private handleWatchStreamUpgrade(
1059
+ req: Request,
1060
+ server: ReturnType<typeof Bun.serve>,
1061
+ ): Response {
1062
+ if (!isPrivateNetworkPeer(server, req) || !isPrivateNetworkOrigin(req)) {
1063
+ return httpError(
1064
+ "FORBIDDEN",
1065
+ "Direct watch stream access disabled: only private network peers allowed",
1066
+ 403,
1067
+ );
1068
+ }
1069
+
1070
+ const tokenError = this.verifyGatewayServiceToken(req);
1071
+ if (tokenError) {
1072
+ return tokenError;
1073
+ }
1074
+
1075
+ const wsUrl = new URL(req.url);
1076
+ const mimeType = wsUrl.searchParams.get("mimeType");
1077
+ if (!mimeType) {
1078
+ return new Response("Missing required query parameter: mimeType", {
1079
+ status: 400,
1080
+ });
1081
+ }
1082
+
1083
+ const sampleRateRaw = wsUrl.searchParams.get("sampleRate");
1084
+ const sampleRate = sampleRateRaw ? parseInt(sampleRateRaw, 10) : undefined;
1085
+ const conversationId =
1086
+ wsUrl.searchParams.get("conversationId")?.trim() || undefined;
1087
+ const clientId = wsUrl.searchParams.get("clientId")?.trim() || undefined;
1088
+
1089
+ const upgraded = server.upgrade(req, {
1090
+ data: {
1091
+ wsType: "watch-stream",
1092
+ mimeType,
1093
+ sampleRate,
1094
+ conversationId,
1095
+ clientId,
1096
+ sessionId: crypto.randomUUID(),
1097
+ } satisfies WatchStreamWebSocketData,
1098
+ });
1099
+ if (!upgraded) {
1100
+ return new Response("WebSocket upgrade failed", { status: 500 });
1101
+ }
1102
+ // Bun's WebSocket upgrade consumes the request, so no Response is sent.
1103
+ return undefined!;
1104
+ }
1105
+
926
1106
  private async handleTwilioWebhook(
927
1107
  req: Request,
928
1108
  path: string,