@vellumai/assistant 0.12.3 → 0.12.4-dev.202609231620.a15f321

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (209) hide show
  1. package/Dockerfile +7 -0
  2. package/README.md +12 -12
  3. package/docker/mousepad/90-vellum.gschema.override +4 -0
  4. package/node_modules/@vellumai/avatar-manifest/src/__tests__/notification-avatar.test.ts +48 -4
  5. package/node_modules/@vellumai/avatar-manifest/src/notification-avatar.ts +12 -8
  6. package/openapi.yaml +97 -4
  7. package/package.json +1 -1
  8. package/src/__tests__/approval-routes-http.test.ts +4 -2
  9. package/src/__tests__/call-routes-http.test.ts +4 -2
  10. package/src/__tests__/channel-readiness-discord.test.ts +8 -0
  11. package/src/__tests__/channel-readiness-slack-remote.test.ts +8 -0
  12. package/src/__tests__/config-loader-backfill.test.ts +1 -0
  13. package/src/__tests__/conversation-pairing.test.ts +10 -0
  14. package/src/__tests__/conversation-resurface-done.test.ts +289 -0
  15. package/src/__tests__/default-profile-catalog-fallback.test.ts +1 -0
  16. package/src/__tests__/gemini-provider.test.ts +31 -0
  17. package/src/__tests__/inference-profile-validation-jev.test.ts +17 -0
  18. package/src/__tests__/job-handler-registration-guard.test.ts +1 -0
  19. package/src/__tests__/llm-catalog-parity.test.ts +4 -0
  20. package/src/__tests__/llm-resolver-override-or-default.test.ts +43 -0
  21. package/src/__tests__/openai-responses-prompt-cache.test.ts +3 -1
  22. package/src/__tests__/platform-callback-registration.test.ts +46 -1
  23. package/src/__tests__/plugin-api-model-profiles.test.ts +7 -0
  24. package/src/__tests__/plugin-import-boundary-guard.test.ts +3 -0
  25. package/src/__tests__/pricing.test.ts +33 -0
  26. package/src/__tests__/provider-error-scenarios.test.ts +46 -0
  27. package/src/__tests__/provider-platform-proxy-integration.test.ts +45 -15
  28. package/src/__tests__/retry-fallback-escalation.test.ts +22 -0
  29. package/src/__tests__/runtime-attachment-metadata.test.ts +4 -2
  30. package/src/__tests__/scaffold-managed-skill-tool.test.ts +116 -66
  31. package/src/__tests__/secret-routes-platform-proxy.test.ts +1 -0
  32. package/src/__tests__/send-endpoint-busy.test.ts +4 -2
  33. package/src/__tests__/skill-load-inline-command.test.ts +35 -0
  34. package/src/__tests__/skill-load-tool.test.ts +42 -11
  35. package/src/__tests__/skill-tool-factory.test.ts +50 -2
  36. package/src/__tests__/steer-on-enqueue-question.test.ts +265 -34
  37. package/src/__tests__/validate-input.test.ts +73 -0
  38. package/src/__tests__/workspace-migration-159-rename-colliding-jev-profile-name.test.ts +197 -0
  39. package/src/api/ask-question-tool.ts +50 -0
  40. package/src/api/events/tool-result.ts +11 -0
  41. package/src/api/index.ts +7 -0
  42. package/src/api/requests/dictation.ts +2 -2
  43. package/src/api/responses/home.ts +27 -0
  44. package/src/avatar/notification-avatar.test.ts +42 -34
  45. package/src/avatar/notification-avatar.ts +1 -0
  46. package/src/calls/__tests__/voice-session-bridge.test.ts +63 -1
  47. package/src/calls/voice-escalation-judge.ts +6 -5
  48. package/src/calls/voice-screen-annotation.ts +44 -0
  49. package/src/calls/voice-session-bridge.ts +22 -0
  50. package/src/cli/commands/__tests__/plugins.test.ts +29 -1
  51. package/src/cli/commands/channel-verification-sessions.help.ts +12 -2
  52. package/src/cli/commands/platform/index.help.ts +2 -1
  53. package/src/cli/commands/platform/index.ts +8 -2
  54. package/src/cli/commands/plugins.ts +16 -0
  55. package/src/cli/lib/__tests__/inspect-plugin.test.ts +29 -0
  56. package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +92 -2
  57. package/src/cli/lib/__tests__/plugin-catalog-local.test.ts +15 -0
  58. package/src/cli/lib/__tests__/plugin-catalog-platform.test.ts +66 -2
  59. package/src/cli/lib/__tests__/plugin-catalog-resolve.test.ts +21 -0
  60. package/src/cli/lib/__tests__/plugin-catalog-visibility.test.ts +65 -0
  61. package/src/cli/lib/__tests__/plugin-details.test.ts +43 -0
  62. package/src/cli/lib/__tests__/plugins-install-offline.test.ts +26 -0
  63. package/src/cli/lib/bundled-marketplace.json +4 -4
  64. package/src/cli/lib/inspect-plugin.ts +5 -0
  65. package/src/cli/lib/plugin-catalog-cache.ts +41 -4
  66. package/src/cli/lib/plugin-catalog-local.ts +9 -0
  67. package/src/cli/lib/plugin-catalog-platform.ts +48 -9
  68. package/src/cli/lib/plugin-catalog-visibility.ts +41 -0
  69. package/src/cli/lib/plugin-details.ts +10 -7
  70. package/src/cli/lib/search-plugins.ts +2 -0
  71. package/src/config/__tests__/default-profile-catalog.test.ts +61 -1
  72. package/src/config/__tests__/jev-managed-profile.test.ts +82 -0
  73. package/src/config/__tests__/sync-gated-profiles.test.ts +71 -0
  74. package/src/config/bundled-skills/screen-annotation/SKILL.md +6 -3
  75. package/src/config/call-site-defaults.ts +17 -11
  76. package/src/config/default-profile-catalog.ts +118 -19
  77. package/src/config/default-profile-names.ts +69 -0
  78. package/src/config/feature-flag-registry.json +25 -9
  79. package/src/config/inference-profile-validation.ts +7 -6
  80. package/src/config/llm-resolver.ts +16 -0
  81. package/src/config/profile-text-generation.test.ts +9 -0
  82. package/src/config/schemas/call-site-catalog.ts +8 -1
  83. package/src/config/schemas/llm.ts +70 -9
  84. package/src/config/sidebar-done-gate.ts +27 -0
  85. package/src/config/sync-gated-profiles.ts +96 -44
  86. package/src/daemon/__tests__/auto-profile-router.test.ts +228 -0
  87. package/src/daemon/auto-profile-router.ts +220 -0
  88. package/src/daemon/conversation-agent-loop.ts +77 -4
  89. package/src/daemon/conversation-slash.ts +2 -2
  90. package/src/daemon/conversation-turn-finalize.ts +12 -0
  91. package/src/daemon/dictation-text-processing.ts +4 -4
  92. package/src/daemon/handlers/__tests__/conversation-history-search.test.ts +61 -0
  93. package/src/daemon/handlers/config-channels.ts +3 -1
  94. package/src/daemon/handlers/conversation-history.ts +11 -1
  95. package/src/daemon/handlers/conversations.ts +118 -10
  96. package/src/daemon/pkb-reminder-builder.test.ts +10 -3
  97. package/src/daemon/pkb-reminder-builder.ts +1 -1
  98. package/src/desktop/desktop-browser-endpoint.ts +1 -0
  99. package/src/home/feed-types.ts +2 -0
  100. package/src/inbound/platform-callback-registration.ts +19 -6
  101. package/src/ipc/routes/__tests__/conversation-sync-ipc-routes.test.ts +49 -1
  102. package/src/ipc/routes/conversation-sync-ipc-routes.ts +38 -2
  103. package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +27 -0
  104. package/src/live-voice/__tests__/protocol.test.ts +22 -0
  105. package/src/live-voice/continuation-judge.ts +1 -1
  106. package/src/live-voice/live-voice-session.ts +7 -6
  107. package/src/live-voice/protocol.ts +16 -5
  108. package/src/messaging/providers/gmail/client.ts +4 -0
  109. package/src/notifications/README.md +11 -3
  110. package/src/notifications/__tests__/home-feed-side-effect.test.ts +87 -3
  111. package/src/notifications/conversation-pairing.ts +23 -8
  112. package/src/notifications/home-feed-mirror.ts +39 -0
  113. package/src/notifications/home-feed-side-effect.ts +46 -19
  114. package/src/oauth/AGENTS.md +7 -1
  115. package/src/permissions/question-prompter.ts +14 -4
  116. package/src/persistence/conversation-crud.ts +114 -3
  117. package/src/persistence/conversation-queries.ts +48 -3
  118. package/src/persistence/embeddings/__tests__/embedding-local-lifecycle.test.ts +228 -32
  119. package/src/persistence/embeddings/embedding-backend-shutdown.test.ts +47 -0
  120. package/src/persistence/embeddings/embedding-backend.ts +23 -41
  121. package/src/persistence/embeddings/embedding-gemini.test.ts +17 -0
  122. package/src/persistence/embeddings/embedding-gemini.ts +3 -1
  123. package/src/persistence/embeddings/embedding-local.ts +72 -17
  124. package/src/persistence/embeddings/embedding-types.ts +0 -19
  125. package/src/persistence/jobs-store.ts +229 -0
  126. package/src/persistence/migrations/383-normalize-opencode-host-connections.ts +44 -0
  127. package/src/persistence/migrations/__tests__/383-normalize-opencode-host-connections.test.ts +102 -0
  128. package/src/persistence/steps.ts +12 -0
  129. package/src/platform/sync-avatar.test.ts +29 -0
  130. package/src/plugin-api/model-profiles.ts +5 -3
  131. package/src/plugins/defaults/memory/AGENTS.md +5 -3
  132. package/src/plugins/defaults/memory/__tests__/memory-retrospective-skill-card.test.ts +2 -0
  133. package/src/plugins/defaults/memory/__tests__/skill-update-receipt-boundary-guard.test.ts +45 -0
  134. package/src/plugins/defaults/memory/__tests__/skill-update-receipt-job.test.ts +580 -0
  135. package/src/plugins/defaults/memory/graph/tools.ts +9 -5
  136. package/src/plugins/defaults/memory/hooks/conversation-deleted.ts +4 -0
  137. package/src/plugins/defaults/memory/job-handlers.ts +8 -2
  138. package/src/plugins/defaults/memory/memory-retrospective-skill-card.ts +5 -0
  139. package/src/plugins/defaults/memory/skill-update-receipt-job.ts +334 -0
  140. package/src/plugins/defaults/memory/substrate/skill-content.ts +1 -1
  141. package/src/plugins/defaults/memory/tools.test.ts +17 -0
  142. package/src/plugins/defaults/memory/v3/__tests__/render-injection.test.ts +3 -0
  143. package/src/providers/__tests__/connection-model-compat.test.ts +2 -0
  144. package/src/providers/__tests__/search-provider-catalog.test.ts +8 -0
  145. package/src/providers/anthropic/client.ts +2 -2
  146. package/src/providers/gemini/client.ts +17 -1
  147. package/src/providers/inference/auth.ts +25 -0
  148. package/src/providers/jev/ask.test.ts +66 -0
  149. package/src/{calls/typesafe-noul.ts → providers/jev/ask.ts} +75 -36
  150. package/src/providers/malformed-tool-call.ts +55 -0
  151. package/src/providers/model-catalog.ts +257 -4
  152. package/src/providers/openai/__tests__/chat-completions-malformed-tool-call.test.ts +105 -0
  153. package/src/providers/openai/chat-completions-provider.ts +30 -0
  154. package/src/providers/openai/codex-models.ts +2 -0
  155. package/src/providers/platform-proxy/constants.ts +5 -0
  156. package/src/providers/retry.ts +68 -36
  157. package/src/providers/search-provider-catalog.ts +10 -0
  158. package/src/providers/vellum-model-routing.test.ts +1 -0
  159. package/src/runtime/__tests__/agent-wake-archived-gate.test.ts +90 -0
  160. package/src/runtime/__tests__/question-resolution.test.ts +32 -0
  161. package/src/runtime/agent-wake.ts +9 -2
  162. package/src/runtime/auth/__tests__/middleware.test.ts +138 -0
  163. package/src/runtime/auth/middleware.ts +15 -11
  164. package/src/runtime/middleware/auth.ts +7 -1
  165. package/src/runtime/pending-interactions.ts +8 -8
  166. package/src/runtime/question-resolution.ts +54 -8
  167. package/src/runtime/routes/__tests__/conversation-list-routes.test.ts +146 -0
  168. package/src/runtime/routes/__tests__/dictation-processing.test.ts +174 -0
  169. package/src/runtime/routes/__tests__/global-search-routes.test.ts +74 -1
  170. package/src/runtime/routes/__tests__/inference-provider-connection-routes.test.ts +60 -0
  171. package/src/runtime/routes/__tests__/llm-call-sites-routes.test.ts +16 -1
  172. package/src/runtime/routes/__tests__/plugins-routes.test.ts +8 -0
  173. package/src/runtime/routes/conversation-list-routes.ts +16 -14
  174. package/src/runtime/routes/conversation-query-routes.ts +4 -0
  175. package/src/runtime/routes/conversation-routes.ts +4 -2
  176. package/src/runtime/routes/diagnostics-routes.ts +66 -229
  177. package/src/runtime/routes/global-search-routes.ts +22 -7
  178. package/src/runtime/routes/inference-provider-connection-routes.ts +25 -8
  179. package/src/runtime/routes/llm-call-sites-routes.ts +19 -10
  180. package/src/runtime/routes/platform-routes.ts +6 -1
  181. package/src/runtime/routes/plugins-routes.ts +5 -2
  182. package/src/runtime/sync/documents-sidecar-publish.test.ts +50 -4
  183. package/src/runtime/sync/resource-sync-events.ts +11 -0
  184. package/src/runtime/sync/worker-daemon-notify.test.ts +30 -0
  185. package/src/runtime/sync/worker-daemon-notify.ts +44 -0
  186. package/src/runtime/verification-outbound-actions.ts +2 -2
  187. package/src/skills/validate-input.ts +75 -11
  188. package/src/telegram/__tests__/webhook-health.test.ts +8 -0
  189. package/src/tools/ask-question/ask-question-tool.test.ts +38 -0
  190. package/src/tools/ask-question/ask-question-tool.ts +12 -1
  191. package/src/tools/network/__tests__/web-fetch-firecrawl.test.ts +28 -1
  192. package/src/tools/network/__tests__/web-fetch-metadata.test.ts +34 -0
  193. package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +35 -1
  194. package/src/tools/network/__tests__/web-search.test.ts +96 -0
  195. package/src/tools/network/web-fetch.ts +30 -9
  196. package/src/tools/network/web-search.ts +235 -4
  197. package/src/tools/skills/execute.ts +1 -1
  198. package/src/tools/skills/format-input-schema.ts +15 -0
  199. package/src/tools/skills/load.ts +15 -43
  200. package/src/tools/skills/scaffold-managed.ts +86 -71
  201. package/src/tools/skills/skill-tool-factory.ts +2 -1
  202. package/src/tools/workflows/manage-workflows.ts +4 -2
  203. package/src/util/worker-pipe.test.ts +146 -0
  204. package/src/util/worker-pipe.ts +49 -0
  205. package/src/workspace/migrations/158-repoint-opencode-host-provider-fragments.ts +163 -0
  206. package/src/workspace/migrations/159-rename-colliding-jev-profile-name.ts +344 -0
  207. package/src/workspace/migrations/__tests__/158-repoint-opencode-host-provider-fragments.test.ts +188 -0
  208. package/src/workspace/migrations/registry.ts +4 -0
  209. package/src/runtime/routes/__tests__/dictation-cleanup-guard.test.ts +0 -107
@@ -54,6 +54,7 @@ export const WebSearchProviderIdSchema = z.enum([
54
54
  "fastcrw",
55
55
  "searxng",
56
56
  "tinyfish",
57
+ "exa",
57
58
  ]);
58
59
 
59
60
  export type WebSearchProviderId = z.infer<typeof WebSearchProviderIdSchema>;
@@ -112,6 +113,16 @@ export const WebFetchMetadataSchema = z.object({
112
113
  durationMs: z.number(),
113
114
  errorMessage: z.string().optional(),
114
115
  mayRequireJavaScript: z.boolean().optional(),
116
+ /**
117
+ * The requested `start_index` was past the end of the page, so nothing was
118
+ * read. Always set, true or false, by an assistant whose metadata carries
119
+ * every warning a reader acts on (this and `providerWarning` included); a
120
+ * record without it came from one that predates them, whose remaining
121
+ * warnings are only in the result text.
122
+ */
123
+ startIndexPastEnd: z.boolean().optional(),
124
+ /** A hosted provider's own warning about the fetch, verbatim. */
125
+ providerWarning: z.string().optional(),
115
126
  });
116
127
 
117
128
  export type WebFetchMetadata = z.infer<typeof WebFetchMetadataSchema>;
package/src/api/index.ts CHANGED
@@ -1,5 +1,10 @@
1
1
  import { z } from "zod";
2
2
 
3
+ export {
4
+ type AskQuestionInput,
5
+ AskQuestionInputSchema,
6
+ type RecordedQuestion,
7
+ } from "./ask-question-tool.js";
3
8
  export {
4
9
  isComputerUseToolCall,
5
10
  resolveComputerUseToolName,
@@ -832,6 +837,8 @@ export {
832
837
  FeedItemStatusSchema,
833
838
  type FeedItemType,
834
839
  FeedItemTypeSchema,
840
+ type FeedItemUpdate,
841
+ FeedItemUpdateSchema,
835
842
  type FeedItemUrgency,
836
843
  FeedItemUrgencySchema,
837
844
  GUARDIAN_TERMINAL_REASON_SUPERSEDED,
@@ -1,8 +1,8 @@
1
1
  /**
2
2
  * Wire contract for the dictation REST endpoint (`POST /dictation`).
3
3
  *
4
- * The body web/macOS sends when asking the daemon to clean up a raw voice
5
- * transcript and classify it as dictation vs. action. `context` is its own
4
+ * The body web/macOS sends to apply explicit dictation replacements, classify
5
+ * voice input, or edit selected text. `context` is its own
6
6
  * named schema/type so consumers (web client) can import `DictationContext`
7
7
  * directly instead of inferring it from the generated client.
8
8
  *
@@ -167,6 +167,28 @@ export type FeedItemGuardianRequest = z.infer<
167
167
  typeof FeedItemGuardianRequestSchema
168
168
  >;
169
169
 
170
+ /**
171
+ * One background rewrite of a managed skill, as a receipt item lists it.
172
+ *
173
+ * A receipt (`detailPanel.kind === "updatesList"`) collapses every skill
174
+ * update a background pass made within one burst into a single feed item, and
175
+ * `updates` keeps each rewrite whole: the skill it touched, the pass's own
176
+ * account of what changed, and the conversation the change came from, when
177
+ * lineage resolved. The same skill can appear more than once, one entry per
178
+ * rewrite, in the order the rewrites happened. Clients group on `skillId`
179
+ * alone, never on `name` or the summary text.
180
+ */
181
+ export const FeedItemUpdateSchema = z.object({
182
+ skillId: z.string(),
183
+ /** The skill's display name at the time of the rewrite. */
184
+ name: z.string(),
185
+ /** The pass's own account of what it changed. Plain text, not markdown. */
186
+ summary: z.string(),
187
+ /** The conversation the rewrite was distilled from, when it resolved. */
188
+ conversationId: z.string().optional(),
189
+ });
190
+ export type FeedItemUpdate = z.infer<typeof FeedItemUpdateSchema>;
191
+
170
192
  /**
171
193
  * A single item rendered in the Home feed.
172
194
  *
@@ -198,6 +220,11 @@ export const FeedItemSchema = z.object({
198
220
  // Present only on the canonical "Needs attention" item projecting a
199
221
  // guardian request; see FeedItemGuardianRequestSchema.
200
222
  guardianRequest: FeedItemGuardianRequestSchema.optional(),
223
+ // Present only on a skill-update receipt (`detailPanel.kind` is
224
+ // `updatesList`); see FeedItemUpdateSchema. `summary` still carries the
225
+ // same content as a plain list, so a client without the panel loses only
226
+ // the per-skill links.
227
+ updates: z.array(FeedItemUpdateSchema).optional(),
201
228
  category: FeedItemCategorySchema.optional(),
202
229
  noteworthy: z.boolean().optional(),
203
230
  fromAssistant: z.boolean().optional(),
@@ -63,9 +63,6 @@ const TRAITS = { bodyShape: "blob", eyeStyle: "curious", color: "green" };
63
63
  const GREEN_DISC = { r: 0xe6, g: 0xf1, b: 0xe7 };
64
64
  /** #E9642F mixed 14% into white. */
65
65
  const ORANGE_DISC = { r: 0xfc, g: 0xe9, b: 0xe2 };
66
- /** The neutral disc every accent-less avatar wears. */
67
- const FALLBACK_DISC = { r: 0xec, g: 0xef, b: 0xea };
68
-
69
66
  /** On the disc and clear of the inset avatar square, which starts at y = 28.16. */
70
67
  const DISC_SAMPLE = { x: 128, y: 8 };
71
68
  /** Outside the inscribed disc, where the render leaves nothing at all. */
@@ -233,7 +230,7 @@ describe("renderNotificationAvatarPng", () => {
233
230
  );
234
231
 
235
232
  nativeTest(
236
- "renders an uploaded image on the neutral disc when it has no accent",
233
+ "fills the circle with an uploaded image when it has no accent",
237
234
  async () => {
238
235
  await writeUpload("png", null);
239
236
 
@@ -245,7 +242,9 @@ describe("renderNotificationAvatarPng", () => {
245
242
  expect(image.height).toBe(256);
246
243
  expect(image.at(CORNER.x, CORNER.y).a).toBe(0);
247
244
  expect(image.at(DISC_SAMPLE.x, DISC_SAMPLE.y)).toEqual({
248
- ...FALLBACK_DISC,
245
+ r: 0x20,
246
+ g: 0x60,
247
+ b: 0xc0,
249
248
  a: 255,
250
249
  });
251
250
  expect(image.at(128, 128)).toEqual({ r: 0x20, g: 0x60, b: 0xc0, a: 255 });
@@ -253,23 +252,34 @@ describe("renderNotificationAvatarPng", () => {
253
252
  NATIVE_RENDER_TEST_TIMEOUT_MS,
254
253
  );
255
254
 
256
- nativeTest(
257
- "cover-crops a wide upload and clips it to the disc",
258
- async () => {
259
- // 4:1, so a letterboxing render would leave disc colour above and below
260
- // the avatar inside the inset square.
261
- await writeUpload("png", null, { width: 256, height: 64 });
262
-
263
- const image = await pixels((await renderNotificationAvatarPng())!);
264
- // Inside the inset square (which starts at 28.16) near its top edge:
265
- // filled by a cover-crop, disc colour by a letterbox.
266
- expect(image.at(128, 35)).toEqual({ r: 0x20, g: 0x60, b: 0xc0, a: 255 });
267
- // Inside the inset square but outside the disc, so only the clip can
268
- // keep the avatar out of it.
269
- expect(image.at(30, 30).a).toBe(0);
270
- },
271
- NATIVE_RENDER_TEST_TIMEOUT_MS,
272
- );
255
+ for (const [shape, width, height] of [
256
+ ["landscape", 256, 64],
257
+ ["portrait", 64, 256],
258
+ ] as const) {
259
+ nativeTest(
260
+ `cover-crops a ${shape} upload to the full disc`,
261
+ async () => {
262
+ await writeUpload("png", null, { width, height });
263
+
264
+ const image = await pixels((await renderNotificationAvatarPng())!);
265
+ for (const [x, y] of [
266
+ [128, 8],
267
+ [8, 128],
268
+ [128, 248],
269
+ [248, 128],
270
+ ]) {
271
+ expect(image.at(x!, y!)).toEqual({
272
+ r: 0x20,
273
+ g: 0x60,
274
+ b: 0xc0,
275
+ a: 255,
276
+ });
277
+ }
278
+ expect(image.at(30, 30).a).toBe(0);
279
+ },
280
+ NATIVE_RENDER_TEST_TIMEOUT_MS,
281
+ );
282
+ }
273
283
 
274
284
  nativeTest(
275
285
  "transcodes a WebP upload, which resvg cannot decode on its own",
@@ -281,20 +291,18 @@ describe("renderNotificationAvatarPng", () => {
281
291
 
282
292
  const image = await pixels(png!);
283
293
  expect(image.at(CORNER.x, CORNER.y).a).toBe(0);
284
- expect(image.at(DISC_SAMPLE.x, DISC_SAMPLE.y)).toEqual({
285
- ...FALLBACK_DISC,
286
- a: 255,
287
- });
288
294
  // WebP is lossy, so the transcode moves the flat fill by a channel step
289
295
  // or two; what matters is that the avatar is drawn at all.
290
- const centre = image.at(128, 128);
291
- expect(centre.a).toBe(255);
292
- for (const [actual, expected] of [
293
- [centre.r, 0x20],
294
- [centre.g, 0x60],
295
- [centre.b, 0xc0],
296
- ]) {
297
- expect(Math.abs(actual! - expected!)).toBeLessThanOrEqual(4);
296
+ for (const point of [{ x: 128, y: 128 }, DISC_SAMPLE]) {
297
+ const pixel = image.at(point.x, point.y);
298
+ expect(pixel.a).toBe(255);
299
+ for (const [actual, expected] of [
300
+ [pixel.r, 0x20],
301
+ [pixel.g, 0x60],
302
+ [pixel.b, 0xc0],
303
+ ]) {
304
+ expect(Math.abs(actual! - expected!)).toBeLessThanOrEqual(4);
305
+ }
298
306
  }
299
307
  },
300
308
  NATIVE_RENDER_TEST_TIMEOUT_MS,
@@ -222,6 +222,7 @@ export async function renderNotificationAvatarPng(
222
222
  const Resvg = getResvg();
223
223
  const resvg = new Resvg(
224
224
  notificationAvatarSvg({
225
+ kind: state.kind,
225
226
  innerPngBase64: source.bytes.toString("base64"),
226
227
  innerMediaType: source.mediaType,
227
228
  accentHex: resolveNotificationAccentHex(state),
@@ -143,6 +143,10 @@ mock.module("../../persistence/conversation-crud.js", () => ({
143
143
  recordConversationPersistedSeq: () => {},
144
144
  }));
145
145
 
146
+ import {
147
+ clearHubClients,
148
+ registerHubClient,
149
+ } from "../../__tests__/helpers/hub-clients.js";
146
150
  import { setConfig } from "../../__tests__/helpers/set-config.js";
147
151
  import type { PreparedModelCall } from "../../agent/loop.js";
148
152
  import { selectWinningProfile } from "../../config/llm-resolver.js";
@@ -2326,21 +2330,30 @@ describe("front-door leg tool suppression", () => {
2326
2330
  });
2327
2331
 
2328
2332
  describe("desktop skill preactivation", () => {
2333
+ afterEach(() => clearHubClients(assistantEventHub));
2334
+
2329
2335
  async function preactivatedFor(turn: Record<string, unknown>): Promise<{
2330
2336
  skillIds: string[];
2331
2337
  proxyInterfaces: unknown[];
2332
2338
  skillIdsDuringLoop: string[];
2339
+ promptDuringLoop: string | null;
2333
2340
  }> {
2334
2341
  const skillIds: string[] = [];
2335
2342
  const proxyInterfaces: unknown[] = [];
2336
2343
  let skillIdsDuringLoop: string[] = [];
2344
+ let prompt: string | null = null;
2345
+ let promptDuringLoop: string | null = null;
2337
2346
  const fake = makeFakeConversation({
2338
2347
  processing: false,
2339
2348
  runAgentLoop: async () => {
2340
2349
  skillIdsDuringLoop = [...skillIds];
2350
+ promptDuringLoop = prompt;
2341
2351
  },
2342
2352
  });
2343
2353
  Object.assign(fake.conversation, {
2354
+ setVoiceCallControlPrompt: (value: string | null) => {
2355
+ prompt = value;
2356
+ },
2344
2357
  addPreactivatedSkillId: (id: string) => {
2345
2358
  skillIds.push(id);
2346
2359
  },
@@ -2356,9 +2369,58 @@ describe("desktop skill preactivation", () => {
2356
2369
  ...turn,
2357
2370
  });
2358
2371
  await flushMicrotasks();
2359
- return { skillIds, proxyInterfaces, skillIdsDuringLoop };
2372
+ return { skillIds, proxyInterfaces, skillIdsDuringLoop, promptDuringLoop };
2360
2373
  }
2361
2374
 
2375
+ test("a shared-screen turn receives the annotation instructions and schemas before inference", async () => {
2376
+ registerHubClient({
2377
+ hub: assistantEventHub,
2378
+ clientId: "annotation-client",
2379
+ interfaceId: "macos",
2380
+ actorPrincipalId: "user-123",
2381
+ capabilities: ["host_cu", "host_cu_annotate"],
2382
+ });
2383
+ const result = await preactivatedFor({
2384
+ routingLeg: "escalated",
2385
+ macosDesktopSession: true,
2386
+ screenSharing: true,
2387
+ actorPrincipalId: "user-123",
2388
+ });
2389
+ expect(result.skillIdsDuringLoop).toContain("screen-annotation");
2390
+ expect(result.promptDuringLoop).toContain("ID: screen-annotation");
2391
+ expect(result.promptDuringLoop).toContain("Name the thing.");
2392
+ expect(result.promptDuringLoop).toContain("screen_point_at");
2393
+ expect(result.promptDuringLoop).toContain("screen_clear_marks");
2394
+ expect(result.promptDuringLoop).toContain('"target"');
2395
+ expect(result.promptDuringLoop).toContain("skill_execute");
2396
+ });
2397
+
2398
+ test.each([
2399
+ { screenSharing: false },
2400
+ { actorPrincipalId: "other-user" },
2401
+ { routingLeg: "front-door" },
2402
+ { macosDesktopSession: false },
2403
+ ])(
2404
+ "does not preload annotation for an ineligible turn: %j",
2405
+ async (override) => {
2406
+ registerHubClient({
2407
+ hub: assistantEventHub,
2408
+ clientId: "annotation-client",
2409
+ interfaceId: "macos",
2410
+ actorPrincipalId: "user-123",
2411
+ capabilities: ["host_cu", "host_cu_annotate"],
2412
+ });
2413
+ const result = await preactivatedFor({
2414
+ routingLeg: "escalated",
2415
+ macosDesktopSession: true,
2416
+ screenSharing: true,
2417
+ actorPrincipalId: "user-123",
2418
+ ...override,
2419
+ });
2420
+ expect(result.promptDuringLoop).not.toContain("ID: screen-annotation");
2421
+ },
2422
+ );
2423
+
2362
2424
  test("an escalated leg of a macOS desktop session starts with computer use active", async () => {
2363
2425
  const result = await preactivatedFor({
2364
2426
  routingLeg: "escalated",
@@ -9,15 +9,16 @@
9
9
  * correct reply need something the front door cannot do from here? A
10
10
  * confident yes overrules a front-door answer and hands the turn off.
11
11
  *
12
- * The judge runs only when the `voiceEscalationJudge` call site resolves to
13
- * the TypeSafe provider. Every other outcome (no such profile, provider
14
- * error, timeout, unparseable answer) is a "clear" verdict, so the front
15
- * door's own decision stands exactly as it would without the judge.
12
+ * The judge uses managed Jev by default when Vellum-managed inference is
13
+ * available. An explicit TypeSafe call-site override can use a user-owned
14
+ * connection. Every other outcome (unavailable route, provider error,
15
+ * timeout, unparseable answer) is a "clear" verdict, so the front door's own
16
+ * decision stands exactly as it would without the judge.
16
17
  */
17
18
 
19
+ import { askTypesafeNoul } from "../providers/jev/ask.js";
18
20
  import type { Message, Provider } from "../providers/types.js";
19
21
  import { safeStringSlice } from "../util/unicode.js";
20
- import { askTypesafeNoul } from "./typesafe-noul.js";
21
22
 
22
23
  export const VOICE_ESCALATION_JUDGE_CALL_SITE = "voiceEscalationJudge";
23
24
 
@@ -0,0 +1,44 @@
1
+ import { shouldAttachHostProxyForCapability } from "../daemon/host-proxy-preactivation.js";
2
+ import { skillLoadTool } from "../tools/skills/load.js";
3
+ import type { ToolContext } from "../tools/types.js";
4
+ import { getLogger } from "../util/logger.js";
5
+
6
+ const log = getLogger("voice-screen-annotation");
7
+
8
+ export async function loadVoiceScreenAnnotation(
9
+ context: ToolContext,
10
+ ): Promise<string | null> {
11
+ if (
12
+ !shouldAttachHostProxyForCapability(
13
+ "host_cu_annotate",
14
+ context.transportInterface,
15
+ context.sourceActorPrincipalId,
16
+ )
17
+ ) {
18
+ return null;
19
+ }
20
+ try {
21
+ const result = await skillLoadTool.execute(
22
+ { skill: "screen-annotation" },
23
+ context,
24
+ { readOnly: true },
25
+ );
26
+ if (result.isError) {
27
+ log.warn(
28
+ { conversationId: context.conversationId, error: result.content },
29
+ "Could not preload screen annotation",
30
+ );
31
+ return null;
32
+ }
33
+ return [
34
+ "The user is sharing their screen. skill_load has automatically loaded screen-annotation for this turn. Use its tools through skill_execute when pointing at the shared screen.",
35
+ result.content,
36
+ ].join("\n\n");
37
+ } catch (err) {
38
+ log.warn(
39
+ { err, conversationId: context.conversationId },
40
+ "Could not preload screen annotation",
41
+ );
42
+ return null;
43
+ }
44
+ }
@@ -32,7 +32,9 @@ import { CONVERSATION_BUSY_MESSAGE } from "../daemon/conversation-messaging.js";
32
32
  import type { ModeSessionSourceHandle } from "../daemon/conversation-mode-session.js";
33
33
  import { resolveChannelCapabilities } from "../daemon/conversation-runtime-assembly.js";
34
34
  import { getOrCreateConversation } from "../daemon/conversation-store.js";
35
+ import { getEffectiveEnabledPluginSet } from "../daemon/conversation-tool-setup.js";
35
36
  import { preactivateHostProxySkills } from "../daemon/host-proxy-preactivation.js";
37
+ import { resolveTrustClass } from "../daemon/trust-context.js";
36
38
  import type { TrustContext } from "../daemon/trust-context-types.js";
37
39
  import {
38
40
  newestPersistedSightFrame,
@@ -78,6 +80,7 @@ import {
78
80
  } from "./voice-control-protocol.js";
79
81
  import { judgeEscalation } from "./voice-escalation-judge.js";
80
82
  import type { VoiceEscalationTarget } from "./voice-escalation-target.js";
83
+ import { loadVoiceScreenAnnotation } from "./voice-screen-annotation.js";
81
84
  import {
82
85
  createFrontDoorStreamGate,
83
86
  escalatedContinuationRule,
@@ -452,6 +455,8 @@ export interface VoiceTurnOptions {
452
455
  * session reports `macos` for its channel capabilities, iOS included.
453
456
  */
454
457
  macosDesktopSession?: boolean;
458
+ /** The desktop client currently shares a surface with this voice session. */
459
+ screenSharing?: boolean;
455
460
  /** Whether this is an inbound call (no outbound task). */
456
461
  isInbound: boolean;
457
462
  /** The outbound call task, if any. */
@@ -2108,6 +2113,23 @@ export async function startVoiceTurn(
2108
2113
  sourceInterface,
2109
2114
  sourceActorPrincipalId,
2110
2115
  );
2116
+ if (opts.screenSharing === true) {
2117
+ const annotation = await loadVoiceScreenAnnotation({
2118
+ conversationId: opts.conversationId,
2119
+ workingDir: conversation.workingDir,
2120
+ trustClass: resolveTrustClass(opts.trustContext),
2121
+ transportInterface: sourceInterface,
2122
+ clientOs: "macos",
2123
+ enabledPluginSet: getEffectiveEnabledPluginSet(conversation),
2124
+ sourceActorPrincipalId,
2125
+ signal: opts.signal,
2126
+ });
2127
+ if (annotation !== null) {
2128
+ conversation.setVoiceCallControlPrompt(
2129
+ [voiceCallControlPrompt, annotation].filter(Boolean).join("\n\n"),
2130
+ );
2131
+ }
2132
+ }
2111
2133
  }
2112
2134
  await conversation.runAgentLoop(persistedContent, messageId, {
2113
2135
  ...(opts.subagentNotification?.cronRunId
@@ -66,6 +66,20 @@ let platformInstallCalls: Array<{ name: string; force?: boolean }> = [];
66
66
  let upgradePluginCalls: UpgradePluginOptions[] = [];
67
67
  let catalogMatches: PluginSearchMatch[] = [];
68
68
  let catalogError: Error | null = null;
69
+ const CATALOG_COMMIT = "a".repeat(40);
70
+
71
+ function githubCatalogMatch(name: string): PluginSearchMatch {
72
+ return {
73
+ name,
74
+ path: `github:example-org/${name}@${CATALOG_COMMIT}`,
75
+ category: null,
76
+ source: {
77
+ kind: "github",
78
+ repo: `example-org/${name}`,
79
+ ref: CATALOG_COMMIT,
80
+ },
81
+ };
82
+ }
69
83
 
70
84
  /**
71
85
  * Queued daemon IPC responses. The default (empty queue) is a transport
@@ -269,7 +283,10 @@ beforeEach(() => {
269
283
  installPluginCalls = [];
270
284
  platformInstallCalls = [];
271
285
  upgradePluginCalls = [];
272
- catalogMatches = [];
286
+ catalogMatches = [
287
+ githubCatalogMatch("example"),
288
+ githubCatalogMatch("imessage"),
289
+ ];
273
290
  catalogError = null;
274
291
  ipcResults = [];
275
292
  inspectResult = null;
@@ -485,6 +502,17 @@ describe("plugins install - declared-schedules consent", () => {
485
502
  ]);
486
503
  expect(r.exitCode).toBe(0);
487
504
  });
505
+
506
+ test("does not use the outage fallback for a gated integration", async () => {
507
+ catalogError = new Error("catalog unavailable");
508
+
509
+ const r = await runCommand(["plugins", "install", "gamma"]);
510
+
511
+ expect(installPluginCalls).toHaveLength(0);
512
+ expect(platformInstallCalls).toHaveLength(0);
513
+ expect(r.stderr).toContain("not in the marketplace catalog");
514
+ expect(r.exitCode).toBe(1);
515
+ });
488
516
  });
489
517
 
490
518
  describe("plugins upgrade - declared-schedules consent (local fallback)", () => {
@@ -44,7 +44,11 @@ Examples:
44
44
  description:
45
45
  "Destination address for outbound verification (handle, phone number, user ID, or email address)",
46
46
  },
47
- { flags: "--rebind", description: "Replace existing guardian binding" },
47
+ {
48
+ flags: "--rebind",
49
+ description:
50
+ "Create a code although the channel already has a guardian (see below)",
51
+ },
48
52
  {
49
53
  flags: "--conversation-id <conversationId>",
50
54
  description: "Conversation ID for inbound challenges",
@@ -74,7 +78,13 @@ Routes between three creation modes based on the provided options:
74
78
  2. Outbound: --channel <ch> --destination <dest>
75
79
  Sends a verification code to the given destination. Supports telegram
76
80
  (handle or chat ID), phone (E.164 number), slack and discord (user ID),
77
- and email. Use --rebind to replace an existing guardian binding.
81
+ and email.
82
+
83
+ --rebind creates a code although the channel already has a guardian. On
84
+ telegram, slack, discord and email, only the account already bound can redeem
85
+ that code, which is how the same guardian verifies again. A code never moves
86
+ the guardian to a different account there: run "revoke" first, then create
87
+ without --rebind. On phone, an outbound code replaces the bound number.
78
88
 
79
89
  3. Inbound: --channel <ch> (no --destination)
80
90
  Generates a challenge secret for the guardian to send back on the
@@ -55,7 +55,8 @@ Fields:
55
55
  isPlatform Whether IS_PLATFORM is set (boolean)
56
56
  baseUrl VELLUM_PLATFORM_URL — the platform gateway base URL
57
57
  assistantId This assistant's platform UUID
58
- hasAssistantApiKey Whether a stored assistant API key is available
58
+ hasAssistantApiKey Whether a stored assistant API key is available; null
59
+ when the credential store could not be read
59
60
  hasWebhookSecret Whether a stored webhook secret is available (needed
60
61
  for email and other inbound webhook channels)
61
62
  available Whether callback registration prerequisites are satisfied
@@ -24,7 +24,7 @@ interface PlatformStatusResult {
24
24
  isPlatform: boolean;
25
25
  baseUrl: string;
26
26
  assistantId: string;
27
- hasAssistantApiKey: boolean;
27
+ hasAssistantApiKey: boolean | null;
28
28
  hasWebhookSecret: boolean;
29
29
  available: boolean;
30
30
  organizationId: string | null;
@@ -95,7 +95,13 @@ export function registerPlatformCommand(program: Command): void {
95
95
  log.info(`Base URL: ${result.baseUrl || "(not set)"}`);
96
96
  log.info(`Assistant ID: ${result.assistantId || "(not set)"}`);
97
97
  log.info(
98
- `Assistant API key: ${result.hasAssistantApiKey ? "set" : "not set"}`,
98
+ `Assistant API key: ${
99
+ result.hasAssistantApiKey === null
100
+ ? "unknown (credential store unreachable)"
101
+ : result.hasAssistantApiKey
102
+ ? "set"
103
+ : "not set"
104
+ }`,
99
105
  );
100
106
  log.info(
101
107
  `Webhook secret: ${result.hasWebhookSecret ? "set" : "not set (run ensure-registration to provision)"}`,
@@ -238,7 +238,15 @@ export function registerPluginsCommand(program: Command): void {
238
238
  const platformEnabled =
239
239
  libs.catalogLocal.arePlatformFeaturesEnabled();
240
240
  if (platformEnabled) {
241
+ if (!libs.catalogLocal.isPluginCatalogEntryVisible(nameOrUrl)) {
242
+ console.error(
243
+ `Plugin "${nameOrUrl}" is not in the marketplace catalog.`,
244
+ );
245
+ process.exitCode = 1;
246
+ return;
247
+ }
241
248
  let match;
249
+ let catalogResolved = false;
242
250
  try {
243
251
  match = (
244
252
  await libs.catalogCache.getPluginCatalog(
@@ -246,10 +254,18 @@ export function registerPluginsCommand(program: Command): void {
246
254
  { fetch: globalThis.fetch.bind(globalThis) },
247
255
  )
248
256
  ).matches.find((candidate) => candidate.name === nameOrUrl);
257
+ catalogResolved = true;
249
258
  } catch {
250
259
  // The install endpoint remains authoritative and can succeed
251
260
  // independently when catalog discovery is unavailable.
252
261
  }
262
+ if (!match && catalogResolved) {
263
+ console.error(
264
+ `Plugin "${nameOrUrl}" is not in the marketplace catalog.`,
265
+ );
266
+ process.exitCode = 1;
267
+ return;
268
+ }
253
269
  if (match?.source.kind === "local") {
254
270
  result = await libs.installGitHub.installPlugin(
255
271
  {
@@ -306,6 +306,35 @@ describe("inspectPlugin", () => {
306
306
  expect(result.remote?.commit).toBe(SHA_B);
307
307
  });
308
308
 
309
+ test("hides an uninstalled gated integration without fetching the marketplace", async () => {
310
+ let fetchCalls = 0;
311
+ const fetch = (async () => {
312
+ fetchCalls += 1;
313
+ return new Response("unexpected request", { status: 500 });
314
+ }) as FetchLike;
315
+
316
+ await expect(
317
+ inspectPlugin(
318
+ { name: "gamma" },
319
+ { fetch, workspacePluginsDir: workspace },
320
+ ),
321
+ ).rejects.toBeInstanceOf(PluginInspectNotFoundError);
322
+ expect(fetchCalls).toBe(0);
323
+ });
324
+
325
+ test("keeps an installed gated integration inspectable", async () => {
326
+ installPlugin(workspace, "gamma", { sidecar: null });
327
+ const fetch = makeFetch({ marketplace: manifestWith("gamma", SHA_B) });
328
+
329
+ const result = await inspectPlugin(
330
+ { name: "gamma" },
331
+ { fetch, workspacePluginsDir: workspace },
332
+ );
333
+
334
+ expect(result.installed).toBe(true);
335
+ expect(result.local?.description).toBe("Installed copy.");
336
+ });
337
+
309
338
  test("previews an uninstalled bundled provider offline without fetching GitHub", async () => {
310
339
  const savedDisablePlatform = process.env.VELLUM_DISABLE_PLATFORM;
311
340
  const savedIsPlatform = process.env.IS_PLATFORM;