@vellumai/assistant 0.12.3-dev.202609222113.8a21e65 → 0.12.3-dev.202609222313.c4916bb

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/node_modules/@vellumai/avatar-manifest/src/__tests__/notification-avatar.test.ts +48 -4
  2. package/node_modules/@vellumai/avatar-manifest/src/notification-avatar.ts +12 -8
  3. package/package.json +1 -1
  4. package/src/__tests__/skill-load-inline-command.test.ts +35 -0
  5. package/src/avatar/notification-avatar.test.ts +42 -34
  6. package/src/avatar/notification-avatar.ts +1 -0
  7. package/src/calls/__tests__/voice-session-bridge.test.ts +63 -1
  8. package/src/calls/voice-screen-annotation.ts +44 -0
  9. package/src/calls/voice-session-bridge.ts +22 -0
  10. package/src/cli/commands/__tests__/plugins.test.ts +29 -1
  11. package/src/cli/commands/plugins.ts +16 -0
  12. package/src/cli/lib/__tests__/inspect-plugin.test.ts +29 -0
  13. package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +32 -2
  14. package/src/cli/lib/__tests__/plugin-catalog-local.test.ts +15 -0
  15. package/src/cli/lib/__tests__/plugin-catalog-resolve.test.ts +21 -0
  16. package/src/cli/lib/__tests__/plugin-catalog-visibility.test.ts +65 -0
  17. package/src/cli/lib/__tests__/plugin-details.test.ts +43 -0
  18. package/src/cli/lib/__tests__/plugins-install-offline.test.ts +26 -0
  19. package/src/cli/lib/inspect-plugin.ts +5 -0
  20. package/src/cli/lib/plugin-catalog-cache.ts +24 -2
  21. package/src/cli/lib/plugin-catalog-local.ts +9 -0
  22. package/src/cli/lib/plugin-catalog-visibility.ts +41 -0
  23. package/src/cli/lib/plugin-details.ts +10 -7
  24. package/src/cli/lib/search-plugins.ts +2 -0
  25. package/src/config/bundled-skills/screen-annotation/SKILL.md +6 -3
  26. package/src/config/feature-flag-registry.json +9 -1
  27. package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +27 -0
  28. package/src/live-voice/__tests__/protocol.test.ts +22 -0
  29. package/src/live-voice/live-voice-session.ts +7 -6
  30. package/src/live-voice/protocol.ts +16 -5
  31. package/src/platform/sync-avatar.test.ts +29 -0
  32. package/src/runtime/routes/__tests__/plugins-routes.test.ts +8 -0
  33. package/src/runtime/routes/plugins-routes.ts +5 -2
  34. package/src/tools/skills/load.ts +13 -17
@@ -67,8 +67,31 @@ describe("notificationAvatarDiscHex", () => {
67
67
  });
68
68
 
69
69
  describe("notificationAvatarSvg", () => {
70
+ test("fills the entire circular crop with a custom image", () => {
71
+ const svg = notificationAvatarSvg({
72
+ kind: "image",
73
+ innerPngBase64: PNG_BASE64,
74
+ accentHex: "#E9642F",
75
+ });
76
+
77
+ expect(attributes("image", svg)).toMatchObject({
78
+ x: "0",
79
+ y: "0",
80
+ width: "256",
81
+ height: "256",
82
+ preserveAspectRatio: "xMidYMid slice",
83
+ "clip-path": "url(#notification-avatar-disc)",
84
+ });
85
+ expect(allAttributes("circle", svg)[1]).toEqual({
86
+ cx: "128",
87
+ cy: "128",
88
+ r: "128",
89
+ });
90
+ });
91
+
70
92
  test("draws one disc and one inset avatar at the default size", () => {
71
93
  const svg = notificationAvatarSvg({
94
+ kind: "character",
72
95
  innerPngBase64: PNG_BASE64,
73
96
  accentHex: "#E9642F",
74
97
  });
@@ -100,6 +123,7 @@ describe("notificationAvatarSvg", () => {
100
123
  // upload fill the square instead of letterboxing is the pair of attributes
101
124
  // asserted here.
102
125
  const svg = notificationAvatarSvg({
126
+ kind: "character",
103
127
  innerPngBase64: PNG_BASE64,
104
128
  accentHex: "#E9642F",
105
129
  });
@@ -121,6 +145,7 @@ describe("notificationAvatarSvg", () => {
121
145
 
122
146
  test("scales the geometry to a custom size", () => {
123
147
  const svg = notificationAvatarSvg({
148
+ kind: "character",
124
149
  innerPngBase64: PNG_BASE64,
125
150
  accentHex: "#E9642F",
126
151
  size: 100,
@@ -142,6 +167,7 @@ describe("notificationAvatarSvg", () => {
142
167
 
143
168
  test("carries a non-PNG inner raster with its own media type", () => {
144
169
  const svg = notificationAvatarSvg({
170
+ kind: "character",
145
171
  innerPngBase64: PNG_BASE64,
146
172
  innerMediaType: "image/jpeg",
147
173
  accentHex: "#E9642F",
@@ -154,6 +180,7 @@ describe("notificationAvatarSvg", () => {
154
180
 
155
181
  test("paints the fallback disc when there is no accent", () => {
156
182
  const svg = notificationAvatarSvg({
183
+ kind: "character",
157
184
  innerPngBase64: PNG_BASE64,
158
185
  accentHex: null,
159
186
  });
@@ -164,8 +191,21 @@ describe("notificationAvatarSvg", () => {
164
191
  });
165
192
 
166
193
  describe("notificationAvatarGeometry", () => {
194
+ test("custom images reach the disc edge at any output size", () => {
195
+ expect(notificationAvatarGeometry("image")).toEqual({
196
+ radius: 128,
197
+ offset: 0,
198
+ inner: 256,
199
+ });
200
+ expect(notificationAvatarGeometry("image", 100)).toEqual({
201
+ radius: 50,
202
+ offset: 0,
203
+ inner: 100,
204
+ });
205
+ });
206
+
167
207
  test("derives the disc, the border and the avatar edge from the size", () => {
168
- expect(notificationAvatarGeometry(100)).toEqual({
208
+ expect(notificationAvatarGeometry("character", 100)).toEqual({
169
209
  radius: 50,
170
210
  offset: 11,
171
211
  inner: 78,
@@ -173,15 +213,19 @@ describe("notificationAvatarGeometry", () => {
173
213
  });
174
214
 
175
215
  test("measures the default size when given none", () => {
176
- expect(notificationAvatarGeometry()).toEqual(
177
- notificationAvatarGeometry(NOTIFICATION_AVATAR_SIZE),
216
+ expect(notificationAvatarGeometry("character")).toEqual(
217
+ notificationAvatarGeometry("character", NOTIFICATION_AVATAR_SIZE),
178
218
  );
179
219
  });
180
220
 
181
221
  test("is the geometry the SVG is drawn with", () => {
182
222
  const size = 100;
183
- const { radius, offset, inner } = notificationAvatarGeometry(size);
223
+ const { radius, offset, inner } = notificationAvatarGeometry(
224
+ "character",
225
+ size,
226
+ );
184
227
  const svg = notificationAvatarSvg({
228
+ kind: "character",
185
229
  innerPngBase64: PNG_BASE64,
186
230
  accentHex: null,
187
231
  size,
@@ -15,11 +15,12 @@
15
15
  */
16
16
 
17
17
  import { isAvatarAccentHex } from "./accent.js";
18
+ import type { AvatarKind } from "./manifest.js";
18
19
 
19
20
  /** Output edge in pixels: large enough for an iOS notification thumbnail. */
20
21
  export const NOTIFICATION_AVATAR_SIZE = 256;
21
22
 
22
- /** Free disc on each side, as a fraction of the edge; the avatar gets the rest. */
23
+ /** Free disc on each side of a character avatar, as a fraction of the edge. */
23
24
  export const NOTIFICATION_AVATAR_INSET = 0.11;
24
25
 
25
26
  /** The disc when the assistant has no accent. */
@@ -48,7 +49,7 @@ export const NOTIFICATION_AVATAR_MAX_LOCAL_BYTES = 512 * 1024;
48
49
  * Bumped whenever the drawing above changes, so a sync keyed on it re-uploads
49
50
  * a disc rendered by an older spec even when the avatar itself is unchanged.
50
51
  */
51
- export const NOTIFICATION_AVATAR_SPEC_VERSION = 1;
52
+ export const NOTIFICATION_AVATAR_SPEC_VERSION = 2;
52
53
 
53
54
  /** The id the inner raster's clip path is referenced by inside the document. */
54
55
  const CLIP_ID = "notification-avatar-disc";
@@ -74,24 +75,26 @@ export function notificationAvatarDiscHex(accentHex: string | null): string {
74
75
 
75
76
  /** Raster formats an `<image>` href carries here; what resvg and a canvas both decode. */
76
77
  export type NotificationAvatarMediaType =
77
- | "image/png"
78
- | "image/jpeg"
79
- | "image/gif";
78
+ "image/png" | "image/jpeg" | "image/gif";
80
79
 
81
80
  /**
82
81
  * The three measurements the drawing derives from the edge: the disc's radius,
83
82
  * which is also its centre; the free border around the avatar; and the
84
83
  * avatar's own edge. Every rasterizer reads them from here, so the daemon's
85
84
  * SVG and the desktop canvas cannot drift.
85
+ * Custom images fill the disc; character avatars retain space around their
86
+ * silhouette.
86
87
  */
87
88
  export function notificationAvatarGeometry(
89
+ kind: Exclude<AvatarKind, "none">,
88
90
  size: number = NOTIFICATION_AVATAR_SIZE,
89
91
  ): { radius: number; offset: number; inner: number } {
90
- const offset = size * NOTIFICATION_AVATAR_INSET;
92
+ const offset = kind === "character" ? size * NOTIFICATION_AVATAR_INSET : 0;
91
93
  return { radius: size / 2, offset, inner: size - 2 * offset };
92
94
  }
93
95
 
94
96
  export interface NotificationAvatarSvgOptions {
97
+ kind: Exclude<AvatarKind, "none">;
95
98
  /** The avatar raster to draw inside the disc, base64 with no data prefix. */
96
99
  innerPngBase64: string;
97
100
  /** What `innerPngBase64` holds; PNG unless an upload arrived as a JPEG or a GIF. */
@@ -107,7 +110,7 @@ function px(value: number): string {
107
110
 
108
111
  /**
109
112
  * The notification avatar as an SVG document: a filled disc with the avatar
110
- * drawn inset into it.
113
+ * filling it for custom images or inset for character avatars.
111
114
  *
112
115
  * The inner raster is cover-cropped (`xMidYMid slice`), matching what the web
113
116
  * canvas does, so a portrait or landscape upload fills the square instead of
@@ -119,12 +122,13 @@ function px(value: number): string {
119
122
  * rasterizer is known to render it.
120
123
  */
121
124
  export function notificationAvatarSvg({
125
+ kind,
122
126
  innerPngBase64,
123
127
  innerMediaType = "image/png",
124
128
  accentHex,
125
129
  size = NOTIFICATION_AVATAR_SIZE,
126
130
  }: NotificationAvatarSvgOptions): string {
127
- const { radius, offset, inner } = notificationAvatarGeometry(size);
131
+ const { radius, offset, inner } = notificationAvatarGeometry(kind, size);
128
132
  const href = `data:${innerMediaType};base64,${innerPngBase64}`;
129
133
  const disc = `cx="${px(radius)}" cy="${px(radius)}" r="${px(radius)}"`;
130
134
  return (
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vellumai/assistant",
3
- "version": "0.12.3-dev.202609222113.8a21e65",
3
+ "version": "0.12.3-dev.202609222313.c4916bb",
4
4
  "license": "MIT",
5
5
  "type": "module",
6
6
  "exports": {
@@ -109,6 +109,41 @@ async function executeSkillLoad(
109
109
  // ── Tests ─────────────────────────────────────────────────────────────────
110
110
 
111
111
  describe("skill_load inline command expansion", () => {
112
+ test("read-only preloading returns instructions without running inline commands", async () => {
113
+ writeSkill(
114
+ "readonly-example",
115
+ "readonly-example",
116
+ "Example skill",
117
+ "Context: !`echo example`",
118
+ );
119
+ const result = await skillLoadTool.execute(
120
+ { skill: "readonly-example" },
121
+ {
122
+ conversationId: "conversation-1",
123
+ workingDir: TEST_DIR,
124
+ trustClass: "guardian",
125
+ },
126
+ { readOnly: true },
127
+ );
128
+ expect(result.isError).toBe(false);
129
+ expect(result.content).toContain("[inline command not executed]");
130
+ expect(runInlineCommandCalls).toHaveLength(0);
131
+ expect(mockAutoInstall).not.toHaveBeenCalled();
132
+ });
133
+
134
+ test("read-only preloading does not install missing skills", async () => {
135
+ const result = await skillLoadTool.execute(
136
+ { skill: "missing-readonly-skill" },
137
+ {
138
+ conversationId: "conversation-1",
139
+ workingDir: TEST_DIR,
140
+ trustClass: "guardian",
141
+ },
142
+ { readOnly: true },
143
+ );
144
+ expect(result.isError).toBe(true);
145
+ expect(mockAutoInstall).not.toHaveBeenCalled();
146
+ });
112
147
  beforeEach(() => {
113
148
  mkdirSync(join(TEST_DIR, "skills"), { recursive: true });
114
149
  runInlineCommandCalls.length = 0;
@@ -63,9 +63,6 @@ const TRAITS = { bodyShape: "blob", eyeStyle: "curious", color: "green" };
63
63
  const GREEN_DISC = { r: 0xe6, g: 0xf1, b: 0xe7 };
64
64
  /** #E9642F mixed 14% into white. */
65
65
  const ORANGE_DISC = { r: 0xfc, g: 0xe9, b: 0xe2 };
66
- /** The neutral disc every accent-less avatar wears. */
67
- const FALLBACK_DISC = { r: 0xec, g: 0xef, b: 0xea };
68
-
69
66
  /** On the disc and clear of the inset avatar square, which starts at y = 28.16. */
70
67
  const DISC_SAMPLE = { x: 128, y: 8 };
71
68
  /** Outside the inscribed disc, where the render leaves nothing at all. */
@@ -233,7 +230,7 @@ describe("renderNotificationAvatarPng", () => {
233
230
  );
234
231
 
235
232
  nativeTest(
236
- "renders an uploaded image on the neutral disc when it has no accent",
233
+ "fills the circle with an uploaded image when it has no accent",
237
234
  async () => {
238
235
  await writeUpload("png", null);
239
236
 
@@ -245,7 +242,9 @@ describe("renderNotificationAvatarPng", () => {
245
242
  expect(image.height).toBe(256);
246
243
  expect(image.at(CORNER.x, CORNER.y).a).toBe(0);
247
244
  expect(image.at(DISC_SAMPLE.x, DISC_SAMPLE.y)).toEqual({
248
- ...FALLBACK_DISC,
245
+ r: 0x20,
246
+ g: 0x60,
247
+ b: 0xc0,
249
248
  a: 255,
250
249
  });
251
250
  expect(image.at(128, 128)).toEqual({ r: 0x20, g: 0x60, b: 0xc0, a: 255 });
@@ -253,23 +252,34 @@ describe("renderNotificationAvatarPng", () => {
253
252
  NATIVE_RENDER_TEST_TIMEOUT_MS,
254
253
  );
255
254
 
256
- nativeTest(
257
- "cover-crops a wide upload and clips it to the disc",
258
- async () => {
259
- // 4:1, so a letterboxing render would leave disc colour above and below
260
- // the avatar inside the inset square.
261
- await writeUpload("png", null, { width: 256, height: 64 });
262
-
263
- const image = await pixels((await renderNotificationAvatarPng())!);
264
- // Inside the inset square (which starts at 28.16) near its top edge:
265
- // filled by a cover-crop, disc colour by a letterbox.
266
- expect(image.at(128, 35)).toEqual({ r: 0x20, g: 0x60, b: 0xc0, a: 255 });
267
- // Inside the inset square but outside the disc, so only the clip can
268
- // keep the avatar out of it.
269
- expect(image.at(30, 30).a).toBe(0);
270
- },
271
- NATIVE_RENDER_TEST_TIMEOUT_MS,
272
- );
255
+ for (const [shape, width, height] of [
256
+ ["landscape", 256, 64],
257
+ ["portrait", 64, 256],
258
+ ] as const) {
259
+ nativeTest(
260
+ `cover-crops a ${shape} upload to the full disc`,
261
+ async () => {
262
+ await writeUpload("png", null, { width, height });
263
+
264
+ const image = await pixels((await renderNotificationAvatarPng())!);
265
+ for (const [x, y] of [
266
+ [128, 8],
267
+ [8, 128],
268
+ [128, 248],
269
+ [248, 128],
270
+ ]) {
271
+ expect(image.at(x!, y!)).toEqual({
272
+ r: 0x20,
273
+ g: 0x60,
274
+ b: 0xc0,
275
+ a: 255,
276
+ });
277
+ }
278
+ expect(image.at(30, 30).a).toBe(0);
279
+ },
280
+ NATIVE_RENDER_TEST_TIMEOUT_MS,
281
+ );
282
+ }
273
283
 
274
284
  nativeTest(
275
285
  "transcodes a WebP upload, which resvg cannot decode on its own",
@@ -281,20 +291,18 @@ describe("renderNotificationAvatarPng", () => {
281
291
 
282
292
  const image = await pixels(png!);
283
293
  expect(image.at(CORNER.x, CORNER.y).a).toBe(0);
284
- expect(image.at(DISC_SAMPLE.x, DISC_SAMPLE.y)).toEqual({
285
- ...FALLBACK_DISC,
286
- a: 255,
287
- });
288
294
  // WebP is lossy, so the transcode moves the flat fill by a channel step
289
295
  // or two; what matters is that the avatar is drawn at all.
290
- const centre = image.at(128, 128);
291
- expect(centre.a).toBe(255);
292
- for (const [actual, expected] of [
293
- [centre.r, 0x20],
294
- [centre.g, 0x60],
295
- [centre.b, 0xc0],
296
- ]) {
297
- expect(Math.abs(actual! - expected!)).toBeLessThanOrEqual(4);
296
+ for (const point of [{ x: 128, y: 128 }, DISC_SAMPLE]) {
297
+ const pixel = image.at(point.x, point.y);
298
+ expect(pixel.a).toBe(255);
299
+ for (const [actual, expected] of [
300
+ [pixel.r, 0x20],
301
+ [pixel.g, 0x60],
302
+ [pixel.b, 0xc0],
303
+ ]) {
304
+ expect(Math.abs(actual! - expected!)).toBeLessThanOrEqual(4);
305
+ }
298
306
  }
299
307
  },
300
308
  NATIVE_RENDER_TEST_TIMEOUT_MS,
@@ -222,6 +222,7 @@ export async function renderNotificationAvatarPng(
222
222
  const Resvg = getResvg();
223
223
  const resvg = new Resvg(
224
224
  notificationAvatarSvg({
225
+ kind: state.kind,
225
226
  innerPngBase64: source.bytes.toString("base64"),
226
227
  innerMediaType: source.mediaType,
227
228
  accentHex: resolveNotificationAccentHex(state),
@@ -143,6 +143,10 @@ mock.module("../../persistence/conversation-crud.js", () => ({
143
143
  recordConversationPersistedSeq: () => {},
144
144
  }));
145
145
 
146
+ import {
147
+ clearHubClients,
148
+ registerHubClient,
149
+ } from "../../__tests__/helpers/hub-clients.js";
146
150
  import { setConfig } from "../../__tests__/helpers/set-config.js";
147
151
  import type { PreparedModelCall } from "../../agent/loop.js";
148
152
  import { selectWinningProfile } from "../../config/llm-resolver.js";
@@ -2326,21 +2330,30 @@ describe("front-door leg tool suppression", () => {
2326
2330
  });
2327
2331
 
2328
2332
  describe("desktop skill preactivation", () => {
2333
+ afterEach(() => clearHubClients(assistantEventHub));
2334
+
2329
2335
  async function preactivatedFor(turn: Record<string, unknown>): Promise<{
2330
2336
  skillIds: string[];
2331
2337
  proxyInterfaces: unknown[];
2332
2338
  skillIdsDuringLoop: string[];
2339
+ promptDuringLoop: string | null;
2333
2340
  }> {
2334
2341
  const skillIds: string[] = [];
2335
2342
  const proxyInterfaces: unknown[] = [];
2336
2343
  let skillIdsDuringLoop: string[] = [];
2344
+ let prompt: string | null = null;
2345
+ let promptDuringLoop: string | null = null;
2337
2346
  const fake = makeFakeConversation({
2338
2347
  processing: false,
2339
2348
  runAgentLoop: async () => {
2340
2349
  skillIdsDuringLoop = [...skillIds];
2350
+ promptDuringLoop = prompt;
2341
2351
  },
2342
2352
  });
2343
2353
  Object.assign(fake.conversation, {
2354
+ setVoiceCallControlPrompt: (value: string | null) => {
2355
+ prompt = value;
2356
+ },
2344
2357
  addPreactivatedSkillId: (id: string) => {
2345
2358
  skillIds.push(id);
2346
2359
  },
@@ -2356,9 +2369,58 @@ describe("desktop skill preactivation", () => {
2356
2369
  ...turn,
2357
2370
  });
2358
2371
  await flushMicrotasks();
2359
- return { skillIds, proxyInterfaces, skillIdsDuringLoop };
2372
+ return { skillIds, proxyInterfaces, skillIdsDuringLoop, promptDuringLoop };
2360
2373
  }
2361
2374
 
2375
+ test("a shared-screen turn receives the annotation instructions and schemas before inference", async () => {
2376
+ registerHubClient({
2377
+ hub: assistantEventHub,
2378
+ clientId: "annotation-client",
2379
+ interfaceId: "macos",
2380
+ actorPrincipalId: "user-123",
2381
+ capabilities: ["host_cu", "host_cu_annotate"],
2382
+ });
2383
+ const result = await preactivatedFor({
2384
+ routingLeg: "escalated",
2385
+ macosDesktopSession: true,
2386
+ screenSharing: true,
2387
+ actorPrincipalId: "user-123",
2388
+ });
2389
+ expect(result.skillIdsDuringLoop).toContain("screen-annotation");
2390
+ expect(result.promptDuringLoop).toContain("ID: screen-annotation");
2391
+ expect(result.promptDuringLoop).toContain("Name the thing.");
2392
+ expect(result.promptDuringLoop).toContain("screen_point_at");
2393
+ expect(result.promptDuringLoop).toContain("screen_clear_marks");
2394
+ expect(result.promptDuringLoop).toContain('"target"');
2395
+ expect(result.promptDuringLoop).toContain("skill_execute");
2396
+ });
2397
+
2398
+ test.each([
2399
+ { screenSharing: false },
2400
+ { actorPrincipalId: "other-user" },
2401
+ { routingLeg: "front-door" },
2402
+ { macosDesktopSession: false },
2403
+ ])(
2404
+ "does not preload annotation for an ineligible turn: %j",
2405
+ async (override) => {
2406
+ registerHubClient({
2407
+ hub: assistantEventHub,
2408
+ clientId: "annotation-client",
2409
+ interfaceId: "macos",
2410
+ actorPrincipalId: "user-123",
2411
+ capabilities: ["host_cu", "host_cu_annotate"],
2412
+ });
2413
+ const result = await preactivatedFor({
2414
+ routingLeg: "escalated",
2415
+ macosDesktopSession: true,
2416
+ screenSharing: true,
2417
+ actorPrincipalId: "user-123",
2418
+ ...override,
2419
+ });
2420
+ expect(result.promptDuringLoop).not.toContain("ID: screen-annotation");
2421
+ },
2422
+ );
2423
+
2362
2424
  test("an escalated leg of a macOS desktop session starts with computer use active", async () => {
2363
2425
  const result = await preactivatedFor({
2364
2426
  routingLeg: "escalated",
@@ -0,0 +1,44 @@
1
+ import { shouldAttachHostProxyForCapability } from "../daemon/host-proxy-preactivation.js";
2
+ import { skillLoadTool } from "../tools/skills/load.js";
3
+ import type { ToolContext } from "../tools/types.js";
4
+ import { getLogger } from "../util/logger.js";
5
+
6
+ const log = getLogger("voice-screen-annotation");
7
+
8
+ export async function loadVoiceScreenAnnotation(
9
+ context: ToolContext,
10
+ ): Promise<string | null> {
11
+ if (
12
+ !shouldAttachHostProxyForCapability(
13
+ "host_cu_annotate",
14
+ context.transportInterface,
15
+ context.sourceActorPrincipalId,
16
+ )
17
+ ) {
18
+ return null;
19
+ }
20
+ try {
21
+ const result = await skillLoadTool.execute(
22
+ { skill: "screen-annotation" },
23
+ context,
24
+ { readOnly: true },
25
+ );
26
+ if (result.isError) {
27
+ log.warn(
28
+ { conversationId: context.conversationId, error: result.content },
29
+ "Could not preload screen annotation",
30
+ );
31
+ return null;
32
+ }
33
+ return [
34
+ "The user is sharing their screen. skill_load has automatically loaded screen-annotation for this turn. Use its tools through skill_execute when pointing at the shared screen.",
35
+ result.content,
36
+ ].join("\n\n");
37
+ } catch (err) {
38
+ log.warn(
39
+ { err, conversationId: context.conversationId },
40
+ "Could not preload screen annotation",
41
+ );
42
+ return null;
43
+ }
44
+ }
@@ -32,7 +32,9 @@ import { CONVERSATION_BUSY_MESSAGE } from "../daemon/conversation-messaging.js";
32
32
  import type { ModeSessionSourceHandle } from "../daemon/conversation-mode-session.js";
33
33
  import { resolveChannelCapabilities } from "../daemon/conversation-runtime-assembly.js";
34
34
  import { getOrCreateConversation } from "../daemon/conversation-store.js";
35
+ import { getEffectiveEnabledPluginSet } from "../daemon/conversation-tool-setup.js";
35
36
  import { preactivateHostProxySkills } from "../daemon/host-proxy-preactivation.js";
37
+ import { resolveTrustClass } from "../daemon/trust-context.js";
36
38
  import type { TrustContext } from "../daemon/trust-context-types.js";
37
39
  import {
38
40
  newestPersistedSightFrame,
@@ -78,6 +80,7 @@ import {
78
80
  } from "./voice-control-protocol.js";
79
81
  import { judgeEscalation } from "./voice-escalation-judge.js";
80
82
  import type { VoiceEscalationTarget } from "./voice-escalation-target.js";
83
+ import { loadVoiceScreenAnnotation } from "./voice-screen-annotation.js";
81
84
  import {
82
85
  createFrontDoorStreamGate,
83
86
  escalatedContinuationRule,
@@ -452,6 +455,8 @@ export interface VoiceTurnOptions {
452
455
  * session reports `macos` for its channel capabilities, iOS included.
453
456
  */
454
457
  macosDesktopSession?: boolean;
458
+ /** The desktop client currently shares a surface with this voice session. */
459
+ screenSharing?: boolean;
455
460
  /** Whether this is an inbound call (no outbound task). */
456
461
  isInbound: boolean;
457
462
  /** The outbound call task, if any. */
@@ -2108,6 +2113,23 @@ export async function startVoiceTurn(
2108
2113
  sourceInterface,
2109
2114
  sourceActorPrincipalId,
2110
2115
  );
2116
+ if (opts.screenSharing === true) {
2117
+ const annotation = await loadVoiceScreenAnnotation({
2118
+ conversationId: opts.conversationId,
2119
+ workingDir: conversation.workingDir,
2120
+ trustClass: resolveTrustClass(opts.trustContext),
2121
+ transportInterface: sourceInterface,
2122
+ clientOs: "macos",
2123
+ enabledPluginSet: getEffectiveEnabledPluginSet(conversation),
2124
+ sourceActorPrincipalId,
2125
+ signal: opts.signal,
2126
+ });
2127
+ if (annotation !== null) {
2128
+ conversation.setVoiceCallControlPrompt(
2129
+ [voiceCallControlPrompt, annotation].filter(Boolean).join("\n\n"),
2130
+ );
2131
+ }
2132
+ }
2111
2133
  }
2112
2134
  await conversation.runAgentLoop(persistedContent, messageId, {
2113
2135
  ...(opts.subagentNotification?.cronRunId
@@ -66,6 +66,20 @@ let platformInstallCalls: Array<{ name: string; force?: boolean }> = [];
66
66
  let upgradePluginCalls: UpgradePluginOptions[] = [];
67
67
  let catalogMatches: PluginSearchMatch[] = [];
68
68
  let catalogError: Error | null = null;
69
+ const CATALOG_COMMIT = "a".repeat(40);
70
+
71
+ function githubCatalogMatch(name: string): PluginSearchMatch {
72
+ return {
73
+ name,
74
+ path: `github:example-org/${name}@${CATALOG_COMMIT}`,
75
+ category: null,
76
+ source: {
77
+ kind: "github",
78
+ repo: `example-org/${name}`,
79
+ ref: CATALOG_COMMIT,
80
+ },
81
+ };
82
+ }
69
83
 
70
84
  /**
71
85
  * Queued daemon IPC responses. The default (empty queue) is a transport
@@ -269,7 +283,10 @@ beforeEach(() => {
269
283
  installPluginCalls = [];
270
284
  platformInstallCalls = [];
271
285
  upgradePluginCalls = [];
272
- catalogMatches = [];
286
+ catalogMatches = [
287
+ githubCatalogMatch("example"),
288
+ githubCatalogMatch("imessage"),
289
+ ];
273
290
  catalogError = null;
274
291
  ipcResults = [];
275
292
  inspectResult = null;
@@ -485,6 +502,17 @@ describe("plugins install - declared-schedules consent", () => {
485
502
  ]);
486
503
  expect(r.exitCode).toBe(0);
487
504
  });
505
+
506
+ test("does not use the outage fallback for a gated integration", async () => {
507
+ catalogError = new Error("catalog unavailable");
508
+
509
+ const r = await runCommand(["plugins", "install", "gamma"]);
510
+
511
+ expect(installPluginCalls).toHaveLength(0);
512
+ expect(platformInstallCalls).toHaveLength(0);
513
+ expect(r.stderr).toContain("not in the marketplace catalog");
514
+ expect(r.exitCode).toBe(1);
515
+ });
488
516
  });
489
517
 
490
518
  describe("plugins upgrade - declared-schedules consent (local fallback)", () => {
@@ -238,7 +238,15 @@ export function registerPluginsCommand(program: Command): void {
238
238
  const platformEnabled =
239
239
  libs.catalogLocal.arePlatformFeaturesEnabled();
240
240
  if (platformEnabled) {
241
+ if (!libs.catalogLocal.isPluginCatalogEntryVisible(nameOrUrl)) {
242
+ console.error(
243
+ `Plugin "${nameOrUrl}" is not in the marketplace catalog.`,
244
+ );
245
+ process.exitCode = 1;
246
+ return;
247
+ }
241
248
  let match;
249
+ let catalogResolved = false;
242
250
  try {
243
251
  match = (
244
252
  await libs.catalogCache.getPluginCatalog(
@@ -246,10 +254,18 @@ export function registerPluginsCommand(program: Command): void {
246
254
  { fetch: globalThis.fetch.bind(globalThis) },
247
255
  )
248
256
  ).matches.find((candidate) => candidate.name === nameOrUrl);
257
+ catalogResolved = true;
249
258
  } catch {
250
259
  // The install endpoint remains authoritative and can succeed
251
260
  // independently when catalog discovery is unavailable.
252
261
  }
262
+ if (!match && catalogResolved) {
263
+ console.error(
264
+ `Plugin "${nameOrUrl}" is not in the marketplace catalog.`,
265
+ );
266
+ process.exitCode = 1;
267
+ return;
268
+ }
253
269
  if (match?.source.kind === "local") {
254
270
  result = await libs.installGitHub.installPlugin(
255
271
  {
@@ -306,6 +306,35 @@ describe("inspectPlugin", () => {
306
306
  expect(result.remote?.commit).toBe(SHA_B);
307
307
  });
308
308
 
309
+ test("hides an uninstalled gated integration without fetching the marketplace", async () => {
310
+ let fetchCalls = 0;
311
+ const fetch = (async () => {
312
+ fetchCalls += 1;
313
+ return new Response("unexpected request", { status: 500 });
314
+ }) as FetchLike;
315
+
316
+ await expect(
317
+ inspectPlugin(
318
+ { name: "gamma" },
319
+ { fetch, workspacePluginsDir: workspace },
320
+ ),
321
+ ).rejects.toBeInstanceOf(PluginInspectNotFoundError);
322
+ expect(fetchCalls).toBe(0);
323
+ });
324
+
325
+ test("keeps an installed gated integration inspectable", async () => {
326
+ installPlugin(workspace, "gamma", { sidecar: null });
327
+ const fetch = makeFetch({ marketplace: manifestWith("gamma", SHA_B) });
328
+
329
+ const result = await inspectPlugin(
330
+ { name: "gamma" },
331
+ { fetch, workspacePluginsDir: workspace },
332
+ );
333
+
334
+ expect(result.installed).toBe(true);
335
+ expect(result.local?.description).toBe("Installed copy.");
336
+ });
337
+
309
338
  test("previews an uninstalled bundled provider offline without fetching GitHub", async () => {
310
339
  const savedDisablePlatform = process.env.VELLUM_DISABLE_PLATFORM;
311
340
  const savedIsPlatform = process.env.IS_PLATFORM;