@vellumai/assistant 0.12.3-dev.202609222113.8a21e65 → 0.12.3-dev.202609222313.c4916bb
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/node_modules/@vellumai/avatar-manifest/src/__tests__/notification-avatar.test.ts +48 -4
- package/node_modules/@vellumai/avatar-manifest/src/notification-avatar.ts +12 -8
- package/package.json +1 -1
- package/src/__tests__/skill-load-inline-command.test.ts +35 -0
- package/src/avatar/notification-avatar.test.ts +42 -34
- package/src/avatar/notification-avatar.ts +1 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +63 -1
- package/src/calls/voice-screen-annotation.ts +44 -0
- package/src/calls/voice-session-bridge.ts +22 -0
- package/src/cli/commands/__tests__/plugins.test.ts +29 -1
- package/src/cli/commands/plugins.ts +16 -0
- package/src/cli/lib/__tests__/inspect-plugin.test.ts +29 -0
- package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +32 -2
- package/src/cli/lib/__tests__/plugin-catalog-local.test.ts +15 -0
- package/src/cli/lib/__tests__/plugin-catalog-resolve.test.ts +21 -0
- package/src/cli/lib/__tests__/plugin-catalog-visibility.test.ts +65 -0
- package/src/cli/lib/__tests__/plugin-details.test.ts +43 -0
- package/src/cli/lib/__tests__/plugins-install-offline.test.ts +26 -0
- package/src/cli/lib/inspect-plugin.ts +5 -0
- package/src/cli/lib/plugin-catalog-cache.ts +24 -2
- package/src/cli/lib/plugin-catalog-local.ts +9 -0
- package/src/cli/lib/plugin-catalog-visibility.ts +41 -0
- package/src/cli/lib/plugin-details.ts +10 -7
- package/src/cli/lib/search-plugins.ts +2 -0
- package/src/config/bundled-skills/screen-annotation/SKILL.md +6 -3
- package/src/config/feature-flag-registry.json +9 -1
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +27 -0
- package/src/live-voice/__tests__/protocol.test.ts +22 -0
- package/src/live-voice/live-voice-session.ts +7 -6
- package/src/live-voice/protocol.ts +16 -5
- package/src/platform/sync-avatar.test.ts +29 -0
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +8 -0
- package/src/runtime/routes/plugins-routes.ts +5 -2
- package/src/tools/skills/load.ts +13 -17
|
@@ -67,8 +67,31 @@ describe("notificationAvatarDiscHex", () => {
|
|
|
67
67
|
});
|
|
68
68
|
|
|
69
69
|
describe("notificationAvatarSvg", () => {
|
|
70
|
+
test("fills the entire circular crop with a custom image", () => {
|
|
71
|
+
const svg = notificationAvatarSvg({
|
|
72
|
+
kind: "image",
|
|
73
|
+
innerPngBase64: PNG_BASE64,
|
|
74
|
+
accentHex: "#E9642F",
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
expect(attributes("image", svg)).toMatchObject({
|
|
78
|
+
x: "0",
|
|
79
|
+
y: "0",
|
|
80
|
+
width: "256",
|
|
81
|
+
height: "256",
|
|
82
|
+
preserveAspectRatio: "xMidYMid slice",
|
|
83
|
+
"clip-path": "url(#notification-avatar-disc)",
|
|
84
|
+
});
|
|
85
|
+
expect(allAttributes("circle", svg)[1]).toEqual({
|
|
86
|
+
cx: "128",
|
|
87
|
+
cy: "128",
|
|
88
|
+
r: "128",
|
|
89
|
+
});
|
|
90
|
+
});
|
|
91
|
+
|
|
70
92
|
test("draws one disc and one inset avatar at the default size", () => {
|
|
71
93
|
const svg = notificationAvatarSvg({
|
|
94
|
+
kind: "character",
|
|
72
95
|
innerPngBase64: PNG_BASE64,
|
|
73
96
|
accentHex: "#E9642F",
|
|
74
97
|
});
|
|
@@ -100,6 +123,7 @@ describe("notificationAvatarSvg", () => {
|
|
|
100
123
|
// upload fill the square instead of letterboxing is the pair of attributes
|
|
101
124
|
// asserted here.
|
|
102
125
|
const svg = notificationAvatarSvg({
|
|
126
|
+
kind: "character",
|
|
103
127
|
innerPngBase64: PNG_BASE64,
|
|
104
128
|
accentHex: "#E9642F",
|
|
105
129
|
});
|
|
@@ -121,6 +145,7 @@ describe("notificationAvatarSvg", () => {
|
|
|
121
145
|
|
|
122
146
|
test("scales the geometry to a custom size", () => {
|
|
123
147
|
const svg = notificationAvatarSvg({
|
|
148
|
+
kind: "character",
|
|
124
149
|
innerPngBase64: PNG_BASE64,
|
|
125
150
|
accentHex: "#E9642F",
|
|
126
151
|
size: 100,
|
|
@@ -142,6 +167,7 @@ describe("notificationAvatarSvg", () => {
|
|
|
142
167
|
|
|
143
168
|
test("carries a non-PNG inner raster with its own media type", () => {
|
|
144
169
|
const svg = notificationAvatarSvg({
|
|
170
|
+
kind: "character",
|
|
145
171
|
innerPngBase64: PNG_BASE64,
|
|
146
172
|
innerMediaType: "image/jpeg",
|
|
147
173
|
accentHex: "#E9642F",
|
|
@@ -154,6 +180,7 @@ describe("notificationAvatarSvg", () => {
|
|
|
154
180
|
|
|
155
181
|
test("paints the fallback disc when there is no accent", () => {
|
|
156
182
|
const svg = notificationAvatarSvg({
|
|
183
|
+
kind: "character",
|
|
157
184
|
innerPngBase64: PNG_BASE64,
|
|
158
185
|
accentHex: null,
|
|
159
186
|
});
|
|
@@ -164,8 +191,21 @@ describe("notificationAvatarSvg", () => {
|
|
|
164
191
|
});
|
|
165
192
|
|
|
166
193
|
describe("notificationAvatarGeometry", () => {
|
|
194
|
+
test("custom images reach the disc edge at any output size", () => {
|
|
195
|
+
expect(notificationAvatarGeometry("image")).toEqual({
|
|
196
|
+
radius: 128,
|
|
197
|
+
offset: 0,
|
|
198
|
+
inner: 256,
|
|
199
|
+
});
|
|
200
|
+
expect(notificationAvatarGeometry("image", 100)).toEqual({
|
|
201
|
+
radius: 50,
|
|
202
|
+
offset: 0,
|
|
203
|
+
inner: 100,
|
|
204
|
+
});
|
|
205
|
+
});
|
|
206
|
+
|
|
167
207
|
test("derives the disc, the border and the avatar edge from the size", () => {
|
|
168
|
-
expect(notificationAvatarGeometry(100)).toEqual({
|
|
208
|
+
expect(notificationAvatarGeometry("character", 100)).toEqual({
|
|
169
209
|
radius: 50,
|
|
170
210
|
offset: 11,
|
|
171
211
|
inner: 78,
|
|
@@ -173,15 +213,19 @@ describe("notificationAvatarGeometry", () => {
|
|
|
173
213
|
});
|
|
174
214
|
|
|
175
215
|
test("measures the default size when given none", () => {
|
|
176
|
-
expect(notificationAvatarGeometry()).toEqual(
|
|
177
|
-
notificationAvatarGeometry(NOTIFICATION_AVATAR_SIZE),
|
|
216
|
+
expect(notificationAvatarGeometry("character")).toEqual(
|
|
217
|
+
notificationAvatarGeometry("character", NOTIFICATION_AVATAR_SIZE),
|
|
178
218
|
);
|
|
179
219
|
});
|
|
180
220
|
|
|
181
221
|
test("is the geometry the SVG is drawn with", () => {
|
|
182
222
|
const size = 100;
|
|
183
|
-
const { radius, offset, inner } = notificationAvatarGeometry(
|
|
223
|
+
const { radius, offset, inner } = notificationAvatarGeometry(
|
|
224
|
+
"character",
|
|
225
|
+
size,
|
|
226
|
+
);
|
|
184
227
|
const svg = notificationAvatarSvg({
|
|
228
|
+
kind: "character",
|
|
185
229
|
innerPngBase64: PNG_BASE64,
|
|
186
230
|
accentHex: null,
|
|
187
231
|
size,
|
|
@@ -15,11 +15,12 @@
|
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
17
|
import { isAvatarAccentHex } from "./accent.js";
|
|
18
|
+
import type { AvatarKind } from "./manifest.js";
|
|
18
19
|
|
|
19
20
|
/** Output edge in pixels: large enough for an iOS notification thumbnail. */
|
|
20
21
|
export const NOTIFICATION_AVATAR_SIZE = 256;
|
|
21
22
|
|
|
22
|
-
/** Free disc on each side, as a fraction of the edge
|
|
23
|
+
/** Free disc on each side of a character avatar, as a fraction of the edge. */
|
|
23
24
|
export const NOTIFICATION_AVATAR_INSET = 0.11;
|
|
24
25
|
|
|
25
26
|
/** The disc when the assistant has no accent. */
|
|
@@ -48,7 +49,7 @@ export const NOTIFICATION_AVATAR_MAX_LOCAL_BYTES = 512 * 1024;
|
|
|
48
49
|
* Bumped whenever the drawing above changes, so a sync keyed on it re-uploads
|
|
49
50
|
* a disc rendered by an older spec even when the avatar itself is unchanged.
|
|
50
51
|
*/
|
|
51
|
-
export const NOTIFICATION_AVATAR_SPEC_VERSION =
|
|
52
|
+
export const NOTIFICATION_AVATAR_SPEC_VERSION = 2;
|
|
52
53
|
|
|
53
54
|
/** The id the inner raster's clip path is referenced by inside the document. */
|
|
54
55
|
const CLIP_ID = "notification-avatar-disc";
|
|
@@ -74,24 +75,26 @@ export function notificationAvatarDiscHex(accentHex: string | null): string {
|
|
|
74
75
|
|
|
75
76
|
/** Raster formats an `<image>` href carries here; what resvg and a canvas both decode. */
|
|
76
77
|
export type NotificationAvatarMediaType =
|
|
77
|
-
| "image/
|
|
78
|
-
| "image/jpeg"
|
|
79
|
-
| "image/gif";
|
|
78
|
+
"image/png" | "image/jpeg" | "image/gif";
|
|
80
79
|
|
|
81
80
|
/**
|
|
82
81
|
* The three measurements the drawing derives from the edge: the disc's radius,
|
|
83
82
|
* which is also its centre; the free border around the avatar; and the
|
|
84
83
|
* avatar's own edge. Every rasterizer reads them from here, so the daemon's
|
|
85
84
|
* SVG and the desktop canvas cannot drift.
|
|
85
|
+
* Custom images fill the disc; character avatars retain space around their
|
|
86
|
+
* silhouette.
|
|
86
87
|
*/
|
|
87
88
|
export function notificationAvatarGeometry(
|
|
89
|
+
kind: Exclude<AvatarKind, "none">,
|
|
88
90
|
size: number = NOTIFICATION_AVATAR_SIZE,
|
|
89
91
|
): { radius: number; offset: number; inner: number } {
|
|
90
|
-
const offset = size * NOTIFICATION_AVATAR_INSET;
|
|
92
|
+
const offset = kind === "character" ? size * NOTIFICATION_AVATAR_INSET : 0;
|
|
91
93
|
return { radius: size / 2, offset, inner: size - 2 * offset };
|
|
92
94
|
}
|
|
93
95
|
|
|
94
96
|
export interface NotificationAvatarSvgOptions {
|
|
97
|
+
kind: Exclude<AvatarKind, "none">;
|
|
95
98
|
/** The avatar raster to draw inside the disc, base64 with no data prefix. */
|
|
96
99
|
innerPngBase64: string;
|
|
97
100
|
/** What `innerPngBase64` holds; PNG unless an upload arrived as a JPEG or a GIF. */
|
|
@@ -107,7 +110,7 @@ function px(value: number): string {
|
|
|
107
110
|
|
|
108
111
|
/**
|
|
109
112
|
* The notification avatar as an SVG document: a filled disc with the avatar
|
|
110
|
-
*
|
|
113
|
+
* filling it for custom images or inset for character avatars.
|
|
111
114
|
*
|
|
112
115
|
* The inner raster is cover-cropped (`xMidYMid slice`), matching what the web
|
|
113
116
|
* canvas does, so a portrait or landscape upload fills the square instead of
|
|
@@ -119,12 +122,13 @@ function px(value: number): string {
|
|
|
119
122
|
* rasterizer is known to render it.
|
|
120
123
|
*/
|
|
121
124
|
export function notificationAvatarSvg({
|
|
125
|
+
kind,
|
|
122
126
|
innerPngBase64,
|
|
123
127
|
innerMediaType = "image/png",
|
|
124
128
|
accentHex,
|
|
125
129
|
size = NOTIFICATION_AVATAR_SIZE,
|
|
126
130
|
}: NotificationAvatarSvgOptions): string {
|
|
127
|
-
const { radius, offset, inner } = notificationAvatarGeometry(size);
|
|
131
|
+
const { radius, offset, inner } = notificationAvatarGeometry(kind, size);
|
|
128
132
|
const href = `data:${innerMediaType};base64,${innerPngBase64}`;
|
|
129
133
|
const disc = `cx="${px(radius)}" cy="${px(radius)}" r="${px(radius)}"`;
|
|
130
134
|
return (
|
package/package.json
CHANGED
|
@@ -109,6 +109,41 @@ async function executeSkillLoad(
|
|
|
109
109
|
// ── Tests ─────────────────────────────────────────────────────────────────
|
|
110
110
|
|
|
111
111
|
describe("skill_load inline command expansion", () => {
|
|
112
|
+
test("read-only preloading returns instructions without running inline commands", async () => {
|
|
113
|
+
writeSkill(
|
|
114
|
+
"readonly-example",
|
|
115
|
+
"readonly-example",
|
|
116
|
+
"Example skill",
|
|
117
|
+
"Context: !`echo example`",
|
|
118
|
+
);
|
|
119
|
+
const result = await skillLoadTool.execute(
|
|
120
|
+
{ skill: "readonly-example" },
|
|
121
|
+
{
|
|
122
|
+
conversationId: "conversation-1",
|
|
123
|
+
workingDir: TEST_DIR,
|
|
124
|
+
trustClass: "guardian",
|
|
125
|
+
},
|
|
126
|
+
{ readOnly: true },
|
|
127
|
+
);
|
|
128
|
+
expect(result.isError).toBe(false);
|
|
129
|
+
expect(result.content).toContain("[inline command not executed]");
|
|
130
|
+
expect(runInlineCommandCalls).toHaveLength(0);
|
|
131
|
+
expect(mockAutoInstall).not.toHaveBeenCalled();
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
test("read-only preloading does not install missing skills", async () => {
|
|
135
|
+
const result = await skillLoadTool.execute(
|
|
136
|
+
{ skill: "missing-readonly-skill" },
|
|
137
|
+
{
|
|
138
|
+
conversationId: "conversation-1",
|
|
139
|
+
workingDir: TEST_DIR,
|
|
140
|
+
trustClass: "guardian",
|
|
141
|
+
},
|
|
142
|
+
{ readOnly: true },
|
|
143
|
+
);
|
|
144
|
+
expect(result.isError).toBe(true);
|
|
145
|
+
expect(mockAutoInstall).not.toHaveBeenCalled();
|
|
146
|
+
});
|
|
112
147
|
beforeEach(() => {
|
|
113
148
|
mkdirSync(join(TEST_DIR, "skills"), { recursive: true });
|
|
114
149
|
runInlineCommandCalls.length = 0;
|
|
@@ -63,9 +63,6 @@ const TRAITS = { bodyShape: "blob", eyeStyle: "curious", color: "green" };
|
|
|
63
63
|
const GREEN_DISC = { r: 0xe6, g: 0xf1, b: 0xe7 };
|
|
64
64
|
/** #E9642F mixed 14% into white. */
|
|
65
65
|
const ORANGE_DISC = { r: 0xfc, g: 0xe9, b: 0xe2 };
|
|
66
|
-
/** The neutral disc every accent-less avatar wears. */
|
|
67
|
-
const FALLBACK_DISC = { r: 0xec, g: 0xef, b: 0xea };
|
|
68
|
-
|
|
69
66
|
/** On the disc and clear of the inset avatar square, which starts at y = 28.16. */
|
|
70
67
|
const DISC_SAMPLE = { x: 128, y: 8 };
|
|
71
68
|
/** Outside the inscribed disc, where the render leaves nothing at all. */
|
|
@@ -233,7 +230,7 @@ describe("renderNotificationAvatarPng", () => {
|
|
|
233
230
|
);
|
|
234
231
|
|
|
235
232
|
nativeTest(
|
|
236
|
-
"
|
|
233
|
+
"fills the circle with an uploaded image when it has no accent",
|
|
237
234
|
async () => {
|
|
238
235
|
await writeUpload("png", null);
|
|
239
236
|
|
|
@@ -245,7 +242,9 @@ describe("renderNotificationAvatarPng", () => {
|
|
|
245
242
|
expect(image.height).toBe(256);
|
|
246
243
|
expect(image.at(CORNER.x, CORNER.y).a).toBe(0);
|
|
247
244
|
expect(image.at(DISC_SAMPLE.x, DISC_SAMPLE.y)).toEqual({
|
|
248
|
-
|
|
245
|
+
r: 0x20,
|
|
246
|
+
g: 0x60,
|
|
247
|
+
b: 0xc0,
|
|
249
248
|
a: 255,
|
|
250
249
|
});
|
|
251
250
|
expect(image.at(128, 128)).toEqual({ r: 0x20, g: 0x60, b: 0xc0, a: 255 });
|
|
@@ -253,23 +252,34 @@ describe("renderNotificationAvatarPng", () => {
|
|
|
253
252
|
NATIVE_RENDER_TEST_TIMEOUT_MS,
|
|
254
253
|
);
|
|
255
254
|
|
|
256
|
-
|
|
257
|
-
"
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
255
|
+
for (const [shape, width, height] of [
|
|
256
|
+
["landscape", 256, 64],
|
|
257
|
+
["portrait", 64, 256],
|
|
258
|
+
] as const) {
|
|
259
|
+
nativeTest(
|
|
260
|
+
`cover-crops a ${shape} upload to the full disc`,
|
|
261
|
+
async () => {
|
|
262
|
+
await writeUpload("png", null, { width, height });
|
|
263
|
+
|
|
264
|
+
const image = await pixels((await renderNotificationAvatarPng())!);
|
|
265
|
+
for (const [x, y] of [
|
|
266
|
+
[128, 8],
|
|
267
|
+
[8, 128],
|
|
268
|
+
[128, 248],
|
|
269
|
+
[248, 128],
|
|
270
|
+
]) {
|
|
271
|
+
expect(image.at(x!, y!)).toEqual({
|
|
272
|
+
r: 0x20,
|
|
273
|
+
g: 0x60,
|
|
274
|
+
b: 0xc0,
|
|
275
|
+
a: 255,
|
|
276
|
+
});
|
|
277
|
+
}
|
|
278
|
+
expect(image.at(30, 30).a).toBe(0);
|
|
279
|
+
},
|
|
280
|
+
NATIVE_RENDER_TEST_TIMEOUT_MS,
|
|
281
|
+
);
|
|
282
|
+
}
|
|
273
283
|
|
|
274
284
|
nativeTest(
|
|
275
285
|
"transcodes a WebP upload, which resvg cannot decode on its own",
|
|
@@ -281,20 +291,18 @@ describe("renderNotificationAvatarPng", () => {
|
|
|
281
291
|
|
|
282
292
|
const image = await pixels(png!);
|
|
283
293
|
expect(image.at(CORNER.x, CORNER.y).a).toBe(0);
|
|
284
|
-
expect(image.at(DISC_SAMPLE.x, DISC_SAMPLE.y)).toEqual({
|
|
285
|
-
...FALLBACK_DISC,
|
|
286
|
-
a: 255,
|
|
287
|
-
});
|
|
288
294
|
// WebP is lossy, so the transcode moves the flat fill by a channel step
|
|
289
295
|
// or two; what matters is that the avatar is drawn at all.
|
|
290
|
-
const
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
[
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
296
|
+
for (const point of [{ x: 128, y: 128 }, DISC_SAMPLE]) {
|
|
297
|
+
const pixel = image.at(point.x, point.y);
|
|
298
|
+
expect(pixel.a).toBe(255);
|
|
299
|
+
for (const [actual, expected] of [
|
|
300
|
+
[pixel.r, 0x20],
|
|
301
|
+
[pixel.g, 0x60],
|
|
302
|
+
[pixel.b, 0xc0],
|
|
303
|
+
]) {
|
|
304
|
+
expect(Math.abs(actual! - expected!)).toBeLessThanOrEqual(4);
|
|
305
|
+
}
|
|
298
306
|
}
|
|
299
307
|
},
|
|
300
308
|
NATIVE_RENDER_TEST_TIMEOUT_MS,
|
|
@@ -222,6 +222,7 @@ export async function renderNotificationAvatarPng(
|
|
|
222
222
|
const Resvg = getResvg();
|
|
223
223
|
const resvg = new Resvg(
|
|
224
224
|
notificationAvatarSvg({
|
|
225
|
+
kind: state.kind,
|
|
225
226
|
innerPngBase64: source.bytes.toString("base64"),
|
|
226
227
|
innerMediaType: source.mediaType,
|
|
227
228
|
accentHex: resolveNotificationAccentHex(state),
|
|
@@ -143,6 +143,10 @@ mock.module("../../persistence/conversation-crud.js", () => ({
|
|
|
143
143
|
recordConversationPersistedSeq: () => {},
|
|
144
144
|
}));
|
|
145
145
|
|
|
146
|
+
import {
|
|
147
|
+
clearHubClients,
|
|
148
|
+
registerHubClient,
|
|
149
|
+
} from "../../__tests__/helpers/hub-clients.js";
|
|
146
150
|
import { setConfig } from "../../__tests__/helpers/set-config.js";
|
|
147
151
|
import type { PreparedModelCall } from "../../agent/loop.js";
|
|
148
152
|
import { selectWinningProfile } from "../../config/llm-resolver.js";
|
|
@@ -2326,21 +2330,30 @@ describe("front-door leg tool suppression", () => {
|
|
|
2326
2330
|
});
|
|
2327
2331
|
|
|
2328
2332
|
describe("desktop skill preactivation", () => {
|
|
2333
|
+
afterEach(() => clearHubClients(assistantEventHub));
|
|
2334
|
+
|
|
2329
2335
|
async function preactivatedFor(turn: Record<string, unknown>): Promise<{
|
|
2330
2336
|
skillIds: string[];
|
|
2331
2337
|
proxyInterfaces: unknown[];
|
|
2332
2338
|
skillIdsDuringLoop: string[];
|
|
2339
|
+
promptDuringLoop: string | null;
|
|
2333
2340
|
}> {
|
|
2334
2341
|
const skillIds: string[] = [];
|
|
2335
2342
|
const proxyInterfaces: unknown[] = [];
|
|
2336
2343
|
let skillIdsDuringLoop: string[] = [];
|
|
2344
|
+
let prompt: string | null = null;
|
|
2345
|
+
let promptDuringLoop: string | null = null;
|
|
2337
2346
|
const fake = makeFakeConversation({
|
|
2338
2347
|
processing: false,
|
|
2339
2348
|
runAgentLoop: async () => {
|
|
2340
2349
|
skillIdsDuringLoop = [...skillIds];
|
|
2350
|
+
promptDuringLoop = prompt;
|
|
2341
2351
|
},
|
|
2342
2352
|
});
|
|
2343
2353
|
Object.assign(fake.conversation, {
|
|
2354
|
+
setVoiceCallControlPrompt: (value: string | null) => {
|
|
2355
|
+
prompt = value;
|
|
2356
|
+
},
|
|
2344
2357
|
addPreactivatedSkillId: (id: string) => {
|
|
2345
2358
|
skillIds.push(id);
|
|
2346
2359
|
},
|
|
@@ -2356,9 +2369,58 @@ describe("desktop skill preactivation", () => {
|
|
|
2356
2369
|
...turn,
|
|
2357
2370
|
});
|
|
2358
2371
|
await flushMicrotasks();
|
|
2359
|
-
return { skillIds, proxyInterfaces, skillIdsDuringLoop };
|
|
2372
|
+
return { skillIds, proxyInterfaces, skillIdsDuringLoop, promptDuringLoop };
|
|
2360
2373
|
}
|
|
2361
2374
|
|
|
2375
|
+
test("a shared-screen turn receives the annotation instructions and schemas before inference", async () => {
|
|
2376
|
+
registerHubClient({
|
|
2377
|
+
hub: assistantEventHub,
|
|
2378
|
+
clientId: "annotation-client",
|
|
2379
|
+
interfaceId: "macos",
|
|
2380
|
+
actorPrincipalId: "user-123",
|
|
2381
|
+
capabilities: ["host_cu", "host_cu_annotate"],
|
|
2382
|
+
});
|
|
2383
|
+
const result = await preactivatedFor({
|
|
2384
|
+
routingLeg: "escalated",
|
|
2385
|
+
macosDesktopSession: true,
|
|
2386
|
+
screenSharing: true,
|
|
2387
|
+
actorPrincipalId: "user-123",
|
|
2388
|
+
});
|
|
2389
|
+
expect(result.skillIdsDuringLoop).toContain("screen-annotation");
|
|
2390
|
+
expect(result.promptDuringLoop).toContain("ID: screen-annotation");
|
|
2391
|
+
expect(result.promptDuringLoop).toContain("Name the thing.");
|
|
2392
|
+
expect(result.promptDuringLoop).toContain("screen_point_at");
|
|
2393
|
+
expect(result.promptDuringLoop).toContain("screen_clear_marks");
|
|
2394
|
+
expect(result.promptDuringLoop).toContain('"target"');
|
|
2395
|
+
expect(result.promptDuringLoop).toContain("skill_execute");
|
|
2396
|
+
});
|
|
2397
|
+
|
|
2398
|
+
test.each([
|
|
2399
|
+
{ screenSharing: false },
|
|
2400
|
+
{ actorPrincipalId: "other-user" },
|
|
2401
|
+
{ routingLeg: "front-door" },
|
|
2402
|
+
{ macosDesktopSession: false },
|
|
2403
|
+
])(
|
|
2404
|
+
"does not preload annotation for an ineligible turn: %j",
|
|
2405
|
+
async (override) => {
|
|
2406
|
+
registerHubClient({
|
|
2407
|
+
hub: assistantEventHub,
|
|
2408
|
+
clientId: "annotation-client",
|
|
2409
|
+
interfaceId: "macos",
|
|
2410
|
+
actorPrincipalId: "user-123",
|
|
2411
|
+
capabilities: ["host_cu", "host_cu_annotate"],
|
|
2412
|
+
});
|
|
2413
|
+
const result = await preactivatedFor({
|
|
2414
|
+
routingLeg: "escalated",
|
|
2415
|
+
macosDesktopSession: true,
|
|
2416
|
+
screenSharing: true,
|
|
2417
|
+
actorPrincipalId: "user-123",
|
|
2418
|
+
...override,
|
|
2419
|
+
});
|
|
2420
|
+
expect(result.promptDuringLoop).not.toContain("ID: screen-annotation");
|
|
2421
|
+
},
|
|
2422
|
+
);
|
|
2423
|
+
|
|
2362
2424
|
test("an escalated leg of a macOS desktop session starts with computer use active", async () => {
|
|
2363
2425
|
const result = await preactivatedFor({
|
|
2364
2426
|
routingLeg: "escalated",
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { shouldAttachHostProxyForCapability } from "../daemon/host-proxy-preactivation.js";
|
|
2
|
+
import { skillLoadTool } from "../tools/skills/load.js";
|
|
3
|
+
import type { ToolContext } from "../tools/types.js";
|
|
4
|
+
import { getLogger } from "../util/logger.js";
|
|
5
|
+
|
|
6
|
+
const log = getLogger("voice-screen-annotation");
|
|
7
|
+
|
|
8
|
+
export async function loadVoiceScreenAnnotation(
|
|
9
|
+
context: ToolContext,
|
|
10
|
+
): Promise<string | null> {
|
|
11
|
+
if (
|
|
12
|
+
!shouldAttachHostProxyForCapability(
|
|
13
|
+
"host_cu_annotate",
|
|
14
|
+
context.transportInterface,
|
|
15
|
+
context.sourceActorPrincipalId,
|
|
16
|
+
)
|
|
17
|
+
) {
|
|
18
|
+
return null;
|
|
19
|
+
}
|
|
20
|
+
try {
|
|
21
|
+
const result = await skillLoadTool.execute(
|
|
22
|
+
{ skill: "screen-annotation" },
|
|
23
|
+
context,
|
|
24
|
+
{ readOnly: true },
|
|
25
|
+
);
|
|
26
|
+
if (result.isError) {
|
|
27
|
+
log.warn(
|
|
28
|
+
{ conversationId: context.conversationId, error: result.content },
|
|
29
|
+
"Could not preload screen annotation",
|
|
30
|
+
);
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
return [
|
|
34
|
+
"The user is sharing their screen. skill_load has automatically loaded screen-annotation for this turn. Use its tools through skill_execute when pointing at the shared screen.",
|
|
35
|
+
result.content,
|
|
36
|
+
].join("\n\n");
|
|
37
|
+
} catch (err) {
|
|
38
|
+
log.warn(
|
|
39
|
+
{ err, conversationId: context.conversationId },
|
|
40
|
+
"Could not preload screen annotation",
|
|
41
|
+
);
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
@@ -32,7 +32,9 @@ import { CONVERSATION_BUSY_MESSAGE } from "../daemon/conversation-messaging.js";
|
|
|
32
32
|
import type { ModeSessionSourceHandle } from "../daemon/conversation-mode-session.js";
|
|
33
33
|
import { resolveChannelCapabilities } from "../daemon/conversation-runtime-assembly.js";
|
|
34
34
|
import { getOrCreateConversation } from "../daemon/conversation-store.js";
|
|
35
|
+
import { getEffectiveEnabledPluginSet } from "../daemon/conversation-tool-setup.js";
|
|
35
36
|
import { preactivateHostProxySkills } from "../daemon/host-proxy-preactivation.js";
|
|
37
|
+
import { resolveTrustClass } from "../daemon/trust-context.js";
|
|
36
38
|
import type { TrustContext } from "../daemon/trust-context-types.js";
|
|
37
39
|
import {
|
|
38
40
|
newestPersistedSightFrame,
|
|
@@ -78,6 +80,7 @@ import {
|
|
|
78
80
|
} from "./voice-control-protocol.js";
|
|
79
81
|
import { judgeEscalation } from "./voice-escalation-judge.js";
|
|
80
82
|
import type { VoiceEscalationTarget } from "./voice-escalation-target.js";
|
|
83
|
+
import { loadVoiceScreenAnnotation } from "./voice-screen-annotation.js";
|
|
81
84
|
import {
|
|
82
85
|
createFrontDoorStreamGate,
|
|
83
86
|
escalatedContinuationRule,
|
|
@@ -452,6 +455,8 @@ export interface VoiceTurnOptions {
|
|
|
452
455
|
* session reports `macos` for its channel capabilities, iOS included.
|
|
453
456
|
*/
|
|
454
457
|
macosDesktopSession?: boolean;
|
|
458
|
+
/** The desktop client currently shares a surface with this voice session. */
|
|
459
|
+
screenSharing?: boolean;
|
|
455
460
|
/** Whether this is an inbound call (no outbound task). */
|
|
456
461
|
isInbound: boolean;
|
|
457
462
|
/** The outbound call task, if any. */
|
|
@@ -2108,6 +2113,23 @@ export async function startVoiceTurn(
|
|
|
2108
2113
|
sourceInterface,
|
|
2109
2114
|
sourceActorPrincipalId,
|
|
2110
2115
|
);
|
|
2116
|
+
if (opts.screenSharing === true) {
|
|
2117
|
+
const annotation = await loadVoiceScreenAnnotation({
|
|
2118
|
+
conversationId: opts.conversationId,
|
|
2119
|
+
workingDir: conversation.workingDir,
|
|
2120
|
+
trustClass: resolveTrustClass(opts.trustContext),
|
|
2121
|
+
transportInterface: sourceInterface,
|
|
2122
|
+
clientOs: "macos",
|
|
2123
|
+
enabledPluginSet: getEffectiveEnabledPluginSet(conversation),
|
|
2124
|
+
sourceActorPrincipalId,
|
|
2125
|
+
signal: opts.signal,
|
|
2126
|
+
});
|
|
2127
|
+
if (annotation !== null) {
|
|
2128
|
+
conversation.setVoiceCallControlPrompt(
|
|
2129
|
+
[voiceCallControlPrompt, annotation].filter(Boolean).join("\n\n"),
|
|
2130
|
+
);
|
|
2131
|
+
}
|
|
2132
|
+
}
|
|
2111
2133
|
}
|
|
2112
2134
|
await conversation.runAgentLoop(persistedContent, messageId, {
|
|
2113
2135
|
...(opts.subagentNotification?.cronRunId
|
|
@@ -66,6 +66,20 @@ let platformInstallCalls: Array<{ name: string; force?: boolean }> = [];
|
|
|
66
66
|
let upgradePluginCalls: UpgradePluginOptions[] = [];
|
|
67
67
|
let catalogMatches: PluginSearchMatch[] = [];
|
|
68
68
|
let catalogError: Error | null = null;
|
|
69
|
+
const CATALOG_COMMIT = "a".repeat(40);
|
|
70
|
+
|
|
71
|
+
function githubCatalogMatch(name: string): PluginSearchMatch {
|
|
72
|
+
return {
|
|
73
|
+
name,
|
|
74
|
+
path: `github:example-org/${name}@${CATALOG_COMMIT}`,
|
|
75
|
+
category: null,
|
|
76
|
+
source: {
|
|
77
|
+
kind: "github",
|
|
78
|
+
repo: `example-org/${name}`,
|
|
79
|
+
ref: CATALOG_COMMIT,
|
|
80
|
+
},
|
|
81
|
+
};
|
|
82
|
+
}
|
|
69
83
|
|
|
70
84
|
/**
|
|
71
85
|
* Queued daemon IPC responses. The default (empty queue) is a transport
|
|
@@ -269,7 +283,10 @@ beforeEach(() => {
|
|
|
269
283
|
installPluginCalls = [];
|
|
270
284
|
platformInstallCalls = [];
|
|
271
285
|
upgradePluginCalls = [];
|
|
272
|
-
catalogMatches = [
|
|
286
|
+
catalogMatches = [
|
|
287
|
+
githubCatalogMatch("example"),
|
|
288
|
+
githubCatalogMatch("imessage"),
|
|
289
|
+
];
|
|
273
290
|
catalogError = null;
|
|
274
291
|
ipcResults = [];
|
|
275
292
|
inspectResult = null;
|
|
@@ -485,6 +502,17 @@ describe("plugins install - declared-schedules consent", () => {
|
|
|
485
502
|
]);
|
|
486
503
|
expect(r.exitCode).toBe(0);
|
|
487
504
|
});
|
|
505
|
+
|
|
506
|
+
test("does not use the outage fallback for a gated integration", async () => {
|
|
507
|
+
catalogError = new Error("catalog unavailable");
|
|
508
|
+
|
|
509
|
+
const r = await runCommand(["plugins", "install", "gamma"]);
|
|
510
|
+
|
|
511
|
+
expect(installPluginCalls).toHaveLength(0);
|
|
512
|
+
expect(platformInstallCalls).toHaveLength(0);
|
|
513
|
+
expect(r.stderr).toContain("not in the marketplace catalog");
|
|
514
|
+
expect(r.exitCode).toBe(1);
|
|
515
|
+
});
|
|
488
516
|
});
|
|
489
517
|
|
|
490
518
|
describe("plugins upgrade - declared-schedules consent (local fallback)", () => {
|
|
@@ -238,7 +238,15 @@ export function registerPluginsCommand(program: Command): void {
|
|
|
238
238
|
const platformEnabled =
|
|
239
239
|
libs.catalogLocal.arePlatformFeaturesEnabled();
|
|
240
240
|
if (platformEnabled) {
|
|
241
|
+
if (!libs.catalogLocal.isPluginCatalogEntryVisible(nameOrUrl)) {
|
|
242
|
+
console.error(
|
|
243
|
+
`Plugin "${nameOrUrl}" is not in the marketplace catalog.`,
|
|
244
|
+
);
|
|
245
|
+
process.exitCode = 1;
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
241
248
|
let match;
|
|
249
|
+
let catalogResolved = false;
|
|
242
250
|
try {
|
|
243
251
|
match = (
|
|
244
252
|
await libs.catalogCache.getPluginCatalog(
|
|
@@ -246,10 +254,18 @@ export function registerPluginsCommand(program: Command): void {
|
|
|
246
254
|
{ fetch: globalThis.fetch.bind(globalThis) },
|
|
247
255
|
)
|
|
248
256
|
).matches.find((candidate) => candidate.name === nameOrUrl);
|
|
257
|
+
catalogResolved = true;
|
|
249
258
|
} catch {
|
|
250
259
|
// The install endpoint remains authoritative and can succeed
|
|
251
260
|
// independently when catalog discovery is unavailable.
|
|
252
261
|
}
|
|
262
|
+
if (!match && catalogResolved) {
|
|
263
|
+
console.error(
|
|
264
|
+
`Plugin "${nameOrUrl}" is not in the marketplace catalog.`,
|
|
265
|
+
);
|
|
266
|
+
process.exitCode = 1;
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
253
269
|
if (match?.source.kind === "local") {
|
|
254
270
|
result = await libs.installGitHub.installPlugin(
|
|
255
271
|
{
|
|
@@ -306,6 +306,35 @@ describe("inspectPlugin", () => {
|
|
|
306
306
|
expect(result.remote?.commit).toBe(SHA_B);
|
|
307
307
|
});
|
|
308
308
|
|
|
309
|
+
test("hides an uninstalled gated integration without fetching the marketplace", async () => {
|
|
310
|
+
let fetchCalls = 0;
|
|
311
|
+
const fetch = (async () => {
|
|
312
|
+
fetchCalls += 1;
|
|
313
|
+
return new Response("unexpected request", { status: 500 });
|
|
314
|
+
}) as FetchLike;
|
|
315
|
+
|
|
316
|
+
await expect(
|
|
317
|
+
inspectPlugin(
|
|
318
|
+
{ name: "gamma" },
|
|
319
|
+
{ fetch, workspacePluginsDir: workspace },
|
|
320
|
+
),
|
|
321
|
+
).rejects.toBeInstanceOf(PluginInspectNotFoundError);
|
|
322
|
+
expect(fetchCalls).toBe(0);
|
|
323
|
+
});
|
|
324
|
+
|
|
325
|
+
test("keeps an installed gated integration inspectable", async () => {
|
|
326
|
+
installPlugin(workspace, "gamma", { sidecar: null });
|
|
327
|
+
const fetch = makeFetch({ marketplace: manifestWith("gamma", SHA_B) });
|
|
328
|
+
|
|
329
|
+
const result = await inspectPlugin(
|
|
330
|
+
{ name: "gamma" },
|
|
331
|
+
{ fetch, workspacePluginsDir: workspace },
|
|
332
|
+
);
|
|
333
|
+
|
|
334
|
+
expect(result.installed).toBe(true);
|
|
335
|
+
expect(result.local?.description).toBe("Installed copy.");
|
|
336
|
+
});
|
|
337
|
+
|
|
309
338
|
test("previews an uninstalled bundled provider offline without fetching GitHub", async () => {
|
|
310
339
|
const savedDisablePlatform = process.env.VELLUM_DISABLE_PLATFORM;
|
|
311
340
|
const savedIsPlatform = process.env.IS_PLATFORM;
|