@vellumai/assistant 0.4.22 → 0.4.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/bun.lock +3 -0
  2. package/package.json +2 -1
  3. package/scripts/ipc/check-swift-decoder-drift.ts +55 -44
  4. package/src/__tests__/__snapshots__/ipc-snapshot.test.ts.snap +0 -90
  5. package/src/__tests__/assistant-events-sse-hardening.test.ts +9 -3
  6. package/src/__tests__/config-schema.test.ts +38 -178
  7. package/src/__tests__/conversation-routes-guardian-reply.test.ts +4 -1
  8. package/src/__tests__/credential-security-invariants.test.ts +0 -2
  9. package/src/__tests__/guardian-verify-setup-skill-regression.test.ts +2 -2
  10. package/src/__tests__/headless-browser-interactions.test.ts +0 -4
  11. package/src/__tests__/ipc-snapshot.test.ts +0 -63
  12. package/src/__tests__/onboarding-template-contract.test.ts +10 -20
  13. package/src/__tests__/relay-server.test.ts +3 -3
  14. package/src/__tests__/resolve-guardian-trust-class.test.ts +61 -0
  15. package/src/__tests__/runtime-events-sse-parity.test.ts +10 -0
  16. package/src/__tests__/runtime-events-sse.test.ts +7 -0
  17. package/src/__tests__/session-init.benchmark.test.ts +0 -4
  18. package/src/__tests__/session-runtime-assembly.test.ts +34 -8
  19. package/src/__tests__/system-prompt.test.ts +7 -1
  20. package/src/__tests__/trusted-contact-approval-notifier.test.ts +12 -8
  21. package/src/__tests__/twilio-routes-twiml.test.ts +2 -2
  22. package/src/__tests__/twilio-routes.test.ts +2 -3
  23. package/src/__tests__/voice-quality.test.ts +21 -132
  24. package/src/calls/relay-server.ts +11 -5
  25. package/src/calls/twilio-routes.ts +4 -38
  26. package/src/calls/voice-quality.ts +7 -63
  27. package/src/config/bundled-skills/guardian-verify-setup/SKILL.md +7 -10
  28. package/src/config/bundled-skills/messaging/SKILL.md +3 -5
  29. package/src/config/bundled-skills/phone-calls/SKILL.md +143 -82
  30. package/src/config/bundled-skills/sms-setup/SKILL.md +0 -20
  31. package/src/config/bundled-skills/twilio-setup/SKILL.md +9 -17
  32. package/src/config/bundled-skills/voice-setup/SKILL.md +36 -1
  33. package/src/config/bundled-skills/voice-setup/icon.svg +20 -0
  34. package/src/config/calls-schema.ts +3 -53
  35. package/src/config/elevenlabs-schema.ts +33 -0
  36. package/src/config/schema.ts +183 -137
  37. package/src/config/types.ts +0 -1
  38. package/src/daemon/daemon-control.ts +3 -0
  39. package/src/daemon/handlers/browser.ts +2 -53
  40. package/src/daemon/ipc-contract/browser.ts +5 -84
  41. package/src/daemon/ipc-contract/surfaces.ts +51 -48
  42. package/src/daemon/ipc-contract-inventory.json +0 -9
  43. package/src/daemon/session-agent-loop-handlers.ts +3 -0
  44. package/src/daemon/session-agent-loop.ts +2 -1
  45. package/src/daemon/session-runtime-assembly.ts +9 -7
  46. package/src/daemon/session-tool-setup.ts +27 -13
  47. package/src/mcp/client.ts +2 -1
  48. package/src/memory/conversation-crud.ts +339 -166
  49. package/src/memory/migrations/102-alter-table-columns.ts +254 -37
  50. package/src/memory/schema.ts +1227 -1035
  51. package/src/runtime/routes/events-routes.ts +7 -0
  52. package/src/runtime/routes/inbound-message-handler.ts +3 -4
  53. package/src/schedule/scheduler.ts +159 -45
  54. package/src/security/secure-keys.ts +3 -3
  55. package/src/tools/browser/browser-execution.ts +314 -331
  56. package/src/tools/browser/browser-handoff.ts +11 -37
  57. package/src/tools/browser/browser-manager.ts +203 -352
  58. package/src/tools/browser/browser-screencast.ts +15 -76
  59. package/src/tools/network/script-proxy/certs.ts +7 -237
  60. package/src/tools/network/script-proxy/connect-tunnel.ts +1 -82
  61. package/src/tools/network/script-proxy/http-forwarder.ts +2 -151
  62. package/src/tools/network/script-proxy/logging.ts +12 -196
  63. package/src/tools/network/script-proxy/mitm-handler.ts +2 -270
  64. package/src/tools/network/script-proxy/policy.ts +4 -152
  65. package/src/tools/network/script-proxy/router.ts +2 -60
  66. package/src/tools/network/script-proxy/server.ts +5 -137
  67. package/src/tools/network/script-proxy/types.ts +19 -125
  68. package/src/tools/system/voice-config.ts +23 -1
  69. package/src/util/logger.ts +4 -1
  70. package/src/__tests__/elevenlabs-config.test.ts +0 -95
  71. package/src/__tests__/twilio-routes-elevenlabs.test.ts +0 -407
  72. package/src/calls/elevenlabs-config.ts +0 -32
@@ -2,14 +2,27 @@
2
2
 
3
3
  // === Surface type definitions ===
4
4
 
5
- export type SurfaceType = 'card' | 'form' | 'list' | 'table' | 'confirmation' | 'dynamic_page' | 'file_upload' | 'browser_view' | 'document_preview';
6
-
7
- export const INTERACTIVE_SURFACE_TYPES: SurfaceType[] = ['form', 'confirmation', 'dynamic_page', 'file_upload'];
5
+ export type SurfaceType =
6
+ | "card"
7
+ | "form"
8
+ | "list"
9
+ | "table"
10
+ | "confirmation"
11
+ | "dynamic_page"
12
+ | "file_upload"
13
+ | "document_preview";
14
+
15
+ export const INTERACTIVE_SURFACE_TYPES: SurfaceType[] = [
16
+ "form",
17
+ "confirmation",
18
+ "dynamic_page",
19
+ "file_upload",
20
+ ];
8
21
 
9
22
  export interface SurfaceAction {
10
23
  id: string;
11
24
  label: string;
12
- style?: 'primary' | 'secondary' | 'destructive';
25
+ style?: "primary" | "secondary" | "destructive";
13
26
  }
14
27
 
15
28
  export interface CardSurfaceData {
@@ -25,7 +38,7 @@ export interface CardSurfaceData {
25
38
 
26
39
  export interface FormField {
27
40
  id: string;
28
- type: 'text' | 'textarea' | 'select' | 'toggle' | 'number' | 'password';
41
+ type: "text" | "textarea" | "select" | "toggle" | "number" | "password";
29
42
  label: string;
30
43
  placeholder?: string;
31
44
  required?: boolean;
@@ -58,7 +71,7 @@ export interface ListItem {
58
71
 
59
72
  export interface ListSurfaceData {
60
73
  items: ListItem[];
61
- selectionMode: 'single' | 'multiple' | 'none';
74
+ selectionMode: "single" | "multiple" | "none";
62
75
  }
63
76
 
64
77
  export interface ConfirmationSurfaceData {
@@ -76,8 +89,8 @@ export interface DynamicPagePreview {
76
89
  description?: string;
77
90
  icon?: string;
78
91
  metrics?: Array<{ label: string; value: string }>;
79
- context?: 'app_create' | 'general';
80
- previewImage?: string; // base64 PNG
92
+ context?: "app_create" | "general";
93
+ previewImage?: string; // base64 PNG
81
94
  }
82
95
 
83
96
  export interface DynamicPageSurfaceData {
@@ -106,8 +119,8 @@ export interface TableColumn {
106
119
 
107
120
  export interface TableCellValue {
108
121
  text: string;
109
- icon?: string; // SF Symbol name
110
- iconColor?: string; // semantic token: "success" | "warning" | "error" | "muted"
122
+ icon?: string; // SF Symbol name
123
+ iconColor?: string; // semantic token: "success" | "warning" | "error" | "muted"
111
124
  }
112
125
 
113
126
  export interface TableRow {
@@ -120,32 +133,30 @@ export interface TableRow {
120
133
  export interface TableSurfaceData {
121
134
  columns: TableColumn[];
122
135
  rows: TableRow[];
123
- selectionMode?: 'none' | 'single' | 'multiple';
136
+ selectionMode?: "none" | "single" | "multiple";
124
137
  caption?: string;
125
138
  }
126
139
 
127
- export interface BrowserViewSurfaceData {
128
- sessionId: string;
129
- currentUrl: string;
130
- status: 'navigating' | 'idle' | 'interacting';
131
- frame?: string; // base64 JPEG
132
- actionText?: string; // "Clicking 'Submit' button"
133
- highlights?: Array<{ x: number; y: number; w: number; h: number; label: string }>;
134
- pages?: Array<{ id: string; title: string; url: string; active: boolean }>;
135
- }
136
-
137
140
  export interface DocumentPreviewSurfaceData {
138
141
  title: string;
139
- surfaceId: string; // the doc's real surfaceId, for focusing the panel
142
+ surfaceId: string; // the doc's real surfaceId, for focusing the panel
140
143
  subtitle?: string;
141
144
  }
142
145
 
143
- export type SurfaceData = CardSurfaceData | FormSurfaceData | ListSurfaceData | TableSurfaceData | ConfirmationSurfaceData | DynamicPageSurfaceData | FileUploadSurfaceData | BrowserViewSurfaceData | DocumentPreviewSurfaceData;
146
+ export type SurfaceData =
147
+ | CardSurfaceData
148
+ | FormSurfaceData
149
+ | ListSurfaceData
150
+ | TableSurfaceData
151
+ | ConfirmationSurfaceData
152
+ | DynamicPageSurfaceData
153
+ | FileUploadSurfaceData
154
+ | DocumentPreviewSurfaceData;
144
155
 
145
156
  // === Client → Server ===
146
157
 
147
158
  export interface UiSurfaceAction {
148
- type: 'ui_surface_action';
159
+ type: "ui_surface_action";
149
160
  sessionId: string;
150
161
  surfaceId: string;
151
162
  actionId: string;
@@ -153,7 +164,7 @@ export interface UiSurfaceAction {
153
164
  }
154
165
 
155
166
  export interface UiSurfaceUndoRequest {
156
- type: 'ui_surface_undo';
167
+ type: "ui_surface_undo";
157
168
  sessionId: string;
158
169
  surfaceId: string;
159
170
  }
@@ -162,58 +173,53 @@ export interface UiSurfaceUndoRequest {
162
173
 
163
174
  /** Common fields shared by all UiSurfaceShow variants. */
164
175
  interface UiSurfaceShowBase {
165
- type: 'ui_surface_show';
176
+ type: "ui_surface_show";
166
177
  sessionId: string;
167
178
  surfaceId: string;
168
179
  title?: string;
169
180
  actions?: SurfaceAction[];
170
- display?: 'inline' | 'panel';
181
+ display?: "inline" | "panel";
171
182
  /** The message ID that this surface belongs to (for history loading). */
172
183
  messageId?: string;
173
184
  }
174
185
 
175
186
  export interface UiSurfaceShowCard extends UiSurfaceShowBase {
176
- surfaceType: 'card';
187
+ surfaceType: "card";
177
188
  data: CardSurfaceData;
178
189
  }
179
190
 
180
191
  export interface UiSurfaceShowForm extends UiSurfaceShowBase {
181
- surfaceType: 'form';
192
+ surfaceType: "form";
182
193
  data: FormSurfaceData;
183
194
  }
184
195
 
185
196
  export interface UiSurfaceShowList extends UiSurfaceShowBase {
186
- surfaceType: 'list';
197
+ surfaceType: "list";
187
198
  data: ListSurfaceData;
188
199
  }
189
200
 
190
201
  export interface UiSurfaceShowConfirmation extends UiSurfaceShowBase {
191
- surfaceType: 'confirmation';
202
+ surfaceType: "confirmation";
192
203
  data: ConfirmationSurfaceData;
193
204
  }
194
205
 
195
206
  export interface UiSurfaceShowDynamicPage extends UiSurfaceShowBase {
196
- surfaceType: 'dynamic_page';
207
+ surfaceType: "dynamic_page";
197
208
  data: DynamicPageSurfaceData;
198
209
  }
199
210
 
200
211
  export interface UiSurfaceShowTable extends UiSurfaceShowBase {
201
- surfaceType: 'table';
212
+ surfaceType: "table";
202
213
  data: TableSurfaceData;
203
214
  }
204
215
 
205
216
  export interface UiSurfaceShowFileUpload extends UiSurfaceShowBase {
206
- surfaceType: 'file_upload';
217
+ surfaceType: "file_upload";
207
218
  data: FileUploadSurfaceData;
208
219
  }
209
220
 
210
- export interface UiSurfaceShowBrowserView extends UiSurfaceShowBase {
211
- surfaceType: 'browser_view';
212
- data: BrowserViewSurfaceData;
213
- }
214
-
215
221
  export interface UiSurfaceShowDocumentPreview extends UiSurfaceShowBase {
216
- surfaceType: 'document_preview';
222
+ surfaceType: "document_preview";
217
223
  data: DocumentPreviewSurfaceData;
218
224
  }
219
225
 
@@ -225,24 +231,23 @@ export type UiSurfaceShow =
225
231
  | UiSurfaceShowConfirmation
226
232
  | UiSurfaceShowDynamicPage
227
233
  | UiSurfaceShowFileUpload
228
- | UiSurfaceShowBrowserView
229
234
  | UiSurfaceShowDocumentPreview;
230
235
 
231
236
  export interface UiSurfaceUpdate {
232
- type: 'ui_surface_update';
237
+ type: "ui_surface_update";
233
238
  sessionId: string;
234
239
  surfaceId: string;
235
240
  data: Partial<SurfaceData>;
236
241
  }
237
242
 
238
243
  export interface UiSurfaceDismiss {
239
- type: 'ui_surface_dismiss';
244
+ type: "ui_surface_dismiss";
240
245
  sessionId: string;
241
246
  surfaceId: string;
242
247
  }
243
248
 
244
249
  export interface UiSurfaceComplete {
245
- type: 'ui_surface_complete';
250
+ type: "ui_surface_complete";
246
251
  sessionId: string;
247
252
  surfaceId: string;
248
253
  summary: string;
@@ -250,7 +255,7 @@ export interface UiSurfaceComplete {
250
255
  }
251
256
 
252
257
  export interface UiSurfaceUndoResult {
253
- type: 'ui_surface_undo_result';
258
+ type: "ui_surface_undo_result";
254
259
  sessionId: string;
255
260
  surfaceId: string;
256
261
  success: boolean;
@@ -260,9 +265,7 @@ export interface UiSurfaceUndoResult {
260
265
 
261
266
  // --- Domain-level union aliases (consumed by the barrel file) ---
262
267
 
263
- export type _SurfacesClientMessages =
264
- | UiSurfaceAction
265
- | UiSurfaceUndoRequest;
268
+ export type _SurfacesClientMessages = UiSurfaceAction | UiSurfaceUndoRequest;
266
269
 
267
270
  export type _SurfacesServerMessages =
268
271
  | UiSurfaceShow
@@ -62,11 +62,6 @@
62
62
  "apps_list",
63
63
  "assistant_inbox_escalation",
64
64
  "auth",
65
- "browser_cdp_response",
66
- "browser_interactive_mode",
67
- "browser_user_click",
68
- "browser_user_keypress",
69
- "browser_user_scroll",
70
65
  "bundle_app",
71
66
  "cancel",
72
67
  "confirmation_response",
@@ -210,10 +205,6 @@
210
205
  "assistant_thinking_delta",
211
206
  "auth_result",
212
207
  "avatar_updated",
213
- "browser_cdp_request",
214
- "browser_frame",
215
- "browser_handoff_request",
216
- "browser_interactive_mode_changed",
217
208
  "bundle_app_response",
218
209
  "client_settings_update",
219
210
  "confirmation_request",
@@ -150,6 +150,9 @@ const TOOL_FRIENDLY_NAMES: Record<string, string> = {
150
150
  browser_wait: 'browser',
151
151
  app_create: 'app',
152
152
  app_update: 'app',
153
+ skill_load: 'skill',
154
+ app_file_edit: 'app file',
155
+ app_file_write: 'app file',
153
156
  };
154
157
 
155
158
  function friendlyToolName(name: string): string {
@@ -65,6 +65,7 @@ import {
65
65
  stripInjectedContext,
66
66
  } from './session-runtime-assembly.js';
67
67
  import type { SkillProjectionCache } from './session-skill-tools.js';
68
+ import { resolveGuardianTrustClass } from './session-tool-setup.js';
68
69
  import { recordUsage } from './session-usage.js';
69
70
  import type { TraceEmitter } from './trace-emitter.js';
70
71
 
@@ -319,7 +320,7 @@ export async function runAgentLoopImpl(
319
320
  conflictGate: ctx.conflictGate,
320
321
  scopeId: ctx.memoryPolicy.scopeId,
321
322
  includeDefaultFallback: ctx.memoryPolicy.includeDefaultFallback,
322
- guardianTrustClass: ctx.guardianContext?.trustClass ?? 'guardian',
323
+ guardianTrustClass: resolveGuardianTrustClass(ctx.guardianContext),
323
324
  isInteractive: options?.isInteractive ?? (!ctx.hasNoClient && !ctx.headlessLock),
324
325
  },
325
326
  content,
@@ -314,7 +314,7 @@ export function resolveChannelCapabilities(
314
314
  return {
315
315
  channel,
316
316
  dashboardCapable: supportsDesktopUi,
317
- supportsDynamicUi: supportsDesktopUi,
317
+ supportsDynamicUi: supportsDesktopUi || iface === "vellum",
318
318
  supportsVoiceInput: supportsDesktopUi,
319
319
  pttActivationKey: sanitizePttActivationKey(
320
320
  pttMetadata?.pttActivationKey,
@@ -592,12 +592,14 @@ export function injectChannelCapabilityContext(
592
592
  lines.push(
593
593
  "- Do NOT reference the dashboard UI, settings panels, or visual preference pickers.",
594
594
  );
595
- lines.push(
596
- "- Do NOT use ui_show, ui_update, or app_create — this channel cannot render them.",
597
- );
598
- lines.push(
599
- "- Present information as well-formatted text instead of dynamic UI.",
600
- );
595
+ if (!caps.supportsDynamicUi) {
596
+ lines.push(
597
+ "- Do NOT use ui_show, ui_update, or app_create — this channel cannot render them.",
598
+ );
599
+ lines.push(
600
+ "- Present information as well-formatted text instead of dynamic UI.",
601
+ );
602
+ }
601
603
  lines.push(
602
604
  "- Defer dashboard-specific actions (e.g. accent color selection) by telling the user",
603
605
  );
@@ -6,6 +6,7 @@
6
6
  * keeping the constructor body focused on wiring.
7
7
  */
8
8
 
9
+ import { isHttpAuthDisabled } from "../config/env.js";
9
10
  import {
10
11
  generateAllowlistOptions,
11
12
  generateScopeOptions,
@@ -19,30 +20,28 @@ import {
19
20
  } from "../permissions/trust-store.js";
20
21
  import { isAllowDecision } from "../permissions/types.js";
21
22
  import type { Message, ToolDefinition } from "../providers/types.js";
23
+ import type { TrustClass } from "../runtime/actor-trust-resolver.js";
22
24
  import { getEffectiveMode } from "../runtime/session-approval-overrides.js";
25
+ import { coreAppProxyTools } from "../tools/apps/definitions.js";
26
+ import { registerSessionSender } from "../tools/browser/browser-screencast.js";
27
+ import { requestComputerControlTool } from "../tools/computer-use/request-computer-control.js";
23
28
  import type { ToolExecutor } from "../tools/executor.js";
29
+ import type {
30
+ ProxyApprovalCallback,
31
+ ProxyApprovalRequest,
32
+ } from "../tools/network/script-proxy/index.js";
33
+ import { getAllToolDefinitions } from "../tools/registry.js";
24
34
  import type {
25
35
  ToolExecutionResult,
26
36
  ToolLifecycleEventHandler,
27
37
  } from "../tools/types.js";
38
+ import { allUiSurfaceTools } from "../tools/ui-surface/definitions.js";
28
39
  import { getLogger } from "../util/logger.js";
29
40
  import {
30
41
  isDoordashCommand,
31
42
  markDoordashStepInProgress,
32
43
  } from "./doordash-steps.js";
33
44
  import type { ServerMessage, UiSurfaceShow } from "./ipc-protocol.js";
34
- import { runPostExecutionSideEffects } from "./tool-side-effects.js";
35
-
36
- const log = getLogger("session-tool-setup");
37
- import { coreAppProxyTools } from "../tools/apps/definitions.js";
38
- import { registerSessionSender } from "../tools/browser/browser-screencast.js";
39
- import { requestComputerControlTool } from "../tools/computer-use/request-computer-control.js";
40
- import type {
41
- ProxyApprovalCallback,
42
- ProxyApprovalRequest,
43
- } from "../tools/network/script-proxy/index.js";
44
- import { getAllToolDefinitions } from "../tools/registry.js";
45
- import { allUiSurfaceTools } from "../tools/ui-surface/definitions.js";
46
45
  import type { GuardianRuntimeContext } from "./session-runtime-assembly.js";
47
46
  import {
48
47
  projectSkillTools,
@@ -50,6 +49,21 @@ import {
50
49
  } from "./session-skill-tools.js";
51
50
  import type { SurfaceSessionContext } from "./session-surfaces.js";
52
51
  import { surfaceProxyResolver } from "./session-surfaces.js";
52
+ import { runPostExecutionSideEffects } from "./tool-side-effects.js";
53
+
54
+ const log = getLogger("session-tool-setup");
55
+
56
+ /**
57
+ * Resolve the effective guardian trust class for tool execution.
58
+ * When HTTP auth is disabled (dev bypass), always treat the actor as
59
+ * guardian so that control-plane gates don't block local development.
60
+ */
61
+ export function resolveGuardianTrustClass(
62
+ guardianContext: GuardianRuntimeContext | undefined,
63
+ ): TrustClass {
64
+ if (isHttpAuthDisabled()) return "guardian";
65
+ return guardianContext?.trustClass ?? "guardian";
66
+ }
53
67
 
54
68
  // ── Context Interface ────────────────────────────────────────────────
55
69
 
@@ -137,7 +151,7 @@ export function createToolExecutor(
137
151
  assistantId: ctx.assistantId,
138
152
  requestId: ctx.currentRequestId,
139
153
  taskRunId: ctx.taskRunId,
140
- guardianTrustClass: ctx.guardianContext?.trustClass ?? "guardian",
154
+ guardianTrustClass: resolveGuardianTrustClass(ctx.guardianContext),
141
155
  executionChannel: ctx.guardianContext?.sourceChannel,
142
156
  callSessionId: ctx.callSessionId,
143
157
  triggeredBySurfaceAction:
package/src/mcp/client.ts CHANGED
@@ -75,7 +75,8 @@ export class McpClient {
75
75
 
76
76
  if (isHttpTransport) {
77
77
  const isAuthError = err instanceof UnauthorizedError
78
- || (err instanceof Error && /\b(401|403|unauthorized|forbidden)\b/i.test(err.message));
78
+ || (err instanceof Error && /\b(401|403|unauthorized|forbidden)\b/i.test(err.message))
79
+ || (err != null && typeof err === 'object' && 'code' in err && (err.code === 401 || err.code === 403));
79
80
 
80
81
  if (isAuthError) {
81
82
  // Auth-related — user can run `vellum mcp auth <name>` to authenticate.