@vellumai/assistant 0.3.3 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +2 -0
- package/README.md +45 -18
- package/package.json +1 -1
- package/scripts/ipc/generate-swift.ts +13 -0
- package/src/__tests__/__snapshots__/ipc-snapshot.test.ts.snap +100 -0
- package/src/__tests__/approval-hardcoded-copy-guard.test.ts +41 -0
- package/src/__tests__/approval-message-composer.test.ts +253 -0
- package/src/__tests__/call-domain.test.ts +12 -2
- package/src/__tests__/call-orchestrator.test.ts +391 -1
- package/src/__tests__/call-routes-http.test.ts +27 -2
- package/src/__tests__/channel-approval-routes.test.ts +397 -135
- package/src/__tests__/channel-approvals.test.ts +99 -3
- package/src/__tests__/channel-delivery-store.test.ts +30 -4
- package/src/__tests__/channel-guardian.test.ts +261 -22
- package/src/__tests__/channel-readiness-service.test.ts +257 -0
- package/src/__tests__/config-schema.test.ts +2 -1
- package/src/__tests__/credential-security-invariants.test.ts +1 -0
- package/src/__tests__/daemon-lifecycle.test.ts +636 -0
- package/src/__tests__/dictation-mode-detection.test.ts +63 -0
- package/src/__tests__/entity-search.test.ts +615 -0
- package/src/__tests__/gateway-only-enforcement.test.ts +19 -13
- package/src/__tests__/handlers-twilio-config.test.ts +480 -0
- package/src/__tests__/ipc-snapshot.test.ts +63 -0
- package/src/__tests__/messaging-send-tool.test.ts +65 -0
- package/src/__tests__/run-orchestrator-assistant-events.test.ts +4 -0
- package/src/__tests__/run-orchestrator.test.ts +22 -0
- package/src/__tests__/secret-scanner.test.ts +223 -0
- package/src/__tests__/session-runtime-assembly.test.ts +85 -1
- package/src/__tests__/shell-parser-property.test.ts +357 -2
- package/src/__tests__/sms-messaging-provider.test.ts +125 -0
- package/src/__tests__/system-prompt.test.ts +25 -1
- package/src/__tests__/tool-executor-lifecycle-events.test.ts +34 -1
- package/src/__tests__/twilio-routes.test.ts +39 -3
- package/src/__tests__/twitter-cli-error-shaping.test.ts +2 -2
- package/src/__tests__/user-reference.test.ts +68 -0
- package/src/__tests__/web-search.test.ts +1 -1
- package/src/__tests__/work-item-output.test.ts +110 -0
- package/src/calls/call-domain.ts +8 -5
- package/src/calls/call-orchestrator.ts +85 -22
- package/src/calls/twilio-config.ts +17 -11
- package/src/calls/twilio-rest.ts +276 -0
- package/src/calls/twilio-routes.ts +39 -1
- package/src/cli/map.ts +6 -0
- package/src/commands/__tests__/cc-command-registry.test.ts +67 -0
- package/src/commands/cc-command-registry.ts +14 -1
- package/src/config/bundled-skills/claude-code/TOOLS.json +10 -3
- package/src/config/bundled-skills/knowledge-graph/SKILL.md +15 -0
- package/src/config/bundled-skills/knowledge-graph/TOOLS.json +56 -0
- package/src/config/bundled-skills/knowledge-graph/tools/graph-query.ts +185 -0
- package/src/config/bundled-skills/media-processing/SKILL.md +199 -0
- package/src/config/bundled-skills/media-processing/TOOLS.json +320 -0
- package/src/config/bundled-skills/media-processing/services/capability-registry.ts +137 -0
- package/src/config/bundled-skills/media-processing/services/event-detection-service.ts +280 -0
- package/src/config/bundled-skills/media-processing/services/feedback-aggregation.ts +144 -0
- package/src/config/bundled-skills/media-processing/services/feedback-store.ts +136 -0
- package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +261 -0
- package/src/config/bundled-skills/media-processing/services/retrieval-service.ts +95 -0
- package/src/config/bundled-skills/media-processing/services/timeline-service.ts +267 -0
- package/src/config/bundled-skills/media-processing/tools/analyze-keyframes.ts +301 -0
- package/src/config/bundled-skills/media-processing/tools/detect-events.ts +110 -0
- package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +190 -0
- package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +195 -0
- package/src/config/bundled-skills/media-processing/tools/ingest-media.ts +197 -0
- package/src/config/bundled-skills/media-processing/tools/media-diagnostics.ts +166 -0
- package/src/config/bundled-skills/media-processing/tools/media-status.ts +75 -0
- package/src/config/bundled-skills/media-processing/tools/query-media-events.ts +300 -0
- package/src/config/bundled-skills/media-processing/tools/recalibrate.ts +235 -0
- package/src/config/bundled-skills/media-processing/tools/select-tracking-profile.ts +142 -0
- package/src/config/bundled-skills/media-processing/tools/submit-feedback.ts +150 -0
- package/src/config/bundled-skills/messaging/SKILL.md +24 -5
- package/src/config/bundled-skills/messaging/tools/messaging-send.ts +5 -1
- package/src/config/bundled-skills/phone-calls/SKILL.md +2 -2
- package/src/config/bundled-skills/twitter/SKILL.md +19 -3
- package/src/config/defaults.ts +2 -1
- package/src/config/schema.ts +9 -3
- package/src/config/skills.ts +5 -32
- package/src/config/system-prompt.ts +40 -0
- package/src/config/templates/IDENTITY.md +2 -2
- package/src/config/user-reference.ts +29 -0
- package/src/config/vellum-skills/catalog.json +58 -0
- package/src/config/vellum-skills/google-oauth-setup/SKILL.md +3 -3
- package/src/config/vellum-skills/slack-oauth-setup/SKILL.md +3 -3
- package/src/config/vellum-skills/sms-setup/SKILL.md +118 -0
- package/src/config/vellum-skills/telegram-setup/SKILL.md +6 -1
- package/src/config/vellum-skills/twilio-setup/SKILL.md +76 -6
- package/src/daemon/auth-manager.ts +103 -0
- package/src/daemon/computer-use-session.ts +8 -1
- package/src/daemon/config-watcher.ts +253 -0
- package/src/daemon/handlers/config.ts +819 -22
- package/src/daemon/handlers/dictation.ts +182 -0
- package/src/daemon/handlers/identity.ts +14 -23
- package/src/daemon/handlers/index.ts +2 -0
- package/src/daemon/handlers/sessions.ts +2 -0
- package/src/daemon/handlers/shared.ts +3 -0
- package/src/daemon/handlers/skills.ts +6 -7
- package/src/daemon/handlers/work-items.ts +15 -7
- package/src/daemon/ipc-contract-inventory.json +10 -0
- package/src/daemon/ipc-contract.ts +114 -4
- package/src/daemon/ipc-handler.ts +87 -0
- package/src/daemon/lifecycle.ts +18 -4
- package/src/daemon/ride-shotgun-handler.ts +11 -1
- package/src/daemon/server.ts +111 -504
- package/src/daemon/session-agent-loop.ts +10 -15
- package/src/daemon/session-runtime-assembly.ts +115 -44
- package/src/daemon/session-tool-setup.ts +2 -0
- package/src/daemon/session.ts +19 -2
- package/src/inbound/public-ingress-urls.ts +3 -3
- package/src/memory/channel-guardian-store.ts +2 -1
- package/src/memory/db-connection.ts +28 -0
- package/src/memory/db-init.ts +1163 -0
- package/src/memory/db.ts +2 -2007
- package/src/memory/embedding-backend.ts +79 -11
- package/src/memory/indexer.ts +2 -0
- package/src/memory/job-handlers/media-processing.ts +100 -0
- package/src/memory/job-utils.ts +64 -4
- package/src/memory/jobs-store.ts +2 -1
- package/src/memory/jobs-worker.ts +11 -1
- package/src/memory/media-store.ts +759 -0
- package/src/memory/recall-cache.ts +107 -0
- package/src/memory/retriever.ts +36 -2
- package/src/memory/schema-migration.ts +984 -0
- package/src/memory/schema.ts +99 -0
- package/src/memory/search/entity.ts +208 -25
- package/src/memory/search/ranking.ts +6 -1
- package/src/memory/search/types.ts +26 -0
- package/src/messaging/provider-types.ts +2 -0
- package/src/messaging/providers/sms/adapter.ts +204 -0
- package/src/messaging/providers/sms/client.ts +93 -0
- package/src/messaging/providers/sms/types.ts +7 -0
- package/src/permissions/checker.ts +16 -2
- package/src/permissions/prompter.ts +14 -3
- package/src/permissions/trust-store.ts +7 -0
- package/src/runtime/approval-message-composer.ts +143 -0
- package/src/runtime/channel-approvals.ts +29 -7
- package/src/runtime/channel-guardian-service.ts +44 -18
- package/src/runtime/channel-readiness-service.ts +292 -0
- package/src/runtime/channel-readiness-types.ts +29 -0
- package/src/runtime/gateway-client.ts +2 -1
- package/src/runtime/http-server.ts +65 -28
- package/src/runtime/http-types.ts +3 -0
- package/src/runtime/routes/call-routes.ts +2 -1
- package/src/runtime/routes/channel-routes.ts +237 -103
- package/src/runtime/routes/run-routes.ts +7 -1
- package/src/runtime/run-orchestrator.ts +43 -3
- package/src/security/secret-scanner.ts +218 -0
- package/src/skills/frontmatter.ts +63 -0
- package/src/skills/slash-commands.ts +23 -0
- package/src/skills/vellum-catalog-remote.ts +107 -0
- package/src/tools/assets/materialize.ts +2 -2
- package/src/tools/browser/auto-navigate.ts +132 -24
- package/src/tools/browser/browser-manager.ts +67 -61
- package/src/tools/calls/call-start.ts +1 -0
- package/src/tools/claude-code/claude-code.ts +55 -3
- package/src/tools/credentials/vault.ts +1 -1
- package/src/tools/execution-target.ts +11 -1
- package/src/tools/executor.ts +10 -2
- package/src/tools/network/web-search.ts +1 -1
- package/src/tools/skills/vellum-catalog.ts +61 -156
- package/src/tools/terminal/parser.ts +21 -5
- package/src/tools/types.ts +2 -0
- package/src/twitter/router.ts +1 -1
- package/src/util/platform.ts +43 -1
- package/src/util/retry.ts +4 -4
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
import * as net from 'node:net';
|
|
2
|
+
import { getConfig } from '../../config/loader.js';
|
|
3
|
+
import { getFailoverProvider, listProviders } from '../../providers/registry.js';
|
|
4
|
+
import type { DictationRequest } from '../ipc-protocol.js';
|
|
5
|
+
import { log, defineHandlers, type HandlerContext } from './shared.js';
|
|
6
|
+
|
|
7
|
+
// Action verbs that signal the user wants a full agent session rather than inline text
|
|
8
|
+
const ACTION_VERBS = ['slack', 'email', 'send', 'create', 'open', 'search', 'find'];
|
|
9
|
+
|
|
10
|
+
const MAX_WINDOW_TITLE_LENGTH = 100;
|
|
11
|
+
|
|
12
|
+
/** Sanitize window title to mitigate prompt injection from attacker-controlled titles (e.g. browser tabs, Slack conversations). */
|
|
13
|
+
function sanitizeWindowTitle(title: string | undefined): string {
|
|
14
|
+
if (!title) return '';
|
|
15
|
+
return title
|
|
16
|
+
.replace(/[<>]/g, '') // strip angle brackets to prevent tag injection
|
|
17
|
+
.slice(0, MAX_WINDOW_TITLE_LENGTH);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Build a delimited app metadata block so the LLM treats it as contextual data, not instructions. */
|
|
21
|
+
function buildAppMetadataBlock(msg: DictationRequest): string {
|
|
22
|
+
const windowTitle = sanitizeWindowTitle(msg.context.windowTitle);
|
|
23
|
+
return [
|
|
24
|
+
'<app_metadata>',
|
|
25
|
+
`App: ${msg.context.appName} (${msg.context.bundleIdentifier})`,
|
|
26
|
+
`Window: ${windowTitle}`,
|
|
27
|
+
'</app_metadata>',
|
|
28
|
+
].join('\n');
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
type DictationMode = 'dictation' | 'command' | 'action';
|
|
32
|
+
|
|
33
|
+
export function detectDictationMode(msg: DictationRequest): DictationMode {
|
|
34
|
+
// Command mode: selected text present — treat transcription as a transformation instruction
|
|
35
|
+
if (msg.context.selectedText && msg.context.selectedText.trim().length > 0) {
|
|
36
|
+
return 'command';
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// Action mode: transcription starts with an action verb
|
|
40
|
+
const firstWord = msg.transcription.trim().split(/\s+/)[0]?.toLowerCase() ?? '';
|
|
41
|
+
if (ACTION_VERBS.includes(firstWord)) {
|
|
42
|
+
return 'action';
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// Dictation mode: cursor is in a text field with no selection — clean up for typing
|
|
46
|
+
if (msg.context.cursorInTextField) {
|
|
47
|
+
return 'dictation';
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// AX focus-role detection in browser editors (for example Gmail compose)
|
|
51
|
+
// is occasionally incomplete. If we default to action here, normal dictation
|
|
52
|
+
// gets misrouted into a new chat task. Treat ambiguous context as dictation.
|
|
53
|
+
return 'dictation';
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function buildDictationPrompt(msg: DictationRequest): string {
|
|
57
|
+
return [
|
|
58
|
+
'You are a dictation assistant. Clean up the following speech transcription for direct insertion into a text field.',
|
|
59
|
+
'',
|
|
60
|
+
'## Rules',
|
|
61
|
+
'- Fix grammar, punctuation, and capitalization',
|
|
62
|
+
'- Remove filler words (um, uh, like, you know)',
|
|
63
|
+
"- Maintain the speaker's intent and meaning",
|
|
64
|
+
'- Do NOT add explanations or commentary',
|
|
65
|
+
'- Return ONLY the cleaned text, nothing else',
|
|
66
|
+
'',
|
|
67
|
+
'## Tone Adaptation',
|
|
68
|
+
'Adapt your output tone based on the active application:',
|
|
69
|
+
'- Email apps (Gmail, Mail): Professional but warm. Use proper greetings and sign-offs if appropriate.',
|
|
70
|
+
'- Slack: Casual and conversational. Match typical chat style.',
|
|
71
|
+
'- Code editors (VS Code, Xcode): Technical and concise. Code comments style.',
|
|
72
|
+
'- Terminal: Command-like, terse.',
|
|
73
|
+
'- Messages/iMessage: Very casual, texting style. Short sentences.',
|
|
74
|
+
'- Notes/Docs: Neutral, clear writing.',
|
|
75
|
+
'- Default: Match the user\'s natural voice.',
|
|
76
|
+
'',
|
|
77
|
+
'## Context Clues',
|
|
78
|
+
'- Window title may contain recipient name (Slack DMs, email compose)',
|
|
79
|
+
'- If you can identify a recipient, adapt formality to the apparent relationship',
|
|
80
|
+
'- Maintain the user\'s natural voice — don\'t over-formalize casual speech',
|
|
81
|
+
'- The user\'s writing patterns and preferences may be available from memory context — follow those when present',
|
|
82
|
+
'',
|
|
83
|
+
buildAppMetadataBlock(msg),
|
|
84
|
+
].join('\n');
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function buildCommandPrompt(msg: DictationRequest): string {
|
|
88
|
+
return [
|
|
89
|
+
'You are a text transformation assistant. The user has selected text and given a voice command to transform it.',
|
|
90
|
+
'',
|
|
91
|
+
'## Rules',
|
|
92
|
+
'- Apply the instruction to the selected text',
|
|
93
|
+
'- Return ONLY the transformed text, nothing else',
|
|
94
|
+
'- Do NOT add explanations or commentary',
|
|
95
|
+
'',
|
|
96
|
+
'## Tone Adaptation',
|
|
97
|
+
'Match the tone to the active application context:',
|
|
98
|
+
'- Email apps (Gmail, Mail): Professional but warm.',
|
|
99
|
+
'- Slack: Casual and conversational.',
|
|
100
|
+
'- Code editors (VS Code, Xcode): Technical and concise.',
|
|
101
|
+
'- Terminal: Command-like, terse.',
|
|
102
|
+
'- Messages/iMessage: Very casual, texting style.',
|
|
103
|
+
'- Notes/Docs: Neutral, clear writing.',
|
|
104
|
+
'- Default: Match the user\'s natural voice.',
|
|
105
|
+
'',
|
|
106
|
+
'## Context Clues',
|
|
107
|
+
'- Window title may contain recipient name (Slack DMs, email compose)',
|
|
108
|
+
'- If you can identify a recipient, adapt formality to the apparent relationship',
|
|
109
|
+
'- Maintain the user\'s natural voice — don\'t over-formalize casual speech',
|
|
110
|
+
'- The user\'s writing patterns and preferences may be available from memory context — follow those when present',
|
|
111
|
+
'',
|
|
112
|
+
buildAppMetadataBlock(msg),
|
|
113
|
+
'',
|
|
114
|
+
'Selected text:',
|
|
115
|
+
msg.context.selectedText ?? '',
|
|
116
|
+
'',
|
|
117
|
+
`Instruction: ${msg.transcription}`,
|
|
118
|
+
].join('\n');
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export async function handleDictationRequest(
|
|
122
|
+
msg: DictationRequest,
|
|
123
|
+
socket: net.Socket,
|
|
124
|
+
ctx: HandlerContext,
|
|
125
|
+
): Promise<void> {
|
|
126
|
+
const mode = detectDictationMode(msg);
|
|
127
|
+
log.info({ mode, transcriptionLength: msg.transcription.length }, 'Dictation request received');
|
|
128
|
+
|
|
129
|
+
// Action mode: return immediately — the client will route to a full agent session
|
|
130
|
+
if (mode === 'action') {
|
|
131
|
+
ctx.send(socket, {
|
|
132
|
+
type: 'dictation_response',
|
|
133
|
+
text: msg.transcription,
|
|
134
|
+
mode: 'action',
|
|
135
|
+
actionPlan: `User wants to: ${msg.transcription}`,
|
|
136
|
+
});
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Dictation / command mode: make a single-turn LLM call for text cleanup or transformation
|
|
141
|
+
const systemPrompt = mode === 'dictation'
|
|
142
|
+
? buildDictationPrompt(msg)
|
|
143
|
+
: buildCommandPrompt(msg);
|
|
144
|
+
|
|
145
|
+
const userText = mode === 'dictation'
|
|
146
|
+
? msg.transcription
|
|
147
|
+
: msg.transcription; // command prompt already embeds the selected text and instruction
|
|
148
|
+
|
|
149
|
+
try {
|
|
150
|
+
const config = getConfig();
|
|
151
|
+
if (!listProviders().includes(config.provider)) {
|
|
152
|
+
log.warn({ provider: config.provider }, 'Dictation: no provider available, returning raw transcription');
|
|
153
|
+
const fallbackText = mode === 'command' ? (msg.context.selectedText ?? msg.transcription) : msg.transcription;
|
|
154
|
+
ctx.send(socket, { type: 'dictation_response', text: fallbackText, mode });
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
const provider = getFailoverProvider(config.provider, config.providerOrder);
|
|
159
|
+
const response = await provider.sendMessage(
|
|
160
|
+
[{ role: 'user', content: [{ type: 'text', text: userText }] }],
|
|
161
|
+
[], // no tools
|
|
162
|
+
systemPrompt,
|
|
163
|
+
{ config: { max_tokens: 1024 } },
|
|
164
|
+
);
|
|
165
|
+
|
|
166
|
+
const textBlock = response.content.find((b) => b.type === 'text');
|
|
167
|
+
const inlineFallback = mode === 'command' ? (msg.context.selectedText ?? msg.transcription) : msg.transcription;
|
|
168
|
+
const cleanedText = textBlock && 'text' in textBlock ? textBlock.text.trim() : inlineFallback;
|
|
169
|
+
|
|
170
|
+
ctx.send(socket, { type: 'dictation_response', text: cleanedText, mode });
|
|
171
|
+
} catch (err) {
|
|
172
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
173
|
+
log.error({ err }, 'Dictation LLM call failed, returning raw transcription');
|
|
174
|
+
const fallbackText = mode === 'command' ? (msg.context.selectedText ?? msg.transcription) : msg.transcription;
|
|
175
|
+
ctx.send(socket, { type: 'dictation_response', text: fallbackText, mode });
|
|
176
|
+
ctx.send(socket, { type: 'error', message: `Dictation cleanup failed: ${message}` });
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
export const dictationHandlers = defineHandlers({
|
|
181
|
+
dictation_request: handleDictationRequest,
|
|
182
|
+
});
|
|
@@ -2,7 +2,7 @@ import * as net from 'node:net';
|
|
|
2
2
|
import { existsSync, readFileSync, statSync } from 'node:fs';
|
|
3
3
|
import { join, dirname } from 'node:path';
|
|
4
4
|
import { fileURLToPath } from 'node:url';
|
|
5
|
-
import { getWorkspacePromptPath } from '../../util/platform.js';
|
|
5
|
+
import { getWorkspacePromptPath, readLockfile } from '../../util/platform.js';
|
|
6
6
|
import { log, defineHandlers, type HandlerContext } from './shared.js';
|
|
7
7
|
|
|
8
8
|
function handleIdentityGet(socket: net.Socket, ctx: HandlerContext): void {
|
|
@@ -68,28 +68,19 @@ function handleIdentityGet(socket: net.Socket, ctx: HandlerContext): void {
|
|
|
68
68
|
let cloud: string | undefined;
|
|
69
69
|
let originSystem: string | undefined;
|
|
70
70
|
try {
|
|
71
|
-
const
|
|
72
|
-
const
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
const dateB = new Date(b.hatchedAt as string || 0).getTime();
|
|
85
|
-
return dateB - dateA;
|
|
86
|
-
});
|
|
87
|
-
const latest = sorted[0];
|
|
88
|
-
assistantId = latest.assistantId as string | undefined;
|
|
89
|
-
cloud = latest.cloud as string | undefined;
|
|
90
|
-
originSystem = cloud === 'local' ? 'local' : cloud;
|
|
91
|
-
}
|
|
92
|
-
break;
|
|
71
|
+
const lockData = readLockfile();
|
|
72
|
+
const assistants = lockData?.assistants as Array<Record<string, unknown>> | undefined;
|
|
73
|
+
if (assistants && assistants.length > 0) {
|
|
74
|
+
// Use the most recently hatched assistant
|
|
75
|
+
const sorted = [...assistants].sort((a, b) => {
|
|
76
|
+
const dateA = new Date(a.hatchedAt as string || 0).getTime();
|
|
77
|
+
const dateB = new Date(b.hatchedAt as string || 0).getTime();
|
|
78
|
+
return dateB - dateA;
|
|
79
|
+
});
|
|
80
|
+
const latest = sorted[0];
|
|
81
|
+
assistantId = latest.assistantId as string | undefined;
|
|
82
|
+
cloud = latest.cloud as string | undefined;
|
|
83
|
+
originSystem = cloud === 'local' ? 'local' : cloud;
|
|
93
84
|
}
|
|
94
85
|
} catch {
|
|
95
86
|
// ignore — lockfile may not exist
|
|
@@ -22,6 +22,7 @@ import { signingHandlers } from './signing.js';
|
|
|
22
22
|
import { twitterAuthHandlers } from './twitter-auth.js';
|
|
23
23
|
import { workspaceFileHandlers } from './workspace-files.js';
|
|
24
24
|
import { identityHandlers } from './identity.js';
|
|
25
|
+
import { dictationHandlers } from './dictation.js';
|
|
25
26
|
|
|
26
27
|
// Re-export types and utilities for backwards compatibility
|
|
27
28
|
export type {
|
|
@@ -107,6 +108,7 @@ const handlers = {
|
|
|
107
108
|
...twitterAuthHandlers,
|
|
108
109
|
...workspaceFileHandlers,
|
|
109
110
|
...identityHandlers,
|
|
111
|
+
...dictationHandlers,
|
|
110
112
|
...inlineHandlers,
|
|
111
113
|
} satisfies DispatchMap;
|
|
112
114
|
|
|
@@ -114,6 +114,8 @@ export async function handleUserMessage(
|
|
|
114
114
|
}
|
|
115
115
|
|
|
116
116
|
rlog.info('Processing user message');
|
|
117
|
+
session.setAssistantId('self');
|
|
118
|
+
session.setGuardianContext(null);
|
|
117
119
|
// Fire-and-forget: don't block the IPC handler so the connection can
|
|
118
120
|
// continue receiving messages (e.g. cancel, confirmations, or
|
|
119
121
|
// additional user_message that will be queued by the session).
|
|
@@ -9,6 +9,7 @@ import type { ClientMessage, CuSessionCreate, ServerMessage, SessionTransportMet
|
|
|
9
9
|
import type { SecretPromptResult } from '../../permissions/secret-prompter.js';
|
|
10
10
|
import { getConfig } from '../../config/loader.js';
|
|
11
11
|
import type { DebouncerMap } from '../../util/debounce.js';
|
|
12
|
+
import type { GuardianRuntimeContext } from '../session-runtime-assembly.js';
|
|
12
13
|
|
|
13
14
|
const log = getLogger('handlers');
|
|
14
15
|
|
|
@@ -99,6 +100,8 @@ export interface SessionCreateOptions {
|
|
|
99
100
|
systemPromptOverride?: string;
|
|
100
101
|
maxResponseTokens?: number;
|
|
101
102
|
transport?: SessionTransportMetadata;
|
|
103
|
+
assistantId?: string;
|
|
104
|
+
guardianContext?: GuardianRuntimeContext;
|
|
102
105
|
memoryScopeId?: string;
|
|
103
106
|
isPrivateThread?: boolean;
|
|
104
107
|
strictPrivateSideEffects?: boolean;
|
|
@@ -7,7 +7,7 @@ import { resolveSkillStates } from '../../config/skill-state.js';
|
|
|
7
7
|
import { getWorkspaceSkillsDir } from '../../util/platform.js';
|
|
8
8
|
import { clawhubInstall, clawhubUpdate, clawhubSearch, clawhubCheckUpdates, clawhubInspect, type ClawhubSearchResultItem } from '../../skills/clawhub.js';
|
|
9
9
|
import { removeSkillsIndexEntry, deleteManagedSkill, validateManagedSkillId } from '../../skills/managed-store.js';
|
|
10
|
-
import { listCatalogEntries, installFromVellumCatalog } from '../../tools/skills/vellum-catalog.js';
|
|
10
|
+
import { listCatalogEntries, installFromVellumCatalog, checkVellumSkill } from '../../tools/skills/vellum-catalog.js';
|
|
11
11
|
import type {
|
|
12
12
|
SkillDetailRequest,
|
|
13
13
|
SkillsEnableRequest,
|
|
@@ -188,14 +188,13 @@ export async function handleSkillsInstall(
|
|
|
188
188
|
): Promise<void> {
|
|
189
189
|
try {
|
|
190
190
|
// Check if the slug matches a vellum-skills catalog entry first
|
|
191
|
-
const
|
|
192
|
-
const isVellumSkill = catalogEntries.some((e) => e.id === msg.slug);
|
|
191
|
+
const isVellumSkill = await checkVellumSkill(msg.slug);
|
|
193
192
|
|
|
194
193
|
let skillId: string;
|
|
195
194
|
|
|
196
195
|
if (isVellumSkill) {
|
|
197
|
-
// Install from vellum-skills catalog (
|
|
198
|
-
const result = installFromVellumCatalog(msg.slug);
|
|
196
|
+
// Install from vellum-skills catalog (remote with bundled fallback)
|
|
197
|
+
const result = await installFromVellumCatalog(msg.slug);
|
|
199
198
|
if (!result.success) {
|
|
200
199
|
ctx.send(socket, {
|
|
201
200
|
type: 'skills_operation_response',
|
|
@@ -424,8 +423,8 @@ export async function handleSkillsSearch(
|
|
|
424
423
|
ctx: HandlerContext,
|
|
425
424
|
): Promise<void> {
|
|
426
425
|
try {
|
|
427
|
-
// Search vellum-skills catalog
|
|
428
|
-
const catalogEntries = listCatalogEntries();
|
|
426
|
+
// Search vellum-skills catalog (remote with bundled fallback)
|
|
427
|
+
const catalogEntries = await listCatalogEntries();
|
|
429
428
|
const query = (msg.query ?? '').toLowerCase();
|
|
430
429
|
const matchingCatalog = catalogEntries.filter((e) => {
|
|
431
430
|
if (!query) return true;
|
|
@@ -148,15 +148,23 @@ function broadcastWorkItemStatus(ctx: HandlerContext, id: string): void {
|
|
|
148
148
|
}
|
|
149
149
|
}
|
|
150
150
|
|
|
151
|
-
/**
|
|
152
|
-
|
|
151
|
+
/**
|
|
152
|
+
* Extract only the latest assistant text block from stored content.
|
|
153
|
+
* Consolidation merges multiple assistant messages into one DB row; scanning
|
|
154
|
+
* from the end keeps task output focused on the final assistant response.
|
|
155
|
+
*/
|
|
156
|
+
function extractLatestTextFromContent(content: string): string {
|
|
153
157
|
try {
|
|
154
158
|
const parsed = JSON.parse(content);
|
|
155
159
|
if (Array.isArray(parsed)) {
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
.
|
|
159
|
-
.
|
|
160
|
+
for (let i = parsed.length - 1; i >= 0; i--) {
|
|
161
|
+
const block = parsed[i] as { type?: unknown; text?: unknown };
|
|
162
|
+
if (block.type !== 'text') continue;
|
|
163
|
+
if (typeof block.text !== 'string') continue;
|
|
164
|
+
if (!block.text.trim()) continue;
|
|
165
|
+
return block.text;
|
|
166
|
+
}
|
|
167
|
+
return '';
|
|
160
168
|
}
|
|
161
169
|
} catch {
|
|
162
170
|
// Plain text content — use as-is
|
|
@@ -299,7 +307,7 @@ export function handleWorkItemOutput(
|
|
|
299
307
|
const m = msgs[i];
|
|
300
308
|
if (m.role !== 'assistant') continue;
|
|
301
309
|
|
|
302
|
-
const text =
|
|
310
|
+
const text = extractLatestTextFromContent(m.content);
|
|
303
311
|
if (!text.trim()) continue;
|
|
304
312
|
|
|
305
313
|
summary = truncate(text, 2000, '');
|
|
@@ -19,12 +19,14 @@
|
|
|
19
19
|
"BrowserUserScroll",
|
|
20
20
|
"BundleAppRequest",
|
|
21
21
|
"CancelRequest",
|
|
22
|
+
"ChannelReadinessRequest",
|
|
22
23
|
"ConfirmationResponse",
|
|
23
24
|
"CuObservation",
|
|
24
25
|
"CuSessionAbort",
|
|
25
26
|
"CuSessionCreate",
|
|
26
27
|
"DeleteQueuedMessage",
|
|
27
28
|
"DiagnosticsExportRequest",
|
|
29
|
+
"DictationRequest",
|
|
28
30
|
"DocumentListRequest",
|
|
29
31
|
"DocumentLoadRequest",
|
|
30
32
|
"DocumentSaveRequest",
|
|
@@ -137,6 +139,7 @@
|
|
|
137
139
|
"BrowserHandoffRequest",
|
|
138
140
|
"BrowserInteractiveModeChanged",
|
|
139
141
|
"BundleAppResponse",
|
|
142
|
+
"ChannelReadinessResponse",
|
|
140
143
|
"ConfirmationRequest",
|
|
141
144
|
"ContextCompacted",
|
|
142
145
|
"CuAction",
|
|
@@ -144,6 +147,7 @@
|
|
|
144
147
|
"CuError",
|
|
145
148
|
"DaemonStatusMessage",
|
|
146
149
|
"DiagnosticsExportResponse",
|
|
150
|
+
"DictationResponse",
|
|
147
151
|
"DocumentEditorShow",
|
|
148
152
|
"DocumentEditorUpdate",
|
|
149
153
|
"DocumentListResponse",
|
|
@@ -179,6 +183,7 @@
|
|
|
179
183
|
"PublishPageResponse",
|
|
180
184
|
"ReminderFired",
|
|
181
185
|
"RemindersListResponse",
|
|
186
|
+
"RideShotgunProgress",
|
|
182
187
|
"RideShotgunResult",
|
|
183
188
|
"ScheduleComplete",
|
|
184
189
|
"SchedulesListResponse",
|
|
@@ -268,12 +273,14 @@
|
|
|
268
273
|
"browser_user_scroll",
|
|
269
274
|
"bundle_app",
|
|
270
275
|
"cancel",
|
|
276
|
+
"channel_readiness",
|
|
271
277
|
"confirmation_response",
|
|
272
278
|
"cu_observation",
|
|
273
279
|
"cu_session_abort",
|
|
274
280
|
"cu_session_create",
|
|
275
281
|
"delete_queued_message",
|
|
276
282
|
"diagnostics_export_request",
|
|
283
|
+
"dictation_request",
|
|
277
284
|
"document_list",
|
|
278
285
|
"document_load",
|
|
279
286
|
"document_save",
|
|
@@ -386,6 +393,7 @@
|
|
|
386
393
|
"browser_handoff_request",
|
|
387
394
|
"browser_interactive_mode_changed",
|
|
388
395
|
"bundle_app_response",
|
|
396
|
+
"channel_readiness_response",
|
|
389
397
|
"confirmation_request",
|
|
390
398
|
"context_compacted",
|
|
391
399
|
"cu_action",
|
|
@@ -393,6 +401,7 @@
|
|
|
393
401
|
"cu_error",
|
|
394
402
|
"daemon_status",
|
|
395
403
|
"diagnostics_export_response",
|
|
404
|
+
"dictation_response",
|
|
396
405
|
"document_editor_show",
|
|
397
406
|
"document_editor_update",
|
|
398
407
|
"document_list_response",
|
|
@@ -428,6 +437,7 @@
|
|
|
428
437
|
"publish_page_response",
|
|
429
438
|
"reminder_fired",
|
|
430
439
|
"reminders_list_response",
|
|
440
|
+
"ride_shotgun_progress",
|
|
431
441
|
"ride_shotgun_result",
|
|
432
442
|
"schedule_complete",
|
|
433
443
|
"schedules_list_response",
|
|
@@ -23,6 +23,14 @@ export interface UsageStats {
|
|
|
23
23
|
estimatedCost: number;
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
+
export interface DictationContext {
|
|
27
|
+
bundleIdentifier: string;
|
|
28
|
+
appName: string;
|
|
29
|
+
windowTitle: string;
|
|
30
|
+
selectedText?: string;
|
|
31
|
+
cursorInTextField: boolean;
|
|
32
|
+
}
|
|
33
|
+
|
|
26
34
|
// === Client → Server messages ===
|
|
27
35
|
|
|
28
36
|
export interface UserMessage {
|
|
@@ -555,13 +563,31 @@ export interface TelegramConfigResponse {
|
|
|
555
563
|
|
|
556
564
|
export interface TwilioConfigRequest {
|
|
557
565
|
type: 'twilio_config';
|
|
558
|
-
action: 'get' | 'set_credentials' | 'clear_credentials' | 'provision_number' | 'assign_number' | 'list_numbers'
|
|
566
|
+
action: 'get' | 'set_credentials' | 'clear_credentials' | 'provision_number' | 'assign_number' | 'list_numbers'
|
|
567
|
+
| 'sms_compliance_status' | 'sms_submit_tollfree_verification' | 'sms_update_tollfree_verification'
|
|
568
|
+
| 'sms_delete_tollfree_verification' | 'release_number' | 'sms_send_test' | 'sms_doctor';
|
|
559
569
|
accountSid?: string; // Only for action: 'set_credentials'
|
|
560
570
|
authToken?: string; // Only for action: 'set_credentials'
|
|
561
|
-
phoneNumber?: string; // Only for action: 'assign_number'
|
|
571
|
+
phoneNumber?: string; // Only for action: 'assign_number' or 'sms_send_test'
|
|
562
572
|
areaCode?: string; // Only for action: 'provision_number'
|
|
563
573
|
country?: string; // Only for action: 'provision_number' (ISO 3166-1 alpha-2, default 'US')
|
|
564
574
|
assistantId?: string; // Scope number assignment/lookup to a specific assistant
|
|
575
|
+
verificationSid?: string; // Only for update/delete verification actions
|
|
576
|
+
verificationParams?: {
|
|
577
|
+
tollfreePhoneNumberSid?: string;
|
|
578
|
+
businessName?: string;
|
|
579
|
+
businessWebsite?: string;
|
|
580
|
+
notificationEmail?: string;
|
|
581
|
+
useCaseCategories?: string[];
|
|
582
|
+
useCaseSummary?: string;
|
|
583
|
+
productionMessageSample?: string;
|
|
584
|
+
optInImageUrls?: string[];
|
|
585
|
+
optInType?: string;
|
|
586
|
+
messageVolume?: string;
|
|
587
|
+
businessType?: string;
|
|
588
|
+
customerProfileSid?: string;
|
|
589
|
+
};
|
|
590
|
+
text?: string; // Only for action: 'sms_send_test' (default: "Test SMS from your Vellum assistant")
|
|
565
591
|
}
|
|
566
592
|
|
|
567
593
|
export interface TwilioConfigResponse {
|
|
@@ -573,6 +599,56 @@ export interface TwilioConfigResponse {
|
|
|
573
599
|
error?: string;
|
|
574
600
|
/** Non-fatal warning message (e.g. webhook sync failure that did not prevent the primary operation). */
|
|
575
601
|
warning?: string;
|
|
602
|
+
compliance?: {
|
|
603
|
+
numberType?: string;
|
|
604
|
+
verificationSid?: string;
|
|
605
|
+
verificationStatus?: string;
|
|
606
|
+
rejectionReason?: string;
|
|
607
|
+
rejectionReasons?: string[];
|
|
608
|
+
errorCode?: string;
|
|
609
|
+
editAllowed?: boolean;
|
|
610
|
+
editExpiration?: string;
|
|
611
|
+
};
|
|
612
|
+
/** Present when action is 'sms_send_test'. */
|
|
613
|
+
testResult?: {
|
|
614
|
+
messageSid: string;
|
|
615
|
+
to: string;
|
|
616
|
+
initialStatus: string;
|
|
617
|
+
finalStatus: string;
|
|
618
|
+
errorCode?: string;
|
|
619
|
+
errorMessage?: string;
|
|
620
|
+
};
|
|
621
|
+
/** Present when action is 'sms_doctor'. */
|
|
622
|
+
diagnostics?: {
|
|
623
|
+
readiness: { ready: boolean; issues: string[] };
|
|
624
|
+
compliance: { status: string; detail?: string; remediation?: string };
|
|
625
|
+
lastSend?: { status: string; errorCode?: string; remediation?: string };
|
|
626
|
+
overallStatus: 'healthy' | 'degraded' | 'broken';
|
|
627
|
+
actionItems: string[];
|
|
628
|
+
};
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
export interface ChannelReadinessRequest {
|
|
632
|
+
type: 'channel_readiness';
|
|
633
|
+
action: 'get' | 'refresh';
|
|
634
|
+
channel?: string;
|
|
635
|
+
assistantId?: string;
|
|
636
|
+
includeRemote?: boolean;
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
export interface ChannelReadinessResponse {
|
|
640
|
+
type: 'channel_readiness_response';
|
|
641
|
+
success: boolean;
|
|
642
|
+
snapshots?: Array<{
|
|
643
|
+
channel: string;
|
|
644
|
+
ready: boolean;
|
|
645
|
+
checkedAt: number;
|
|
646
|
+
stale: boolean;
|
|
647
|
+
reasons: Array<{ code: string; text: string }>;
|
|
648
|
+
localChecks: Array<{ name: string; passed: boolean; message: string }>;
|
|
649
|
+
remoteChecks?: Array<{ name: string; passed: boolean; message: string }>;
|
|
650
|
+
}>;
|
|
651
|
+
error?: string;
|
|
576
652
|
}
|
|
577
653
|
|
|
578
654
|
export interface GuardianVerificationRequest {
|
|
@@ -591,6 +667,16 @@ export interface GuardianVerificationResponse {
|
|
|
591
667
|
/** Present when action is 'status'. */
|
|
592
668
|
bound?: boolean;
|
|
593
669
|
guardianExternalUserId?: string;
|
|
670
|
+
/** The channel this status pertains to (e.g. "telegram", "sms"). Present when action is 'status'. */
|
|
671
|
+
channel?: string;
|
|
672
|
+
/** The assistant ID scoped to this status. Present when action is 'status'. */
|
|
673
|
+
assistantId?: string;
|
|
674
|
+
/** The delivery chat ID for the guardian (e.g. Telegram chat ID). Present when action is 'status' and bound is true. */
|
|
675
|
+
guardianDeliveryChatId?: string;
|
|
676
|
+
/** Optional channel username/handle for the bound guardian (for UI display). */
|
|
677
|
+
guardianUsername?: string;
|
|
678
|
+
/** Optional display name for the bound guardian (for UI display). */
|
|
679
|
+
guardianDisplayName?: string;
|
|
594
680
|
error?: string;
|
|
595
681
|
}
|
|
596
682
|
|
|
@@ -653,6 +739,12 @@ export interface IpcBlobProbe {
|
|
|
653
739
|
nonceSha256: string;
|
|
654
740
|
}
|
|
655
741
|
|
|
742
|
+
export interface DictationRequest {
|
|
743
|
+
type: 'dictation_request';
|
|
744
|
+
transcription: string;
|
|
745
|
+
context: DictationContext;
|
|
746
|
+
}
|
|
747
|
+
|
|
656
748
|
// === Surface types ===
|
|
657
749
|
|
|
658
750
|
export type SurfaceType = 'card' | 'form' | 'list' | 'table' | 'confirmation' | 'dynamic_page' | 'file_upload' | 'browser_view' | 'document_preview';
|
|
@@ -1078,6 +1170,7 @@ export type ClientMessage =
|
|
|
1078
1170
|
| TwitterIntegrationConfigRequest
|
|
1079
1171
|
| TelegramConfigRequest
|
|
1080
1172
|
| TwilioConfigRequest
|
|
1173
|
+
| ChannelReadinessRequest
|
|
1081
1174
|
| GuardianVerificationRequest
|
|
1082
1175
|
| TwitterAuthStartRequest
|
|
1083
1176
|
| TwitterAuthStatusRequest
|
|
@@ -1123,7 +1216,8 @@ export type ClientMessage =
|
|
|
1123
1216
|
| WorkspaceFileReadRequest
|
|
1124
1217
|
| IdentityGetRequest
|
|
1125
1218
|
| ToolPermissionSimulateRequest
|
|
1126
|
-
| ToolNamesListRequest
|
|
1219
|
+
| ToolNamesListRequest
|
|
1220
|
+
| DictationRequest;
|
|
1127
1221
|
|
|
1128
1222
|
export interface IntegrationListRequest {
|
|
1129
1223
|
type: 'integration_list';
|
|
@@ -1522,6 +1616,12 @@ export interface TaskRouted {
|
|
|
1522
1616
|
escalatedFrom?: string;
|
|
1523
1617
|
}
|
|
1524
1618
|
|
|
1619
|
+
export interface RideShotgunProgress {
|
|
1620
|
+
type: 'ride_shotgun_progress';
|
|
1621
|
+
watchId: string;
|
|
1622
|
+
message: string;
|
|
1623
|
+
}
|
|
1624
|
+
|
|
1525
1625
|
export interface RideShotgunResult {
|
|
1526
1626
|
type: 'ride_shotgun_result';
|
|
1527
1627
|
sessionId: string;
|
|
@@ -2360,6 +2460,13 @@ export interface ToolNamesListResponse {
|
|
|
2360
2460
|
schemas?: Record<string, ToolInputSchema>;
|
|
2361
2461
|
}
|
|
2362
2462
|
|
|
2463
|
+
export interface DictationResponse {
|
|
2464
|
+
type: 'dictation_response';
|
|
2465
|
+
text: string;
|
|
2466
|
+
mode: 'dictation' | 'command' | 'action';
|
|
2467
|
+
actionPlan?: string;
|
|
2468
|
+
}
|
|
2469
|
+
|
|
2363
2470
|
export type ServerMessage =
|
|
2364
2471
|
| AuthResult
|
|
2365
2472
|
| UserMessageEcho
|
|
@@ -2394,6 +2501,7 @@ export type ServerMessage =
|
|
|
2394
2501
|
| CuError
|
|
2395
2502
|
| SessionErrorMessage
|
|
2396
2503
|
| TaskRouted
|
|
2504
|
+
| RideShotgunProgress
|
|
2397
2505
|
| RideShotgunResult
|
|
2398
2506
|
| UiSurfaceShow
|
|
2399
2507
|
| UiSurfaceUpdate
|
|
@@ -2446,6 +2554,7 @@ export type ServerMessage =
|
|
|
2446
2554
|
| TwitterIntegrationConfigResponse
|
|
2447
2555
|
| TelegramConfigResponse
|
|
2448
2556
|
| TwilioConfigResponse
|
|
2557
|
+
| ChannelReadinessResponse
|
|
2449
2558
|
| GuardianVerificationResponse
|
|
2450
2559
|
| TwitterAuthResult
|
|
2451
2560
|
| TwitterAuthStatusResponse
|
|
@@ -2489,7 +2598,8 @@ export type ServerMessage =
|
|
|
2489
2598
|
| WorkspaceFileReadResponse
|
|
2490
2599
|
| IdentityGetResponse
|
|
2491
2600
|
| ToolPermissionSimulateResponse
|
|
2492
|
-
| ToolNamesListResponse
|
|
2601
|
+
| ToolNamesListResponse
|
|
2602
|
+
| DictationResponse;
|
|
2493
2603
|
|
|
2494
2604
|
// === Subagent IPC ─────────────────────────────────────────────────────
|
|
2495
2605
|
|