@vellumai/assistant 0.3.13 → 0.3.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +17 -3
- package/Dockerfile +1 -1
- package/README.md +2 -0
- package/docs/architecture/scheduling.md +81 -0
- package/package.json +1 -1
- package/src/__tests__/__snapshots__/ipc-snapshot.test.ts.snap +22 -0
- package/src/__tests__/channel-policy.test.ts +19 -0
- package/src/__tests__/guardian-control-plane-policy.test.ts +582 -0
- package/src/__tests__/guardian-outbound-http.test.ts +8 -8
- package/src/__tests__/intent-routing.test.ts +22 -0
- package/src/__tests__/ipc-snapshot.test.ts +10 -0
- package/src/__tests__/notification-routing-intent.test.ts +185 -0
- package/src/__tests__/recording-handler.test.ts +191 -31
- package/src/__tests__/recording-intent-fallback.test.ts +180 -0
- package/src/__tests__/recording-intent-handler.test.ts +597 -74
- package/src/__tests__/recording-intent.test.ts +738 -342
- package/src/__tests__/recording-state-machine.test.ts +1109 -0
- package/src/__tests__/reminder-store.test.ts +20 -18
- package/src/__tests__/reminder.test.ts +2 -1
- package/src/channels/config.ts +1 -1
- package/src/config/bundled-skills/phone-calls/SKILL.md +1 -11
- package/src/config/bundled-skills/screen-recording/SKILL.md +91 -12
- package/src/config/system-prompt.ts +5 -0
- package/src/config/vellum-skills/guardian-verify-setup/SKILL.md +1 -0
- package/src/daemon/handlers/config-channels.ts +6 -6
- package/src/daemon/handlers/index.ts +1 -1
- package/src/daemon/handlers/misc.ts +258 -102
- package/src/daemon/handlers/recording.ts +417 -5
- package/src/daemon/handlers/sessions.ts +142 -68
- package/src/daemon/ipc-contract/computer-use.ts +23 -3
- package/src/daemon/ipc-contract/messages.ts +3 -1
- package/src/daemon/ipc-contract/shared.ts +6 -0
- package/src/daemon/ipc-contract-inventory.json +2 -0
- package/src/daemon/lifecycle.ts +2 -0
- package/src/daemon/recording-executor.ts +180 -0
- package/src/daemon/recording-intent-fallback.ts +132 -0
- package/src/daemon/recording-intent.ts +306 -15
- package/src/daemon/session-tool-setup.ts +4 -0
- package/src/memory/conversation-attention-store.ts +5 -5
- package/src/notifications/README.md +69 -1
- package/src/notifications/adapters/sms.ts +80 -0
- package/src/notifications/broadcaster.ts +1 -0
- package/src/notifications/copy-composer.ts +3 -3
- package/src/notifications/decision-engine.ts +70 -1
- package/src/notifications/decisions-store.ts +24 -0
- package/src/notifications/destination-resolver.ts +2 -1
- package/src/notifications/emit-signal.ts +35 -3
- package/src/notifications/signal.ts +6 -0
- package/src/notifications/types.ts +3 -0
- package/src/runtime/guardian-outbound-actions.ts +9 -9
- package/src/runtime/http-server.ts +7 -7
- package/src/runtime/routes/conversation-attention-routes.ts +3 -3
- package/src/runtime/routes/integration-routes.ts +5 -5
- package/src/schedule/scheduler.ts +15 -3
- package/src/tools/executor.ts +29 -0
- package/src/tools/guardian-control-plane-policy.ts +141 -0
- package/src/tools/types.ts +2 -0
|
@@ -51,22 +51,6 @@ describe('reminder-store', () => {
|
|
|
51
51
|
expect(r.updatedAt).toBeGreaterThan(0);
|
|
52
52
|
});
|
|
53
53
|
|
|
54
|
-
test('insertReminder defaults routingIntent to single_channel when omitted', () => {
|
|
55
|
-
const r = insertReminder({
|
|
56
|
-
label: 'No routing',
|
|
57
|
-
message: 'Should default',
|
|
58
|
-
fireAt: Date.now() + 60_000,
|
|
59
|
-
mode: 'notify',
|
|
60
|
-
});
|
|
61
|
-
|
|
62
|
-
expect(r.routingIntent).toBe('single_channel');
|
|
63
|
-
expect(r.routingHints).toEqual({});
|
|
64
|
-
|
|
65
|
-
const fetched = getReminder(r.id);
|
|
66
|
-
expect(fetched!.routingIntent).toBe('single_channel');
|
|
67
|
-
expect(fetched!.routingHints).toEqual({});
|
|
68
|
-
});
|
|
69
|
-
|
|
70
54
|
test('insertReminder persists routing metadata', () => {
|
|
71
55
|
const r = insertReminder({
|
|
72
56
|
label: 'Multi-channel',
|
|
@@ -85,6 +69,22 @@ describe('reminder-store', () => {
|
|
|
85
69
|
expect(fetched!.routingHints).toEqual({ preferred: ['telegram', 'sms'] });
|
|
86
70
|
});
|
|
87
71
|
|
|
72
|
+
test('insertReminder defaults routingIntent to single_channel when omitted', () => {
|
|
73
|
+
const r = insertReminder({
|
|
74
|
+
label: 'No routing',
|
|
75
|
+
message: 'Should default',
|
|
76
|
+
fireAt: Date.now() + 60_000,
|
|
77
|
+
mode: 'notify',
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
expect(r.routingIntent).toBe('single_channel');
|
|
81
|
+
expect(r.routingHints).toEqual({});
|
|
82
|
+
|
|
83
|
+
const fetched = getReminder(r.id);
|
|
84
|
+
expect(fetched!.routingIntent).toBe('single_channel');
|
|
85
|
+
expect(fetched!.routingHints).toEqual({});
|
|
86
|
+
});
|
|
87
|
+
|
|
88
88
|
// ── getReminder ─────────────────────────────────────────────────────
|
|
89
89
|
|
|
90
90
|
test('getReminder returns null for nonexistent ID', () => {
|
|
@@ -157,9 +157,9 @@ describe('reminder-store', () => {
|
|
|
157
157
|
|
|
158
158
|
// ── claimDueReminders ──────────────────────────────────────────────
|
|
159
159
|
|
|
160
|
-
test('claimDueReminders claims reminders where fireAt <= now', () => {
|
|
160
|
+
test('claimDueReminders claims reminders where fireAt <= now and preserves routing', () => {
|
|
161
161
|
const now = Date.now();
|
|
162
|
-
insertReminder({ label: 'Past', message: 'x', fireAt: now - 5000, mode: 'notify' });
|
|
162
|
+
insertReminder({ label: 'Past', message: 'x', fireAt: now - 5000, mode: 'notify', routingIntent: 'all_channels', routingHints: { foo: 'bar' } });
|
|
163
163
|
insertReminder({ label: 'Future', message: 'y', fireAt: now + 60_000, mode: 'notify' });
|
|
164
164
|
|
|
165
165
|
const claimed = claimDueReminders(now);
|
|
@@ -167,6 +167,8 @@ describe('reminder-store', () => {
|
|
|
167
167
|
expect(claimed[0].label).toBe('Past');
|
|
168
168
|
expect(claimed[0].status).toBe('firing');
|
|
169
169
|
expect(claimed[0].firedAt).toBe(now);
|
|
170
|
+
expect(claimed[0].routingIntent).toBe('all_channels');
|
|
171
|
+
expect(claimed[0].routingHints).toEqual({ foo: 'bar' });
|
|
170
172
|
});
|
|
171
173
|
|
|
172
174
|
test('claimDueReminders skips already-fired reminders', () => {
|
|
@@ -246,13 +246,14 @@ describe('reminder tool', () => {
|
|
|
246
246
|
fire_at: future,
|
|
247
247
|
label: 'Test reminder',
|
|
248
248
|
message: 'Test message',
|
|
249
|
+
routing_intent: 'all_channels',
|
|
249
250
|
});
|
|
250
251
|
|
|
251
252
|
const result = executeReminderList();
|
|
252
253
|
expect(result.isError).toBe(false);
|
|
253
254
|
expect(result.content).toContain('Test reminder');
|
|
254
255
|
expect(result.content).toContain('pending');
|
|
255
|
-
expect(result.content).toContain('routing:
|
|
256
|
+
expect(result.content).toContain('routing: all_channels');
|
|
256
257
|
});
|
|
257
258
|
|
|
258
259
|
// ── cancel ──────────────────────────────────────────────────────────
|
package/src/channels/config.ts
CHANGED
|
@@ -364,17 +364,7 @@ No additional configuration is needed beyond the standard Twilio setup (Steps 1-
|
|
|
364
364
|
|
|
365
365
|
### Guardian voice verification for inbound calls
|
|
366
366
|
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
- Call `vellum_skills_catalog` with `action: "install"` and `skill_id: "guardian-verify-setup"`.
|
|
370
|
-
- Then call `skill_load` with `skill: "guardian-verify-setup"`.
|
|
371
|
-
|
|
372
|
-
When invoked for the `voice` channel, the guardian-verify-setup skill:
|
|
373
|
-
1. Collects the user's phone number as the destination
|
|
374
|
-
2. Starts an outbound verification session via `POST /v1/integrations/guardian/outbound/start` with `channel: "voice"`
|
|
375
|
-
3. The API initiates a phone call to the user's number. The response includes a `secret` field with the verification code -- the code is shared with the user BEFORE the call connects so they know what to enter
|
|
376
|
-
4. When the user answers, they enter the verification code via their phone's keypad (DTMF)
|
|
377
|
-
5. If the code matches, a guardian binding is created for the voice channel
|
|
367
|
+
For guardian verification setup, first install the skill via `vellum_skills_catalog` with `action: "install"` and `skill_id: "guardian-verify-setup"`, then load it with `skill_load` using `skill: "guardian-verify-setup"`. This skill handles the full outbound verification flow; `phone-calls` does not orchestrate it inline.
|
|
378
368
|
|
|
379
369
|
Once a guardian binding exists for the voice channel, inbound callers may be prompted for verification before calls proceed. The relay server detects pending challenges and prompts callers: "Please enter your six-digit verification code using your keypad, or speak the digits now." If verification fails after 3 attempts, the call ends with "Verification failed. Goodbye."
|
|
380
370
|
|
|
@@ -26,35 +26,114 @@ This skill activates when the user asks to record their screen. Common phrases:
|
|
|
26
26
|
- "finish recording"
|
|
27
27
|
- "halt recording"
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
**Restart recording:**
|
|
30
|
+
- "restart recording"
|
|
31
|
+
- "redo the recording"
|
|
32
|
+
- "stop recording and start a new one"
|
|
33
|
+
- "stop recording and start a new recording"
|
|
34
|
+
- "stop and restart the recording"
|
|
30
35
|
|
|
31
|
-
|
|
36
|
+
**Pause recording:**
|
|
37
|
+
- "pause recording"
|
|
38
|
+
- "pause the recording"
|
|
32
39
|
|
|
33
|
-
|
|
34
|
-
-
|
|
35
|
-
-
|
|
36
|
-
|
|
40
|
+
**Resume recording:**
|
|
41
|
+
- "resume recording"
|
|
42
|
+
- "unpause the recording"
|
|
43
|
+
|
|
44
|
+
## Intent Classification
|
|
45
|
+
|
|
46
|
+
Recording prompts are classified by `resolveRecordingIntent` into one of 11 intent kinds:
|
|
47
|
+
|
|
48
|
+
### Pure commands (handled by the standalone recording route)
|
|
49
|
+
|
|
50
|
+
- **`start_only`** — Pure start request with no additional task (e.g., "record my screen").
|
|
51
|
+
- **`stop_only`** — Pure stop request (e.g., "stop recording").
|
|
52
|
+
- **`restart_only`** — Pure restart request (e.g., "restart recording", "stop recording and start a new one").
|
|
53
|
+
- **`pause_only`** — Pure pause request (e.g., "pause the recording").
|
|
54
|
+
- **`resume_only`** — Pure resume request (e.g., "resume the recording").
|
|
55
|
+
|
|
56
|
+
### Recording + additional task (recording action is deferred and executed alongside the task)
|
|
57
|
+
|
|
58
|
+
- **`start_with_remainder`** — Start recording embedded in a broader task. The recording clause is stripped, and the remainder is processed as a separate task. Example: "open Safari and record my screen" produces remainder "open Safari".
|
|
59
|
+
- **`stop_with_remainder`** — Stop recording embedded in a broader task. Example: "close the browser and stop recording" produces remainder "close the browser".
|
|
60
|
+
- **`restart_with_remainder`** — Restart recording embedded in a broader task. Example: "restart recording and open Safari" produces remainder "open Safari".
|
|
61
|
+
|
|
62
|
+
### Both start and stop detected
|
|
63
|
+
|
|
64
|
+
- **`start_and_stop_only`** — Both start and stop patterns present with no additional task (e.g., "stop recording and record my screen").
|
|
65
|
+
- **`start_and_stop_with_remainder`** — Both start and stop patterns present alongside additional task text.
|
|
66
|
+
|
|
67
|
+
### No recording intent
|
|
68
|
+
|
|
69
|
+
- **`none`** — No recording intent detected. Normal routing.
|
|
37
70
|
|
|
38
71
|
Dynamic name prefixes (from IDENTITY.md) are stripped during classification, so "Nova, record my screen" classifies the same as "record my screen".
|
|
39
72
|
|
|
40
73
|
## Routing
|
|
41
74
|
|
|
42
|
-
Recording
|
|
75
|
+
Recording intent resolution follows a precedence chain:
|
|
76
|
+
|
|
77
|
+
### 1. `commandIntent` (structured IPC) — highest priority
|
|
78
|
+
|
|
79
|
+
The macOS client can send structured intents with `domain: 'screen_recording'` and `action: 'start' | 'stop' | 'restart' | 'pause' | 'resume'`. These bypass text parsing entirely. The daemon checks for `commandIntent` before any text analysis.
|
|
80
|
+
|
|
81
|
+
### 2. Deterministic text resolver (`resolveRecordingIntent`)
|
|
82
|
+
|
|
83
|
+
A regex-based pipeline that classifies the user's text. The pipeline:
|
|
84
|
+
1. Strips dynamic assistant names (leading vocative like "Nova, ...")
|
|
85
|
+
2. Strips leading polite wrappers ("please", "can you", etc.)
|
|
86
|
+
3. Applies the interrogative guard — WH-questions return `none`
|
|
87
|
+
4. Checks restart compound patterns (before independent start/stop, so "stop recording and start a new one" is recognized as restart)
|
|
88
|
+
5. Checks pause/resume patterns
|
|
89
|
+
6. Checks start and stop patterns independently
|
|
90
|
+
7. Determines if the intent is pure or has a remainder by stripping recording clauses and checking for substantive remaining content
|
|
91
|
+
|
|
92
|
+
### 3. Normal processing
|
|
93
|
+
|
|
94
|
+
If no recording intent is detected (kind is `none`), the message flows to the classifier and computer-use session as usual.
|
|
95
|
+
|
|
96
|
+
## Interrogative Guard
|
|
97
|
+
|
|
98
|
+
Questions about recording are NOT treated as commands. The resolver filters out WH-questions to prevent side effects from informational queries.
|
|
99
|
+
|
|
100
|
+
**Filtered (no recording action triggered):**
|
|
101
|
+
- "how do I stop recording?"
|
|
102
|
+
- "what does screen recording do?"
|
|
103
|
+
- "why is the recording paused?"
|
|
104
|
+
- "when should I stop recording?"
|
|
105
|
+
|
|
106
|
+
**Preserved as commands (recording action IS triggered):**
|
|
107
|
+
- "can you stop recording?" — polite imperative
|
|
108
|
+
- "could you record my screen?" — polite imperative
|
|
109
|
+
- "please stop recording" — direct command with filler
|
|
110
|
+
|
|
111
|
+
The guard checks for WH-question starters (how, what, why, when, where, who, which) at the beginning of the text, after stripping dynamic names and polite prefixes.
|
|
112
|
+
|
|
113
|
+
## Mixed-Intent Examples
|
|
114
|
+
|
|
115
|
+
When a recording intent is combined with another task, the recording clause is stripped from the text, and both parts are handled:
|
|
116
|
+
|
|
117
|
+
- **"open Safari and record my screen"** — `start_with_remainder` with remainder "open Safari". Recording starts alongside the Safari task.
|
|
118
|
+
- **"stop recording and start a new one and open Safari"** — `restart_with_remainder` with remainder "open Safari". Restart executes and the remainder is processed separately.
|
|
119
|
+
- **"close the browser and stop recording"** — `stop_with_remainder` with remainder "close the browser". Stop executes and the remainder is processed.
|
|
43
120
|
|
|
44
|
-
|
|
45
|
-
- When the user says "stop recording", the daemon intercepts and stops the active recording for the current conversation.
|
|
46
|
-
- The recording is saved as a video file and attached to the conversation thread.
|
|
121
|
+
The remainder preserves the user's original phrasing (stripping is applied to the original text, not the normalized form).
|
|
47
122
|
|
|
48
123
|
## Behavior Rules
|
|
49
124
|
|
|
50
125
|
1. **Do not invoke computer use** for recording-only requests. The daemon handles these directly.
|
|
51
126
|
2. **One recording at a time.** If a recording is already active, starting another returns an "already recording" message.
|
|
52
|
-
3. **Conversation-
|
|
127
|
+
3. **Conversation-linked.** Each recording is linked to the conversation that started it for attachment purposes. However, since only one recording can be active at a time, stop commands from any conversation will stop the active recording regardless of which conversation started it.
|
|
53
128
|
4. **Permission required.** Screen recording requires macOS Screen Recording permission. If denied, the user sees actionable guidance to enable it in System Settings.
|
|
54
|
-
5. **Mixed-intent prompts** (recording + other task) are NOT intercepted by the standalone route.
|
|
129
|
+
5. **Mixed-intent prompts** (recording + other task) are NOT intercepted by the standalone route — the recording action is deferred and executed alongside the task.
|
|
130
|
+
6. **Restart always reopens the source picker** and requires source reselection.
|
|
131
|
+
7. **Restart cancel** (user closes the source picker) leaves state idle — no false "recording started" message.
|
|
132
|
+
8. **Pause/resume toggle the recording** without stopping it. The HUD shows paused state.
|
|
55
133
|
|
|
56
134
|
## What This Skill Does NOT Do
|
|
57
135
|
|
|
58
136
|
- This skill does not contain recorder logic — the `RecordingManager` and `ScreenRecorder` in the macOS app handle the actual recording.
|
|
59
137
|
- This skill does not provide shell commands or scripts for recording.
|
|
60
138
|
- This skill does not fall back to computer use for recording tasks.
|
|
139
|
+
- This skill does not handle informational questions about recording — those flow through to normal AI response.
|
|
@@ -209,6 +209,11 @@ export function buildGuardianVerificationRoutingSection(): string {
|
|
|
209
209
|
'4. Guide the user through code entry, resend, or cancel',
|
|
210
210
|
'',
|
|
211
211
|
'Load with: `skill_load` using `skill: "guardian-verify-setup"`',
|
|
212
|
+
'',
|
|
213
|
+
'### Exclusivity rules',
|
|
214
|
+
'- Guardian verification intents must only be handled by `guardian-verify-setup` — load it exclusively.',
|
|
215
|
+
'- Do NOT load `phone-calls` for guardian verification intent routing. The phone-calls skill does not orchestrate verification flows.',
|
|
216
|
+
'- If the user has already explicitly specified a channel (e.g., "verify my phone for SMS", "verify my Telegram"), do not re-ask which channel unless the input is contradictory. Note: "verify my phone number" alone is ambiguous (phone numbers apply to both sms and voice) — ask which channel.',
|
|
212
217
|
].join('\n');
|
|
213
218
|
}
|
|
214
219
|
|
|
@@ -133,3 +133,4 @@ If not yet bound, offer to resend (Step 4) or generate a new session (Step 3).
|
|
|
133
133
|
- The resend cooldown is 15 seconds between sends, with a maximum of 5 sends per session.
|
|
134
134
|
- Per-destination rate limiting allows up to 10 sends within a 1-hour rolling window.
|
|
135
135
|
- Guardian verification is identity-bound: the code can only be consumed by the identity matching the destination provided at start time.
|
|
136
|
+
- **Missing `secret` guardrail**: For voice and Telegram chat-ID flows, the API response MUST include a `secret` field. If `secret` is unexpectedly absent from a start or resend response that otherwise indicates success, treat this as a control-plane error. Do NOT fabricate a code or tell the user to proceed without one. Instead, tell the user something went wrong and ask them to retry the start (Step 3) or resend (Step 4).
|
|
@@ -1,22 +1,22 @@
|
|
|
1
1
|
import * as net from 'node:net';
|
|
2
2
|
|
|
3
|
+
import type { ChannelId } from '../../channels/types.js';
|
|
3
4
|
import * as externalConversationStore from '../../memory/external-conversation-store.js';
|
|
4
5
|
import {
|
|
5
6
|
createVerificationChallenge,
|
|
7
|
+
findActiveSession,
|
|
6
8
|
getGuardianBinding,
|
|
7
9
|
getPendingChallenge,
|
|
8
10
|
revokeBinding as revokeGuardianBinding,
|
|
9
11
|
revokePendingChallenges,
|
|
10
|
-
findActiveSession,
|
|
11
12
|
} from '../../runtime/channel-guardian-service.js';
|
|
12
13
|
import { type ChannelReadinessService, createReadinessService } from '../../runtime/channel-readiness-service.js';
|
|
13
14
|
import {
|
|
14
|
-
startOutbound,
|
|
15
|
-
resendOutbound,
|
|
16
15
|
cancelOutbound,
|
|
16
|
+
resendOutbound,
|
|
17
|
+
startOutbound,
|
|
17
18
|
} from '../../runtime/guardian-outbound-actions.js';
|
|
18
19
|
import { normalizeAssistantId } from '../../util/platform.js';
|
|
19
|
-
import type { ChannelId } from '../../channels/types.js';
|
|
20
20
|
import type {
|
|
21
21
|
ChannelReadinessRequest,
|
|
22
22
|
GuardianVerificationRequest,
|
|
@@ -34,10 +34,10 @@ export type GuardianVerificationResult = Omit<GuardianVerificationResponse, 'typ
|
|
|
34
34
|
// ---------------------------------------------------------------------------
|
|
35
35
|
|
|
36
36
|
export {
|
|
37
|
+
DESTINATION_RATE_WINDOW_MS,
|
|
38
|
+
MAX_SENDS_PER_DESTINATION_WINDOW,
|
|
37
39
|
MAX_SENDS_PER_SESSION,
|
|
38
40
|
RESEND_COOLDOWN_MS,
|
|
39
|
-
MAX_SENDS_PER_DESTINATION_WINDOW,
|
|
40
|
-
DESTINATION_RATE_WINDOW_MS,
|
|
41
41
|
} from '../../runtime/guardian-outbound-actions.js';
|
|
42
42
|
|
|
43
43
|
// ---------------------------------------------------------------------------
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as net from 'node:net';
|
|
2
2
|
|
|
3
|
-
import {
|
|
3
|
+
import { type Confidence, recordConversationSeenSignal, type SignalType } from '../../memory/conversation-attention-store.js';
|
|
4
4
|
import { updateDeliveryClientOutcome } from '../../notifications/deliveries-store.js';
|
|
5
5
|
import type { ClientMessage } from '../ipc-protocol.js';
|
|
6
6
|
import { handleRideShotgunStart, handleRideShotgunStop } from '../ride-shotgun-handler.js';
|