@vellumai/assistant 0.12.2-staging.3 → 0.12.2-staging.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/acp/session-snapshot.ts +260 -0
- package/src/config/bundled-skills/acp/SKILL.md +1 -1
- package/src/config/bundled-skills/acp/TOOLS.json +2 -2
- package/src/config/bundled-skills/skill-management/TOOLS.json +1 -1
- package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +374 -0
- package/src/live-voice/__tests__/live-voice-vad.test.ts +34 -0
- package/src/live-voice/__tests__/protocol.test.ts +48 -0
- package/src/live-voice/__tests__/session-controls.test.ts +25 -0
- package/src/live-voice/live-voice-session.ts +210 -12
- package/src/live-voice/protocol.ts +20 -0
- package/src/live-voice/session-controls.ts +55 -1
- package/src/notifications/__tests__/decision-engine.test.ts +175 -0
- package/src/notifications/decision-engine.ts +43 -8
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +10 -0
- package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +1 -1
- package/src/runtime/routes/__tests__/acp-routes.test.ts +19 -0
- package/src/runtime/routes/acp-routes.ts +17 -161
- package/src/skills/managed-store.ts +56 -0
- package/src/tools/acp/status.test.ts +276 -21
- package/src/tools/acp/status.ts +98 -32
- package/src/tools/skills/find-similar-skills.test.ts +211 -2
- package/src/tools/skills/find-similar-skills.ts +62 -9
package/package.json
CHANGED
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
import { desc, eq } from "drizzle-orm";
|
|
2
|
+
|
|
3
|
+
import { getDb } from "../persistence/db-connection.js";
|
|
4
|
+
import { acpSessionHistory } from "../persistence/schema/index.js";
|
|
5
|
+
import { getLogger } from "../util/logger.js";
|
|
6
|
+
import { acpAuthMarkerStillCurrent } from "./acp-auth-marker-store.js";
|
|
7
|
+
import { getAcpSessionManager } from "./index.js";
|
|
8
|
+
import type { AcpSessionManager } from "./session-manager.js";
|
|
9
|
+
import type { AcpSessionState } from "./types.js";
|
|
10
|
+
|
|
11
|
+
const log = getLogger("acp:session-snapshot");
|
|
12
|
+
|
|
13
|
+
export interface AcpSessionSnapshot {
|
|
14
|
+
id: string;
|
|
15
|
+
agentId: string;
|
|
16
|
+
acpSessionId: string;
|
|
17
|
+
parentConversationId: string;
|
|
18
|
+
status: string;
|
|
19
|
+
startedAt: number;
|
|
20
|
+
completedAt?: number | null;
|
|
21
|
+
error?: string | null;
|
|
22
|
+
stopReason?: string | null;
|
|
23
|
+
task?: string;
|
|
24
|
+
parentToolUseId?: string;
|
|
25
|
+
authErrorCode?: string;
|
|
26
|
+
authErrorCredential?: string;
|
|
27
|
+
model?: string;
|
|
28
|
+
availableModels?: AcpSessionState["availableModels"];
|
|
29
|
+
modelRevisionEpoch?: string;
|
|
30
|
+
modelRevision?: number;
|
|
31
|
+
usedTokens?: number;
|
|
32
|
+
contextSize?: number;
|
|
33
|
+
costAmount?: number;
|
|
34
|
+
costCurrency?: string;
|
|
35
|
+
inputTokens?: number;
|
|
36
|
+
outputTokens?: number;
|
|
37
|
+
eventLog?: unknown[];
|
|
38
|
+
source: "live" | "history";
|
|
39
|
+
resumable: boolean;
|
|
40
|
+
cwd?: string | null;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export interface AcpSessionSnapshotPage {
|
|
44
|
+
sessions: AcpSessionSnapshot[];
|
|
45
|
+
sawEveryHistoryRow: boolean;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
interface SnapshotOptions {
|
|
49
|
+
includeEventLog?: boolean;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function fromLiveState(
|
|
53
|
+
state: AcpSessionState,
|
|
54
|
+
manager: AcpSessionManager,
|
|
55
|
+
opts: SnapshotOptions,
|
|
56
|
+
): AcpSessionSnapshot {
|
|
57
|
+
return {
|
|
58
|
+
id: state.id,
|
|
59
|
+
agentId: state.agentId,
|
|
60
|
+
acpSessionId: state.acpSessionId,
|
|
61
|
+
parentConversationId: state.parentConversationId,
|
|
62
|
+
status: state.status,
|
|
63
|
+
startedAt: state.startedAt,
|
|
64
|
+
completedAt: state.completedAt ?? null,
|
|
65
|
+
error: state.error ?? null,
|
|
66
|
+
stopReason: state.stopReason ?? null,
|
|
67
|
+
task: state.task,
|
|
68
|
+
parentToolUseId: state.parentToolUseId,
|
|
69
|
+
authErrorCode: state.authErrorCode,
|
|
70
|
+
authErrorCredential: state.authErrorCredential,
|
|
71
|
+
model: state.model,
|
|
72
|
+
availableModels: state.availableModels,
|
|
73
|
+
modelRevisionEpoch: state.modelRevisionEpoch,
|
|
74
|
+
modelRevision: state.modelRevision,
|
|
75
|
+
usedTokens: state.latestUsage?.usedTokens,
|
|
76
|
+
contextSize: state.latestUsage?.contextSize,
|
|
77
|
+
costAmount: state.latestUsage?.costAmount,
|
|
78
|
+
costCurrency: state.latestUsage?.costCurrency,
|
|
79
|
+
inputTokens: state.latestUsage?.inputTokens,
|
|
80
|
+
outputTokens: state.latestUsage?.outputTokens,
|
|
81
|
+
eventLog: opts.includeEventLog
|
|
82
|
+
? manager.getBufferedUpdates(state.id)
|
|
83
|
+
: undefined,
|
|
84
|
+
source: "live",
|
|
85
|
+
resumable: false,
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function isResumableHistoryRow(
|
|
90
|
+
row: typeof acpSessionHistory.$inferSelect,
|
|
91
|
+
): boolean {
|
|
92
|
+
return Boolean(row.cwd && row.acpSessionId);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export function snapshotHistoryRow(
|
|
96
|
+
row: typeof acpSessionHistory.$inferSelect,
|
|
97
|
+
opts: SnapshotOptions = { includeEventLog: true },
|
|
98
|
+
): AcpSessionSnapshot {
|
|
99
|
+
let eventLog: unknown[] | undefined;
|
|
100
|
+
if (opts.includeEventLog !== false) {
|
|
101
|
+
eventLog = [];
|
|
102
|
+
try {
|
|
103
|
+
const parsed = JSON.parse(row.eventLogJson) as unknown;
|
|
104
|
+
if (Array.isArray(parsed)) {
|
|
105
|
+
eventLog = parsed;
|
|
106
|
+
}
|
|
107
|
+
} catch (err) {
|
|
108
|
+
log.warn(
|
|
109
|
+
{ id: row.id, err },
|
|
110
|
+
"Failed to parse event_log_json for ACP session history row",
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
return {
|
|
116
|
+
id: row.id,
|
|
117
|
+
agentId: row.agentId,
|
|
118
|
+
acpSessionId: row.acpSessionId,
|
|
119
|
+
parentConversationId: row.parentConversationId,
|
|
120
|
+
status: row.status,
|
|
121
|
+
startedAt: row.startedAt,
|
|
122
|
+
completedAt: row.completedAt,
|
|
123
|
+
error: row.error,
|
|
124
|
+
stopReason: row.stopReason,
|
|
125
|
+
task: row.task ?? undefined,
|
|
126
|
+
parentToolUseId: row.parentToolUseId ?? undefined,
|
|
127
|
+
authErrorCode: row.authErrorCode ?? undefined,
|
|
128
|
+
authErrorCredential: row.authErrorCredential ?? undefined,
|
|
129
|
+
usedTokens: row.usedTokens ?? undefined,
|
|
130
|
+
contextSize: row.contextSize ?? undefined,
|
|
131
|
+
costAmount: row.costAmount ?? undefined,
|
|
132
|
+
costCurrency: row.costCurrency ?? undefined,
|
|
133
|
+
inputTokens: row.inputTokens ?? undefined,
|
|
134
|
+
outputTokens: row.outputTokens ?? undefined,
|
|
135
|
+
eventLog,
|
|
136
|
+
source: "history",
|
|
137
|
+
resumable: isResumableHistoryRow(row),
|
|
138
|
+
cwd: row.cwd,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Blank `authErrorCode` on any session whose marker no longer describes the
|
|
144
|
+
* credential its agent would resolve.
|
|
145
|
+
*
|
|
146
|
+
* This comparison is what retires a Connect card: the marker no longer
|
|
147
|
+
* describing the credential in use. Applied after merging rather than inside
|
|
148
|
+
* the query, so live sessions and history rows are judged by the same rule.
|
|
149
|
+
*
|
|
150
|
+
* Resolved per agent and memoised across the batch, because precedence is per
|
|
151
|
+
* agent and each resolution costs a vault read.
|
|
152
|
+
*/
|
|
153
|
+
export async function withCurrentAuthMarkers<
|
|
154
|
+
T extends {
|
|
155
|
+
agentId: string;
|
|
156
|
+
authErrorCode?: string;
|
|
157
|
+
authErrorCredential?: string;
|
|
158
|
+
},
|
|
159
|
+
>(
|
|
160
|
+
sessions: readonly T[],
|
|
161
|
+
resolvedFor: (agentId: string) => Promise<string | undefined>,
|
|
162
|
+
): Promise<T[]> {
|
|
163
|
+
if (!sessions.some((session) => session.authErrorCode !== undefined)) {
|
|
164
|
+
return [...sessions];
|
|
165
|
+
}
|
|
166
|
+
const resolvedByAgent = new Map<string, string | undefined>();
|
|
167
|
+
const resolve = async (agentId: string) => {
|
|
168
|
+
if (!resolvedByAgent.has(agentId)) {
|
|
169
|
+
resolvedByAgent.set(agentId, await resolvedFor(agentId));
|
|
170
|
+
}
|
|
171
|
+
return resolvedByAgent.get(agentId);
|
|
172
|
+
};
|
|
173
|
+
const judged: T[] = [];
|
|
174
|
+
for (const session of sessions) {
|
|
175
|
+
if (session.authErrorCode === undefined) {
|
|
176
|
+
judged.push(session);
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
const current = acpAuthMarkerStillCurrent(
|
|
180
|
+
session.authErrorCredential,
|
|
181
|
+
await resolve(session.agentId),
|
|
182
|
+
);
|
|
183
|
+
judged.push(
|
|
184
|
+
current ? session : { ...session, authErrorCode: undefined },
|
|
185
|
+
);
|
|
186
|
+
}
|
|
187
|
+
return judged;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
export function getAcpSessionSnapshot(
|
|
191
|
+
acpSessionId: string,
|
|
192
|
+
opts: SnapshotOptions = {},
|
|
193
|
+
): AcpSessionSnapshot | undefined {
|
|
194
|
+
const manager = getAcpSessionManager();
|
|
195
|
+
const snapshotOptions = {
|
|
196
|
+
includeEventLog: opts.includeEventLog ?? true,
|
|
197
|
+
};
|
|
198
|
+
const live = (manager.getStatus() as AcpSessionState[]).find(
|
|
199
|
+
(state) => state.id === acpSessionId,
|
|
200
|
+
);
|
|
201
|
+
if (live) {
|
|
202
|
+
return fromLiveState(live, manager, snapshotOptions);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
const row = getDb()
|
|
206
|
+
.select()
|
|
207
|
+
.from(acpSessionHistory)
|
|
208
|
+
.where(eq(acpSessionHistory.id, acpSessionId))
|
|
209
|
+
.get();
|
|
210
|
+
return row ? snapshotHistoryRow(row, snapshotOptions) : undefined;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
export function listAcpSessionSnapshots(opts: {
|
|
214
|
+
limit: number;
|
|
215
|
+
conversationId?: string;
|
|
216
|
+
includeEventLog?: boolean;
|
|
217
|
+
}): AcpSessionSnapshotPage {
|
|
218
|
+
const manager = getAcpSessionManager();
|
|
219
|
+
const inMemory = manager.getStatus() as AcpSessionState[];
|
|
220
|
+
const snapshotOptions = {
|
|
221
|
+
includeEventLog: opts.includeEventLog ?? true,
|
|
222
|
+
};
|
|
223
|
+
|
|
224
|
+
const merged = new Map<string, AcpSessionSnapshot>();
|
|
225
|
+
for (const state of inMemory) {
|
|
226
|
+
if (
|
|
227
|
+
opts.conversationId &&
|
|
228
|
+
state.parentConversationId !== opts.conversationId
|
|
229
|
+
) {
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
merged.set(state.id, fromLiveState(state, manager, snapshotOptions));
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
const db = getDb();
|
|
236
|
+
const baseQuery = db.select().from(acpSessionHistory);
|
|
237
|
+
const filtered = opts.conversationId
|
|
238
|
+
? baseQuery.where(
|
|
239
|
+
eq(acpSessionHistory.parentConversationId, opts.conversationId),
|
|
240
|
+
)
|
|
241
|
+
: baseQuery;
|
|
242
|
+
const historyLimit = opts.limit + merged.size;
|
|
243
|
+
const historyRows = filtered
|
|
244
|
+
.orderBy(desc(acpSessionHistory.startedAt))
|
|
245
|
+
.limit(historyLimit)
|
|
246
|
+
.all();
|
|
247
|
+
|
|
248
|
+
for (const row of historyRows) {
|
|
249
|
+
if (!merged.has(row.id)) {
|
|
250
|
+
merged.set(row.id, snapshotHistoryRow(row, snapshotOptions));
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
return {
|
|
255
|
+
sessions: Array.from(merged.values()).sort(
|
|
256
|
+
(a, b) => b.startedAt - a.startedAt,
|
|
257
|
+
),
|
|
258
|
+
sawEveryHistoryRow: historyRows.length < historyLimit,
|
|
259
|
+
};
|
|
260
|
+
}
|
|
@@ -125,5 +125,5 @@ Default to the conversation's current working directory when spawning an agent.
|
|
|
125
125
|
|
|
126
126
|
- The spawned agent runs autonomously with its own tools, file editing, and terminal access.
|
|
127
127
|
- Results are streamed back and injected into the conversation when the agent completes.
|
|
128
|
-
- Use `acp_status` to
|
|
128
|
+
- Use `acp_status` to inspect running and idle agents. Use `acp_steer` to attempt follow-up work on an idle session; do not replace it with a new `acp_spawn`.
|
|
129
129
|
- The `cwd` parameter controls where the agent works - set it to the project root the user wants the agent to operate in.
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
},
|
|
34
34
|
{
|
|
35
35
|
"name": "acp_status",
|
|
36
|
-
"description": "Get the status of a specific ACP session or list
|
|
36
|
+
"description": "Get the status of a specific ACP session or list recent ACP sessions. Cleanly completed sessions with durable resume metadata are returned as idle; use `acp_steer` to attempt follow-up work on them instead of starting over with `acp_spawn`. Only use this when the user explicitly asks about ACP session status - do NOT poll automatically, as you will be notified when sessions complete.",
|
|
37
37
|
"category": "orchestration",
|
|
38
38
|
"risk": "low",
|
|
39
39
|
"input_schema": {
|
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
"properties": {
|
|
42
42
|
"acp_session_id": {
|
|
43
43
|
"type": "string",
|
|
44
|
-
"description": "Optional ACP session ID to query. If omitted, returns
|
|
44
|
+
"description": "Optional ACP session ID to query. If omitted, returns recent ACP sessions, including active and resumable idle sessions."
|
|
45
45
|
}
|
|
46
46
|
},
|
|
47
47
|
"required": []
|
|
@@ -118,7 +118,7 @@
|
|
|
118
118
|
},
|
|
119
119
|
{
|
|
120
120
|
"name": "find_similar_skills",
|
|
121
|
-
"description": "Find the existing skills most similar to a goal. Scores the goal against the skill catalog's capability pages and returns a ranked shortlist of nearest skills, each with its name, description, source (bundled, managed, plugin, workspace, or extra), and similarity score. For a managed hit it also returns `author
|
|
121
|
+
"description": "Find the existing skills most similar to a goal. Scores the goal against the skill catalog's capability pages and returns a ranked shortlist of nearest skills, each with its name, description, source (bundled, managed, plugin, workspace, or extra), and similarity score. For a managed hit it also returns `author`: \"assistant\" if the assistant authored that skill (so it may be overwritten) or \"user\" if a person did (off-limits); `author` is omitted for non-managed sources and for managed skills with no recorded author. For a background memory pass, a hit it may refine (managed, assistant-authored) also carries `current`: the skill as it is now, in scaffold_managed_skill's own argument names (name, description, emoji, category, includes, activation_hints, avoid_when, body_markdown), so a refinement can be written from the real skill and carry forward what it is not changing. Read-only: it never creates, edits, or deletes anything.",
|
|
122
122
|
"category": "skills",
|
|
123
123
|
"risk": "low",
|
|
124
124
|
"input_schema": {
|
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Answering a look (`[LOOK:SCREEN]`, `[LOOK:CAMERA]`) on a turn of its own.
|
|
3
|
+
*
|
|
4
|
+
* The reply that asks for a look only acknowledges it. A client that declared
|
|
5
|
+
* `lookFrames` sends a fresh frame once it has carried the look out, and the
|
|
6
|
+
* session answers from that frame without waiting for the user to speak again.
|
|
7
|
+
* Without the declaration, or without the frame, nothing extra runs.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { describe, expect, mock, test } from "bun:test";
|
|
11
|
+
|
|
12
|
+
import {
|
|
13
|
+
createMockProvider,
|
|
14
|
+
textResponse,
|
|
15
|
+
} from "../../__tests__/helpers/mock-provider.js";
|
|
16
|
+
import { setConfig } from "../../__tests__/helpers/set-config.js";
|
|
17
|
+
import { waitFor } from "../../__tests__/helpers/wait-for.js";
|
|
18
|
+
|
|
19
|
+
setConfig("memory", { enabled: false });
|
|
20
|
+
|
|
21
|
+
import type {
|
|
22
|
+
VoiceTurnCallbacks,
|
|
23
|
+
VoiceTurnOptions,
|
|
24
|
+
} from "../../calls/voice-session-bridge.js";
|
|
25
|
+
import { Conversation } from "../../daemon/conversation.js";
|
|
26
|
+
import {
|
|
27
|
+
deleteConversation,
|
|
28
|
+
setConversation,
|
|
29
|
+
} from "../../daemon/conversation-registry.js";
|
|
30
|
+
import { uploadAttachment } from "../../persistence/attachments-store.js";
|
|
31
|
+
import {
|
|
32
|
+
createConversation,
|
|
33
|
+
getMessages,
|
|
34
|
+
} from "../../persistence/conversation-crud.js";
|
|
35
|
+
import { initializeDb } from "../../persistence/db-init.js";
|
|
36
|
+
import type {
|
|
37
|
+
StreamingTranscriber,
|
|
38
|
+
SttStreamServerEvent,
|
|
39
|
+
} from "../../stt/types.js";
|
|
40
|
+
import {
|
|
41
|
+
LiveVoiceSession,
|
|
42
|
+
type LiveVoiceTtsStreamer,
|
|
43
|
+
} from "../live-voice-session.js";
|
|
44
|
+
import type { LiveVoiceSessionFactoryContext } from "../live-voice-session-manager.js";
|
|
45
|
+
import type { LiveVoiceTtsOptions } from "../live-voice-tts.js";
|
|
46
|
+
import {
|
|
47
|
+
createLiveVoiceServerFrameSequencer,
|
|
48
|
+
type LiveVoiceServerFrame,
|
|
49
|
+
} from "../protocol.js";
|
|
50
|
+
import {
|
|
51
|
+
LOOK_FOLLOW_UP_CONTENT,
|
|
52
|
+
LOOK_FRAME_REASON,
|
|
53
|
+
} from "../session-controls.js";
|
|
54
|
+
|
|
55
|
+
await initializeDb();
|
|
56
|
+
|
|
57
|
+
const IMAGE_BASE64 =
|
|
58
|
+
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk";
|
|
59
|
+
|
|
60
|
+
/** Hands over one spoken sentence each time a capture is released. */
|
|
61
|
+
class MockStreamingTranscriber implements StreamingTranscriber {
|
|
62
|
+
readonly providerId = "deepgram" as const;
|
|
63
|
+
readonly boundaryId = "daemon-streaming" as const;
|
|
64
|
+
private onEvent: ((event: SttStreamServerEvent) => void) | null = null;
|
|
65
|
+
|
|
66
|
+
async start(onEvent: (event: SttStreamServerEvent) => void): Promise<void> {
|
|
67
|
+
this.onEvent = onEvent;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
sendAudio(): void {}
|
|
71
|
+
|
|
72
|
+
stop(): void {
|
|
73
|
+
this.onEvent?.({ type: "final", text: "look at my screen" });
|
|
74
|
+
this.onEvent?.({ type: "closed" });
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function createHarness(options: { lookFrames: boolean }) {
|
|
79
|
+
const conversation = createConversation("Look follow-up");
|
|
80
|
+
const { provider } = createMockProvider([textResponse("")]);
|
|
81
|
+
const activeConversation = new Conversation(
|
|
82
|
+
conversation.id,
|
|
83
|
+
provider,
|
|
84
|
+
"system prompt",
|
|
85
|
+
() => {},
|
|
86
|
+
"/tmp",
|
|
87
|
+
{ maxTokens: 4096 },
|
|
88
|
+
);
|
|
89
|
+
activeConversation.setTrustContext({
|
|
90
|
+
trustClass: "guardian",
|
|
91
|
+
sourceChannel: "vellum",
|
|
92
|
+
});
|
|
93
|
+
setConversation(conversation.id, activeConversation);
|
|
94
|
+
|
|
95
|
+
const sequencer = createLiveVoiceServerFrameSequencer();
|
|
96
|
+
const frames: LiveVoiceServerFrame[] = [];
|
|
97
|
+
const context: LiveVoiceSessionFactoryContext = {
|
|
98
|
+
sessionId: "session-look",
|
|
99
|
+
startFrame: {
|
|
100
|
+
type: "start",
|
|
101
|
+
conversationId: conversation.id,
|
|
102
|
+
audio: { mimeType: "audio/pcm", sampleRate: 24_000, channels: 1 },
|
|
103
|
+
textInput: true,
|
|
104
|
+
sessionControls: ["look_screen", "look_camera", "look_stop"],
|
|
105
|
+
...(options.lookFrames ? { lookFrames: true } : {}),
|
|
106
|
+
},
|
|
107
|
+
sendFrame: mock(async (payload) => {
|
|
108
|
+
const frame = sequencer.next(payload);
|
|
109
|
+
frames.push(frame);
|
|
110
|
+
return frame;
|
|
111
|
+
}),
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
const turns: VoiceTurnOptions[] = [];
|
|
115
|
+
const startVoiceTurn = mock(async (turnOptions: VoiceTurnOptions) => {
|
|
116
|
+
turns.push(turnOptions);
|
|
117
|
+
return { turnId: `bridge-turn-${turns.length}`, abort: mock() };
|
|
118
|
+
});
|
|
119
|
+
const streamTtsAudio: LiveVoiceTtsStreamer = mock(
|
|
120
|
+
async (ttsOptions: LiveVoiceTtsOptions) => ({
|
|
121
|
+
provider: "fish-audio" as const,
|
|
122
|
+
contentType: "audio/pcm",
|
|
123
|
+
sampleRate: 24_000,
|
|
124
|
+
chunks: 1,
|
|
125
|
+
bytes: Buffer.byteLength(ttsOptions.text),
|
|
126
|
+
}),
|
|
127
|
+
);
|
|
128
|
+
|
|
129
|
+
let turnCount = 0;
|
|
130
|
+
const session = new LiveVoiceSession(context, {
|
|
131
|
+
resolveTranscriber: mock(async () => new MockStreamingTranscriber()),
|
|
132
|
+
startVoiceTurn,
|
|
133
|
+
streamTtsAudio,
|
|
134
|
+
createTurnId: () => `live-turn-${++turnCount}`,
|
|
135
|
+
emitMetrics: false,
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
const callbacks = (index: number): VoiceTurnCallbacks | undefined =>
|
|
139
|
+
turns[index]?.callbacks;
|
|
140
|
+
|
|
141
|
+
/** Finish turn `index` with `text` and wait for its speech to drain. */
|
|
142
|
+
const reply = async (index: number, text: string): Promise<void> => {
|
|
143
|
+
const turnCallbacks = callbacks(index);
|
|
144
|
+
turnCallbacks?.assistant_text_delta?.({
|
|
145
|
+
type: "assistant_text_delta",
|
|
146
|
+
text,
|
|
147
|
+
conversationId: conversation.id,
|
|
148
|
+
});
|
|
149
|
+
turnCallbacks?.message_complete?.({
|
|
150
|
+
type: "message_complete",
|
|
151
|
+
conversationId: conversation.id,
|
|
152
|
+
messageId: `assistant-message-${index}`,
|
|
153
|
+
});
|
|
154
|
+
const doneCount = index + 1;
|
|
155
|
+
await waitFor(
|
|
156
|
+
() =>
|
|
157
|
+
frames.filter((frame) => frame.type === "tts_done").length >= doneCount,
|
|
158
|
+
{ message: `Timed out waiting for turn ${index} to drain` },
|
|
159
|
+
);
|
|
160
|
+
};
|
|
161
|
+
|
|
162
|
+
/** Ask for a look, and wait for the control to reach the client. */
|
|
163
|
+
const askForLook = async (): Promise<void> => {
|
|
164
|
+
await session.start();
|
|
165
|
+
await session.handleClientFrame({ type: "ptt_release" });
|
|
166
|
+
await waitFor(() => turns.length === 1, {
|
|
167
|
+
message: "Timed out waiting for the spoken turn",
|
|
168
|
+
});
|
|
169
|
+
await reply(0, "Taking a look. [LOOK:SCREEN]");
|
|
170
|
+
await waitFor(
|
|
171
|
+
() => frames.some((frame) => frame.type === "session_control"),
|
|
172
|
+
{ message: "Timed out waiting for the look control" },
|
|
173
|
+
);
|
|
174
|
+
};
|
|
175
|
+
|
|
176
|
+
/** Send a real kept frame with the given timing reason. */
|
|
177
|
+
const sendFrame = async (reason: string): Promise<void> => {
|
|
178
|
+
const attachment = await uploadAttachment(
|
|
179
|
+
"frame.png",
|
|
180
|
+
"image/png",
|
|
181
|
+
IMAGE_BASE64,
|
|
182
|
+
);
|
|
183
|
+
const before = getMessages(conversation.id).length;
|
|
184
|
+
await session.handleClientFrame({
|
|
185
|
+
type: "sight_frame",
|
|
186
|
+
attachmentId: attachment.id,
|
|
187
|
+
timing: {
|
|
188
|
+
reason,
|
|
189
|
+
keepToEncodedMs: 1,
|
|
190
|
+
encodedToUploadedMs: 1,
|
|
191
|
+
uploadedToSentMs: 0,
|
|
192
|
+
bytes: 1,
|
|
193
|
+
},
|
|
194
|
+
});
|
|
195
|
+
await waitFor(() => getMessages(conversation.id).length > before, {
|
|
196
|
+
message: "Timed out waiting for the frame to persist",
|
|
197
|
+
});
|
|
198
|
+
};
|
|
199
|
+
|
|
200
|
+
return {
|
|
201
|
+
frames,
|
|
202
|
+
session,
|
|
203
|
+
turns,
|
|
204
|
+
reply,
|
|
205
|
+
askForLook,
|
|
206
|
+
sendFrame,
|
|
207
|
+
dispose: async () => {
|
|
208
|
+
await session.close("client_end");
|
|
209
|
+
deleteConversation(conversation.id);
|
|
210
|
+
activeConversation.dispose();
|
|
211
|
+
},
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/** Long enough for a follow-up that was going to start to have started. */
|
|
216
|
+
async function settle(): Promise<void> {
|
|
217
|
+
await new Promise((resolve) => setTimeout(resolve, 600));
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
describe("live-voice look follow-up", () => {
|
|
221
|
+
test("answers the look from its frame on a hidden turn of its own", async () => {
|
|
222
|
+
const harness = createHarness({ lookFrames: true });
|
|
223
|
+
try {
|
|
224
|
+
await harness.askForLook();
|
|
225
|
+
expect(harness.turns).toHaveLength(1);
|
|
226
|
+
|
|
227
|
+
await harness.sendFrame(LOOK_FRAME_REASON);
|
|
228
|
+
await waitFor(() => harness.turns.length === 2, {
|
|
229
|
+
message: "Timed out waiting for the look to be answered",
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
const followUp = harness.turns[1];
|
|
233
|
+
expect(followUp?.content).toBe(LOOK_FOLLOW_UP_CONTENT);
|
|
234
|
+
expect(followUp?.hiddenSyntheticPrompt).toBe(true);
|
|
235
|
+
expect(followUp?.voiceControlPrompt).toContain(
|
|
236
|
+
"You just took a fresh look at their screen",
|
|
237
|
+
);
|
|
238
|
+
} finally {
|
|
239
|
+
await harness.dispose();
|
|
240
|
+
}
|
|
241
|
+
});
|
|
242
|
+
|
|
243
|
+
test("the reply that asks for a look is taught to only acknowledge it", async () => {
|
|
244
|
+
const harness = createHarness({ lookFrames: true });
|
|
245
|
+
try {
|
|
246
|
+
await harness.askForLook();
|
|
247
|
+
expect(harness.turns[0]?.voiceControlPrompt).toContain(
|
|
248
|
+
"Use it even when their screen is already shared with you",
|
|
249
|
+
);
|
|
250
|
+
} finally {
|
|
251
|
+
await harness.dispose();
|
|
252
|
+
}
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
test("a client that did not declare lookFrames keeps the old look", async () => {
|
|
256
|
+
const harness = createHarness({ lookFrames: false });
|
|
257
|
+
try {
|
|
258
|
+
await harness.askForLook();
|
|
259
|
+
expect(harness.turns[0]?.voiceControlPrompt).toContain(
|
|
260
|
+
"say you will take it from their next words",
|
|
261
|
+
);
|
|
262
|
+
|
|
263
|
+
await harness.sendFrame(LOOK_FRAME_REASON);
|
|
264
|
+
await settle();
|
|
265
|
+
expect(harness.turns).toHaveLength(1);
|
|
266
|
+
} finally {
|
|
267
|
+
await harness.dispose();
|
|
268
|
+
}
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
test("an ambient frame does not answer the look", async () => {
|
|
272
|
+
const harness = createHarness({ lookFrames: true });
|
|
273
|
+
try {
|
|
274
|
+
await harness.askForLook();
|
|
275
|
+
await harness.sendFrame("heartbeat");
|
|
276
|
+
await settle();
|
|
277
|
+
expect(harness.turns).toHaveLength(1);
|
|
278
|
+
} finally {
|
|
279
|
+
await harness.dispose();
|
|
280
|
+
}
|
|
281
|
+
});
|
|
282
|
+
|
|
283
|
+
// The turn started before the frame was in the conversation, so it could
|
|
284
|
+
// not have read it: the look is still owed an answer once that turn is done.
|
|
285
|
+
test("a turn that started before the frame landed does not answer the look", async () => {
|
|
286
|
+
const harness = createHarness({ lookFrames: true });
|
|
287
|
+
try {
|
|
288
|
+
await harness.askForLook();
|
|
289
|
+
await harness.session.handleClientFrame({
|
|
290
|
+
type: "text",
|
|
291
|
+
text: "the second dropdown",
|
|
292
|
+
});
|
|
293
|
+
await waitFor(() => harness.turns.length === 2, {
|
|
294
|
+
message: "Timed out waiting for the typed turn",
|
|
295
|
+
});
|
|
296
|
+
|
|
297
|
+
await harness.sendFrame(LOOK_FRAME_REASON);
|
|
298
|
+
await harness.reply(1, "Which one do you mean?");
|
|
299
|
+
await waitFor(() => harness.turns.length === 3, {
|
|
300
|
+
timeoutMs: 2_000,
|
|
301
|
+
message: "Timed out waiting for the look to be answered",
|
|
302
|
+
});
|
|
303
|
+
expect(harness.turns[2]?.content).toBe(LOOK_FOLLOW_UP_CONTENT);
|
|
304
|
+
} finally {
|
|
305
|
+
await harness.dispose();
|
|
306
|
+
}
|
|
307
|
+
});
|
|
308
|
+
|
|
309
|
+
test("a turn that starts after the frame landed is not answered twice", async () => {
|
|
310
|
+
const harness = createHarness({ lookFrames: true });
|
|
311
|
+
try {
|
|
312
|
+
await harness.askForLook();
|
|
313
|
+
await harness.session.handleClientFrame({
|
|
314
|
+
type: "text",
|
|
315
|
+
text: "the second dropdown",
|
|
316
|
+
});
|
|
317
|
+
await waitFor(() => harness.turns.length === 2, {
|
|
318
|
+
message: "Timed out waiting for the typed turn",
|
|
319
|
+
});
|
|
320
|
+
|
|
321
|
+
// Lands while the typed turn holds the floor, so the look waits.
|
|
322
|
+
await harness.sendFrame(LOOK_FRAME_REASON);
|
|
323
|
+
await harness.reply(1, "Which one do you mean?");
|
|
324
|
+
// The user's next turn starts before the look's wait checks again, and
|
|
325
|
+
// it reads the frame that is now in the conversation.
|
|
326
|
+
await harness.session.handleClientFrame({
|
|
327
|
+
type: "text",
|
|
328
|
+
text: "the plan picker",
|
|
329
|
+
});
|
|
330
|
+
await waitFor(() => harness.turns.length === 3, {
|
|
331
|
+
message: "Timed out waiting for the second typed turn",
|
|
332
|
+
});
|
|
333
|
+
await harness.reply(2, "That one sets the billing plan.");
|
|
334
|
+
await settle();
|
|
335
|
+
|
|
336
|
+
expect(harness.turns).toHaveLength(3);
|
|
337
|
+
expect(harness.turns[2]?.content).toBe("the plan picker");
|
|
338
|
+
} finally {
|
|
339
|
+
await harness.dispose();
|
|
340
|
+
}
|
|
341
|
+
});
|
|
342
|
+
|
|
343
|
+
test("an interrupt drops a look still waiting on its frame", async () => {
|
|
344
|
+
const harness = createHarness({ lookFrames: true });
|
|
345
|
+
try {
|
|
346
|
+
await harness.askForLook();
|
|
347
|
+
await harness.session.handleClientFrame({ type: "interrupt" });
|
|
348
|
+
|
|
349
|
+
await harness.sendFrame(LOOK_FRAME_REASON);
|
|
350
|
+
await settle();
|
|
351
|
+
expect(harness.turns).toHaveLength(1);
|
|
352
|
+
} finally {
|
|
353
|
+
await harness.dispose();
|
|
354
|
+
}
|
|
355
|
+
});
|
|
356
|
+
|
|
357
|
+
test("the turn that answers a look cannot chain another look", async () => {
|
|
358
|
+
const harness = createHarness({ lookFrames: true });
|
|
359
|
+
try {
|
|
360
|
+
await harness.askForLook();
|
|
361
|
+
await harness.sendFrame(LOOK_FRAME_REASON);
|
|
362
|
+
await waitFor(() => harness.turns.length === 2, {
|
|
363
|
+
message: "Timed out waiting for the look to be answered",
|
|
364
|
+
});
|
|
365
|
+
|
|
366
|
+
await harness.reply(1, "It is the plan picker. [LOOK:SCREEN]");
|
|
367
|
+
await harness.sendFrame(LOOK_FRAME_REASON);
|
|
368
|
+
await settle();
|
|
369
|
+
expect(harness.turns).toHaveLength(2);
|
|
370
|
+
} finally {
|
|
371
|
+
await harness.dispose();
|
|
372
|
+
}
|
|
373
|
+
});
|
|
374
|
+
});
|