@vellumai/assistant 0.4.21 → 0.4.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/ipc/check-swift-decoder-drift.ts +55 -44
- package/src/__tests__/__snapshots__/ipc-snapshot.test.ts.snap +0 -75
- package/src/__tests__/headless-browser-interactions.test.ts +0 -4
- package/src/__tests__/ipc-snapshot.test.ts +0 -54
- package/src/__tests__/resolve-guardian-trust-class.test.ts +61 -0
- package/src/__tests__/session-init.benchmark.test.ts +0 -4
- package/src/config/system-prompt.ts +1 -0
- package/src/config/templates/BOOTSTRAP.md +21 -31
- package/src/config/templates/SOUL.md +19 -9
- package/src/daemon/computer-use-session.ts +5 -3
- package/src/daemon/daemon-control.ts +3 -0
- package/src/daemon/handlers/browser.ts +2 -48
- package/src/daemon/handlers/config-voice.ts +155 -33
- package/src/daemon/handlers/dictation.ts +361 -214
- package/src/daemon/ipc-contract/browser.ts +4 -74
- package/src/daemon/ipc-contract/surfaces.ts +51 -48
- package/src/daemon/ipc-contract-inventory.json +0 -7
- package/src/daemon/session-agent-loop.ts +2 -1
- package/src/daemon/session-runtime-assembly.ts +477 -247
- package/src/daemon/session-surfaces.ts +5 -3
- package/src/daemon/session-tool-setup.ts +27 -13
- package/src/memory/migrations/102-alter-table-columns.ts +254 -37
- package/src/memory/schema.ts +1227 -1035
- package/src/tools/browser/browser-execution.ts +314 -331
- package/src/tools/browser/browser-handoff.ts +11 -37
- package/src/tools/browser/browser-manager.ts +271 -264
- package/src/tools/browser/browser-screencast.ts +19 -75
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import type { ServerMessage } from
|
|
2
|
-
import { browserManager
|
|
1
|
+
import type { ServerMessage } from "../../daemon/ipc-contract.js";
|
|
2
|
+
import { browserManager } from "./browser-manager.js";
|
|
3
3
|
|
|
4
|
-
// Track which sessions have an active browser page
|
|
5
|
-
// watches the actual browser window directly).
|
|
4
|
+
// Track which sessions have an active browser page.
|
|
6
5
|
const activeBrowserSessions = new Set<string>();
|
|
7
6
|
|
|
8
7
|
// Registry of sendToClient callbacks per session
|
|
@@ -12,9 +11,15 @@ const sessionSenders = new Map<string, (msg: ServerMessage) => void>();
|
|
|
12
11
|
* Register a sendToClient callback for a session.
|
|
13
12
|
* Called from session-tool-setup when the session is created.
|
|
14
13
|
*/
|
|
15
|
-
export function registerSessionSender(
|
|
14
|
+
export function registerSessionSender(
|
|
15
|
+
sessionId: string,
|
|
16
|
+
sendToClient: (msg: ServerMessage) => void,
|
|
17
|
+
): void {
|
|
16
18
|
sessionSenders.set(sessionId, sendToClient);
|
|
17
|
-
browserManager.registerSender(
|
|
19
|
+
browserManager.registerSender(
|
|
20
|
+
sessionId,
|
|
21
|
+
sendToClient as (msg: { type: string; sessionId: string }) => void,
|
|
22
|
+
);
|
|
18
23
|
}
|
|
19
24
|
|
|
20
25
|
/**
|
|
@@ -25,14 +30,13 @@ export function unregisterSessionSender(sessionId: string): void {
|
|
|
25
30
|
browserManager.unregisterSender(sessionId);
|
|
26
31
|
}
|
|
27
32
|
|
|
28
|
-
function getSender(
|
|
33
|
+
function getSender(
|
|
34
|
+
sessionId: string,
|
|
35
|
+
): ((msg: ServerMessage) => void) | undefined {
|
|
29
36
|
return sessionSenders.get(sessionId);
|
|
30
37
|
}
|
|
31
38
|
|
|
32
|
-
export async function ensureScreencast(
|
|
33
|
-
sessionId: string,
|
|
34
|
-
_sendToClient: (msg: ServerMessage) => void,
|
|
35
|
-
): Promise<void> {
|
|
39
|
+
export async function ensureScreencast(sessionId: string): Promise<void> {
|
|
36
40
|
if (activeBrowserSessions.has(sessionId)) return;
|
|
37
41
|
|
|
38
42
|
activeBrowserSessions.add(sessionId);
|
|
@@ -40,9 +44,6 @@ export async function ensureScreencast(
|
|
|
40
44
|
try {
|
|
41
45
|
// Ensure the page exists (may trigger browser launch/connect)
|
|
42
46
|
await browserManager.getOrCreateSessionPage(sessionId);
|
|
43
|
-
|
|
44
|
-
// No PiP surface or CDP screencast — the user watches the actual
|
|
45
|
-
// browser window directly (positioned in top-right via positionWindowSidebar).
|
|
46
47
|
} catch (err) {
|
|
47
48
|
// Roll back so future calls can retry
|
|
48
49
|
activeBrowserSessions.delete(sessionId);
|
|
@@ -50,30 +51,7 @@ export async function ensureScreencast(
|
|
|
50
51
|
}
|
|
51
52
|
}
|
|
52
53
|
|
|
53
|
-
export function
|
|
54
|
-
sessionId: string,
|
|
55
|
-
_sendToClient: (msg: ServerMessage) => void,
|
|
56
|
-
_status: 'navigating' | 'idle' | 'interacting',
|
|
57
|
-
_actionText?: string,
|
|
58
|
-
_currentUrl?: string,
|
|
59
|
-
): void {
|
|
60
|
-
// No-op: PiP surface was removed so there is no ui_surface to update.
|
|
61
|
-
// The function signature is preserved to avoid churn at callsites.
|
|
62
|
-
if (!activeBrowserSessions.has(sessionId)) return;
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
export async function updatePagesList(
|
|
66
|
-
sessionId: string,
|
|
67
|
-
_sendToClient: (msg: ServerMessage) => void,
|
|
68
|
-
): Promise<void> {
|
|
69
|
-
// No-op: PiP surface was removed so there is no ui_surface to update.
|
|
70
|
-
if (!activeBrowserSessions.has(sessionId)) return;
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
export async function stopBrowserScreencast(
|
|
74
|
-
sessionId: string,
|
|
75
|
-
_sendToClient: (msg: ServerMessage) => void,
|
|
76
|
-
): Promise<void> {
|
|
54
|
+
export async function stopBrowserScreencast(sessionId: string): Promise<void> {
|
|
77
55
|
if (!activeBrowserSessions.has(sessionId)) return;
|
|
78
56
|
|
|
79
57
|
// Safe no-op if CDP screencast was never started
|
|
@@ -82,47 +60,13 @@ export async function stopBrowserScreencast(
|
|
|
82
60
|
activeBrowserSessions.delete(sessionId);
|
|
83
61
|
}
|
|
84
62
|
|
|
85
|
-
export async function getElementBounds(
|
|
86
|
-
sessionId: string,
|
|
87
|
-
selector: string,
|
|
88
|
-
): Promise<{ x: number; y: number; w: number; h: number } | null> {
|
|
89
|
-
try {
|
|
90
|
-
const page = await browserManager.getOrCreateSessionPage(sessionId);
|
|
91
|
-
const result = await page.evaluate(`
|
|
92
|
-
(() => {
|
|
93
|
-
const el = document.querySelector(${JSON.stringify(selector)});
|
|
94
|
-
if (!el) return null;
|
|
95
|
-
const rect = el.getBoundingClientRect();
|
|
96
|
-
return { x: rect.x, y: rect.y, w: rect.width, h: rect.height, vw: window.innerWidth, vh: window.innerHeight };
|
|
97
|
-
})()
|
|
98
|
-
`) as { x: number; y: number; w: number; h: number; vw: number; vh: number } | null;
|
|
99
|
-
if (!result) return null;
|
|
100
|
-
const scale = Math.min(SCREENCAST_WIDTH / result.vw, SCREENCAST_HEIGHT / result.vh);
|
|
101
|
-
return {
|
|
102
|
-
x: result.x * scale,
|
|
103
|
-
y: result.y * scale,
|
|
104
|
-
w: result.w * scale,
|
|
105
|
-
h: result.h * scale,
|
|
106
|
-
};
|
|
107
|
-
} catch {
|
|
108
|
-
return null;
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
export function updateHighlights(
|
|
113
|
-
sessionId: string,
|
|
114
|
-
_sendToClient: (msg: ServerMessage) => void,
|
|
115
|
-
_highlights: Array<{ x: number; y: number; w: number; h: number; label: string }>,
|
|
116
|
-
): void {
|
|
117
|
-
// No-op: PiP surface was removed so there is no ui_surface to update.
|
|
118
|
-
if (!activeBrowserSessions.has(sessionId)) return;
|
|
119
|
-
}
|
|
120
|
-
|
|
121
63
|
export async function stopAllScreencasts(): Promise<void> {
|
|
122
64
|
for (const sessionId of activeBrowserSessions) {
|
|
123
65
|
try {
|
|
124
66
|
await browserManager.stopScreencast(sessionId);
|
|
125
|
-
} catch {
|
|
67
|
+
} catch {
|
|
68
|
+
/* best-effort */
|
|
69
|
+
}
|
|
126
70
|
}
|
|
127
71
|
activeBrowserSessions.clear();
|
|
128
72
|
}
|