@aexol/spectral 0.9.211 → 0.9.215
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/agents.d.ts.map +1 -1
- package/dist/agent/agents.js +40 -0
- package/dist/backend/generated-images.d.ts +90 -0
- package/dist/backend/generated-images.d.ts.map +1 -0
- package/dist/backend/generated-images.js +214 -0
- package/dist/backend/machine-routing.d.ts +25 -0
- package/dist/backend/machine-routing.d.ts.map +1 -0
- package/dist/backend/machine-routing.js +35 -0
- package/dist/commands/serve.d.ts.map +1 -1
- package/dist/commands/serve.js +31 -6
- package/dist/extensions/browser/browser-service.d.ts +75 -1
- package/dist/extensions/browser/browser-service.d.ts.map +1 -1
- package/dist/extensions/browser/browser-service.js +600 -34
- package/dist/extensions/browser/index.d.ts +10 -4
- package/dist/extensions/browser/index.d.ts.map +1 -1
- package/dist/extensions/browser/index.js +44 -5
- package/dist/extensions/browser/stream/stream-tool.d.ts +10 -0
- package/dist/extensions/browser/stream/stream-tool.d.ts.map +1 -1
- package/dist/extensions/browser/stream/stream-tool.js +42 -14
- package/dist/extensions/browser/tools/core.d.ts.map +1 -1
- package/dist/extensions/browser/tools/core.js +63 -2
- package/dist/extensions/desktop-control/index.d.ts +55 -0
- package/dist/extensions/desktop-control/index.d.ts.map +1 -0
- package/dist/extensions/desktop-control/index.js +633 -0
- package/dist/extensions/desktop-control/keys.d.ts +33 -0
- package/dist/extensions/desktop-control/keys.d.ts.map +1 -0
- package/dist/extensions/desktop-control/keys.js +170 -0
- package/dist/extensions/desktop-control/linux.d.ts +26 -0
- package/dist/extensions/desktop-control/linux.d.ts.map +1 -0
- package/dist/extensions/desktop-control/linux.js +195 -0
- package/dist/extensions/desktop-control/macos.d.ts +53 -0
- package/dist/extensions/desktop-control/macos.d.ts.map +1 -0
- package/dist/extensions/desktop-control/macos.js +382 -0
- package/dist/extensions/desktop-control/safety.d.ts +89 -0
- package/dist/extensions/desktop-control/safety.d.ts.map +1 -0
- package/dist/extensions/desktop-control/safety.js +157 -0
- package/dist/extensions/desktop-control/types.d.ts +79 -0
- package/dist/extensions/desktop-control/types.d.ts.map +1 -0
- package/dist/extensions/desktop-control/types.js +10 -0
- package/dist/extensions/desktop-control/windows.d.ts +31 -0
- package/dist/extensions/desktop-control/windows.d.ts.map +1 -0
- package/dist/extensions/desktop-control/windows.js +276 -0
- package/dist/extensions/desktop-screenshot/index.d.ts +103 -0
- package/dist/extensions/desktop-screenshot/index.d.ts.map +1 -1
- package/dist/extensions/desktop-screenshot/index.js +363 -17
- package/dist/extensions/image-generation/index.d.ts.map +1 -1
- package/dist/extensions/image-generation/index.js +133 -34
- package/dist/extensions/seo/index.d.ts.map +1 -1
- package/dist/extensions/seo/index.js +4 -0
- package/dist/extensions/seo/tools/backend-bridge.d.ts +4 -0
- package/dist/extensions/seo/tools/backend-bridge.d.ts.map +1 -0
- package/dist/extensions/seo/tools/backend-bridge.js +378 -0
- package/dist/extensions/seo/tools/full-audit.d.ts +2 -0
- package/dist/extensions/seo/tools/full-audit.d.ts.map +1 -1
- package/dist/extensions/seo/tools/full-audit.js +1 -1
- package/dist/extensions/seo/tools/report.d.ts +10 -0
- package/dist/extensions/seo/tools/report.d.ts.map +1 -1
- package/dist/extensions/seo/tools/report.js +1 -1
- package/dist/extensions/spectral-vision-fallback.d.ts.map +1 -1
- package/dist/extensions/spectral-vision-fallback.js +11 -1
- package/dist/generated/zeus/const.d.ts.map +1 -1
- package/dist/generated/zeus/const.js +148 -4
- package/dist/generated/zeus/index.d.ts +649 -0
- package/dist/generated/zeus/index.d.ts.map +1 -1
- package/dist/mcp/tool-registrar.d.ts.map +1 -1
- package/dist/mcp/tool-registrar.js +5 -2
- package/dist/mcp/types.d.ts +5 -0
- package/dist/mcp/types.d.ts.map +1 -1
- package/dist/relay/client.d.ts +11 -0
- package/dist/relay/client.d.ts.map +1 -1
- package/dist/relay/client.js +24 -6
- package/dist/relay/dispatcher.d.ts +12 -1
- package/dist/relay/dispatcher.d.ts.map +1 -1
- package/dist/relay/dispatcher.js +44 -42
- package/dist/sdk/ai/providers/faux.js +1 -1
- package/dist/sdk/ai/providers/openai-completions.d.ts.map +1 -1
- package/dist/sdk/ai/providers/openai-completions.js +57 -16
- package/dist/sdk/ai/providers/openrouter-images.d.ts.map +1 -1
- package/dist/sdk/ai/providers/openrouter-images.js +45 -9
- package/dist/sdk/ai/providers/simple-options.d.ts.map +1 -1
- package/dist/sdk/ai/providers/simple-options.js +2 -0
- package/dist/sdk/ai/types.d.ts +37 -1
- package/dist/sdk/ai/types.d.ts.map +1 -1
- package/dist/sdk/ai/utils/image-source.d.ts +121 -0
- package/dist/sdk/ai/utils/image-source.d.ts.map +1 -0
- package/dist/sdk/ai/utils/image-source.js +410 -0
- package/dist/sdk/coding-agent/core/extensions/native-extensions.d.ts.map +1 -1
- package/dist/sdk/coding-agent/core/extensions/native-extensions.js +14 -0
- package/dist/sdk/coding-agent/core/extensions/types.d.ts +9 -0
- package/dist/sdk/coding-agent/core/extensions/types.d.ts.map +1 -1
- package/dist/sdk/coding-agent/core/system-prompt.d.ts.map +1 -1
- package/dist/sdk/coding-agent/core/system-prompt.js +15 -9
- package/dist/sdk/coding-agent/modes/rpc/rpc-mode.d.ts.map +1 -1
- package/dist/sdk/coding-agent/modes/rpc/rpc-mode.js +16 -3
- package/dist/sdk/coding-agent/utils/image-resize.d.ts.map +1 -1
- package/dist/sdk/coding-agent/utils/image-resize.js +7 -3
- package/dist/server/agent-bridge.d.ts +14 -0
- package/dist/server/agent-bridge.d.ts.map +1 -1
- package/dist/server/agent-bridge.js +105 -26
- package/dist/server/handlers/queue.d.ts.map +1 -1
- package/dist/server/handlers/queue.js +11 -34
- package/dist/server/image-attachments.d.ts +109 -0
- package/dist/server/image-attachments.d.ts.map +1 -0
- package/dist/server/image-attachments.js +195 -0
- package/dist/server/session-stream.d.ts +69 -0
- package/dist/server/session-stream.d.ts.map +1 -1
- package/dist/server/session-stream.js +388 -32
- package/dist/server/storage.d.ts.map +1 -1
- package/dist/server/storage.js +10 -31
- package/dist/server/ui/terminal-confirm.d.ts +145 -0
- package/dist/server/ui/terminal-confirm.d.ts.map +1 -0
- package/dist/server/ui/terminal-confirm.js +333 -0
- package/dist/server/wire.d.ts +30 -5
- package/dist/server/wire.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -6,16 +6,19 @@
|
|
|
6
6
|
* control a real browser: navigate, click, type, evaluate JS, take
|
|
7
7
|
* screenshots, inspect network traffic, and more.
|
|
8
8
|
*
|
|
9
|
-
* All
|
|
9
|
+
* All 31 tools match the same API surface as playwright-mcp, but without
|
|
10
10
|
* the MCP protocol layer — they call Playwright directly from Node.js.
|
|
11
11
|
*
|
|
12
12
|
* Architecture:
|
|
13
13
|
* - BrowserService (singleton) — one browser, one context, multiple pages
|
|
14
14
|
* - Tools register via ext.registerTool() — same pattern as aexol-mcp
|
|
15
|
-
* - Browser auto-closes on session_shutdown
|
|
15
|
+
* - Browser auto-closes on session_shutdown (stream + recording stopped
|
|
16
|
+
* first, then the browser, then a SIGKILL fallback for the Chromium
|
|
17
|
+
* process so a visible window is never orphaned)
|
|
16
18
|
*
|
|
17
|
-
* Registered tools (
|
|
18
|
-
* Core: browser_navigate, browser_navigate_back, browser_close, browser_resize
|
|
19
|
+
* Registered tools (31):
|
|
20
|
+
* Core: browser_navigate, browser_navigate_back, browser_close, browser_resize,
|
|
21
|
+
* browser_quit
|
|
19
22
|
* Tabs: browser_tabs
|
|
20
23
|
* Input: browser_click, browser_type, browser_press_key, browser_hover,
|
|
21
24
|
* browser_drag, browser_select_option, browser_fill_form, browser_file_upload
|
|
@@ -26,6 +29,9 @@
|
|
|
26
29
|
* Dialog: browser_handle_dialog
|
|
27
30
|
* Wait: browser_wait_for
|
|
28
31
|
* Drop: browser_drop
|
|
32
|
+
* Stream: browser_stream_start, browser_stream_stop, browser_stream_status
|
|
33
|
+
* Record: browser_record_start, browser_record_stop, browser_record_status
|
|
34
|
+
* Scenario: browser_run_scenario
|
|
29
35
|
*
|
|
30
36
|
* Usage:
|
|
31
37
|
* spectral --extension ./cli-node/src/extensions/browser/index.ts
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/extensions/browser/index.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/extensions/browser/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAiBpE,wBAA8B,gBAAgB,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAkG/E"}
|
|
@@ -6,16 +6,19 @@
|
|
|
6
6
|
* control a real browser: navigate, click, type, evaluate JS, take
|
|
7
7
|
* screenshots, inspect network traffic, and more.
|
|
8
8
|
*
|
|
9
|
-
* All
|
|
9
|
+
* All 31 tools match the same API surface as playwright-mcp, but without
|
|
10
10
|
* the MCP protocol layer — they call Playwright directly from Node.js.
|
|
11
11
|
*
|
|
12
12
|
* Architecture:
|
|
13
13
|
* - BrowserService (singleton) — one browser, one context, multiple pages
|
|
14
14
|
* - Tools register via ext.registerTool() — same pattern as aexol-mcp
|
|
15
|
-
* - Browser auto-closes on session_shutdown
|
|
15
|
+
* - Browser auto-closes on session_shutdown (stream + recording stopped
|
|
16
|
+
* first, then the browser, then a SIGKILL fallback for the Chromium
|
|
17
|
+
* process so a visible window is never orphaned)
|
|
16
18
|
*
|
|
17
|
-
* Registered tools (
|
|
18
|
-
* Core: browser_navigate, browser_navigate_back, browser_close, browser_resize
|
|
19
|
+
* Registered tools (31):
|
|
20
|
+
* Core: browser_navigate, browser_navigate_back, browser_close, browser_resize,
|
|
21
|
+
* browser_quit
|
|
19
22
|
* Tabs: browser_tabs
|
|
20
23
|
* Input: browser_click, browser_type, browser_press_key, browser_hover,
|
|
21
24
|
* browser_drag, browser_select_option, browser_fill_form, browser_file_upload
|
|
@@ -26,6 +29,9 @@
|
|
|
26
29
|
* Dialog: browser_handle_dialog
|
|
27
30
|
* Wait: browser_wait_for
|
|
28
31
|
* Drop: browser_drop
|
|
32
|
+
* Stream: browser_stream_start, browser_stream_stop, browser_stream_status
|
|
33
|
+
* Record: browser_record_start, browser_record_stop, browser_record_status
|
|
34
|
+
* Scenario: browser_run_scenario
|
|
29
35
|
*
|
|
30
36
|
* Usage:
|
|
31
37
|
* spectral --extension ./cli-node/src/extensions/browser/index.ts
|
|
@@ -42,6 +48,7 @@ import { registerDialogTools } from "./tools/dialog.js";
|
|
|
42
48
|
import { registerWaitTools } from "./tools/wait.js";
|
|
43
49
|
import { registerDropTools } from "./tools/drop.js";
|
|
44
50
|
import { registerStreamTools } from "./stream/stream-tool.js";
|
|
51
|
+
import { stopActiveStream } from "./stream/stream-tool.js";
|
|
45
52
|
import { registerRecordTools } from "./record/record-tool.js";
|
|
46
53
|
import { registerScenarioTools } from "./scenario/scenario-tool.js";
|
|
47
54
|
export default async function browserExtension(ext) {
|
|
@@ -61,8 +68,31 @@ export default async function browserExtension(ext) {
|
|
|
61
68
|
return;
|
|
62
69
|
}
|
|
63
70
|
const bs = BrowserService.getInstance();
|
|
64
|
-
// Auto-close browser when the session ends
|
|
71
|
+
// Auto-close browser when the session ends. Every step is best-effort: a
|
|
72
|
+
// failing step must never prevent the next one (and must never prevent the
|
|
73
|
+
// Chromium process from being killed).
|
|
65
74
|
ext.on("session_shutdown", async () => {
|
|
75
|
+
// 1. Stop an in-flight WebRTC stream (never re-starts signaling).
|
|
76
|
+
try {
|
|
77
|
+
const stopped = await stopActiveStream();
|
|
78
|
+
if (stopped.stopped) {
|
|
79
|
+
process.stderr.write("[browser] Stream stopped on shutdown.\n");
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
catch (err) {
|
|
83
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
84
|
+
process.stderr.write(`[browser] Error stopping stream: ${msg}\n`);
|
|
85
|
+
}
|
|
86
|
+
// 2. Finalize an in-flight recording (context.close() writes the .webm).
|
|
87
|
+
try {
|
|
88
|
+
// relaunch:false — never spawn a fresh Chromium just to close it again.
|
|
89
|
+
await bs.stopRecording({ relaunch: false });
|
|
90
|
+
process.stderr.write("[browser] Recording finalized on shutdown.\n");
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
// No recording active (the common case) — ignore.
|
|
94
|
+
}
|
|
95
|
+
// 3. Close the browser.
|
|
66
96
|
try {
|
|
67
97
|
await bs.close();
|
|
68
98
|
process.stderr.write("[browser] Browser closed.\n");
|
|
@@ -71,6 +101,14 @@ export default async function browserExtension(ext) {
|
|
|
71
101
|
const msg = err instanceof Error ? err.message : String(err);
|
|
72
102
|
process.stderr.write(`[browser] Error closing browser: ${msg}\n`);
|
|
73
103
|
}
|
|
104
|
+
// 4. Belt and braces: if anything above bailed out early, SIGKILL the
|
|
105
|
+
// Chromium process tree instead of leaving it behind.
|
|
106
|
+
try {
|
|
107
|
+
bs.forceKillBrowserProcess();
|
|
108
|
+
}
|
|
109
|
+
catch {
|
|
110
|
+
// ignore
|
|
111
|
+
}
|
|
74
112
|
});
|
|
75
113
|
// Register all tool groups
|
|
76
114
|
const registrars = [
|
|
@@ -112,6 +150,7 @@ function countRegisteredTools(ext) {
|
|
|
112
150
|
let count = 0;
|
|
113
151
|
const knownTools = [
|
|
114
152
|
"browser_navigate", "browser_navigate_back", "browser_close", "browser_resize",
|
|
153
|
+
"browser_quit",
|
|
115
154
|
"browser_tabs",
|
|
116
155
|
"browser_click", "browser_type", "browser_press_key", "browser_hover",
|
|
117
156
|
"browser_drag", "browser_select_option", "browser_fill_form", "browser_file_upload",
|
|
@@ -6,5 +6,15 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import type { ExtensionAPI } from "../../../sdk/coding-agent/index.js";
|
|
8
8
|
import { BrowserService } from "../browser-service.js";
|
|
9
|
+
export interface StopActiveStreamResult {
|
|
10
|
+
stopped: boolean;
|
|
11
|
+
sessionId?: string;
|
|
12
|
+
durationSeconds?: number;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Tear down the module-level stream (WebRTC + injected page code) if one is
|
|
16
|
+
* running. Never throws, never re-starts signaling. Safe to call at shutdown.
|
|
17
|
+
*/
|
|
18
|
+
export declare function stopActiveStream(): Promise<StopActiveStreamResult>;
|
|
9
19
|
export declare function registerStreamTools(ext: ExtensionAPI, bs: BrowserService): void;
|
|
10
20
|
//# sourceMappingURL=stream-tool.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stream-tool.d.ts","sourceRoot":"","sources":["../../../../src/extensions/browser/stream/stream-tool.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAkB,MAAM,oCAAoC,CAAC;AACvF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"stream-tool.d.ts","sourceRoot":"","sources":["../../../../src/extensions/browser/stream/stream-tool.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAkB,MAAM,oCAAoC,CAAC;AACvF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAmBvD,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;GAGG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,sBAAsB,CAAC,CAsBxE;AAED,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,YAAY,EAAE,EAAE,EAAE,cAAc,GAAG,IAAI,CA+K/E"}
|
|
@@ -7,7 +7,46 @@
|
|
|
7
7
|
import { SignalingClient } from "./signaling-client.js";
|
|
8
8
|
import { injectStreamingCode, stopStreaming } from "./stream-injector.js";
|
|
9
9
|
let activeStream = null;
|
|
10
|
+
/**
|
|
11
|
+
* Service the stream was registered with. Kept so `stopActiveStream()` can be
|
|
12
|
+
* called from outside the tool (e.g. the `session_shutdown` handler) without
|
|
13
|
+
* having to reach for `BrowserService.getInstance()` — which would resurrect
|
|
14
|
+
* a fresh instance after the singleton was already torn down.
|
|
15
|
+
*/
|
|
16
|
+
let activeService = null;
|
|
17
|
+
/**
|
|
18
|
+
* Tear down the module-level stream (WebRTC + injected page code) if one is
|
|
19
|
+
* running. Never throws, never re-starts signaling. Safe to call at shutdown.
|
|
20
|
+
*/
|
|
21
|
+
export async function stopActiveStream() {
|
|
22
|
+
const stream = activeStream;
|
|
23
|
+
if (!stream)
|
|
24
|
+
return { stopped: false };
|
|
25
|
+
// Clear first so a concurrent/second call (or a throwing stop) cannot
|
|
26
|
+
// double-close the same state.
|
|
27
|
+
activeStream = null;
|
|
28
|
+
try {
|
|
29
|
+
const page = activeService?.getActivePage();
|
|
30
|
+
if (page)
|
|
31
|
+
await stopStreaming(page);
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
/* ignore page-level errors — the page may already be gone */
|
|
35
|
+
}
|
|
36
|
+
try {
|
|
37
|
+
stream.signalingClient.close();
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
/* ignore */
|
|
41
|
+
}
|
|
42
|
+
return {
|
|
43
|
+
stopped: true,
|
|
44
|
+
sessionId: stream.sessionId,
|
|
45
|
+
durationSeconds: Math.round((Date.now() - stream.startedAt) / 1000),
|
|
46
|
+
};
|
|
47
|
+
}
|
|
10
48
|
export function registerStreamTools(ext, bs) {
|
|
49
|
+
activeService = bs;
|
|
11
50
|
const startTool = {
|
|
12
51
|
name: "browser_stream_start",
|
|
13
52
|
label: "Start Browser Stream",
|
|
@@ -111,28 +150,17 @@ export function registerStreamTools(ext, bs) {
|
|
|
111
150
|
properties: {},
|
|
112
151
|
},
|
|
113
152
|
async execute() {
|
|
114
|
-
|
|
153
|
+
const result = await stopActiveStream();
|
|
154
|
+
if (!result.stopped) {
|
|
115
155
|
return {
|
|
116
156
|
content: [{ type: "text", text: "No active stream." }],
|
|
117
157
|
details: {},
|
|
118
158
|
};
|
|
119
159
|
}
|
|
120
|
-
try {
|
|
121
|
-
const page = bs.getActivePage();
|
|
122
|
-
await stopStreaming(page);
|
|
123
|
-
}
|
|
124
|
-
catch { /* ignore page-level errors */ }
|
|
125
|
-
try {
|
|
126
|
-
activeStream.signalingClient.close();
|
|
127
|
-
}
|
|
128
|
-
catch { /* ignore */ }
|
|
129
|
-
const duration = Math.round((Date.now() - activeStream.startedAt) / 1000);
|
|
130
|
-
const sessionId = activeStream.sessionId;
|
|
131
|
-
activeStream = null;
|
|
132
160
|
return {
|
|
133
161
|
content: [{
|
|
134
162
|
type: "text",
|
|
135
|
-
text: `✓ Stream stopped.\nSession: ${sessionId}\nDuration: ${
|
|
163
|
+
text: `✓ Stream stopped.\nSession: ${result.sessionId}\nDuration: ${result.durationSeconds}s`,
|
|
136
164
|
}],
|
|
137
165
|
details: {},
|
|
138
166
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"core.d.ts","sourceRoot":"","sources":["../../../../src/extensions/browser/tools/core.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAkB,MAAM,oCAAoC,CAAC;AACvF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"core.d.ts","sourceRoot":"","sources":["../../../../src/extensions/browser/tools/core.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAkB,MAAM,oCAAoC,CAAC;AACvF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAwBvD,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,YAAY,EAAE,EAAE,EAAE,cAAc,GAAG,IAAI,CAoJ7E"}
|
|
@@ -1,6 +1,29 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Core browser tools: navigate, navigate_back, close, resize.
|
|
3
3
|
*/
|
|
4
|
+
import { stopActiveStream } from "../stream/stream-tool.js";
|
|
5
|
+
/**
|
|
6
|
+
* Best-effort pre-close teardown shared by `browser_quit`: stop an in-flight
|
|
7
|
+
* WebRTC stream (it keeps a signaling WS and injected page code alive) and
|
|
8
|
+
* finalize a running recording — `context.close()` is what writes the .webm,
|
|
9
|
+
* so closing the browser first would lose the file. Every step is wrapped in
|
|
10
|
+
* its own try/catch: a failure must never stop the browser from being closed.
|
|
11
|
+
*/
|
|
12
|
+
async function stopStreamAndRecording(bs) {
|
|
13
|
+
try {
|
|
14
|
+
await stopActiveStream();
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
/* no stream active or the page is already gone — ignore */
|
|
18
|
+
}
|
|
19
|
+
try {
|
|
20
|
+
// relaunch:false — never spawn a fresh Chromium just to close it again.
|
|
21
|
+
await bs.stopRecording({ relaunch: false });
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
// No recording active (the common case) — ignore.
|
|
25
|
+
}
|
|
26
|
+
}
|
|
4
27
|
export function registerCoreTools(ext, bs) {
|
|
5
28
|
const navigateTool = {
|
|
6
29
|
name: "browser_navigate",
|
|
@@ -61,7 +84,7 @@ export function registerCoreTools(ext, bs) {
|
|
|
61
84
|
const closeTool = {
|
|
62
85
|
name: "browser_close",
|
|
63
86
|
label: "Browser Close Page",
|
|
64
|
-
description: "
|
|
87
|
+
description: "Closes only the current page (tab) — it does NOT close the browser. A blank page is opened if it was the last one. Use browser_quit to close the whole browser.",
|
|
65
88
|
promptSnippet: "`browser_close` — close the current browser page",
|
|
66
89
|
parameters: {
|
|
67
90
|
type: "object",
|
|
@@ -78,6 +101,44 @@ export function registerCoreTools(ext, bs) {
|
|
|
78
101
|
}
|
|
79
102
|
},
|
|
80
103
|
};
|
|
104
|
+
const quitTool = {
|
|
105
|
+
name: "browser_quit",
|
|
106
|
+
label: "Browser Quit",
|
|
107
|
+
description: "Close the WHOLE browser: closes the browser context, the browser itself and kills the Chromium process. Unlike browser_close (which only closes the current tab), this tears the browser down completely. Use it when you are done working with the browser or when the browser window does not want to close.",
|
|
108
|
+
promptSnippet: "`browser_quit` — close the whole browser and kill the Chromium process",
|
|
109
|
+
parameters: {
|
|
110
|
+
type: "object",
|
|
111
|
+
properties: {},
|
|
112
|
+
},
|
|
113
|
+
async execute() {
|
|
114
|
+
try {
|
|
115
|
+
// Same order as the `session_shutdown` handler in ../index.ts: stream
|
|
116
|
+
// → recording → browser → SIGKILL fallback.
|
|
117
|
+
await stopStreamAndRecording(bs);
|
|
118
|
+
await bs.close();
|
|
119
|
+
// close() already SIGKILLs a process that refuses to exit; this is a
|
|
120
|
+
// belt-and-braces fallback for the case where close() returned early.
|
|
121
|
+
bs.forceKillBrowserProcess();
|
|
122
|
+
return { content: [{ type: "text", text: "✓ Browser closed." }], details: {} };
|
|
123
|
+
}
|
|
124
|
+
catch (err) {
|
|
125
|
+
try {
|
|
126
|
+
await stopStreamAndRecording(bs);
|
|
127
|
+
}
|
|
128
|
+
catch {
|
|
129
|
+
/* ignore — already best-effort */
|
|
130
|
+
}
|
|
131
|
+
try {
|
|
132
|
+
bs.forceKillBrowserProcess();
|
|
133
|
+
}
|
|
134
|
+
catch {
|
|
135
|
+
/* ignore */
|
|
136
|
+
}
|
|
137
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
138
|
+
return { content: [{ type: "text", text: `Browser quit failed: ${msg}` }], details: { isError: true } };
|
|
139
|
+
}
|
|
140
|
+
},
|
|
141
|
+
};
|
|
81
142
|
const resizeTool = {
|
|
82
143
|
name: "browser_resize",
|
|
83
144
|
label: "Browser Resize",
|
|
@@ -103,7 +164,7 @@ export function registerCoreTools(ext, bs) {
|
|
|
103
164
|
}
|
|
104
165
|
},
|
|
105
166
|
};
|
|
106
|
-
for (const tool of [navigateTool, navigateBackTool, closeTool, resizeTool]) {
|
|
167
|
+
for (const tool of [navigateTool, navigateBackTool, closeTool, quitTool, resizeTool]) {
|
|
107
168
|
try {
|
|
108
169
|
ext.registerTool(tool);
|
|
109
170
|
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Desktop Control Extension (computer-use with a safety layer)
|
|
3
|
+
*
|
|
4
|
+
* Registers snake_case `desktop_*` tools that inject REAL mouse/keyboard
|
|
5
|
+
* input via native OS commands:
|
|
6
|
+
* - macOS: JXA (osascript -l JavaScript) + CoreGraphics CGEventPost
|
|
7
|
+
* - Windows: PowerShell + Add-Type user32 SendInput/SetCursorPos
|
|
8
|
+
* - Linux: xdotool (X11) / wtype (Wayland, keyboard-only)
|
|
9
|
+
*
|
|
10
|
+
* Tools are registered ONLY when the platform/session is capable. Every
|
|
11
|
+
* mutating action is wrapped by the safety layer:
|
|
12
|
+
* 1. By default there is NO confirmation prompt: enabling the extension
|
|
13
|
+
* (enabledNativeExtensions contains "desktop-control") is the explicit
|
|
14
|
+
* user consent, so actions run in interactive, headless, serve and RPC
|
|
15
|
+
* sessions alike. Opt-in SPECTRAL_DESKTOP_REQUIRE_CONFIRM=1|true restores
|
|
16
|
+
* the per-session interactive confirmation (ctx.hasUI; headless
|
|
17
|
+
* auto-denies single actions without locking the session).
|
|
18
|
+
* 2. Optional SPECTRAL_DESKTOP_ALLOWLIST restricts actions to matching
|
|
19
|
+
* frontmost apps.
|
|
20
|
+
* 3. Screenshot BEFORE -> execute -> screenshot AFTER; both images are
|
|
21
|
+
* returned as image content so the model verifies the outcome
|
|
22
|
+
* (spectral-vision-fallback analyzes them automatically). A pointer
|
|
23
|
+
* read-back and a "no visible change" hint expose silent injection
|
|
24
|
+
* failures (CGEventPost never errors).
|
|
25
|
+
* 4. SPECTRAL_DESKTOP_DRY_RUN=1 returns action plans without executing.
|
|
26
|
+
*
|
|
27
|
+
* This extension is registered with defaultEnabled: false — it must be
|
|
28
|
+
* opted in via Settings.enabledNativeExtensions: ["desktop-control"].
|
|
29
|
+
*
|
|
30
|
+
* Zero external npm dependencies beyond cross-spawn (Windows PATH is trimmed
|
|
31
|
+
* under Electron). No native modules.
|
|
32
|
+
*/
|
|
33
|
+
import type { ExtensionAPI, ToolDefinition } from "../../sdk/coding-agent/index.js";
|
|
34
|
+
import { type ScreenshotFormat, type WindowInfo } from "../desktop-screenshot/index.js";
|
|
35
|
+
import { DesktopActionPolicy } from "./safety.js";
|
|
36
|
+
import type { DesktopAdapter } from "./types.js";
|
|
37
|
+
export declare function createAdapter(env?: NodeJS.ProcessEnv): Promise<{
|
|
38
|
+
adapter: DesktopAdapter | null;
|
|
39
|
+
reason?: string;
|
|
40
|
+
}>;
|
|
41
|
+
export interface DesktopControlDeps {
|
|
42
|
+
adapter: DesktopAdapter;
|
|
43
|
+
/** Screenshot capture (defaults to desktop-screenshot's full-screen capture). */
|
|
44
|
+
capture?: (options?: {
|
|
45
|
+
format?: ScreenshotFormat;
|
|
46
|
+
}) => Promise<Buffer>;
|
|
47
|
+
/** Window listing (defaults to desktop-screenshot's listWindows). */
|
|
48
|
+
listWindowsFn?: () => Promise<WindowInfo[]>;
|
|
49
|
+
env?: NodeJS.ProcessEnv;
|
|
50
|
+
policy?: DesktopActionPolicy;
|
|
51
|
+
}
|
|
52
|
+
/** Build the 8 desktop tools supported by the adapter's capabilities. */
|
|
53
|
+
export declare function createDesktopTools(deps: DesktopControlDeps): Array<ToolDefinition<Record<string, unknown>, Record<string, unknown>>>;
|
|
54
|
+
export default function desktopControlExtension(ext: ExtensionAPI): Promise<void>;
|
|
55
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/extensions/desktop-control/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,KAAK,EAEX,YAAY,EAEZ,cAAc,EACd,MAAM,iCAAiC,CAAC;AAGzC,OAAO,EAIN,KAAK,gBAAgB,EACrB,KAAK,UAAU,EACf,MAAM,gCAAgC,CAAC;AAKxC,OAAO,EACN,mBAAmB,EAQnB,MAAM,aAAa,CAAC;AAErB,OAAO,KAAK,EAA6B,cAAc,EAAE,MAAM,YAAY,CAAC;AAQ5E,wBAAsB,aAAa,CAClC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAClC,OAAO,CAAC;IAAE,OAAO,EAAE,cAAc,GAAG,IAAI,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAW9D;AAMD,MAAM,WAAW,kBAAkB;IAClC,OAAO,EAAE,cAAc,CAAC;IACxB,iFAAiF;IACjF,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,gBAAgB,CAAA;KAAE,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IACvE,qEAAqE;IACrE,aAAa,CAAC,EAAE,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IAC5C,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB,MAAM,CAAC,EAAE,mBAAmB,CAAC;CAC7B;AAmQD,yEAAyE;AACzE,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,kBAAkB,GAAG,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CA0SpI;AAMD,wBAA8B,uBAAuB,CAAC,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CA+BtF"}
|