@testdriverai/mcp 7.10.0-canary.0 → 7.10.1-test
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +17 -0
- package/agent/index.js +116 -71
- package/agent/interface.js +13 -1
- package/agent/lib/commands.js +35 -15
- package/agent/lib/provision-commands.js +92 -11
- package/agent/lib/sandbox.js +263 -30
- package/agent/lib/sdk.js +16 -12
- package/agent/lib/system.js +107 -31
- package/ai/.claude-plugin/plugin.json +89 -4
- package/ai/agents/testdriver.md +4 -8
- package/ai/skills/testdriver-assert/SKILL.md +0 -1
- package/ai/skills/testdriver-aws-setup/SKILL.md +1 -1
- package/ai/skills/testdriver-caching/SKILL.md +13 -6
- package/ai/skills/testdriver-ci-cd/SKILL.md +57 -14
- package/ai/skills/testdriver-client/SKILL.md +10 -6
- package/ai/skills/testdriver-customizing-devices/SKILL.md +7 -5
- package/ai/skills/testdriver-dashcam/SKILL.md +36 -3
- package/ai/skills/testdriver-debugging-with-screenshots/SKILL.md +28 -4
- package/ai/skills/testdriver-device-config/SKILL.md +5 -5
- package/ai/skills/testdriver-find/SKILL.md +50 -11
- package/ai/skills/testdriver-generating-tests/SKILL.md +163 -8
- package/ai/skills/testdriver-hosted/SKILL.md +1 -1
- package/ai/skills/testdriver-machine-setup/SKILL.md +331 -0
- package/ai/skills/testdriver-making-assertions/SKILL.md +78 -2
- package/ai/skills/testdriver-mcp-workflow/SKILL.md +2 -2
- package/ai/skills/testdriver-parse/SKILL.md +1 -1
- package/ai/skills/testdriver-performing-actions/SKILL.md +97 -3
- package/ai/skills/testdriver-provision/SKILL.md +0 -20
- package/ai/skills/testdriver-quickstart/SKILL.md +299 -21
- package/ai/skills/testdriver-reusable-code/SKILL.md +3 -3
- package/ai/skills/testdriver-running-tests/SKILL.md +1 -1
- package/ai/skills/testdriver-screenshot/SKILL.md +5 -5
- package/ai/skills/testdriver-scroll/SKILL.md +30 -160
- package/ai/skills/testdriver-test-writer/SKILL.md +3 -5
- package/ai/skills/testdriver-testdriver/SKILL.md +3 -7
- package/ai/skills/testdriver-wait/SKILL.md +1 -1
- package/{docs/v7/ai.mdx → ai/skills/testdriver:ai/SKILL.md} +3 -4
- package/ai/skills/testdriver:assert/SKILL.md +315 -0
- package/ai/skills/testdriver:aws-setup/SKILL.md +448 -0
- package/ai/skills/testdriver:cache/SKILL.md +221 -0
- package/ai/skills/testdriver:caching/SKILL.md +124 -0
- package/ai/skills/testdriver:captcha/SKILL.md +158 -0
- package/ai/skills/testdriver:ci-cd/SKILL.md +659 -0
- package/ai/skills/testdriver:click/SKILL.md +286 -0
- package/ai/skills/testdriver:client/SKILL.md +477 -0
- package/ai/skills/testdriver:customizing-devices/SKILL.md +319 -0
- package/ai/skills/testdriver:dashcam/SKILL.md +451 -0
- package/ai/skills/testdriver:debugging-with-screenshots/SKILL.md +415 -0
- package/{docs/v7/device-config.mdx → ai/skills/testdriver:device-config/SKILL.md} +8 -8
- package/ai/skills/testdriver:double-click/SKILL.md +102 -0
- package/ai/skills/testdriver:elements/SKILL.md +605 -0
- package/ai/skills/testdriver:enterprise/SKILL.md +7 -0
- package/ai/skills/testdriver:errors/SKILL.md +246 -0
- package/ai/skills/testdriver:events/SKILL.md +356 -0
- package/ai/skills/testdriver:exec/SKILL.md +317 -0
- package/ai/skills/testdriver:find/SKILL.md +860 -0
- package/ai/skills/testdriver:focus-application/SKILL.md +293 -0
- package/ai/skills/testdriver:generating-tests/SKILL.md +25 -0
- package/ai/skills/testdriver:hosted/SKILL.md +156 -0
- package/ai/skills/testdriver:hover/SKILL.md +278 -0
- package/{docs/v7/locating-elements.mdx → ai/skills/testdriver:locating-elements/SKILL.md} +3 -3
- package/ai/skills/testdriver:making-assertions/SKILL.md +32 -0
- package/ai/skills/testdriver:mcp/SKILL.md +7 -0
- package/ai/skills/testdriver:mouse-down/SKILL.md +161 -0
- package/ai/skills/testdriver:mouse-up/SKILL.md +164 -0
- package/ai/skills/testdriver:parse/SKILL.md +236 -0
- package/ai/skills/testdriver:performing-actions/SKILL.md +53 -0
- package/ai/skills/testdriver:press-keys/SKILL.md +348 -0
- package/ai/skills/testdriver:provision/SKILL.md +331 -0
- package/ai/skills/testdriver:quickstart/SKILL.md +172 -0
- package/ai/skills/testdriver:redraw/SKILL.md +214 -0
- package/ai/skills/testdriver:reusable-code/SKILL.md +249 -0
- package/ai/skills/testdriver:right-click/SKILL.md +123 -0
- package/{docs/v7/running-tests.mdx → ai/skills/testdriver:running-tests/SKILL.md} +4 -4
- package/ai/skills/testdriver:screenshot/SKILL.md +248 -0
- package/ai/skills/testdriver:screenshots/SKILL.md +184 -0
- package/ai/skills/testdriver:scroll/SKILL.md +205 -0
- package/ai/skills/testdriver:secrets/SKILL.md +115 -0
- package/ai/skills/testdriver:self-hosted/SKILL.md +147 -0
- package/ai/skills/testdriver:test-results-json/SKILL.md +257 -0
- package/ai/skills/testdriver:testdriver/SKILL.md +624 -0
- package/ai/skills/testdriver:type/SKILL.md +357 -0
- package/ai/skills/testdriver:variables/SKILL.md +111 -0
- package/ai/skills/testdriver:wait/SKILL.md +50 -0
- package/{docs/v7/waiting-for-elements.mdx → ai/skills/testdriver:waiting-for-elements/SKILL.md} +3 -3
- package/ai/skills/testdriver:what-is-testdriver/SKILL.md +54 -0
- package/debugger/index.html +12 -0
- package/docs/_data/examples-manifest.json +54 -46
- package/docs/_data/examples-manifest.schema.json +12 -2
- package/docs/_scripts/extract-example-urls.js +21 -3
- package/docs/_scripts/generate-examples.js +64 -16
- package/docs/changelog.mdx +29 -2
- package/docs/claude-mcp-plugin.mdx +65 -18
- package/docs/docs.json +16 -33
- package/docs/snippets/tests/scroll-yaml.mdx +2 -2
- package/docs/v6/commands/scroll-until-image.mdx +1 -1
- package/docs/v6/commands/scroll-until-text.mdx +2 -2
- package/docs/v6/commands/scroll.mdx +2 -2
- package/docs/v7/_drafts/agents.mdx +0 -4
- package/docs/v7/_drafts/best-practices.mdx +0 -12
- package/docs/v7/_drafts/commands/scroll-until-image.mdx +1 -1
- package/docs/v7/_drafts/commands/scroll-until-text.mdx +2 -2
- package/docs/v7/_drafts/commands/scroll.mdx +1 -1
- package/docs/v7/_drafts/configuration.mdx +0 -12
- package/docs/v7/_drafts/faq.mdx +2 -10
- package/docs/v7/_drafts/readme.mdx +0 -1
- package/docs/v7/_drafts/troubleshooting.mdx +1 -4
- package/docs/v7/assert.mdx +0 -1
- package/docs/v7/aws-setup.mdx +1 -1
- package/docs/v7/caching.mdx +14 -10
- package/docs/v7/ci-cd.mdx +98 -14
- package/docs/v7/client.mdx +10 -6
- package/docs/v7/copilot/auto-healing.mdx +167 -18
- package/docs/v7/copilot/running-tests.mdx +915 -54
- package/docs/v7/customizing-devices.mdx +7 -5
- package/docs/v7/dashcam.mdx +36 -3
- package/docs/v7/debugging-with-screenshots.mdx +31 -7
- package/docs/v7/examples/ai.mdx +13 -8
- package/docs/v7/examples/assert.mdx +1 -1
- package/docs/v7/examples/chrome-extension.mdx +30 -13
- package/docs/v7/examples/element-not-found.mdx +1 -1
- package/docs/v7/examples/findall-coffee-icons.mdx +3 -2
- package/docs/v7/examples/formatted-logging.mdx +74 -0
- package/docs/v7/examples/hover-image.mdx +1 -1
- package/docs/v7/examples/hover-text-with-description.mdx +2 -2
- package/docs/v7/examples/hover-text.mdx +1 -1
- package/docs/v7/examples/installer.mdx +1 -1
- package/docs/v7/examples/launch-vscode-linux.mdx +27 -27
- package/docs/v7/examples/match-image.mdx +1 -1
- package/docs/v7/examples/parse.mdx +3 -2
- package/docs/v7/examples/press-keys.mdx +2 -2
- package/docs/v7/examples/scroll-keyboard.mdx +2 -2
- package/docs/v7/examples/scroll-until-text.mdx +1 -1
- package/docs/v7/examples/scroll.mdx +1 -1
- package/docs/v7/examples/type.mdx +1 -1
- package/docs/v7/find.mdx +50 -11
- package/docs/v7/generating-tests.mdx +166 -10
- package/docs/v7/hosted.mdx +1 -1
- package/docs/v7/making-assertions.mdx +81 -4
- package/docs/v7/parse.mdx +1 -1
- package/docs/v7/performing-actions.mdx +100 -5
- package/docs/v7/provision.mdx +0 -20
- package/docs/v7/quickstart.mdx +299 -21
- package/docs/v7/reusable-code.mdx +3 -3
- package/docs/v7/screenshot.mdx +5 -5
- package/docs/v7/scroll.mdx +30 -160
- package/docs/v7/wait.mdx +1 -1
- package/examples/chrome-extension.test.mjs +29 -12
- package/examples/config.mjs +1 -1
- package/examples/launch-vscode-linux.test.mjs +25 -25
- package/interfaces/cli/commands/init.js +84 -2
- package/interfaces/vitest-plugin.mjs +51 -61
- package/lib/core/Dashcam.js +43 -2
- package/lib/core/index.d.ts +13 -1
- package/lib/github-comment.mjs +11 -5
- package/lib/init-project.js +98 -44
- package/lib/install-clients.js +470 -0
- package/lib/provision.js +749 -0
- package/lib/resolve-channel.js +2 -2
- package/lib/sentry.js +15 -2
- package/lib/vitest/hooks.mjs +44 -34
- package/mcp-server/README.md +32 -0
- package/mcp-server/dist/core/actions.d.ts +176 -0
- package/mcp-server/dist/core/actions.js +753 -0
- package/mcp-server/dist/env-utils.d.ts +45 -0
- package/mcp-server/dist/env-utils.js +63 -0
- package/mcp-server/dist/provision-types.d.ts +6 -2
- package/mcp-server/dist/provision-types.js +3 -1
- package/mcp-server/dist/server.mjs +449 -751
- package/mcp-server/package-lock.json +384 -1
- package/mcp-server/package.json +5 -2
- package/mcp-server/src/core/actions.ts +912 -0
- package/mcp-server/src/env-utils.test.ts +82 -0
- package/mcp-server/src/env-utils.ts +77 -0
- package/mcp-server/src/provision-types.ts +4 -1
- package/mcp-server/src/server.ts +533 -954
- package/mcp-server/tsconfig.json +1 -1
- package/mcp-server/vitest.config.ts +7 -0
- package/package.json +15 -3
- package/sdk.d.ts +11 -9
- package/sdk.js +116 -746
- package/setup/aws/spawn-runner.sh +24 -3
- package/docs/v7/_drafts/caching-ai.mdx +0 -215
- package/docs/v7/_drafts/caching.mdx +0 -366
- package/docs/v7/_drafts/prompt-cache.mdx +0 -200
- package/docs/v7/copilot/creating-tests.mdx +0 -156
- package/docs/v7/copilot/github.mdx +0 -143
- package/docs/v7/copilot/setup.mdx +0 -143
- package/docs/v7/examples/exec-output.mdx +0 -84
- package/docs/v7/examples/exec-pwsh.mdx +0 -82
- package/docs/v7/examples/focus-window.mdx +0 -61
- package/docs/v7/examples/scroll-until-image.mdx +0 -82
- package/docs/v7/examples/windows-installer.mdx +0 -95
- package/examples/formatted-logging.test.mjs +0 -27
- package/examples/match-image.test.mjs +0 -55
- package/manual/exec-stream-logs.test.mjs +0 -25
package/mcp-server/src/server.ts
CHANGED
|
@@ -12,8 +12,12 @@ process.env.TD_DEBUG = "true";
|
|
|
12
12
|
import { registerAppResource, registerAppTool, RESOURCE_MIME_TYPE } from "@modelcontextprotocol/ext-apps/server";
|
|
13
13
|
import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
14
14
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
15
|
+
import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
|
|
16
|
+
import { randomUUID } from "crypto";
|
|
17
|
+
import * as http from "http";
|
|
18
|
+
import type { RequestHandlerExtra } from "@modelcontextprotocol/sdk/shared/protocol.js";
|
|
15
19
|
import type { Variables } from "@modelcontextprotocol/sdk/shared/uriTemplate.js";
|
|
16
|
-
import type { CallToolResult, ReadResourceResult } from "@modelcontextprotocol/sdk/types.js";
|
|
20
|
+
import type { CallToolResult, ReadResourceResult, ServerNotification, ServerRequest } from "@modelcontextprotocol/sdk/types.js";
|
|
17
21
|
import * as Sentry from "@sentry/node";
|
|
18
22
|
import * as fs from "fs";
|
|
19
23
|
import * as os from "os";
|
|
@@ -21,8 +25,10 @@ import * as path from "path";
|
|
|
21
25
|
import { fileURLToPath, pathToFileURL } from "url";
|
|
22
26
|
import { z } from "zod";
|
|
23
27
|
|
|
24
|
-
import
|
|
25
|
-
import {
|
|
28
|
+
import * as core from "./core/actions.js";
|
|
29
|
+
import { NoActiveSessionError, type ActionResult } from "./core/actions.js";
|
|
30
|
+
import { resolveE2bTemplateId, resolveOs } from "./env-utils.js";
|
|
31
|
+
import { SessionStartInputSchema, type SessionStartInput } from "./provision-types.js";
|
|
26
32
|
import { sessionManager, type SessionState } from "./session.js";
|
|
27
33
|
|
|
28
34
|
// =============================================================================
|
|
@@ -55,8 +61,7 @@ function resolveChannel(ver: string): string {
|
|
|
55
61
|
}
|
|
56
62
|
|
|
57
63
|
function resolveSentryEnvironment(ver: string): string {
|
|
58
|
-
|
|
59
|
-
return CHANNEL_TO_ENV[resolveChannel(ver)] || "production";
|
|
64
|
+
return resolveChannel(ver);
|
|
60
65
|
}
|
|
61
66
|
|
|
62
67
|
const activeChannel = resolveChannel(version);
|
|
@@ -77,8 +82,8 @@ if (isSentryEnabled()) {
|
|
|
77
82
|
"https://452bd5a00dbd83a38ee8813e11c57694@o4510262629236736.ingest.us.sentry.io/4510480443637760",
|
|
78
83
|
environment: sentryEnvironment,
|
|
79
84
|
release: version,
|
|
80
|
-
sampleRate:
|
|
81
|
-
tracesSampleRate:
|
|
85
|
+
sampleRate: 0.01,
|
|
86
|
+
tracesSampleRate: 0.01,
|
|
82
87
|
sendDefaultPii: true,
|
|
83
88
|
integrations: [Sentry.httpIntegration(), Sentry.nodeContextIntegration()],
|
|
84
89
|
initialScope: {
|
|
@@ -210,12 +215,6 @@ const RESOURCE_URI = "ui://testdriver/mcp-app.html";
|
|
|
210
215
|
const SCREENSHOT_RESOURCE_BASE = "screenshot://testdriver/screenshot";
|
|
211
216
|
const CROPPED_IMAGE_RESOURCE_BASE = "screenshot://testdriver/cropped";
|
|
212
217
|
|
|
213
|
-
// SDK instance (will be initialized on session start)
|
|
214
|
-
let sdk: any = null;
|
|
215
|
-
|
|
216
|
-
// Last screenshot base64 for check comparisons
|
|
217
|
-
let lastScreenshotBase64: string | null = null;
|
|
218
|
-
|
|
219
218
|
// =============================================================================
|
|
220
219
|
// Image Store - Stores images with unique IDs for reload persistence
|
|
221
220
|
// =============================================================================
|
|
@@ -283,61 +282,189 @@ function getSessionData(session: SessionState | null) {
|
|
|
283
282
|
}
|
|
284
283
|
|
|
285
284
|
/**
|
|
286
|
-
*
|
|
287
|
-
*
|
|
288
|
-
*
|
|
289
|
-
* Auto-extends the session on each successful check to prevent expiry during active use
|
|
285
|
+
* Map a {@link NoActiveSessionError} thrown by the core into the exact same
|
|
286
|
+
* "no/expired session" tool result the server's old `requireActiveSession`
|
|
287
|
+
* produced. Kept byte-identical so external behavior is unchanged.
|
|
290
288
|
*/
|
|
291
|
-
function
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
action: "session_start",
|
|
304
|
-
message: "No sandbox session exists. Call session_start to create one."
|
|
305
|
-
}
|
|
306
|
-
)
|
|
307
|
-
};
|
|
289
|
+
function noSessionResult(err: NoActiveSessionError): CallToolResult {
|
|
290
|
+
if (err.code === "SESSION_EXPIRED") {
|
|
291
|
+
return createToolResult(
|
|
292
|
+
false,
|
|
293
|
+
"ERROR: Session has expired or timed out. The sandbox is no longer available. You must call session_start again to create a new sandbox session before continuing.",
|
|
294
|
+
{
|
|
295
|
+
error: "SESSION_EXPIRED",
|
|
296
|
+
action: "session_start",
|
|
297
|
+
message: "The previous sandbox session has expired. Call session_start to create a new one.",
|
|
298
|
+
expiredSessionId: err.expiredSessionId,
|
|
299
|
+
}
|
|
300
|
+
);
|
|
308
301
|
}
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
302
|
+
return createToolResult(
|
|
303
|
+
false,
|
|
304
|
+
"ERROR: No active session. You must call session_start first to create a sandbox before using any other tools.",
|
|
305
|
+
{
|
|
306
|
+
error: "NO_SESSION",
|
|
307
|
+
action: "session_start",
|
|
308
|
+
message: "No sandbox session exists. Call session_start to create one.",
|
|
309
|
+
}
|
|
310
|
+
);
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* Map a core {@link ActionResult} into an MCP CallToolResult, storing images as
|
|
315
|
+
* resource URIs (the core returns BARE base64, which is what `storeImage` wants).
|
|
316
|
+
*/
|
|
317
|
+
function resultToMcp(r: ActionResult): CallToolResult {
|
|
318
|
+
const data: Record<string, unknown> = { ...r.data };
|
|
319
|
+
for (const img of r.images ?? []) {
|
|
320
|
+
const uri = storeImage(img.base64, img.kind);
|
|
321
|
+
if (img.kind === "cropped") data.croppedImageResourceUri = uri;
|
|
322
|
+
else data.screenshotResourceUri = uri;
|
|
323
|
+
}
|
|
324
|
+
return createToolResult(r.ok, r.text, data, r.code);
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
// =============================================================================
|
|
328
|
+
// Progress reporting (MCP `notifications/progress`)
|
|
329
|
+
// =============================================================================
|
|
330
|
+
|
|
331
|
+
/** The `extra` argument every tool callback receives from the MCP SDK. */
|
|
332
|
+
type ToolExtra = RequestHandlerExtra<ServerRequest, ServerNotification>;
|
|
333
|
+
|
|
334
|
+
/**
|
|
335
|
+
* Reports progress for a long-running tool back to the client.
|
|
336
|
+
*
|
|
337
|
+
* Per the MCP spec, progress is only sent when the caller included a
|
|
338
|
+
* `progressToken` in the request's `_meta`. When no token is present this is a
|
|
339
|
+
* no-op, so callers can report unconditionally without branching.
|
|
340
|
+
*
|
|
341
|
+
* Each `report()` call increments an internal counter (indeterminate progress —
|
|
342
|
+
* we rarely know a real total ahead of time) and forwards a human-readable
|
|
343
|
+
* `message`. A `heartbeat()` helper keeps the client's idle timeout alive while
|
|
344
|
+
* a single long SDK call is in flight (e.g. provisioning a sandbox, polling a
|
|
345
|
+
* find), which is the situation that was tripping the `session_start` timeout.
|
|
346
|
+
*/
|
|
347
|
+
interface ProgressReporter {
|
|
348
|
+
/** Emit one progress step with an optional human-readable message. */
|
|
349
|
+
report(message?: string): void;
|
|
350
|
+
/**
|
|
351
|
+
* Emit a progress tick every `intervalMs` until the returned stop function is
|
|
352
|
+
* called. Use this around a single long `await` so the client keeps receiving
|
|
353
|
+
* activity. Always pair with the returned `stop()` in a `finally`.
|
|
354
|
+
*/
|
|
355
|
+
heartbeat(message: string, intervalMs?: number): () => void;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
const DEFAULT_HEARTBEAT_MS = 3000;
|
|
359
|
+
|
|
360
|
+
function makeProgressReporter(extra: ToolExtra): ProgressReporter {
|
|
361
|
+
const progressToken = extra?._meta?.progressToken;
|
|
362
|
+
|
|
363
|
+
// No token → client did not opt into progress. Return a no-op reporter.
|
|
364
|
+
if (progressToken === undefined || progressToken === null) {
|
|
314
365
|
return {
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
false,
|
|
318
|
-
"ERROR: Session has expired or timed out. The sandbox is no longer available. You must call session_start again to create a new sandbox session before continuing.",
|
|
319
|
-
{
|
|
320
|
-
error: "SESSION_EXPIRED",
|
|
321
|
-
action: "session_start",
|
|
322
|
-
message: "The previous sandbox session has expired. Call session_start to create a new one.",
|
|
323
|
-
expiredSessionId: session.sessionId
|
|
324
|
-
}
|
|
325
|
-
)
|
|
366
|
+
report: () => {},
|
|
367
|
+
heartbeat: () => () => {},
|
|
326
368
|
};
|
|
327
369
|
}
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
370
|
+
|
|
371
|
+
let progress = 0;
|
|
372
|
+
|
|
373
|
+
const send = (message?: string) => {
|
|
374
|
+
progress += 1;
|
|
375
|
+
// Fire-and-forget: a failed notification must never break the tool call.
|
|
376
|
+
void extra
|
|
377
|
+
.sendNotification({
|
|
378
|
+
method: "notifications/progress",
|
|
379
|
+
params: { progressToken, progress, message },
|
|
380
|
+
})
|
|
381
|
+
.catch((err) => {
|
|
382
|
+
logger.debug("progress: sendNotification failed", { error: String(err) });
|
|
383
|
+
});
|
|
384
|
+
};
|
|
385
|
+
|
|
386
|
+
return {
|
|
387
|
+
report: (message) => send(message),
|
|
388
|
+
heartbeat: (message, intervalMs = DEFAULT_HEARTBEAT_MS) => {
|
|
389
|
+
send(message);
|
|
390
|
+
const timer = setInterval(() => send(message), intervalMs);
|
|
391
|
+
// Don't let the heartbeat keep the event loop alive on its own.
|
|
392
|
+
timer.unref?.();
|
|
393
|
+
return () => clearInterval(timer);
|
|
394
|
+
},
|
|
395
|
+
};
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
// =============================================================================
|
|
399
|
+
// Cancellation (MCP `notifications/cancelled` → `extra.signal`)
|
|
400
|
+
// =============================================================================
|
|
401
|
+
|
|
402
|
+
/** Thrown when a tool call is aborted by the client. */
|
|
403
|
+
class ToolAbortError extends Error {
|
|
404
|
+
constructor(tool: string) {
|
|
405
|
+
super(`${tool} was cancelled by the client`);
|
|
406
|
+
this.name = "ToolAbortError";
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
/** Reject as soon as `signal` aborts. Used to race against long SDK calls. */
|
|
411
|
+
function rejectOnAbort(signal: AbortSignal | undefined, tool: string): { promise: Promise<never>; cleanup: () => void } {
|
|
412
|
+
if (!signal) {
|
|
413
|
+
// Never-resolving promise with a no-op cleanup — Promise.race ignores it.
|
|
414
|
+
return { promise: new Promise<never>(() => {}), cleanup: () => {} };
|
|
415
|
+
}
|
|
416
|
+
let onAbort: () => void = () => {};
|
|
417
|
+
const promise = new Promise<never>((_, reject) => {
|
|
418
|
+
onAbort = () => reject(new ToolAbortError(tool));
|
|
419
|
+
if (signal.aborted) {
|
|
420
|
+
onAbort();
|
|
421
|
+
} else {
|
|
422
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
423
|
+
}
|
|
424
|
+
});
|
|
425
|
+
return { promise, cleanup: () => signal.removeEventListener("abort", onAbort) };
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
/**
|
|
429
|
+
* Run a long-running SDK call, but settle as soon as the client aborts.
|
|
430
|
+
*
|
|
431
|
+
* The wrapped SDK methods are not themselves signal-aware, so on abort the
|
|
432
|
+
* underlying work keeps running to completion in the background — but the tool
|
|
433
|
+
* call returns promptly with a `ToolAbortError` instead of blocking the client.
|
|
434
|
+
* Callers that hold cleanable resources (e.g. `session_start`) should catch
|
|
435
|
+
* `ToolAbortError` and tear them down.
|
|
436
|
+
*/
|
|
437
|
+
async function raceAbort<T>(signal: AbortSignal | undefined, tool: string, work: Promise<T>): Promise<T> {
|
|
438
|
+
if (signal?.aborted) {
|
|
439
|
+
throw new ToolAbortError(tool);
|
|
440
|
+
}
|
|
441
|
+
const { promise, cleanup } = rejectOnAbort(signal, tool);
|
|
442
|
+
try {
|
|
443
|
+
return await Promise.race([work, promise]);
|
|
444
|
+
} finally {
|
|
445
|
+
cleanup();
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
/**
|
|
450
|
+
* If `error` is a client cancellation, return a "cancelled" tool result so the
|
|
451
|
+
* caller can `return` it; otherwise return null so normal error handling (log +
|
|
452
|
+
* Sentry + rethrow) proceeds. Keeps abort out of error reporting — a user
|
|
453
|
+
* cancelling is not a failure.
|
|
454
|
+
*/
|
|
455
|
+
function cancelledResultOrNull(error: unknown, tool: string): CallToolResult | null {
|
|
456
|
+
if (error instanceof ToolAbortError) {
|
|
457
|
+
logger.info(`${tool}: Cancelled by client`);
|
|
458
|
+
return createToolResult(false, `${tool} was cancelled.`, { action: tool, cancelled: true });
|
|
459
|
+
}
|
|
460
|
+
return null;
|
|
334
461
|
}
|
|
335
462
|
|
|
336
463
|
/**
|
|
337
464
|
* Create tool result with structured content for MCP App
|
|
338
465
|
* Images: imageUrl (data URL) goes to structuredContent for UI to display
|
|
339
466
|
* The croppedImage from find() is small (~10KB) so it's acceptable as data URL
|
|
340
|
-
*
|
|
467
|
+
*
|
|
341
468
|
* If generatedCode is provided, it's appended to the text response with instructions
|
|
342
469
|
* for the agent to write it to the test file.
|
|
343
470
|
*/
|
|
@@ -399,10 +526,6 @@ const server = isSentryEnabled()
|
|
|
399
526
|
version: version,
|
|
400
527
|
});
|
|
401
528
|
|
|
402
|
-
// Element reference storage (for click/hover after find)
|
|
403
|
-
// Stores actual Element instances - no raw coordinates as input
|
|
404
|
-
const elementRefs = new Map<string, { element: any; description: string; coords: { x: number; y: number; centerX: number; centerY: number } }>();
|
|
405
|
-
|
|
406
529
|
// =============================================================================
|
|
407
530
|
// Register UI Resource
|
|
408
531
|
// =============================================================================
|
|
@@ -524,226 +647,96 @@ Debug mode (connect to existing sandbox):
|
|
|
524
647
|
inputSchema: SessionStartInputSchema as any,
|
|
525
648
|
_meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
|
|
526
649
|
},
|
|
527
|
-
async (params: SessionStartInput): Promise<CallToolResult> => {
|
|
650
|
+
async (params: SessionStartInput, extra: ToolExtra): Promise<CallToolResult> => {
|
|
528
651
|
const startTime = Date.now();
|
|
529
|
-
|
|
530
|
-
|
|
652
|
+
const progress = makeProgressReporter(extra);
|
|
653
|
+
|
|
654
|
+
// Resolve OS with priority: explicit param > TD_OS env var > "linux" default
|
|
655
|
+
// This mirrors the behavior of the Vitest hooks (hooks.mjs) which also reads TD_OS
|
|
656
|
+
const { os: resolvedOs, warning: osWarning } = resolveOs(params.os);
|
|
657
|
+
if (osWarning) {
|
|
658
|
+
logger.warn(`session_start: ${osWarning}`);
|
|
659
|
+
} else if (!params.os && resolvedOs !== "linux") {
|
|
660
|
+
logger.info("session_start: Using TD_OS environment variable", { os: resolvedOs });
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
// Resolve E2B template ID with priority: explicit param > TD_E2B_TEMPLATE_ID env var
|
|
664
|
+
// This mirrors the behavior of the Vitest hooks (hooks.mjs) which also reads TD_E2B_TEMPLATE_ID
|
|
665
|
+
const resolvedE2bTemplateId = resolveE2bTemplateId(params.e2bTemplateId);
|
|
666
|
+
if (!params.e2bTemplateId && resolvedE2bTemplateId) {
|
|
667
|
+
logger.info("session_start: Using TD_E2B_TEMPLATE_ID environment variable", { e2bTemplateId: resolvedE2bTemplateId });
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
logger.info("session_start: Starting", {
|
|
671
|
+
type: params.type,
|
|
531
672
|
url: params.url,
|
|
532
|
-
os:
|
|
673
|
+
os: resolvedOs,
|
|
533
674
|
reconnect: params.reconnect,
|
|
534
675
|
sandboxId: params.sandboxId,
|
|
535
676
|
});
|
|
536
677
|
|
|
537
678
|
try {
|
|
538
|
-
//
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
679
|
+
// The core owns session creation, SDK init, connect, provisioning and the
|
|
680
|
+
// initial screenshot. We keep the abort race + heartbeat scaffolding here:
|
|
681
|
+
// a heartbeat ticks while the long provisioning await is in flight, and the
|
|
682
|
+
// whole core call is raced against the client's abort signal so cancellation
|
|
683
|
+
// still returns promptly. Progress messages from the core are forwarded.
|
|
684
|
+
const stopHeartbeat = progress.heartbeat(
|
|
685
|
+
params.sandboxId
|
|
686
|
+
? `Connecting to existing sandbox ${params.sandboxId}...`
|
|
687
|
+
: "Starting session..."
|
|
688
|
+
);
|
|
689
|
+
let result: ActionResult;
|
|
690
|
+
try {
|
|
691
|
+
result = await raceAbort(
|
|
692
|
+
extra.signal,
|
|
693
|
+
"session_start",
|
|
694
|
+
core.sessionStart(
|
|
695
|
+
params,
|
|
696
|
+
{ os: resolvedOs, e2bTemplateId: resolvedE2bTemplateId },
|
|
697
|
+
{ onProgress: (m) => progress.report(m) }
|
|
698
|
+
)
|
|
699
|
+
);
|
|
700
|
+
} finally {
|
|
701
|
+
stopHeartbeat();
|
|
546
702
|
}
|
|
547
703
|
|
|
548
|
-
|
|
549
|
-
const newSession = sessionManager.createSession({
|
|
550
|
-
os: params.os,
|
|
551
|
-
keepAlive: params.keepAlive,
|
|
552
|
-
testFile: params.testFile,
|
|
553
|
-
});
|
|
554
|
-
logger.debug("session_start: Session created", { sessionId: newSession.sessionId });
|
|
555
|
-
|
|
556
|
-
// Determine API root
|
|
557
|
-
const apiRoot = params.apiRoot || process.env.TD_API_ROOT || "https://api.testdriver.ai";
|
|
558
|
-
logger.debug("session_start: Using API root", { apiRoot });
|
|
704
|
+
const duration = Date.now() - startTime;
|
|
559
705
|
|
|
560
|
-
//
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
// Determine preview mode from environment variable
|
|
565
|
-
// TD_PREVIEW can be "ide", "browser", or "none"
|
|
566
|
-
// Default to "ide" so the live preview shows within the IDE
|
|
567
|
-
const previewMode = process.env.TD_PREVIEW || "ide";
|
|
568
|
-
logger.debug("session_start: Preview mode", { preview: previewMode });
|
|
569
|
-
|
|
570
|
-
// Get IP from params or environment (for self-hosted instances)
|
|
571
|
-
const instanceIp = params.ip || process.env.TD_IP;
|
|
572
|
-
|
|
573
|
-
// Get API key - check multiple sources for GitHub Copilot coding agent compatibility
|
|
574
|
-
// 1. TD_API_KEY (standard environment variable)
|
|
575
|
-
// 2. COPILOT_MCP_TD_API_KEY (fallback for GitHub Copilot coding agent)
|
|
576
|
-
const apiKey = process.env.TD_API_KEY || process.env.COPILOT_MCP_TD_API_KEY || "";
|
|
577
|
-
|
|
578
|
-
if (!apiKey) {
|
|
579
|
-
logger.error("session_start: No API key found", {
|
|
580
|
-
hasTD_API_KEY: !!process.env.TD_API_KEY,
|
|
581
|
-
hasCOPILOT_MCP_TD_API_KEY: !!process.env.COPILOT_MCP_TD_API_KEY,
|
|
582
|
-
availableEnvVars: Object.keys(process.env).filter(k => k.includes('TD') || k.includes('COPILOT_MCP'))
|
|
583
|
-
});
|
|
584
|
-
return createToolResult(false, "No API key found. Please set TD_API_KEY or COPILOT_MCP_TD_API_KEY environment variable.", {
|
|
585
|
-
error: "Missing API key",
|
|
586
|
-
hint: "For GitHub Copilot coding agent, create a Copilot environment secret named COPILOT_MCP_TD_API_KEY"
|
|
587
|
-
});
|
|
706
|
+
// Set Sentry context once the session id is known.
|
|
707
|
+
if (result.ok && typeof result.data.sessionId === "string") {
|
|
708
|
+
setSessionContext(result.data.sessionId, (result.data.sandboxId as string) || undefined);
|
|
588
709
|
}
|
|
589
|
-
|
|
590
|
-
logger.debug("session_start: API key found", {
|
|
591
|
-
source: process.env.TD_API_KEY ? "TD_API_KEY" : "COPILOT_MCP_TD_API_KEY",
|
|
592
|
-
keyPrefix: apiKey.substring(0, 7) + "..."
|
|
593
|
-
});
|
|
594
|
-
|
|
595
|
-
sdk = new TestDriverSDK(apiKey, {
|
|
596
|
-
os: params.os,
|
|
597
|
-
logging: false,
|
|
598
|
-
apiRoot,
|
|
599
|
-
preview: previewMode as "browser" | "ide" | "none",
|
|
600
|
-
ip: instanceIp,
|
|
601
|
-
});
|
|
602
710
|
|
|
603
|
-
|
|
604
|
-
if (params.sandboxId) {
|
|
605
|
-
logger.info("session_start: Connecting to existing sandbox (debug mode)", { sandboxId: params.sandboxId });
|
|
606
|
-
await sdk.connect({
|
|
607
|
-
sandboxId: params.sandboxId,
|
|
608
|
-
keepAlive: params.keepAlive,
|
|
609
|
-
});
|
|
610
|
-
|
|
611
|
-
// Get sandbox ID
|
|
612
|
-
const instance = sdk.getInstance();
|
|
613
|
-
logger.info("session_start: Connected to existing sandbox", { instanceId: instance?.instanceId });
|
|
614
|
-
sessionManager.activateSession(newSession.sessionId, instance?.instanceId || params.sandboxId);
|
|
615
|
-
|
|
616
|
-
// Set Sentry context for error tracking
|
|
617
|
-
setSessionContext(newSession.sessionId, instance?.instanceId);
|
|
618
|
-
|
|
619
|
-
// Capture screenshot of current state
|
|
620
|
-
logger.debug("session_start: Capturing screenshot of existing sandbox");
|
|
621
|
-
const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
622
|
-
|
|
623
|
-
let screenshotResourceUri: string | undefined;
|
|
624
|
-
if (screenshotBase64) {
|
|
625
|
-
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
626
|
-
lastScreenshotBase64 = screenshotBase64;
|
|
627
|
-
}
|
|
628
|
-
|
|
629
|
-
const duration = Date.now() - startTime;
|
|
630
|
-
logger.info("session_start: Connected to existing sandbox", { duration, sessionId: newSession.sessionId, sandboxId: params.sandboxId });
|
|
631
|
-
|
|
632
|
-
return createToolResult(
|
|
633
|
-
true,
|
|
634
|
-
`Connected to existing sandbox (debug mode)
|
|
635
|
-
Session: ${newSession.sessionId}
|
|
636
|
-
Sandbox: ${params.sandboxId}
|
|
637
|
-
Expires in: ${Math.round(params.keepAlive / 1000)}s
|
|
711
|
+
logger.info("session_start: Completed", { duration, ok: result.ok });
|
|
638
712
|
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
sessionId: newSession.sessionId,
|
|
643
|
-
sandboxId: params.sandboxId,
|
|
644
|
-
debugMode: true,
|
|
645
|
-
screenshotResourceUri,
|
|
646
|
-
duration
|
|
647
|
-
},
|
|
648
|
-
"// Connected to existing sandbox - no provision code needed"
|
|
649
|
-
);
|
|
713
|
+
if (!result.ok) {
|
|
714
|
+
// Validation / missing-key failures: pass through with duration added.
|
|
715
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
650
716
|
}
|
|
651
717
|
|
|
652
|
-
//
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
718
|
+
// Reproduce the server's exact success output (text + data) which differs
|
|
719
|
+
// from the core's neutral text. Images are mapped to resource URIs.
|
|
720
|
+
const data: Record<string, unknown> = { ...result.data, duration };
|
|
721
|
+
for (const img of result.images ?? []) {
|
|
722
|
+
const uri = storeImage(img.base64, img.kind);
|
|
723
|
+
if (img.kind === "cropped") data.croppedImageResourceUri = uri;
|
|
724
|
+
else data.screenshotResourceUri = uri;
|
|
657
725
|
}
|
|
658
|
-
await sdk.connect({
|
|
659
|
-
reconnect: params.reconnect,
|
|
660
|
-
keepAlive: params.keepAlive,
|
|
661
|
-
ip: instanceIp,
|
|
662
|
-
});
|
|
663
726
|
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
setSessionContext(newSession.sessionId, instance?.instanceId);
|
|
671
|
-
|
|
672
|
-
// Get provision-specific options
|
|
673
|
-
const provisionOptions = getProvisionOptions(params);
|
|
674
|
-
let provisionCmd = "";
|
|
675
|
-
|
|
676
|
-
// Provision based on type
|
|
677
|
-
switch (params.type) {
|
|
678
|
-
case "chrome": {
|
|
679
|
-
const chromeOpts = provisionOptions as { url: string; maximized?: boolean; guest?: boolean };
|
|
680
|
-
logger.info("session_start: Provisioning Chrome", { url: chromeOpts.url });
|
|
681
|
-
await sdk.provision.chrome(chromeOpts);
|
|
682
|
-
provisionCmd = "provision.chrome";
|
|
683
|
-
logger.debug("session_start: Chrome provisioned");
|
|
684
|
-
break;
|
|
685
|
-
}
|
|
686
|
-
|
|
687
|
-
case "chromeExtension": {
|
|
688
|
-
const extOpts = provisionOptions as { extensionPath?: string; extensionId?: string; maximized?: boolean };
|
|
689
|
-
logger.info("session_start: Provisioning Chrome Extension", { extensionPath: extOpts.extensionPath, extensionId: extOpts.extensionId });
|
|
690
|
-
await sdk.provision.chromeExtension(extOpts);
|
|
691
|
-
provisionCmd = "provision.chromeExtension";
|
|
692
|
-
logger.debug("session_start: Chrome Extension provisioned");
|
|
693
|
-
break;
|
|
694
|
-
}
|
|
695
|
-
|
|
696
|
-
case "vscode": {
|
|
697
|
-
const vscodeOpts = provisionOptions as { workspace?: string; extensions?: string[] };
|
|
698
|
-
logger.info("session_start: Provisioning VS Code", { workspace: vscodeOpts.workspace });
|
|
699
|
-
await sdk.provision.vscode(vscodeOpts);
|
|
700
|
-
provisionCmd = "provision.vscode";
|
|
701
|
-
logger.debug("session_start: VS Code provisioned");
|
|
702
|
-
break;
|
|
703
|
-
}
|
|
704
|
-
|
|
705
|
-
case "installer": {
|
|
706
|
-
const installerOpts = provisionOptions as { url: string; filename?: string; appName?: string; launch?: boolean };
|
|
707
|
-
logger.info("session_start: Provisioning installer", { url: installerOpts.url });
|
|
708
|
-
await sdk.provision.installer(installerOpts);
|
|
709
|
-
provisionCmd = "provision.installer";
|
|
710
|
-
logger.debug("session_start: Installer provisioned");
|
|
711
|
-
break;
|
|
712
|
-
}
|
|
713
|
-
|
|
714
|
-
case "electron": {
|
|
715
|
-
const electronOpts = provisionOptions as { appPath: string; args?: string[] };
|
|
716
|
-
logger.info("session_start: Provisioning Electron", { appPath: electronOpts.appPath });
|
|
717
|
-
await sdk.provision.electron(electronOpts);
|
|
718
|
-
provisionCmd = "provision.electron";
|
|
719
|
-
logger.debug("session_start: Electron app provisioned");
|
|
720
|
-
break;
|
|
721
|
-
}
|
|
722
|
-
}
|
|
727
|
+
if (result.data.debugMode) {
|
|
728
|
+
// Debug (existing-sandbox) success text — preserve original wording.
|
|
729
|
+
const text = `Connected to existing sandbox (debug mode)
|
|
730
|
+
Session: ${result.data.sessionId}
|
|
731
|
+
Sandbox: ${result.data.sandboxId}
|
|
732
|
+
Expires in: ${Math.round(params.keepAlive / 1000)}s
|
|
723
733
|
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
727
|
-
|
|
728
|
-
let screenshotResourceUri: string | undefined;
|
|
729
|
-
if (screenshotBase64) {
|
|
730
|
-
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
731
|
-
lastScreenshotBase64 = screenshotBase64;
|
|
734
|
+
You are now connected to the sandbox in its current state. Use find, click, type, etc. to interact.`;
|
|
735
|
+
return createToolResult(true, text, data, result.code);
|
|
732
736
|
}
|
|
733
737
|
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
// Generate the code for this provision action
|
|
738
|
-
const generatedCode = generateActionCode(provisionCmd, provisionOptions);
|
|
739
|
-
|
|
740
|
-
// Build debugger URL for the session
|
|
741
|
-
const debuggerUrl = instance?.debuggerUrl || (instanceIp ? `http://${instanceIp}:9222` : null);
|
|
742
|
-
|
|
743
|
-
const connectionType = instanceIp ? `Self-hosted (${instanceIp})` : "Cloud";
|
|
744
|
-
return createToolResult(
|
|
745
|
-
true,
|
|
746
|
-
`Session started: ${newSession.sessionId}\nConnection: ${connectionType}\nType: ${params.type}\nSandbox: ${instance?.instanceId}\nExpires in: ${Math.round(params.keepAlive / 1000)}s
|
|
738
|
+
// Normal provisioning success — append the EXACT dependency guidance block.
|
|
739
|
+
const text = `${result.text}
|
|
747
740
|
|
|
748
741
|
IMPORTANT - If creating a new test project, use these EXACT dependencies in package.json:
|
|
749
742
|
{
|
|
@@ -755,20 +748,21 @@ IMPORTANT - If creating a new test project, use these EXACT dependencies in pack
|
|
|
755
748
|
"scripts": {
|
|
756
749
|
"test": "vitest"
|
|
757
750
|
}
|
|
758
|
-
}
|
|
759
|
-
|
|
760
|
-
action: "session_start",
|
|
761
|
-
sessionId: newSession.sessionId,
|
|
762
|
-
provisionType: params.type,
|
|
763
|
-
selfHosted: !!instanceIp,
|
|
764
|
-
instanceIp: instanceIp || undefined,
|
|
765
|
-
debuggerUrl,
|
|
766
|
-
screenshotResourceUri,
|
|
767
|
-
duration
|
|
768
|
-
},
|
|
769
|
-
generatedCode
|
|
770
|
-
);
|
|
751
|
+
}`;
|
|
752
|
+
return createToolResult(true, text, data, result.code);
|
|
771
753
|
} catch (error) {
|
|
754
|
+
// On client cancellation, tear down the half-provisioned session so we
|
|
755
|
+
// don't leak a connected sandbox. The underlying SDK call may still be
|
|
756
|
+
// running in the background; best-effort cleanup is all we can do.
|
|
757
|
+
if (error instanceof ToolAbortError) {
|
|
758
|
+
logger.info("session_start: Cancelled by client, tearing down session");
|
|
759
|
+
try {
|
|
760
|
+
await core.disconnect();
|
|
761
|
+
} catch (cleanupErr) {
|
|
762
|
+
logger.warn("session_start: Cleanup after cancel failed", { error: String(cleanupErr) });
|
|
763
|
+
}
|
|
764
|
+
return createToolResult(false, "Session start was cancelled.", { action: "session_start", cancelled: true });
|
|
765
|
+
}
|
|
772
766
|
logger.error("session_start: Failed", { error: String(error) });
|
|
773
767
|
captureException(error as Error, { tags: { tool: "session_start" }, extra: { params } });
|
|
774
768
|
throw error;
|
|
@@ -786,27 +780,17 @@ server.registerTool(
|
|
|
786
780
|
async (): Promise<CallToolResult> => {
|
|
787
781
|
const startTime = Date.now();
|
|
788
782
|
logger.info("session_status: Checking");
|
|
789
|
-
const session = sessionManager.getCurrentSession();
|
|
790
|
-
|
|
791
|
-
if (!session) {
|
|
792
|
-
logger.warn("session_status: No active session");
|
|
793
|
-
return createToolResult(false, "No active session", { error: "No active session. Call session_start first." });
|
|
794
|
-
}
|
|
795
783
|
|
|
796
|
-
const
|
|
784
|
+
const result = core.sessionStatus();
|
|
797
785
|
const duration = Date.now() - startTime;
|
|
798
|
-
logger.info("session_status: Completed", {
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
duration
|
|
786
|
+
logger.info("session_status: Completed", {
|
|
787
|
+
ok: result.ok,
|
|
788
|
+
sessionId: result.data.sessionId,
|
|
789
|
+
status: result.data.status,
|
|
790
|
+
duration,
|
|
803
791
|
});
|
|
804
792
|
|
|
805
|
-
return
|
|
806
|
-
true,
|
|
807
|
-
`Session: ${session.sessionId}\nStatus: ${session.status}\nTime remaining: ${Math.round((summary?.timeRemaining || 0) / 1000)}s`,
|
|
808
|
-
{ action: "session_status", ...summary, sessionId: session.sessionId, status: session.status, duration }
|
|
809
|
-
);
|
|
793
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
810
794
|
}
|
|
811
795
|
);
|
|
812
796
|
|
|
@@ -821,22 +805,22 @@ server.registerTool(
|
|
|
821
805
|
},
|
|
822
806
|
async (params) => {
|
|
823
807
|
logger.info("session_extend: Extending", { additionalMs: params.additionalMs });
|
|
824
|
-
const session = sessionManager.getCurrentSession();
|
|
825
808
|
|
|
826
|
-
|
|
809
|
+
const result = core.sessionExtend(params.additionalMs);
|
|
810
|
+
|
|
811
|
+
if (!result.ok) {
|
|
827
812
|
logger.warn("session_extend: No active session");
|
|
828
813
|
return { content: [{ type: "text" as const, text: "No active session" }] };
|
|
829
814
|
}
|
|
830
815
|
|
|
831
|
-
|
|
832
|
-
const newExpiry = sessionManager.getTimeRemaining(session.sessionId);
|
|
833
|
-
logger.info("session_extend: Extended", { sessionId: session.sessionId, newExpiry });
|
|
816
|
+
logger.info("session_extend: Extended", { newExpiry: result.data.newExpiry });
|
|
834
817
|
|
|
818
|
+
// Preserve the original plain content shape (not via createToolResult).
|
|
835
819
|
return {
|
|
836
820
|
content: [
|
|
837
821
|
{
|
|
838
822
|
type: "text" as const,
|
|
839
|
-
text:
|
|
823
|
+
text: result.text,
|
|
840
824
|
},
|
|
841
825
|
],
|
|
842
826
|
};
|
|
@@ -856,107 +840,32 @@ registerAppTool(
|
|
|
856
840
|
}) as any,
|
|
857
841
|
_meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
|
|
858
842
|
},
|
|
859
|
-
async (params: { description: string; timeout?: number }): Promise<CallToolResult> => {
|
|
843
|
+
async (params: { description: string; timeout?: number }, extra: ToolExtra): Promise<CallToolResult> => {
|
|
860
844
|
const startTime = Date.now();
|
|
845
|
+
const progress = makeProgressReporter(extra);
|
|
861
846
|
logger.info("find: Starting", { description: params.description, timeout: params.timeout });
|
|
862
847
|
|
|
863
|
-
const sessionCheck = requireActiveSession();
|
|
864
|
-
if (!sessionCheck.valid) {
|
|
865
|
-
logger.warn("find: No active session");
|
|
866
|
-
return sessionCheck.error;
|
|
867
|
-
}
|
|
868
|
-
|
|
869
848
|
try {
|
|
870
849
|
logger.debug("find: Calling SDK find");
|
|
871
|
-
const
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
if (found && coords) {
|
|
878
|
-
elementRefs.set(elementRef, {
|
|
879
|
-
element: element, // Store the actual Element instance
|
|
880
|
-
description: params.description,
|
|
881
|
-
coords: {
|
|
882
|
-
x: coords.x,
|
|
883
|
-
y: coords.y,
|
|
884
|
-
centerX: coords.centerX,
|
|
885
|
-
centerY: coords.centerY,
|
|
886
|
-
},
|
|
887
|
-
});
|
|
888
|
-
logger.info("find: Element found", {
|
|
889
|
-
description: params.description,
|
|
890
|
-
coords: { x: coords.centerX, y: coords.centerY },
|
|
891
|
-
confidence: element.confidence,
|
|
892
|
-
elementRef
|
|
893
|
-
});
|
|
894
|
-
} else {
|
|
895
|
-
logger.warn("find: Element not found", { description: params.description });
|
|
850
|
+
const stopHeartbeat = progress.heartbeat(`Looking for "${params.description}"...`);
|
|
851
|
+
let result: ActionResult;
|
|
852
|
+
try {
|
|
853
|
+
result = await raceAbort(extra.signal, "find", core.find(params.description, params.timeout));
|
|
854
|
+
} finally {
|
|
855
|
+
stopHeartbeat();
|
|
896
856
|
}
|
|
897
857
|
|
|
898
|
-
// Return raw SDK response directly
|
|
899
|
-
const rawResponse = element._response || {};
|
|
900
858
|
const duration = Date.now() - startTime;
|
|
901
|
-
|
|
902
|
-
// Store cropped image for resource serving (instead of inline data URL)
|
|
903
|
-
let croppedImageResourceUri: string | undefined;
|
|
904
|
-
let screenshotResourceUri: string | undefined;
|
|
905
|
-
const croppedImage = rawResponse.croppedImage;
|
|
906
|
-
if (croppedImage) {
|
|
907
|
-
const imageData = croppedImage.startsWith('data:')
|
|
908
|
-
? croppedImage.replace(/^data:image\/\w+;base64,/, '')
|
|
909
|
-
: croppedImage;
|
|
910
|
-
croppedImageResourceUri = storeImage(imageData, "cropped");
|
|
911
|
-
// Remove croppedImage from response to avoid context bloat
|
|
912
|
-
delete rawResponse.croppedImage;
|
|
913
|
-
} else if (!found) {
|
|
914
|
-
// Element not found and no cropped image - capture a fresh screenshot
|
|
915
|
-
// so the user can see what's currently visible on screen
|
|
916
|
-
try {
|
|
917
|
-
const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
918
|
-
if (screenshotBase64) {
|
|
919
|
-
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
920
|
-
logger.debug("find: Captured screenshot for not-found state");
|
|
921
|
-
}
|
|
922
|
-
} catch (e) {
|
|
923
|
-
logger.warn("find: Failed to capture screenshot for not-found state", { error: String(e) });
|
|
924
|
-
}
|
|
925
|
-
}
|
|
926
|
-
|
|
927
|
-
// Remove extractedText and pixelDiffImage from response to reduce context bloat
|
|
928
|
-
delete rawResponse.extractedText;
|
|
929
|
-
delete rawResponse.pixelDiffImage;
|
|
930
|
-
|
|
931
|
-
// Generate code for this find action
|
|
932
|
-
const generatedCode = found ? generateActionCode("find", { description: params.description }) : undefined;
|
|
933
|
-
|
|
934
|
-
// Build element info for display (cropped image is always centered on element)
|
|
935
|
-
const elementInfo = found ? {
|
|
936
|
-
description: params.description,
|
|
937
|
-
centerX: coords?.centerX,
|
|
938
|
-
centerY: coords?.centerY,
|
|
939
|
-
confidence: element.confidence,
|
|
940
|
-
ref: elementRef,
|
|
941
|
-
} : undefined;
|
|
859
|
+
logger.info("find: Completed", { description: params.description, found: result.ok, duration });
|
|
942
860
|
|
|
943
|
-
return
|
|
944
|
-
found,
|
|
945
|
-
found
|
|
946
|
-
? `Found: "${params.description}" at (${rawResponse.coordinates?.x}, ${rawResponse.coordinates?.y})\nRef: ${elementRef}`
|
|
947
|
-
: `Element not found: "${params.description}"`,
|
|
948
|
-
{
|
|
949
|
-
...rawResponse,
|
|
950
|
-
action: "find",
|
|
951
|
-
element: elementInfo,
|
|
952
|
-
ref: elementRef,
|
|
953
|
-
croppedImageResourceUri,
|
|
954
|
-
screenshotResourceUri,
|
|
955
|
-
duration,
|
|
956
|
-
},
|
|
957
|
-
generatedCode
|
|
958
|
-
);
|
|
861
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
959
862
|
} catch (error) {
|
|
863
|
+
if (error instanceof NoActiveSessionError) {
|
|
864
|
+
logger.warn("find: No active session");
|
|
865
|
+
return noSessionResult(error);
|
|
866
|
+
}
|
|
867
|
+
const cancelled = cancelledResultOrNull(error, "find");
|
|
868
|
+
if (cancelled) return cancelled;
|
|
960
869
|
logger.error("find: Failed", { error: String(error), description: params.description });
|
|
961
870
|
captureException(error as Error, { tags: { tool: "find" }, extra: { description: params.description } });
|
|
962
871
|
throw error;
|
|
@@ -977,115 +886,32 @@ registerAppTool(
|
|
|
977
886
|
}) as any,
|
|
978
887
|
_meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
|
|
979
888
|
},
|
|
980
|
-
async (params: { description: string; timeout?: number }): Promise<CallToolResult> => {
|
|
889
|
+
async (params: { description: string; timeout?: number }, extra: ToolExtra): Promise<CallToolResult> => {
|
|
981
890
|
const startTime = Date.now();
|
|
891
|
+
const progress = makeProgressReporter(extra);
|
|
982
892
|
logger.info("findall: Starting", { description: params.description, timeout: params.timeout });
|
|
983
893
|
|
|
984
|
-
const sessionCheck = requireActiveSession();
|
|
985
|
-
if (!sessionCheck.valid) {
|
|
986
|
-
logger.warn("findall: No active session");
|
|
987
|
-
return sessionCheck.error;
|
|
988
|
-
}
|
|
989
|
-
|
|
990
894
|
try {
|
|
991
895
|
logger.debug("findall: Calling SDK findAll");
|
|
992
|
-
const
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
for (let i = 0; i < elements.length; i++) {
|
|
1000
|
-
const element = elements[i];
|
|
1001
|
-
const coords = element.getCoordinates();
|
|
1002
|
-
const elementRef = `el-${Date.now()}-${i}`;
|
|
1003
|
-
|
|
1004
|
-
if (coords) {
|
|
1005
|
-
elementRefs.set(elementRef, {
|
|
1006
|
-
element: element,
|
|
1007
|
-
description: `${params.description} [${i}]`,
|
|
1008
|
-
coords: {
|
|
1009
|
-
x: coords.x,
|
|
1010
|
-
y: coords.y,
|
|
1011
|
-
centerX: coords.centerX,
|
|
1012
|
-
centerY: coords.centerY,
|
|
1013
|
-
},
|
|
1014
|
-
});
|
|
1015
|
-
refs.push(elementRef);
|
|
1016
|
-
elementInfos.push({
|
|
1017
|
-
ref: elementRef,
|
|
1018
|
-
x: coords.x,
|
|
1019
|
-
y: coords.y,
|
|
1020
|
-
centerX: coords.centerX,
|
|
1021
|
-
centerY: coords.centerY,
|
|
1022
|
-
confidence: element.confidence,
|
|
1023
|
-
});
|
|
1024
|
-
}
|
|
896
|
+
const stopHeartbeat = progress.heartbeat(`Looking for all "${params.description}"...`);
|
|
897
|
+
let result: ActionResult;
|
|
898
|
+
try {
|
|
899
|
+
result = await raceAbort(extra.signal, "findall", core.findAll(params.description, params.timeout));
|
|
900
|
+
} finally {
|
|
901
|
+
stopHeartbeat();
|
|
1025
902
|
}
|
|
1026
903
|
|
|
1027
|
-
logger.info("findall: Elements found", {
|
|
1028
|
-
description: params.description,
|
|
1029
|
-
count,
|
|
1030
|
-
refs
|
|
1031
|
-
});
|
|
1032
|
-
|
|
1033
|
-
// Get the first element's response for the image (shows all highlights)
|
|
1034
|
-
const rawResponse = elements[0]?._response || {};
|
|
1035
904
|
const duration = Date.now() - startTime;
|
|
1036
|
-
|
|
1037
|
-
// Store cropped image for resource serving (instead of inline data URL)
|
|
1038
|
-
let croppedImageResourceUri: string | undefined;
|
|
1039
|
-
let screenshotResourceUri: string | undefined;
|
|
1040
|
-
const croppedImage = rawResponse.croppedImage;
|
|
1041
|
-
if (croppedImage) {
|
|
1042
|
-
const imageData = croppedImage.startsWith('data:')
|
|
1043
|
-
? croppedImage.replace(/^data:image\/\w+;base64,/, '')
|
|
1044
|
-
: croppedImage;
|
|
1045
|
-
croppedImageResourceUri = storeImage(imageData, "cropped");
|
|
1046
|
-
// Remove croppedImage from response to avoid context bloat
|
|
1047
|
-
delete rawResponse.croppedImage;
|
|
1048
|
-
} else if (count === 0) {
|
|
1049
|
-
// No elements found and no cropped image - capture a fresh screenshot
|
|
1050
|
-
// so the user can see what's currently visible on screen
|
|
1051
|
-
try {
|
|
1052
|
-
const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
1053
|
-
if (screenshotBase64) {
|
|
1054
|
-
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
1055
|
-
logger.debug("findall: Captured screenshot for not-found state");
|
|
1056
|
-
}
|
|
1057
|
-
} catch (e) {
|
|
1058
|
-
logger.warn("findall: Failed to capture screenshot for not-found state", { error: String(e) });
|
|
1059
|
-
}
|
|
1060
|
-
}
|
|
1061
|
-
|
|
1062
|
-
// Remove extractedText and pixelDiffImage from response to reduce context bloat
|
|
1063
|
-
delete rawResponse.extractedText;
|
|
1064
|
-
delete rawResponse.pixelDiffImage;
|
|
1065
|
-
|
|
1066
|
-
// Generate code for this findall action
|
|
1067
|
-
const generatedCode = count > 0 ? generateActionCode("findall", { description: params.description }) : undefined;
|
|
1068
|
-
|
|
1069
|
-
// Build refs list for text output
|
|
1070
|
-
const refsList = refs.map((ref, i) => ` [${i}] ${ref}`).join('\n');
|
|
905
|
+
logger.info("findall: Completed", { description: params.description, count: result.data.count, duration });
|
|
1071
906
|
|
|
1072
|
-
return
|
|
1073
|
-
count > 0,
|
|
1074
|
-
count > 0
|
|
1075
|
-
? `Found ${count} elements matching "${params.description}":\n${refsList}`
|
|
1076
|
-
: `No elements found matching: "${params.description}"`,
|
|
1077
|
-
{
|
|
1078
|
-
...rawResponse,
|
|
1079
|
-
count,
|
|
1080
|
-
refs,
|
|
1081
|
-
elements: elementInfos,
|
|
1082
|
-
croppedImageResourceUri,
|
|
1083
|
-
screenshotResourceUri,
|
|
1084
|
-
duration,
|
|
1085
|
-
},
|
|
1086
|
-
generatedCode
|
|
1087
|
-
);
|
|
907
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1088
908
|
} catch (error) {
|
|
909
|
+
if (error instanceof NoActiveSessionError) {
|
|
910
|
+
logger.warn("findall: No active session");
|
|
911
|
+
return noSessionResult(error);
|
|
912
|
+
}
|
|
913
|
+
const cancelled = cancelledResultOrNull(error, "findall");
|
|
914
|
+
if (cancelled) return cancelled;
|
|
1089
915
|
logger.error("findall: Failed", { error: String(error), description: params.description });
|
|
1090
916
|
captureException(error as Error, { tags: { tool: "findall" }, extra: { description: params.description } });
|
|
1091
917
|
throw error;
|
|
@@ -1110,70 +936,20 @@ registerAppTool(
|
|
|
1110
936
|
const startTime = Date.now();
|
|
1111
937
|
logger.info("click: Starting", { elementRef: params.elementRef, action: params.action });
|
|
1112
938
|
|
|
1113
|
-
const sessionCheck = requireActiveSession();
|
|
1114
|
-
if (!sessionCheck.valid) {
|
|
1115
|
-
logger.warn("click: No active session");
|
|
1116
|
-
return sessionCheck.error;
|
|
1117
|
-
}
|
|
1118
|
-
|
|
1119
|
-
// Look up the element reference
|
|
1120
|
-
const ref = elementRefs.get(params.elementRef);
|
|
1121
|
-
if (!ref) {
|
|
1122
|
-
logger.warn("click: Element reference not found", { elementRef: params.elementRef });
|
|
1123
|
-
return createToolResult(false, `Element reference "${params.elementRef}" not found. Use 'find' first to locate the element.`, { error: "Element reference not found" });
|
|
1124
|
-
}
|
|
1125
|
-
|
|
1126
|
-
const { element, description, coords } = ref;
|
|
1127
|
-
|
|
1128
939
|
try {
|
|
1129
|
-
logger.debug("click: Executing click on element", {
|
|
1130
|
-
|
|
1131
|
-
// Use the Element's click method instead of raw coordinates
|
|
1132
|
-
if (params.action === "click") {
|
|
1133
|
-
await element.click();
|
|
1134
|
-
} else if (params.action === "double-click") {
|
|
1135
|
-
await element.doubleClick();
|
|
1136
|
-
} else if (params.action === "right-click") {
|
|
1137
|
-
await element.rightClick();
|
|
1138
|
-
}
|
|
1139
|
-
|
|
1140
|
-
// Capture screenshot after click to show result
|
|
1141
|
-
logger.debug("click: Capturing screenshot after click");
|
|
1142
|
-
const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
1143
|
-
|
|
1144
|
-
let screenshotResourceUri: string | undefined;
|
|
1145
|
-
if (screenshotBase64) {
|
|
1146
|
-
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
1147
|
-
lastScreenshotBase64 = screenshotBase64;
|
|
1148
|
-
}
|
|
940
|
+
logger.debug("click: Executing click on element", { elementRef: params.elementRef, action: params.action });
|
|
941
|
+
const result = await core.click(params.elementRef, params.action);
|
|
1149
942
|
|
|
1150
|
-
const rawResponse = element._response || {};
|
|
1151
|
-
// Remove large data from response to reduce context bloat
|
|
1152
|
-
delete rawResponse.croppedImage;
|
|
1153
|
-
delete rawResponse.extractedText;
|
|
1154
|
-
delete rawResponse.pixelDiffImage;
|
|
1155
|
-
|
|
1156
943
|
const duration = Date.now() - startTime;
|
|
1157
|
-
logger.info("click: Completed", {
|
|
944
|
+
logger.info("click: Completed", { elementRef: params.elementRef, duration });
|
|
1158
945
|
|
|
1159
|
-
|
|
1160
|
-
const generatedCode = generateActionCode("click", { action: params.action });
|
|
1161
|
-
|
|
1162
|
-
return createToolResult(
|
|
1163
|
-
true,
|
|
1164
|
-
`Clicked on "${description}"`,
|
|
1165
|
-
{
|
|
1166
|
-
...rawResponse,
|
|
1167
|
-
action: "click",
|
|
1168
|
-
clickAction: params.action,
|
|
1169
|
-
clickPosition: coords,
|
|
1170
|
-
screenshotResourceUri,
|
|
1171
|
-
duration
|
|
1172
|
-
},
|
|
1173
|
-
generatedCode
|
|
1174
|
-
);
|
|
946
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1175
947
|
} catch (error) {
|
|
1176
|
-
|
|
948
|
+
if (error instanceof NoActiveSessionError) {
|
|
949
|
+
logger.warn("click: No active session");
|
|
950
|
+
return noSessionResult(error);
|
|
951
|
+
}
|
|
952
|
+
logger.error("click: Failed", { error: String(error), elementRef: params.elementRef });
|
|
1177
953
|
captureException(error as Error, { tags: { tool: "click" }, extra: { elementRef: params.elementRef, action: params.action } });
|
|
1178
954
|
throw error;
|
|
1179
955
|
}
|
|
@@ -1196,60 +972,20 @@ registerAppTool(
|
|
|
1196
972
|
const startTime = Date.now();
|
|
1197
973
|
logger.info("hover: Starting", { elementRef: params.elementRef });
|
|
1198
974
|
|
|
1199
|
-
const sessionCheck = requireActiveSession();
|
|
1200
|
-
if (!sessionCheck.valid) {
|
|
1201
|
-
logger.warn("hover: No active session");
|
|
1202
|
-
return sessionCheck.error;
|
|
1203
|
-
}
|
|
1204
|
-
|
|
1205
|
-
// Look up the element reference
|
|
1206
|
-
const ref = elementRefs.get(params.elementRef);
|
|
1207
|
-
if (!ref) {
|
|
1208
|
-
logger.warn("hover: Element reference not found", { elementRef: params.elementRef });
|
|
1209
|
-
return createToolResult(false, `Element reference "${params.elementRef}" not found. Use 'find' first to locate the element.`, { error: "Element reference not found" });
|
|
1210
|
-
}
|
|
1211
|
-
|
|
1212
|
-
const { element, description, coords } = ref;
|
|
1213
|
-
|
|
1214
975
|
try {
|
|
1215
|
-
logger.debug("hover: Executing hover on element", {
|
|
1216
|
-
await
|
|
976
|
+
logger.debug("hover: Executing hover on element", { elementRef: params.elementRef });
|
|
977
|
+
const result = await core.hover(params.elementRef);
|
|
1217
978
|
|
|
1218
|
-
// Capture screenshot after hover to show result
|
|
1219
|
-
logger.debug("hover: Capturing screenshot after hover");
|
|
1220
|
-
const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
1221
|
-
|
|
1222
|
-
let screenshotResourceUri: string | undefined;
|
|
1223
|
-
if (screenshotBase64) {
|
|
1224
|
-
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
1225
|
-
lastScreenshotBase64 = screenshotBase64;
|
|
1226
|
-
}
|
|
1227
|
-
|
|
1228
|
-
const rawResponse = element._response || {};
|
|
1229
|
-
// Remove large data from response to reduce context bloat
|
|
1230
|
-
delete rawResponse.croppedImage;
|
|
1231
|
-
delete rawResponse.extractedText;
|
|
1232
|
-
delete rawResponse.pixelDiffImage;
|
|
1233
|
-
|
|
1234
979
|
const duration = Date.now() - startTime;
|
|
1235
|
-
logger.info("hover: Completed", {
|
|
980
|
+
logger.info("hover: Completed", { elementRef: params.elementRef, duration });
|
|
1236
981
|
|
|
1237
|
-
|
|
1238
|
-
const generatedCode = generateActionCode("hover", {});
|
|
1239
|
-
|
|
1240
|
-
return createToolResult(
|
|
1241
|
-
true,
|
|
1242
|
-
`Hovered over "${description}"`,
|
|
1243
|
-
{
|
|
1244
|
-
...rawResponse,
|
|
1245
|
-
action: "hover",
|
|
1246
|
-
screenshotResourceUri,
|
|
1247
|
-
duration
|
|
1248
|
-
},
|
|
1249
|
-
generatedCode
|
|
1250
|
-
);
|
|
982
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1251
983
|
} catch (error) {
|
|
1252
|
-
|
|
984
|
+
if (error instanceof NoActiveSessionError) {
|
|
985
|
+
logger.warn("hover: No active session");
|
|
986
|
+
return noSessionResult(error);
|
|
987
|
+
}
|
|
988
|
+
logger.error("hover: Failed", { error: String(error), elementRef: params.elementRef });
|
|
1253
989
|
captureException(error as Error, { tags: { tool: "hover" }, extra: { elementRef: params.elementRef } });
|
|
1254
990
|
throw error;
|
|
1255
991
|
}
|
|
@@ -1269,29 +1005,19 @@ server.registerTool(
|
|
|
1269
1005
|
const startTime = Date.now();
|
|
1270
1006
|
logger.info("wait: Starting", { timeout: params.timeout });
|
|
1271
1007
|
|
|
1272
|
-
const sessionCheck = requireActiveSession();
|
|
1273
|
-
if (!sessionCheck.valid) {
|
|
1274
|
-
logger.warn("wait: No active session");
|
|
1275
|
-
return sessionCheck.error;
|
|
1276
|
-
}
|
|
1277
|
-
|
|
1278
1008
|
try {
|
|
1279
1009
|
logger.debug("wait: Waiting", { timeout: params.timeout });
|
|
1280
|
-
await
|
|
1010
|
+
const result = await core.wait(params.timeout);
|
|
1281
1011
|
|
|
1282
1012
|
const duration = Date.now() - startTime;
|
|
1283
1013
|
logger.info("wait: Completed", { timeout: params.timeout, duration });
|
|
1284
1014
|
|
|
1285
|
-
|
|
1286
|
-
const generatedCode = generateActionCode("wait", { timeout: params.timeout });
|
|
1287
|
-
|
|
1288
|
-
return createToolResult(
|
|
1289
|
-
true,
|
|
1290
|
-
`Waited for ${params.timeout}ms`,
|
|
1291
|
-
{ action: "wait", timeout: params.timeout, duration },
|
|
1292
|
-
generatedCode
|
|
1293
|
-
);
|
|
1015
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1294
1016
|
} catch (error) {
|
|
1017
|
+
if (error instanceof NoActiveSessionError) {
|
|
1018
|
+
logger.warn("wait: No active session");
|
|
1019
|
+
return noSessionResult(error);
|
|
1020
|
+
}
|
|
1295
1021
|
logger.error("wait: Failed", { error: String(error) });
|
|
1296
1022
|
captureException(error as Error, { tags: { tool: "wait" }, extra: { timeout: params.timeout } });
|
|
1297
1023
|
throw error;
|
|
@@ -1312,29 +1038,19 @@ server.registerTool(
|
|
|
1312
1038
|
const startTime = Date.now();
|
|
1313
1039
|
logger.info("focus_application: Starting", { name: params.name });
|
|
1314
1040
|
|
|
1315
|
-
const sessionCheck = requireActiveSession();
|
|
1316
|
-
if (!sessionCheck.valid) {
|
|
1317
|
-
logger.warn("focus_application: No active session");
|
|
1318
|
-
return sessionCheck.error;
|
|
1319
|
-
}
|
|
1320
|
-
|
|
1321
1041
|
try {
|
|
1322
1042
|
logger.debug("focus_application: Focusing", { name: params.name });
|
|
1323
|
-
await
|
|
1043
|
+
const result = await core.focusApplication(params.name);
|
|
1324
1044
|
|
|
1325
1045
|
const duration = Date.now() - startTime;
|
|
1326
1046
|
logger.info("focus_application: Completed", { name: params.name, duration });
|
|
1327
1047
|
|
|
1328
|
-
|
|
1329
|
-
const generatedCode = generateActionCode("focus_application", { name: params.name });
|
|
1330
|
-
|
|
1331
|
-
return createToolResult(
|
|
1332
|
-
true,
|
|
1333
|
-
`Focused application: "${params.name}"`,
|
|
1334
|
-
{ action: "focus", name: params.name, duration },
|
|
1335
|
-
generatedCode
|
|
1336
|
-
);
|
|
1048
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1337
1049
|
} catch (error) {
|
|
1050
|
+
if (error instanceof NoActiveSessionError) {
|
|
1051
|
+
logger.warn("focus_application: No active session");
|
|
1052
|
+
return noSessionResult(error);
|
|
1053
|
+
}
|
|
1338
1054
|
logger.error("focus_application: Failed", { error: String(error), name: params.name });
|
|
1339
1055
|
captureException(error as Error, { tags: { tool: "focus_application" }, extra: { name: params.name } });
|
|
1340
1056
|
throw error;
|
|
@@ -1355,144 +1071,32 @@ registerAppTool(
|
|
|
1355
1071
|
}) as any,
|
|
1356
1072
|
_meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
|
|
1357
1073
|
},
|
|
1358
|
-
async (params: { description: string; action: "click" | "double-click" | "right-click" }): Promise<CallToolResult> => {
|
|
1074
|
+
async (params: { description: string; action: "click" | "double-click" | "right-click" }, extra: ToolExtra): Promise<CallToolResult> => {
|
|
1359
1075
|
const startTime = Date.now();
|
|
1076
|
+
const progress = makeProgressReporter(extra);
|
|
1360
1077
|
logger.info("find_and_click: Starting", { description: params.description, action: params.action });
|
|
1361
1078
|
|
|
1362
|
-
const sessionCheck = requireActiveSession();
|
|
1363
|
-
if (!sessionCheck.valid) {
|
|
1364
|
-
logger.warn("find_and_click: No active session");
|
|
1365
|
-
return sessionCheck.error;
|
|
1366
|
-
}
|
|
1367
|
-
|
|
1368
1079
|
try {
|
|
1369
1080
|
logger.debug("find_and_click: Finding element");
|
|
1370
|
-
const
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
// Capture screenshot to show current state even when element not found
|
|
1377
|
-
const rawResponse = element._response || {};
|
|
1378
|
-
const duration = Date.now() - startTime;
|
|
1379
|
-
|
|
1380
|
-
// Store cropped image (screenshot) for resource serving
|
|
1381
|
-
let croppedImageResourceUri: string | undefined;
|
|
1382
|
-
let screenshotResourceUri: string | undefined;
|
|
1383
|
-
const croppedImage = rawResponse.croppedImage;
|
|
1384
|
-
if (croppedImage) {
|
|
1385
|
-
const imageData = croppedImage.startsWith('data:')
|
|
1386
|
-
? croppedImage.replace(/^data:image\/\w+;base64,/, '')
|
|
1387
|
-
: croppedImage;
|
|
1388
|
-
croppedImageResourceUri = storeImage(imageData, "screenshot");
|
|
1389
|
-
delete rawResponse.croppedImage;
|
|
1390
|
-
} else {
|
|
1391
|
-
// No cropped image - capture a fresh screenshot so the user can see
|
|
1392
|
-
// what's currently visible on screen when element was not found
|
|
1393
|
-
try {
|
|
1394
|
-
const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
1395
|
-
if (screenshotBase64) {
|
|
1396
|
-
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
1397
|
-
logger.debug("find_and_click: Captured screenshot for not-found state");
|
|
1398
|
-
}
|
|
1399
|
-
} catch (e) {
|
|
1400
|
-
logger.warn("find_and_click: Failed to capture screenshot for not-found state", { error: String(e) });
|
|
1401
|
-
}
|
|
1402
|
-
}
|
|
1403
|
-
|
|
1404
|
-
// Remove extractedText and pixelDiffImage from response to reduce context bloat
|
|
1405
|
-
delete rawResponse.extractedText;
|
|
1406
|
-
delete rawResponse.pixelDiffImage;
|
|
1407
|
-
|
|
1408
|
-
return createToolResult(
|
|
1409
|
-
false,
|
|
1410
|
-
`Element not found: "${params.description}"`,
|
|
1411
|
-
{
|
|
1412
|
-
...rawResponse,
|
|
1413
|
-
action: "find_and_click",
|
|
1414
|
-
error: "Element not found",
|
|
1415
|
-
croppedImageResourceUri,
|
|
1416
|
-
screenshotResourceUri,
|
|
1417
|
-
duration
|
|
1418
|
-
}
|
|
1419
|
-
);
|
|
1420
|
-
}
|
|
1421
|
-
|
|
1422
|
-
const coords = element.getCoordinates();
|
|
1423
|
-
|
|
1424
|
-
// Store element ref for later use (in case user wants to interact again)
|
|
1425
|
-
const elementRef = `el-${Date.now()}`;
|
|
1426
|
-
if (coords) {
|
|
1427
|
-
elementRefs.set(elementRef, {
|
|
1428
|
-
element: element,
|
|
1429
|
-
description: params.description,
|
|
1430
|
-
coords: {
|
|
1431
|
-
x: coords.x,
|
|
1432
|
-
y: coords.y,
|
|
1433
|
-
centerX: coords.centerX,
|
|
1434
|
-
centerY: coords.centerY,
|
|
1435
|
-
},
|
|
1436
|
-
});
|
|
1081
|
+
const stopHeartbeat = progress.heartbeat(`Looking for "${params.description}"...`);
|
|
1082
|
+
let result: ActionResult;
|
|
1083
|
+
try {
|
|
1084
|
+
result = await raceAbort(extra.signal, "find_and_click", core.findAndClick(params.description, params.action));
|
|
1085
|
+
} finally {
|
|
1086
|
+
stopHeartbeat();
|
|
1437
1087
|
}
|
|
1438
1088
|
|
|
1439
|
-
logger.debug("find_and_click: Element found, clicking", { action: params.action, elementRef });
|
|
1440
|
-
if (params.action === "click") {
|
|
1441
|
-
await element.click();
|
|
1442
|
-
} else if (params.action === "double-click") {
|
|
1443
|
-
await element.doubleClick();
|
|
1444
|
-
} else if (params.action === "right-click") {
|
|
1445
|
-
await element.rightClick();
|
|
1446
|
-
}
|
|
1447
|
-
|
|
1448
|
-
// Return raw SDK response directly
|
|
1449
|
-
const rawResponse = element._response || {};
|
|
1450
1089
|
const duration = Date.now() - startTime;
|
|
1451
|
-
|
|
1452
|
-
// Store cropped image for resource serving (instead of inline data URL)
|
|
1453
|
-
let croppedImageResourceUri: string | undefined;
|
|
1454
|
-
const croppedImage = rawResponse.croppedImage;
|
|
1455
|
-
if (croppedImage) {
|
|
1456
|
-
const imageData = croppedImage.startsWith('data:')
|
|
1457
|
-
? croppedImage.replace(/^data:image\/\w+;base64,/, '')
|
|
1458
|
-
: croppedImage;
|
|
1459
|
-
croppedImageResourceUri = storeImage(imageData, "cropped");
|
|
1460
|
-
// Remove croppedImage from response to avoid context bloat
|
|
1461
|
-
delete rawResponse.croppedImage;
|
|
1462
|
-
}
|
|
1463
|
-
|
|
1464
|
-
// Remove extractedText and pixelDiffImage from response to reduce context bloat
|
|
1465
|
-
delete rawResponse.extractedText;
|
|
1466
|
-
delete rawResponse.pixelDiffImage;
|
|
1467
|
-
|
|
1468
|
-
// Generate code for this find_and_click action
|
|
1469
|
-
const generatedCode = generateActionCode("find_and_click", { description: params.description, action: params.action });
|
|
1470
|
-
|
|
1471
|
-
// Build element info for display (match find action format)
|
|
1472
|
-
const elementInfo = coords ? {
|
|
1473
|
-
description: params.description,
|
|
1474
|
-
centerX: coords.centerX,
|
|
1475
|
-
centerY: coords.centerY,
|
|
1476
|
-
confidence: element.confidence,
|
|
1477
|
-
ref: elementRef,
|
|
1478
|
-
} : undefined;
|
|
1090
|
+
logger.info("find_and_click: Completed", { description: params.description, found: result.ok, duration });
|
|
1479
1091
|
|
|
1480
|
-
return
|
|
1481
|
-
true,
|
|
1482
|
-
`Found and clicked: "${params.description}" at (${rawResponse.coordinates?.x}, ${rawResponse.coordinates?.y})\nRef: ${elementRef}`,
|
|
1483
|
-
{
|
|
1484
|
-
...rawResponse,
|
|
1485
|
-
action: "find_and_click",
|
|
1486
|
-
element: elementInfo,
|
|
1487
|
-
ref: elementRef,
|
|
1488
|
-
clickAction: params.action,
|
|
1489
|
-
clickPosition: coords ? { x: coords.centerX, y: coords.centerY } : undefined,
|
|
1490
|
-
croppedImageResourceUri,
|
|
1491
|
-
duration,
|
|
1492
|
-
},
|
|
1493
|
-
generatedCode
|
|
1494
|
-
);
|
|
1092
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1495
1093
|
} catch (error) {
|
|
1094
|
+
if (error instanceof NoActiveSessionError) {
|
|
1095
|
+
logger.warn("find_and_click: No active session");
|
|
1096
|
+
return noSessionResult(error);
|
|
1097
|
+
}
|
|
1098
|
+
const cancelled = cancelledResultOrNull(error, "find_and_click");
|
|
1099
|
+
if (cancelled) return cancelled;
|
|
1496
1100
|
logger.error("find_and_click: Failed", { error: String(error), description: params.description });
|
|
1497
1101
|
captureException(error as Error, { tags: { tool: "find_and_click" }, extra: { description: params.description, action: params.action } });
|
|
1498
1102
|
throw error;
|
|
@@ -1515,29 +1119,19 @@ server.registerTool(
|
|
|
1515
1119
|
const startTime = Date.now();
|
|
1516
1120
|
logger.info("type: Starting", { textLength: params.text.length, secret: params.secret });
|
|
1517
1121
|
|
|
1518
|
-
const sessionCheck = requireActiveSession();
|
|
1519
|
-
if (!sessionCheck.valid) {
|
|
1520
|
-
logger.warn("type: No active session");
|
|
1521
|
-
return sessionCheck.error;
|
|
1522
|
-
}
|
|
1523
|
-
|
|
1524
1122
|
try {
|
|
1525
1123
|
logger.debug("type: Typing text");
|
|
1526
|
-
await
|
|
1124
|
+
const result = await core.type(params.text, params.secret, params.delay);
|
|
1527
1125
|
|
|
1528
1126
|
const duration = Date.now() - startTime;
|
|
1529
1127
|
logger.info("type: Completed", { duration });
|
|
1530
1128
|
|
|
1531
|
-
|
|
1532
|
-
const generatedCode = generateActionCode("type", { text: params.text, secret: params.secret });
|
|
1533
|
-
|
|
1534
|
-
return createToolResult(
|
|
1535
|
-
true,
|
|
1536
|
-
`Typed: ${params.secret ? "[secret text]" : `"${params.text}"`}`,
|
|
1537
|
-
{ action: "type", text: params.secret ? "[SECRET]" : params.text, duration },
|
|
1538
|
-
generatedCode
|
|
1539
|
-
);
|
|
1129
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1540
1130
|
} catch (error) {
|
|
1131
|
+
if (error instanceof NoActiveSessionError) {
|
|
1132
|
+
logger.warn("type: No active session");
|
|
1133
|
+
return noSessionResult(error);
|
|
1134
|
+
}
|
|
1541
1135
|
logger.error("type: Failed", { error: String(error) });
|
|
1542
1136
|
captureException(error as Error, { tags: { tool: "type" }, extra: { textLength: params.text.length, secret: params.secret } });
|
|
1543
1137
|
throw error;
|
|
@@ -1558,29 +1152,19 @@ server.registerTool(
|
|
|
1558
1152
|
const startTime = Date.now();
|
|
1559
1153
|
logger.info("press_keys: Starting", { keys: params.keys });
|
|
1560
1154
|
|
|
1561
|
-
const sessionCheck = requireActiveSession();
|
|
1562
|
-
if (!sessionCheck.valid) {
|
|
1563
|
-
logger.warn("press_keys: No active session");
|
|
1564
|
-
return sessionCheck.error;
|
|
1565
|
-
}
|
|
1566
|
-
|
|
1567
1155
|
try {
|
|
1568
1156
|
logger.debug("press_keys: Pressing keys");
|
|
1569
|
-
await
|
|
1157
|
+
const result = await core.pressKeys(params.keys);
|
|
1570
1158
|
|
|
1571
1159
|
const duration = Date.now() - startTime;
|
|
1572
1160
|
logger.info("press_keys: Completed", { keys: params.keys, duration });
|
|
1573
1161
|
|
|
1574
|
-
|
|
1575
|
-
const generatedCode = generateActionCode("press_keys", { keys: params.keys });
|
|
1576
|
-
|
|
1577
|
-
return createToolResult(
|
|
1578
|
-
true,
|
|
1579
|
-
`Pressed keys: ${params.keys.join(" + ")}`,
|
|
1580
|
-
{ action: "press_keys", keys: params.keys, duration },
|
|
1581
|
-
generatedCode
|
|
1582
|
-
);
|
|
1162
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1583
1163
|
} catch (error) {
|
|
1164
|
+
if (error instanceof NoActiveSessionError) {
|
|
1165
|
+
logger.warn("press_keys: No active session");
|
|
1166
|
+
return noSessionResult(error);
|
|
1167
|
+
}
|
|
1584
1168
|
logger.error("press_keys: Failed", { error: String(error), keys: params.keys });
|
|
1585
1169
|
captureException(error as Error, { tags: { tool: "press_keys" }, extra: { keys: params.keys } });
|
|
1586
1170
|
throw error;
|
|
@@ -1602,29 +1186,19 @@ server.registerTool(
|
|
|
1602
1186
|
const startTime = Date.now();
|
|
1603
1187
|
logger.info("scroll: Starting", { direction: params.direction, amount: params.amount });
|
|
1604
1188
|
|
|
1605
|
-
const sessionCheck = requireActiveSession();
|
|
1606
|
-
if (!sessionCheck.valid) {
|
|
1607
|
-
logger.warn("scroll: No active session");
|
|
1608
|
-
return sessionCheck.error;
|
|
1609
|
-
}
|
|
1610
|
-
|
|
1611
1189
|
try {
|
|
1612
1190
|
logger.debug("scroll: Scrolling");
|
|
1613
|
-
await
|
|
1191
|
+
const result = await core.scroll(params.direction, params.amount);
|
|
1614
1192
|
|
|
1615
1193
|
const duration = Date.now() - startTime;
|
|
1616
1194
|
logger.info("scroll: Completed", { direction: params.direction, duration });
|
|
1617
1195
|
|
|
1618
|
-
|
|
1619
|
-
const generatedCode = generateActionCode("scroll", { direction: params.direction, amount: params.amount });
|
|
1620
|
-
|
|
1621
|
-
return createToolResult(
|
|
1622
|
-
true,
|
|
1623
|
-
`Scrolled ${params.direction}${params.amount ? ` by ${params.amount}px` : ""}`,
|
|
1624
|
-
{ action: "scroll", scrollDirection: params.direction, direction: params.direction, amount: params.amount, duration },
|
|
1625
|
-
generatedCode
|
|
1626
|
-
);
|
|
1196
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1627
1197
|
} catch (error) {
|
|
1198
|
+
if (error instanceof NoActiveSessionError) {
|
|
1199
|
+
logger.warn("scroll: No active session");
|
|
1200
|
+
return noSessionResult(error);
|
|
1201
|
+
}
|
|
1628
1202
|
logger.error("scroll: Failed", { error: String(error), direction: params.direction });
|
|
1629
1203
|
captureException(error as Error, { tags: { tool: "scroll" }, extra: { direction: params.direction, amount: params.amount } });
|
|
1630
1204
|
throw error;
|
|
@@ -1651,29 +1225,19 @@ Unlike 'check' which is for your understanding during development, 'assert' crea
|
|
|
1651
1225
|
const startTime = Date.now();
|
|
1652
1226
|
logger.info("assert: Starting", { assertion: params.assertion });
|
|
1653
1227
|
|
|
1654
|
-
const sessionCheck = requireActiveSession();
|
|
1655
|
-
if (!sessionCheck.valid) {
|
|
1656
|
-
logger.warn("assert: No active session");
|
|
1657
|
-
return sessionCheck.error;
|
|
1658
|
-
}
|
|
1659
|
-
|
|
1660
1228
|
try {
|
|
1661
1229
|
logger.debug("assert: Running assertion");
|
|
1662
|
-
const result = await
|
|
1230
|
+
const result = await core.assert(params.assertion);
|
|
1663
1231
|
|
|
1664
1232
|
const duration = Date.now() - startTime;
|
|
1665
|
-
logger.info("assert: Completed", { assertion: params.assertion, passed: result, duration });
|
|
1233
|
+
logger.info("assert: Completed", { assertion: params.assertion, passed: result.ok, duration });
|
|
1666
1234
|
|
|
1667
|
-
|
|
1668
|
-
const generatedCode = generateActionCode("assert", { assertion: params.assertion });
|
|
1669
|
-
|
|
1670
|
-
return createToolResult(
|
|
1671
|
-
result,
|
|
1672
|
-
result ? `✓ Assertion passed: "${params.assertion}"` : `✗ Assertion failed: "${params.assertion}"`,
|
|
1673
|
-
{ action: "assert", assertion: params.assertion, passed: result, success: result, duration },
|
|
1674
|
-
generatedCode
|
|
1675
|
-
);
|
|
1235
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1676
1236
|
} catch (error) {
|
|
1237
|
+
if (error instanceof NoActiveSessionError) {
|
|
1238
|
+
logger.warn("assert: No active session");
|
|
1239
|
+
return noSessionResult(error);
|
|
1240
|
+
}
|
|
1677
1241
|
logger.error("assert: Failed", { error: String(error), assertion: params.assertion });
|
|
1678
1242
|
captureException(error as Error, { tags: { tool: "assert" }, extra: { assertion: params.assertion } });
|
|
1679
1243
|
throw error;
|
|
@@ -1712,119 +1276,68 @@ You can optionally provide a reference image URI to compare against a previous s
|
|
|
1712
1276
|
}) as any,
|
|
1713
1277
|
_meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
|
|
1714
1278
|
},
|
|
1715
|
-
async (params: { task: string; referenceImageUri?: string }): Promise<CallToolResult> => {
|
|
1279
|
+
async (params: { task: string; referenceImageUri?: string }, extra: ToolExtra): Promise<CallToolResult> => {
|
|
1716
1280
|
const startTime = Date.now();
|
|
1281
|
+
const progress = makeProgressReporter(extra);
|
|
1717
1282
|
logger.info("check: Starting", { task: params.task, hasReferenceImageUri: !!params.referenceImageUri });
|
|
1718
1283
|
|
|
1719
|
-
const sessionCheck = requireActiveSession();
|
|
1720
|
-
if (!sessionCheck.valid) {
|
|
1721
|
-
logger.warn("check: No active session");
|
|
1722
|
-
return sessionCheck.error;
|
|
1723
|
-
}
|
|
1724
|
-
|
|
1725
1284
|
try {
|
|
1726
|
-
//
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
// Use provided reference image URI, last screenshot as "before" state, or current if no previous screenshot
|
|
1731
|
-
let beforeScreenshot: string;
|
|
1285
|
+
// Resolve the optional reference image from the image store (a server/UI
|
|
1286
|
+
// concern). The core handles "last screenshot" / current-screenshot
|
|
1287
|
+
// fallback internally when no reference image is passed.
|
|
1288
|
+
let referenceImage: string | undefined;
|
|
1732
1289
|
if (params.referenceImageUri) {
|
|
1733
1290
|
// Extract image ID from URI (e.g., "screenshot://testdriver/screenshot/screenshot-1" -> "screenshot-1")
|
|
1734
|
-
const uriParts = params.referenceImageUri.split(
|
|
1291
|
+
const uriParts = params.referenceImageUri.split("/");
|
|
1735
1292
|
const imageId = uriParts[uriParts.length - 1];
|
|
1736
|
-
|
|
1737
|
-
logger.info("check: Looking up reference image", {
|
|
1738
|
-
referenceImageUri: params.referenceImageUri,
|
|
1293
|
+
|
|
1294
|
+
logger.info("check: Looking up reference image", {
|
|
1295
|
+
referenceImageUri: params.referenceImageUri,
|
|
1739
1296
|
extractedImageId: imageId,
|
|
1740
1297
|
imageStoreSize: imageStore.size,
|
|
1741
|
-
availableKeys: Array.from(imageStore.keys())
|
|
1298
|
+
availableKeys: Array.from(imageStore.keys()),
|
|
1742
1299
|
});
|
|
1743
|
-
|
|
1300
|
+
|
|
1744
1301
|
const storedImage = getStoredImage(imageId);
|
|
1745
|
-
|
|
1302
|
+
|
|
1746
1303
|
if (storedImage) {
|
|
1747
|
-
logger.info("check: Found reference image", {
|
|
1748
|
-
imageId,
|
|
1304
|
+
logger.info("check: Found reference image", {
|
|
1305
|
+
imageId,
|
|
1749
1306
|
dataLength: storedImage.data?.length,
|
|
1750
1307
|
type: storedImage.type,
|
|
1751
|
-
hasData: !!storedImage.data
|
|
1308
|
+
hasData: !!storedImage.data,
|
|
1752
1309
|
});
|
|
1753
|
-
|
|
1310
|
+
referenceImage = storedImage.data;
|
|
1754
1311
|
} else {
|
|
1755
|
-
logger.warn("check: Reference image NOT found in store, falling back to last screenshot", {
|
|
1756
|
-
referenceImageUri: params.referenceImageUri,
|
|
1312
|
+
logger.warn("check: Reference image NOT found in store, falling back to last screenshot", {
|
|
1313
|
+
referenceImageUri: params.referenceImageUri,
|
|
1757
1314
|
imageId,
|
|
1758
1315
|
imageStoreSize: imageStore.size,
|
|
1759
|
-
availableKeys: Array.from(imageStore.keys())
|
|
1316
|
+
availableKeys: Array.from(imageStore.keys()),
|
|
1760
1317
|
});
|
|
1761
|
-
beforeScreenshot = lastScreenshotBase64 || currentScreenshot;
|
|
1762
1318
|
}
|
|
1763
|
-
} else {
|
|
1764
|
-
beforeScreenshot = lastScreenshotBase64 || currentScreenshot;
|
|
1765
1319
|
}
|
|
1766
|
-
|
|
1767
|
-
// Update last screenshot for next check
|
|
1768
|
-
lastScreenshotBase64 = currentScreenshot;
|
|
1769
|
-
|
|
1770
|
-
// Get system state
|
|
1771
|
-
const mousePosition = await sdk.agent.system.getMousePosition();
|
|
1772
|
-
const activeWindow = await sdk.agent.system.activeWin();
|
|
1773
|
-
|
|
1774
|
-
// Call the check endpoint
|
|
1775
|
-
logger.info("check: Calling check API endpoint", {
|
|
1776
|
-
hasLastScreenshot: beforeScreenshot !== currentScreenshot,
|
|
1777
|
-
usingReferenceImageUri: !!params.referenceImageUri,
|
|
1778
|
-
beforeScreenshotLength: beforeScreenshot?.length || 0,
|
|
1779
|
-
currentScreenshotLength: currentScreenshot?.length || 0,
|
|
1780
|
-
beforeScreenshotPreview: beforeScreenshot?.substring(0, 50),
|
|
1781
|
-
currentScreenshotPreview: currentScreenshot?.substring(0, 50)
|
|
1782
|
-
});
|
|
1783
|
-
const response = await sdk.agent.sdk.req("check", {
|
|
1784
|
-
tasks: [params.task],
|
|
1785
|
-
images: [beforeScreenshot, currentScreenshot],
|
|
1786
|
-
mousePosition,
|
|
1787
|
-
activeWindow,
|
|
1788
|
-
});
|
|
1789
1320
|
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1321
|
+
progress.report("Capturing screenshot...");
|
|
1322
|
+
const stopHeartbeat = progress.heartbeat(`Checking: "${params.task}"...`);
|
|
1323
|
+
let result: ActionResult;
|
|
1324
|
+
try {
|
|
1325
|
+
result = await raceAbort(extra.signal, "check", core.check(params.task, referenceImage));
|
|
1326
|
+
} finally {
|
|
1327
|
+
stopHeartbeat();
|
|
1796
1328
|
}
|
|
1797
|
-
|
|
1798
|
-
// Determine if the check passed based on the AI response
|
|
1799
|
-
// The AI typically returns markdown with its analysis
|
|
1800
|
-
// We consider it "complete" if the response doesn't contain code blocks (indicating more work needed)
|
|
1801
|
-
const hasCodeBlocks = aiResponse && (
|
|
1802
|
-
aiResponse.includes("```yml") ||
|
|
1803
|
-
aiResponse.includes("```yaml") ||
|
|
1804
|
-
aiResponse.includes("- command:")
|
|
1805
|
-
);
|
|
1806
|
-
const isComplete = !hasCodeBlocks;
|
|
1807
1329
|
|
|
1808
1330
|
const duration = Date.now() - startTime;
|
|
1809
|
-
logger.info("check: Completed", { task: params.task, complete:
|
|
1331
|
+
logger.info("check: Completed", { task: params.task, complete: result.ok, duration });
|
|
1810
1332
|
|
|
1811
|
-
|
|
1812
|
-
return createToolResult(
|
|
1813
|
-
isComplete,
|
|
1814
|
-
isComplete
|
|
1815
|
-
? `✓ Task appears complete: "${params.task}"\n\nAI Analysis:\n${aiResponse}`
|
|
1816
|
-
: `⚠ Task may not be complete: "${params.task}"\n\nAI Analysis:\n${aiResponse}`,
|
|
1817
|
-
{
|
|
1818
|
-
action: "check",
|
|
1819
|
-
task: params.task,
|
|
1820
|
-
complete: isComplete,
|
|
1821
|
-
success: isComplete,
|
|
1822
|
-
aiResponse,
|
|
1823
|
-
screenshotResourceUri,
|
|
1824
|
-
duration
|
|
1825
|
-
}
|
|
1826
|
-
);
|
|
1333
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1827
1334
|
} catch (error) {
|
|
1335
|
+
if (error instanceof NoActiveSessionError) {
|
|
1336
|
+
logger.warn("check: No active session");
|
|
1337
|
+
return noSessionResult(error);
|
|
1338
|
+
}
|
|
1339
|
+
const cancelled = cancelledResultOrNull(error, "check");
|
|
1340
|
+
if (cancelled) return cancelled;
|
|
1828
1341
|
logger.error("check: Failed", { error: String(error), task: params.task });
|
|
1829
1342
|
captureException(error as Error, { tags: { tool: "check" }, extra: { task: params.task } });
|
|
1830
1343
|
throw error;
|
|
@@ -1847,29 +1360,19 @@ server.registerTool(
|
|
|
1847
1360
|
const startTime = Date.now();
|
|
1848
1361
|
logger.info("exec: Starting", { language: params.language, codeLength: params.code.length, timeout: params.timeout });
|
|
1849
1362
|
|
|
1850
|
-
const sessionCheck = requireActiveSession();
|
|
1851
|
-
if (!sessionCheck.valid) {
|
|
1852
|
-
logger.warn("exec: No active session");
|
|
1853
|
-
return sessionCheck.error;
|
|
1854
|
-
}
|
|
1855
|
-
|
|
1856
1363
|
try {
|
|
1857
1364
|
logger.debug("exec: Executing code", { language: params.language });
|
|
1858
|
-
const
|
|
1365
|
+
const result = await core.exec(params.language, params.code, params.timeout);
|
|
1859
1366
|
|
|
1860
1367
|
const duration = Date.now() - startTime;
|
|
1861
|
-
logger.info("exec: Completed", { language: params.language, outputLength: output?.length || 0, duration });
|
|
1368
|
+
logger.info("exec: Completed", { language: params.language, outputLength: (result.data.output as string)?.length || 0, duration });
|
|
1862
1369
|
|
|
1863
|
-
|
|
1864
|
-
const generatedCode = generateActionCode("exec", { language: params.language, code: params.code, timeout: params.timeout });
|
|
1865
|
-
|
|
1866
|
-
return createToolResult(
|
|
1867
|
-
true,
|
|
1868
|
-
`Executed ${params.language} code:\n${output || "(no output)"}`,
|
|
1869
|
-
{ action: "exec", language: params.language, output, duration },
|
|
1870
|
-
generatedCode
|
|
1871
|
-
);
|
|
1370
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1872
1371
|
} catch (error) {
|
|
1372
|
+
if (error instanceof NoActiveSessionError) {
|
|
1373
|
+
logger.warn("exec: No active session");
|
|
1374
|
+
return noSessionResult(error);
|
|
1375
|
+
}
|
|
1873
1376
|
logger.error("exec: Failed", { error: String(error), language: params.language });
|
|
1874
1377
|
captureException(error as Error, { tags: { tool: "exec" }, extra: { language: params.language, codeLength: params.code.length } });
|
|
1875
1378
|
throw error;
|
|
@@ -2249,37 +1752,27 @@ Only use 'screenshot' when you explicitly want to show something to the human us
|
|
|
2249
1752
|
const startTime = Date.now();
|
|
2250
1753
|
logger.info("screenshot: Starting");
|
|
2251
1754
|
|
|
2252
|
-
const sessionCheck = requireActiveSession();
|
|
2253
|
-
if (!sessionCheck.valid) {
|
|
2254
|
-
logger.warn("screenshot: No active session");
|
|
2255
|
-
return sessionCheck.error;
|
|
2256
|
-
}
|
|
2257
|
-
|
|
2258
1755
|
try {
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
1756
|
+
const result = await core.screenshot();
|
|
1757
|
+
|
|
1758
|
+
// Store the captured image as a resource URI (kept OUT of AI context — the
|
|
1759
|
+
// MCP app fetches it via resources/read). Preserve the original user-facing
|
|
1760
|
+
// text rather than the core's neutral text.
|
|
1761
|
+
const data: Record<string, unknown> = { action: "screenshot" };
|
|
1762
|
+
for (const img of result.images ?? []) {
|
|
1763
|
+
data.screenshotResourceUri = storeImage(img.base64, "screenshot");
|
|
2266
1764
|
}
|
|
2267
|
-
|
|
1765
|
+
|
|
2268
1766
|
const duration = Date.now() - startTime;
|
|
2269
|
-
|
|
1767
|
+
data.duration = duration;
|
|
1768
|
+
logger.info("screenshot: Completed", { duration, hasImage: (result.images?.length ?? 0) > 0 });
|
|
2270
1769
|
|
|
2271
|
-
|
|
2272
|
-
// This keeps the base64 image data OUT of AI context
|
|
2273
|
-
return createToolResult(
|
|
2274
|
-
true,
|
|
2275
|
-
"Screenshot captured and displayed to user",
|
|
2276
|
-
{
|
|
2277
|
-
action: "screenshot",
|
|
2278
|
-
screenshotResourceUri,
|
|
2279
|
-
duration
|
|
2280
|
-
}
|
|
2281
|
-
);
|
|
1770
|
+
return createToolResult(true, "Screenshot captured and displayed to user", data);
|
|
2282
1771
|
} catch (error) {
|
|
1772
|
+
if (error instanceof NoActiveSessionError) {
|
|
1773
|
+
logger.warn("screenshot: No active session");
|
|
1774
|
+
return noSessionResult(error);
|
|
1775
|
+
}
|
|
2283
1776
|
logger.error("screenshot: Failed", { error: String(error) });
|
|
2284
1777
|
return createToolResult(false, `Screenshot failed: ${error}`, { error: String(error) });
|
|
2285
1778
|
}
|
|
@@ -2309,6 +1802,7 @@ API Key: The apiKey parameter is optional. If not provided, you'll need to manua
|
|
|
2309
1802
|
directory: z.string().optional().describe("Target directory (defaults to current working directory)"),
|
|
2310
1803
|
apiKey: z.string().optional().describe("TestDriver API key (will be saved to .env)"),
|
|
2311
1804
|
skipInstall: z.boolean().default(false).describe("Skip npm install step"),
|
|
1805
|
+
skipSampleTest: z.boolean().default(false).describe("Skip scaffolding the example test files (tests/example.test.js + tests/login.js). Useful when an agent writes its own tests."),
|
|
2312
1806
|
}),
|
|
2313
1807
|
},
|
|
2314
1808
|
async (params): Promise<CallToolResult> => {
|
|
@@ -2327,6 +1821,7 @@ API Key: The apiKey parameter is optional. If not provided, you'll need to manua
|
|
|
2327
1821
|
targetDir,
|
|
2328
1822
|
apiKey: params.apiKey,
|
|
2329
1823
|
skipInstall: params.skipInstall,
|
|
1824
|
+
skipSampleTest: params.skipSampleTest,
|
|
2330
1825
|
});
|
|
2331
1826
|
|
|
2332
1827
|
const duration = Date.now() - startTime;
|
|
@@ -2376,27 +1871,111 @@ Learn more at https://docs.testdriver.ai/v7/getting-started/
|
|
|
2376
1871
|
);
|
|
2377
1872
|
|
|
2378
1873
|
|
|
1874
|
+
// =============================================================================
|
|
1875
|
+
// HTTP transport (Streamable HTTP, no auth)
|
|
1876
|
+
// =============================================================================
|
|
1877
|
+
|
|
1878
|
+
/**
|
|
1879
|
+
* Serve the MCP server over Streamable HTTP with no authentication.
|
|
1880
|
+
*
|
|
1881
|
+
* This is what the eve agent (and any other Streamable-HTTP MCP client) connects
|
|
1882
|
+
* to. The server holds single-session global state (the active `sdk`, element
|
|
1883
|
+
* refs, image store), so we mirror that with a single long-lived MCP transport:
|
|
1884
|
+
* the first `initialize` request mints a session id and every subsequent request
|
|
1885
|
+
* reuses the same `server` + transport. Concurrent sessions are intentionally not
|
|
1886
|
+
* supported — this server provisions one sandbox at a time.
|
|
1887
|
+
*
|
|
1888
|
+
* No auth is applied here by design (the connection is expected to be local-only
|
|
1889
|
+
* or otherwise protected outside the MCP layer). Do not expose this on a public
|
|
1890
|
+
* network without putting a real authenticating proxy in front of it.
|
|
1891
|
+
*/
|
|
1892
|
+
async function startHttpServer() {
|
|
1893
|
+
const host = process.env.TD_MCP_HOST || "127.0.0.1";
|
|
1894
|
+
const port = Number(process.env.TD_MCP_PORT || process.env.PORT || 8788);
|
|
1895
|
+
const mcpPath = process.env.TD_MCP_PATH || "/mcp";
|
|
1896
|
+
|
|
1897
|
+
// One transport for the lifetime of the process (single-session design).
|
|
1898
|
+
const transport = new StreamableHTTPServerTransport({
|
|
1899
|
+
sessionIdGenerator: () => randomUUID(),
|
|
1900
|
+
});
|
|
1901
|
+
await server.connect(transport);
|
|
1902
|
+
|
|
1903
|
+
const httpServer = http.createServer((req, res) => {
|
|
1904
|
+
const url = new URL(req.url || "/", `http://${req.headers.host}`);
|
|
1905
|
+
|
|
1906
|
+
// Lightweight health check for readiness probes / `eve dev`.
|
|
1907
|
+
if (req.method === "GET" && url.pathname === "/health") {
|
|
1908
|
+
res.writeHead(200, { "content-type": "application/json" });
|
|
1909
|
+
res.end(JSON.stringify({ status: "ok", server: "testdriver", version }));
|
|
1910
|
+
return;
|
|
1911
|
+
}
|
|
1912
|
+
|
|
1913
|
+
if (url.pathname !== mcpPath) {
|
|
1914
|
+
res.writeHead(404, { "content-type": "application/json" });
|
|
1915
|
+
res.end(JSON.stringify({ error: "Not found", hint: `MCP endpoint is ${mcpPath}` }));
|
|
1916
|
+
return;
|
|
1917
|
+
}
|
|
1918
|
+
|
|
1919
|
+
// The Streamable HTTP transport handles POST (requests), GET (SSE stream),
|
|
1920
|
+
// and DELETE (session teardown) on the same path. Body parsing is left to
|
|
1921
|
+
// the transport so it can read the raw stream.
|
|
1922
|
+
transport.handleRequest(req, res).catch((error) => {
|
|
1923
|
+
logger.error("http: handleRequest failed", { error: String(error) });
|
|
1924
|
+
captureException(error as Error, { tags: { phase: "http" } });
|
|
1925
|
+
if (!res.headersSent) {
|
|
1926
|
+
res.writeHead(500, { "content-type": "application/json" });
|
|
1927
|
+
res.end(JSON.stringify({ error: "Internal server error" }));
|
|
1928
|
+
}
|
|
1929
|
+
});
|
|
1930
|
+
});
|
|
1931
|
+
|
|
1932
|
+
await new Promise<void>((resolve) => httpServer.listen(port, host, resolve));
|
|
1933
|
+
logger.info("TestDriver MCP Server running on Streamable HTTP", {
|
|
1934
|
+
url: `http://${host}:${port}${mcpPath}`,
|
|
1935
|
+
});
|
|
1936
|
+
|
|
1937
|
+
const shutdown = async () => {
|
|
1938
|
+
logger.info("Shutting down MCP Server");
|
|
1939
|
+
httpServer.close();
|
|
1940
|
+
await transport.close().catch(() => {});
|
|
1941
|
+
await flushSentry();
|
|
1942
|
+
process.exit(0);
|
|
1943
|
+
};
|
|
1944
|
+
process.on("SIGINT", shutdown);
|
|
1945
|
+
process.on("SIGTERM", shutdown);
|
|
1946
|
+
}
|
|
1947
|
+
|
|
2379
1948
|
// Start the server
|
|
2380
1949
|
async function main() {
|
|
2381
|
-
|
|
1950
|
+
// Transport selection: TD_MCP_TRANSPORT=http serves Streamable HTTP (used by
|
|
1951
|
+
// the eve agent); anything else (default) uses stdio for local CLI clients.
|
|
1952
|
+
const transportMode = (process.env.TD_MCP_TRANSPORT || "stdio").toLowerCase();
|
|
1953
|
+
|
|
1954
|
+
logger.info("Starting TestDriver MCP Server", {
|
|
2382
1955
|
version,
|
|
1956
|
+
transport: transportMode,
|
|
2383
1957
|
logLevel: process.env.TD_LOG_LEVEL || "INFO",
|
|
2384
1958
|
distDir: DIST_DIR,
|
|
2385
1959
|
sentryEnabled: isSentryEnabled(),
|
|
2386
1960
|
});
|
|
2387
|
-
|
|
1961
|
+
|
|
1962
|
+
if (transportMode === "http") {
|
|
1963
|
+
await startHttpServer();
|
|
1964
|
+
return;
|
|
1965
|
+
}
|
|
1966
|
+
|
|
2388
1967
|
const transport = new StdioServerTransport();
|
|
2389
1968
|
await server.connect(transport);
|
|
2390
|
-
|
|
1969
|
+
|
|
2391
1970
|
logger.info("TestDriver MCP Server running on stdio");
|
|
2392
|
-
|
|
1971
|
+
|
|
2393
1972
|
// Handle graceful shutdown
|
|
2394
1973
|
const shutdown = async () => {
|
|
2395
1974
|
logger.info("Shutting down MCP Server");
|
|
2396
1975
|
await flushSentry();
|
|
2397
1976
|
process.exit(0);
|
|
2398
1977
|
};
|
|
2399
|
-
|
|
1978
|
+
|
|
2400
1979
|
process.on("SIGINT", shutdown);
|
|
2401
1980
|
process.on("SIGTERM", shutdown);
|
|
2402
1981
|
}
|