@testdriverai/mcp 7.9.115-test → 7.9.117-test
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent/index.js +38 -17
- package/ai/skills/testdriver-ci-cd/SKILL.md +58 -1
- package/ai/skills/testdriver:ci-cd/SKILL.md +58 -1
- package/docs/_data/examples-manifest.json +46 -46
- package/docs/v7/ci-cd.mdx +58 -1
- package/docs/v7/examples/ai.mdx +1 -1
- package/docs/v7/examples/assert.mdx +1 -1
- package/docs/v7/examples/chrome-extension.mdx +1 -1
- package/docs/v7/examples/element-not-found.mdx +1 -1
- package/docs/v7/examples/findall-coffee-icons.mdx +1 -1
- package/docs/v7/examples/hover-image.mdx +1 -1
- package/docs/v7/examples/hover-text-with-description.mdx +1 -1
- package/docs/v7/examples/hover-text.mdx +1 -1
- package/docs/v7/examples/installer.mdx +1 -1
- package/docs/v7/examples/launch-vscode-linux.mdx +1 -1
- package/docs/v7/examples/parse.mdx +1 -1
- package/docs/v7/examples/press-keys.mdx +1 -1
- package/docs/v7/examples/scroll-keyboard.mdx +1 -1
- package/docs/v7/examples/scroll.mdx +1 -1
- package/docs/v7/examples/type.mdx +1 -1
- package/mcp-server/dist/core/actions.d.ts +176 -0
- package/mcp-server/dist/core/actions.js +753 -0
- package/mcp-server/dist/server.mjs +283 -781
- package/mcp-server/package.json +1 -0
- package/mcp-server/src/core/actions.ts +912 -0
- package/mcp-server/src/server.ts +329 -988
- package/package.json +13 -1
package/mcp-server/src/server.ts
CHANGED
|
@@ -12,6 +12,9 @@ process.env.TD_DEBUG = "true";
|
|
|
12
12
|
import { registerAppResource, registerAppTool, RESOURCE_MIME_TYPE } from "@modelcontextprotocol/ext-apps/server";
|
|
13
13
|
import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
14
14
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
15
|
+
import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
|
|
16
|
+
import { randomUUID } from "crypto";
|
|
17
|
+
import * as http from "http";
|
|
15
18
|
import type { RequestHandlerExtra } from "@modelcontextprotocol/sdk/shared/protocol.js";
|
|
16
19
|
import type { Variables } from "@modelcontextprotocol/sdk/shared/uriTemplate.js";
|
|
17
20
|
import type { CallToolResult, ReadResourceResult, ServerNotification, ServerRequest } from "@modelcontextprotocol/sdk/types.js";
|
|
@@ -22,9 +25,10 @@ import * as path from "path";
|
|
|
22
25
|
import { fileURLToPath, pathToFileURL } from "url";
|
|
23
26
|
import { z } from "zod";
|
|
24
27
|
|
|
25
|
-
import
|
|
28
|
+
import * as core from "./core/actions.js";
|
|
29
|
+
import { NoActiveSessionError, type ActionResult } from "./core/actions.js";
|
|
26
30
|
import { resolveE2bTemplateId, resolveOs } from "./env-utils.js";
|
|
27
|
-
import {
|
|
31
|
+
import { SessionStartInputSchema, type SessionStartInput } from "./provision-types.js";
|
|
28
32
|
import { sessionManager, type SessionState } from "./session.js";
|
|
29
33
|
|
|
30
34
|
// =============================================================================
|
|
@@ -211,12 +215,6 @@ const RESOURCE_URI = "ui://testdriver/mcp-app.html";
|
|
|
211
215
|
const SCREENSHOT_RESOURCE_BASE = "screenshot://testdriver/screenshot";
|
|
212
216
|
const CROPPED_IMAGE_RESOURCE_BASE = "screenshot://testdriver/cropped";
|
|
213
217
|
|
|
214
|
-
// SDK instance (will be initialized on session start)
|
|
215
|
-
let sdk: any = null;
|
|
216
|
-
|
|
217
|
-
// Last screenshot base64 for check comparisons
|
|
218
|
-
let lastScreenshotBase64: string | null = null;
|
|
219
|
-
|
|
220
218
|
// =============================================================================
|
|
221
219
|
// Image Store - Stores images with unique IDs for reload persistence
|
|
222
220
|
// =============================================================================
|
|
@@ -284,54 +282,46 @@ function getSessionData(session: SessionState | null) {
|
|
|
284
282
|
}
|
|
285
283
|
|
|
286
284
|
/**
|
|
287
|
-
*
|
|
288
|
-
*
|
|
289
|
-
*
|
|
290
|
-
* Auto-extends the session on each successful check to prevent expiry during active use
|
|
285
|
+
* Map a {@link NoActiveSessionError} thrown by the core into the exact same
|
|
286
|
+
* "no/expired session" tool result the server's old `requireActiveSession`
|
|
287
|
+
* produced. Kept byte-identical so external behavior is unchanged.
|
|
291
288
|
*/
|
|
292
|
-
function
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
action: "session_start",
|
|
305
|
-
message: "No sandbox session exists. Call session_start to create one."
|
|
306
|
-
}
|
|
307
|
-
)
|
|
308
|
-
};
|
|
289
|
+
function noSessionResult(err: NoActiveSessionError): CallToolResult {
|
|
290
|
+
if (err.code === "SESSION_EXPIRED") {
|
|
291
|
+
return createToolResult(
|
|
292
|
+
false,
|
|
293
|
+
"ERROR: Session has expired or timed out. The sandbox is no longer available. You must call session_start again to create a new sandbox session before continuing.",
|
|
294
|
+
{
|
|
295
|
+
error: "SESSION_EXPIRED",
|
|
296
|
+
action: "session_start",
|
|
297
|
+
message: "The previous sandbox session has expired. Call session_start to create a new one.",
|
|
298
|
+
expiredSessionId: err.expiredSessionId,
|
|
299
|
+
}
|
|
300
|
+
);
|
|
309
301
|
}
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
302
|
+
return createToolResult(
|
|
303
|
+
false,
|
|
304
|
+
"ERROR: No active session. You must call session_start first to create a sandbox before using any other tools.",
|
|
305
|
+
{
|
|
306
|
+
error: "NO_SESSION",
|
|
307
|
+
action: "session_start",
|
|
308
|
+
message: "No sandbox session exists. Call session_start to create one.",
|
|
309
|
+
}
|
|
310
|
+
);
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* Map a core {@link ActionResult} into an MCP CallToolResult, storing images as
|
|
315
|
+
* resource URIs (the core returns BARE base64, which is what `storeImage` wants).
|
|
316
|
+
*/
|
|
317
|
+
function resultToMcp(r: ActionResult): CallToolResult {
|
|
318
|
+
const data: Record<string, unknown> = { ...r.data };
|
|
319
|
+
for (const img of r.images ?? []) {
|
|
320
|
+
const uri = storeImage(img.base64, img.kind);
|
|
321
|
+
if (img.kind === "cropped") data.croppedImageResourceUri = uri;
|
|
322
|
+
else data.screenshotResourceUri = uri;
|
|
328
323
|
}
|
|
329
|
-
|
|
330
|
-
// Auto-extend session on each command to prevent expiry during active use
|
|
331
|
-
// This resets the expiry timer back to the original keepAlive duration
|
|
332
|
-
sessionManager.refreshSession(session.sessionId);
|
|
333
|
-
|
|
334
|
-
return { valid: true };
|
|
324
|
+
return createToolResult(r.ok, r.text, data, r.code);
|
|
335
325
|
}
|
|
336
326
|
|
|
337
327
|
// =============================================================================
|
|
@@ -536,10 +526,6 @@ const server = isSentryEnabled()
|
|
|
536
526
|
version: version,
|
|
537
527
|
});
|
|
538
528
|
|
|
539
|
-
// Element reference storage (for click/hover after find)
|
|
540
|
-
// Stores actual Element instances - no raw coordinates as input
|
|
541
|
-
const elementRefs = new Map<string, { element: any; description: string; coords: { x: number; y: number; centerX: number; centerY: number } }>();
|
|
542
|
-
|
|
543
529
|
// =============================================================================
|
|
544
530
|
// Register UI Resource
|
|
545
531
|
// =============================================================================
|
|
@@ -681,248 +667,76 @@ Debug mode (connect to existing sandbox):
|
|
|
681
667
|
logger.info("session_start: Using TD_E2B_TEMPLATE_ID environment variable", { e2bTemplateId: resolvedE2bTemplateId });
|
|
682
668
|
}
|
|
683
669
|
|
|
684
|
-
logger.info("session_start: Starting", {
|
|
685
|
-
type: params.type,
|
|
670
|
+
logger.info("session_start: Starting", {
|
|
671
|
+
type: params.type,
|
|
686
672
|
url: params.url,
|
|
687
|
-
os: resolvedOs,
|
|
673
|
+
os: resolvedOs,
|
|
688
674
|
reconnect: params.reconnect,
|
|
689
675
|
sandboxId: params.sandboxId,
|
|
690
676
|
});
|
|
691
677
|
|
|
692
678
|
try {
|
|
693
|
-
//
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
679
|
+
// The core owns session creation, SDK init, connect, provisioning and the
|
|
680
|
+
// initial screenshot. We keep the abort race + heartbeat scaffolding here:
|
|
681
|
+
// a heartbeat ticks while the long provisioning await is in flight, and the
|
|
682
|
+
// whole core call is raced against the client's abort signal so cancellation
|
|
683
|
+
// still returns promptly. Progress messages from the core are forwarded.
|
|
684
|
+
const stopHeartbeat = progress.heartbeat(
|
|
685
|
+
params.sandboxId
|
|
686
|
+
? `Connecting to existing sandbox ${params.sandboxId}...`
|
|
687
|
+
: "Starting session..."
|
|
688
|
+
);
|
|
689
|
+
let result: ActionResult;
|
|
690
|
+
try {
|
|
691
|
+
result = await raceAbort(
|
|
692
|
+
extra.signal,
|
|
693
|
+
"session_start",
|
|
694
|
+
core.sessionStart(
|
|
695
|
+
params,
|
|
696
|
+
{ os: resolvedOs, e2bTemplateId: resolvedE2bTemplateId },
|
|
697
|
+
{ onProgress: (m) => progress.report(m) }
|
|
698
|
+
)
|
|
699
|
+
);
|
|
700
|
+
} finally {
|
|
701
|
+
stopHeartbeat();
|
|
701
702
|
}
|
|
702
703
|
|
|
703
|
-
|
|
704
|
-
const newSession = sessionManager.createSession({
|
|
705
|
-
os: resolvedOs,
|
|
706
|
-
keepAlive: params.keepAlive,
|
|
707
|
-
testFile: params.testFile,
|
|
708
|
-
});
|
|
709
|
-
logger.debug("session_start: Session created", { sessionId: newSession.sessionId });
|
|
710
|
-
|
|
711
|
-
// Determine API root
|
|
712
|
-
const apiRoot = params.apiRoot || process.env.TD_API_ROOT || "https://api.testdriver.ai";
|
|
713
|
-
logger.debug("session_start: Using API root", { apiRoot });
|
|
704
|
+
const duration = Date.now() - startTime;
|
|
714
705
|
|
|
715
|
-
//
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
// Determine preview mode from environment variable
|
|
720
|
-
// TD_PREVIEW can be "ide", "browser", or "none"
|
|
721
|
-
// Default to "ide" so the live preview shows within the IDE
|
|
722
|
-
const previewMode = process.env.TD_PREVIEW || "ide";
|
|
723
|
-
logger.debug("session_start: Preview mode", { preview: previewMode });
|
|
724
|
-
|
|
725
|
-
// Get IP from params or environment (for self-hosted instances)
|
|
726
|
-
const instanceIp = params.ip || process.env.TD_IP;
|
|
727
|
-
|
|
728
|
-
// Get API key - check multiple sources for GitHub Copilot coding agent compatibility
|
|
729
|
-
// 1. TD_API_KEY (standard environment variable)
|
|
730
|
-
// 2. COPILOT_MCP_TD_API_KEY (fallback for GitHub Copilot coding agent)
|
|
731
|
-
const apiKey = process.env.TD_API_KEY || process.env.COPILOT_MCP_TD_API_KEY || "";
|
|
732
|
-
|
|
733
|
-
if (!apiKey) {
|
|
734
|
-
logger.error("session_start: No API key found", {
|
|
735
|
-
hasTD_API_KEY: !!process.env.TD_API_KEY,
|
|
736
|
-
hasCOPILOT_MCP_TD_API_KEY: !!process.env.COPILOT_MCP_TD_API_KEY,
|
|
737
|
-
availableEnvVars: Object.keys(process.env).filter(k => k.includes('TD') || k.includes('COPILOT_MCP'))
|
|
738
|
-
});
|
|
739
|
-
return createToolResult(false, "No API key found. Please set TD_API_KEY or COPILOT_MCP_TD_API_KEY environment variable.", {
|
|
740
|
-
error: "Missing API key",
|
|
741
|
-
hint: "For GitHub Copilot coding agent, create a Copilot environment secret named COPILOT_MCP_TD_API_KEY"
|
|
742
|
-
});
|
|
706
|
+
// Set Sentry context once the session id is known.
|
|
707
|
+
if (result.ok && typeof result.data.sessionId === "string") {
|
|
708
|
+
setSessionContext(result.data.sessionId, (result.data.sandboxId as string) || undefined);
|
|
743
709
|
}
|
|
744
|
-
|
|
745
|
-
logger.debug("session_start: API key found", {
|
|
746
|
-
source: process.env.TD_API_KEY ? "TD_API_KEY" : "COPILOT_MCP_TD_API_KEY",
|
|
747
|
-
keyPrefix: apiKey.substring(0, 7) + "..."
|
|
748
|
-
});
|
|
749
|
-
|
|
750
|
-
sdk = new TestDriverSDK(apiKey, {
|
|
751
|
-
os: resolvedOs,
|
|
752
|
-
logging: false,
|
|
753
|
-
apiRoot,
|
|
754
|
-
preview: previewMode as "browser" | "ide" | "none",
|
|
755
|
-
ip: instanceIp,
|
|
756
|
-
e2bTemplateId: resolvedE2bTemplateId,
|
|
757
|
-
});
|
|
758
710
|
|
|
759
|
-
|
|
760
|
-
if (params.sandboxId) {
|
|
761
|
-
logger.info("session_start: Connecting to existing sandbox (debug mode)", { sandboxId: params.sandboxId });
|
|
762
|
-
const stopHeartbeat = progress.heartbeat(`Connecting to existing sandbox ${params.sandboxId}...`);
|
|
763
|
-
try {
|
|
764
|
-
await raceAbort(extra.signal, "session_start", sdk.connect({
|
|
765
|
-
sandboxId: params.sandboxId,
|
|
766
|
-
keepAlive: params.keepAlive,
|
|
767
|
-
}));
|
|
768
|
-
} finally {
|
|
769
|
-
stopHeartbeat();
|
|
770
|
-
}
|
|
771
|
-
|
|
772
|
-
// Get sandbox ID
|
|
773
|
-
const instance = sdk.getInstance();
|
|
774
|
-
logger.info("session_start: Connected to existing sandbox", { instanceId: instance?.instanceId });
|
|
775
|
-
sessionManager.activateSession(newSession.sessionId, instance?.instanceId || params.sandboxId);
|
|
776
|
-
|
|
777
|
-
// Set Sentry context for error tracking
|
|
778
|
-
setSessionContext(newSession.sessionId, instance?.instanceId);
|
|
779
|
-
|
|
780
|
-
// Capture screenshot of current state
|
|
781
|
-
logger.debug("session_start: Capturing screenshot of existing sandbox");
|
|
782
|
-
progress.report("Capturing screenshot...");
|
|
783
|
-
const screenshotBase64 = await raceAbort(extra.signal, "session_start", sdk.agent.system.captureScreenBase64(1, false, true)) as string;
|
|
784
|
-
|
|
785
|
-
let screenshotResourceUri: string | undefined;
|
|
786
|
-
if (screenshotBase64) {
|
|
787
|
-
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
788
|
-
lastScreenshotBase64 = screenshotBase64;
|
|
789
|
-
}
|
|
711
|
+
logger.info("session_start: Completed", { duration, ok: result.ok });
|
|
790
712
|
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
return createToolResult(
|
|
795
|
-
true,
|
|
796
|
-
`Connected to existing sandbox (debug mode)
|
|
797
|
-
Session: ${newSession.sessionId}
|
|
798
|
-
Sandbox: ${params.sandboxId}
|
|
799
|
-
Expires in: ${Math.round(params.keepAlive / 1000)}s
|
|
800
|
-
|
|
801
|
-
You are now connected to the sandbox in its current state. Use find, click, type, etc. to interact.`,
|
|
802
|
-
{
|
|
803
|
-
action: "session_start",
|
|
804
|
-
sessionId: newSession.sessionId,
|
|
805
|
-
sandboxId: params.sandboxId,
|
|
806
|
-
debugMode: true,
|
|
807
|
-
screenshotResourceUri,
|
|
808
|
-
duration
|
|
809
|
-
},
|
|
810
|
-
"// Connected to existing sandbox - no provision code needed"
|
|
811
|
-
);
|
|
713
|
+
if (!result.ok) {
|
|
714
|
+
// Validation / missing-key failures: pass through with duration added.
|
|
715
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
812
716
|
}
|
|
813
717
|
|
|
814
|
-
//
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
const stopHeartbeat = progress.heartbeat(
|
|
822
|
-
instanceIp ? `Connecting to self-hosted instance ${instanceIp}...` : "Connecting to cloud sandbox..."
|
|
823
|
-
);
|
|
824
|
-
try {
|
|
825
|
-
await raceAbort(extra.signal, "session_start", sdk.connect({
|
|
826
|
-
reconnect: params.reconnect,
|
|
827
|
-
keepAlive: params.keepAlive,
|
|
828
|
-
ip: instanceIp,
|
|
829
|
-
}));
|
|
830
|
-
} finally {
|
|
831
|
-
stopHeartbeat();
|
|
832
|
-
}
|
|
718
|
+
// Reproduce the server's exact success output (text + data) which differs
|
|
719
|
+
// from the core's neutral text. Images are mapped to resource URIs.
|
|
720
|
+
const data: Record<string, unknown> = { ...result.data, duration };
|
|
721
|
+
for (const img of result.images ?? []) {
|
|
722
|
+
const uri = storeImage(img.base64, img.kind);
|
|
723
|
+
if (img.kind === "cropped") data.croppedImageResourceUri = uri;
|
|
724
|
+
else data.screenshotResourceUri = uri;
|
|
833
725
|
}
|
|
834
726
|
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
setSessionContext(newSession.sessionId, instance?.instanceId);
|
|
842
|
-
|
|
843
|
-
// Get provision-specific options
|
|
844
|
-
const provisionOptions = getProvisionOptions(params);
|
|
845
|
-
let provisionCmd = "";
|
|
846
|
-
|
|
847
|
-
// Provisioning can take tens of seconds (downloading installers, booting
|
|
848
|
-
// apps). Heartbeat so the client's idle timeout keeps resetting.
|
|
849
|
-
const stopProvisionHeartbeat = progress.heartbeat(`Provisioning ${params.type}...`);
|
|
850
|
-
try {
|
|
851
|
-
// Provision based on type
|
|
852
|
-
switch (params.type) {
|
|
853
|
-
case "chrome": {
|
|
854
|
-
const chromeOpts = provisionOptions as { url: string; maximized?: boolean; guest?: boolean };
|
|
855
|
-
logger.info("session_start: Provisioning Chrome", { url: chromeOpts.url });
|
|
856
|
-
await raceAbort(extra.signal, "session_start", sdk.provision.chrome(chromeOpts));
|
|
857
|
-
provisionCmd = "provision.chrome";
|
|
858
|
-
logger.debug("session_start: Chrome provisioned");
|
|
859
|
-
break;
|
|
860
|
-
}
|
|
861
|
-
|
|
862
|
-
case "chromeExtension": {
|
|
863
|
-
const extOpts = provisionOptions as { extensionPath?: string; extensionId?: string; maximized?: boolean };
|
|
864
|
-
logger.info("session_start: Provisioning Chrome Extension", { extensionPath: extOpts.extensionPath, extensionId: extOpts.extensionId });
|
|
865
|
-
await raceAbort(extra.signal, "session_start", sdk.provision.chromeExtension(extOpts));
|
|
866
|
-
provisionCmd = "provision.chromeExtension";
|
|
867
|
-
logger.debug("session_start: Chrome Extension provisioned");
|
|
868
|
-
break;
|
|
869
|
-
}
|
|
870
|
-
|
|
871
|
-
case "vscode": {
|
|
872
|
-
const vscodeOpts = provisionOptions as { workspace?: string; extensions?: string[] };
|
|
873
|
-
logger.info("session_start: Provisioning VS Code", { workspace: vscodeOpts.workspace });
|
|
874
|
-
await raceAbort(extra.signal, "session_start", sdk.provision.vscode(vscodeOpts));
|
|
875
|
-
provisionCmd = "provision.vscode";
|
|
876
|
-
logger.debug("session_start: VS Code provisioned");
|
|
877
|
-
break;
|
|
878
|
-
}
|
|
879
|
-
|
|
880
|
-
case "installer": {
|
|
881
|
-
const installerOpts = provisionOptions as { url: string; filename?: string; appName?: string; launch?: boolean };
|
|
882
|
-
logger.info("session_start: Provisioning installer", { url: installerOpts.url });
|
|
883
|
-
await raceAbort(extra.signal, "session_start", sdk.provision.installer(installerOpts));
|
|
884
|
-
provisionCmd = "provision.installer";
|
|
885
|
-
logger.debug("session_start: Installer provisioned");
|
|
886
|
-
break;
|
|
887
|
-
}
|
|
888
|
-
|
|
889
|
-
case "electron": {
|
|
890
|
-
const electronOpts = provisionOptions as { appPath: string; args?: string[] };
|
|
891
|
-
logger.info("session_start: Provisioning Electron", { appPath: electronOpts.appPath });
|
|
892
|
-
await raceAbort(extra.signal, "session_start", sdk.provision.electron(electronOpts));
|
|
893
|
-
provisionCmd = "provision.electron";
|
|
894
|
-
logger.debug("session_start: Electron app provisioned");
|
|
895
|
-
break;
|
|
896
|
-
}
|
|
897
|
-
}
|
|
898
|
-
} finally {
|
|
899
|
-
stopProvisionHeartbeat();
|
|
900
|
-
}
|
|
727
|
+
if (result.data.debugMode) {
|
|
728
|
+
// Debug (existing-sandbox) success text — preserve original wording.
|
|
729
|
+
const text = `Connected to existing sandbox (debug mode)
|
|
730
|
+
Session: ${result.data.sessionId}
|
|
731
|
+
Sandbox: ${result.data.sandboxId}
|
|
732
|
+
Expires in: ${Math.round(params.keepAlive / 1000)}s
|
|
901
733
|
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
progress.report("Capturing screenshot...");
|
|
905
|
-
const screenshotBase64 = await raceAbort(extra.signal, "session_start", sdk.agent.system.captureScreenBase64(1, false, true)) as string;
|
|
906
|
-
|
|
907
|
-
let screenshotResourceUri: string | undefined;
|
|
908
|
-
if (screenshotBase64) {
|
|
909
|
-
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
910
|
-
lastScreenshotBase64 = screenshotBase64;
|
|
734
|
+
You are now connected to the sandbox in its current state. Use find, click, type, etc. to interact.`;
|
|
735
|
+
return createToolResult(true, text, data, result.code);
|
|
911
736
|
}
|
|
912
737
|
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
// Generate the code for this provision action
|
|
917
|
-
const generatedCode = generateActionCode(provisionCmd, provisionOptions);
|
|
918
|
-
|
|
919
|
-
// Build debugger URL for the session
|
|
920
|
-
const debuggerUrl = instance?.debuggerUrl || (instanceIp ? `http://${instanceIp}:9222` : null);
|
|
921
|
-
|
|
922
|
-
const connectionType = instanceIp ? `Self-hosted (${instanceIp})` : "Cloud";
|
|
923
|
-
return createToolResult(
|
|
924
|
-
true,
|
|
925
|
-
`Session started: ${newSession.sessionId}\nConnection: ${connectionType}\nType: ${params.type}\nSandbox: ${instance?.instanceId}\nExpires in: ${Math.round(params.keepAlive / 1000)}s
|
|
738
|
+
// Normal provisioning success — append the EXACT dependency guidance block.
|
|
739
|
+
const text = `${result.text}
|
|
926
740
|
|
|
927
741
|
IMPORTANT - If creating a new test project, use these EXACT dependencies in package.json:
|
|
928
742
|
{
|
|
@@ -934,19 +748,8 @@ IMPORTANT - If creating a new test project, use these EXACT dependencies in pack
|
|
|
934
748
|
"scripts": {
|
|
935
749
|
"test": "vitest"
|
|
936
750
|
}
|
|
937
|
-
}
|
|
938
|
-
|
|
939
|
-
action: "session_start",
|
|
940
|
-
sessionId: newSession.sessionId,
|
|
941
|
-
provisionType: params.type,
|
|
942
|
-
selfHosted: !!instanceIp,
|
|
943
|
-
instanceIp: instanceIp || undefined,
|
|
944
|
-
debuggerUrl,
|
|
945
|
-
screenshotResourceUri,
|
|
946
|
-
duration
|
|
947
|
-
},
|
|
948
|
-
generatedCode
|
|
949
|
-
);
|
|
751
|
+
}`;
|
|
752
|
+
return createToolResult(true, text, data, result.code);
|
|
950
753
|
} catch (error) {
|
|
951
754
|
// On client cancellation, tear down the half-provisioned session so we
|
|
952
755
|
// don't leak a connected sandbox. The underlying SDK call may still be
|
|
@@ -954,7 +757,7 @@ IMPORTANT - If creating a new test project, use these EXACT dependencies in pack
|
|
|
954
757
|
if (error instanceof ToolAbortError) {
|
|
955
758
|
logger.info("session_start: Cancelled by client, tearing down session");
|
|
956
759
|
try {
|
|
957
|
-
await
|
|
760
|
+
await core.disconnect();
|
|
958
761
|
} catch (cleanupErr) {
|
|
959
762
|
logger.warn("session_start: Cleanup after cancel failed", { error: String(cleanupErr) });
|
|
960
763
|
}
|
|
@@ -977,27 +780,17 @@ server.registerTool(
|
|
|
977
780
|
async (): Promise<CallToolResult> => {
|
|
978
781
|
const startTime = Date.now();
|
|
979
782
|
logger.info("session_status: Checking");
|
|
980
|
-
const session = sessionManager.getCurrentSession();
|
|
981
783
|
|
|
982
|
-
|
|
983
|
-
logger.warn("session_status: No active session");
|
|
984
|
-
return createToolResult(false, "No active session", { error: "No active session. Call session_start first." });
|
|
985
|
-
}
|
|
986
|
-
|
|
987
|
-
const summary = sessionManager.getSessionSummary(session.sessionId);
|
|
784
|
+
const result = core.sessionStatus();
|
|
988
785
|
const duration = Date.now() - startTime;
|
|
989
|
-
logger.info("session_status: Completed", {
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
duration
|
|
786
|
+
logger.info("session_status: Completed", {
|
|
787
|
+
ok: result.ok,
|
|
788
|
+
sessionId: result.data.sessionId,
|
|
789
|
+
status: result.data.status,
|
|
790
|
+
duration,
|
|
994
791
|
});
|
|
995
792
|
|
|
996
|
-
return
|
|
997
|
-
true,
|
|
998
|
-
`Session: ${session.sessionId}\nStatus: ${session.status}\nTime remaining: ${Math.round((summary?.timeRemaining || 0) / 1000)}s`,
|
|
999
|
-
{ action: "session_status", ...summary, sessionId: session.sessionId, status: session.status, duration }
|
|
1000
|
-
);
|
|
793
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1001
794
|
}
|
|
1002
795
|
);
|
|
1003
796
|
|
|
@@ -1012,22 +805,22 @@ server.registerTool(
|
|
|
1012
805
|
},
|
|
1013
806
|
async (params) => {
|
|
1014
807
|
logger.info("session_extend: Extending", { additionalMs: params.additionalMs });
|
|
1015
|
-
const session = sessionManager.getCurrentSession();
|
|
1016
808
|
|
|
1017
|
-
|
|
809
|
+
const result = core.sessionExtend(params.additionalMs);
|
|
810
|
+
|
|
811
|
+
if (!result.ok) {
|
|
1018
812
|
logger.warn("session_extend: No active session");
|
|
1019
813
|
return { content: [{ type: "text" as const, text: "No active session" }] };
|
|
1020
814
|
}
|
|
1021
815
|
|
|
1022
|
-
|
|
1023
|
-
const newExpiry = sessionManager.getTimeRemaining(session.sessionId);
|
|
1024
|
-
logger.info("session_extend: Extended", { sessionId: session.sessionId, newExpiry });
|
|
816
|
+
logger.info("session_extend: Extended", { newExpiry: result.data.newExpiry });
|
|
1025
817
|
|
|
818
|
+
// Preserve the original plain content shape (not via createToolResult).
|
|
1026
819
|
return {
|
|
1027
820
|
content: [
|
|
1028
821
|
{
|
|
1029
822
|
type: "text" as const,
|
|
1030
|
-
text:
|
|
823
|
+
text: result.text,
|
|
1031
824
|
},
|
|
1032
825
|
],
|
|
1033
826
|
};
|
|
@@ -1052,113 +845,25 @@ registerAppTool(
|
|
|
1052
845
|
const progress = makeProgressReporter(extra);
|
|
1053
846
|
logger.info("find: Starting", { description: params.description, timeout: params.timeout });
|
|
1054
847
|
|
|
1055
|
-
const sessionCheck = requireActiveSession();
|
|
1056
|
-
if (!sessionCheck.valid) {
|
|
1057
|
-
logger.warn("find: No active session");
|
|
1058
|
-
return sessionCheck.error;
|
|
1059
|
-
}
|
|
1060
|
-
|
|
1061
848
|
try {
|
|
1062
849
|
logger.debug("find: Calling SDK find");
|
|
1063
850
|
const stopHeartbeat = progress.heartbeat(`Looking for "${params.description}"...`);
|
|
1064
|
-
let
|
|
851
|
+
let result: ActionResult;
|
|
1065
852
|
try {
|
|
1066
|
-
|
|
1067
|
-
extra.signal,
|
|
1068
|
-
"find",
|
|
1069
|
-
sdk.find(params.description, params.timeout ? { timeout: params.timeout } : undefined)
|
|
1070
|
-
);
|
|
853
|
+
result = await raceAbort(extra.signal, "find", core.find(params.description, params.timeout));
|
|
1071
854
|
} finally {
|
|
1072
855
|
stopHeartbeat();
|
|
1073
856
|
}
|
|
1074
|
-
const found = element.found();
|
|
1075
|
-
const coords = element.getCoordinates();
|
|
1076
|
-
|
|
1077
|
-
// Store element ref for later use (stores actual Element instance)
|
|
1078
|
-
const elementRef = `el-${Date.now()}`;
|
|
1079
|
-
if (found && coords) {
|
|
1080
|
-
elementRefs.set(elementRef, {
|
|
1081
|
-
element: element, // Store the actual Element instance
|
|
1082
|
-
description: params.description,
|
|
1083
|
-
coords: {
|
|
1084
|
-
x: coords.x,
|
|
1085
|
-
y: coords.y,
|
|
1086
|
-
centerX: coords.centerX,
|
|
1087
|
-
centerY: coords.centerY,
|
|
1088
|
-
},
|
|
1089
|
-
});
|
|
1090
|
-
logger.info("find: Element found", {
|
|
1091
|
-
description: params.description,
|
|
1092
|
-
coords: { x: coords.centerX, y: coords.centerY },
|
|
1093
|
-
confidence: element.confidence,
|
|
1094
|
-
elementRef
|
|
1095
|
-
});
|
|
1096
|
-
} else {
|
|
1097
|
-
logger.warn("find: Element not found", { description: params.description });
|
|
1098
|
-
}
|
|
1099
857
|
|
|
1100
|
-
// Return raw SDK response directly
|
|
1101
|
-
const rawResponse = element._response || {};
|
|
1102
858
|
const duration = Date.now() - startTime;
|
|
1103
|
-
|
|
1104
|
-
// Store cropped image for resource serving (instead of inline data URL)
|
|
1105
|
-
let croppedImageResourceUri: string | undefined;
|
|
1106
|
-
let screenshotResourceUri: string | undefined;
|
|
1107
|
-
const croppedImage = rawResponse.croppedImage;
|
|
1108
|
-
if (croppedImage) {
|
|
1109
|
-
const imageData = croppedImage.startsWith('data:')
|
|
1110
|
-
? croppedImage.replace(/^data:image\/\w+;base64,/, '')
|
|
1111
|
-
: croppedImage;
|
|
1112
|
-
croppedImageResourceUri = storeImage(imageData, "cropped");
|
|
1113
|
-
// Remove croppedImage from response to avoid context bloat
|
|
1114
|
-
delete rawResponse.croppedImage;
|
|
1115
|
-
} else if (!found) {
|
|
1116
|
-
// Element not found and no cropped image - capture a fresh screenshot
|
|
1117
|
-
// so the user can see what's currently visible on screen
|
|
1118
|
-
try {
|
|
1119
|
-
const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
1120
|
-
if (screenshotBase64) {
|
|
1121
|
-
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
1122
|
-
logger.debug("find: Captured screenshot for not-found state");
|
|
1123
|
-
}
|
|
1124
|
-
} catch (e) {
|
|
1125
|
-
logger.warn("find: Failed to capture screenshot for not-found state", { error: String(e) });
|
|
1126
|
-
}
|
|
1127
|
-
}
|
|
1128
|
-
|
|
1129
|
-
// Remove extractedText and pixelDiffImage from response to reduce context bloat
|
|
1130
|
-
delete rawResponse.extractedText;
|
|
1131
|
-
delete rawResponse.pixelDiffImage;
|
|
1132
|
-
|
|
1133
|
-
// Generate code for this find action
|
|
1134
|
-
const generatedCode = found ? generateActionCode("find", { description: params.description }) : undefined;
|
|
1135
|
-
|
|
1136
|
-
// Build element info for display (cropped image is always centered on element)
|
|
1137
|
-
const elementInfo = found ? {
|
|
1138
|
-
description: params.description,
|
|
1139
|
-
centerX: coords?.centerX,
|
|
1140
|
-
centerY: coords?.centerY,
|
|
1141
|
-
confidence: element.confidence,
|
|
1142
|
-
ref: elementRef,
|
|
1143
|
-
} : undefined;
|
|
859
|
+
logger.info("find: Completed", { description: params.description, found: result.ok, duration });
|
|
1144
860
|
|
|
1145
|
-
return
|
|
1146
|
-
found,
|
|
1147
|
-
found
|
|
1148
|
-
? `Found: "${params.description}" at (${rawResponse.coordinates?.x}, ${rawResponse.coordinates?.y})\nRef: ${elementRef}`
|
|
1149
|
-
: `Element not found: "${params.description}"`,
|
|
1150
|
-
{
|
|
1151
|
-
...rawResponse,
|
|
1152
|
-
action: "find",
|
|
1153
|
-
element: elementInfo,
|
|
1154
|
-
ref: elementRef,
|
|
1155
|
-
croppedImageResourceUri,
|
|
1156
|
-
screenshotResourceUri,
|
|
1157
|
-
duration,
|
|
1158
|
-
},
|
|
1159
|
-
generatedCode
|
|
1160
|
-
);
|
|
861
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1161
862
|
} catch (error) {
|
|
863
|
+
if (error instanceof NoActiveSessionError) {
|
|
864
|
+
logger.warn("find: No active session");
|
|
865
|
+
return noSessionResult(error);
|
|
866
|
+
}
|
|
1162
867
|
const cancelled = cancelledResultOrNull(error, "find");
|
|
1163
868
|
if (cancelled) return cancelled;
|
|
1164
869
|
logger.error("find: Failed", { error: String(error), description: params.description });
|
|
@@ -1186,121 +891,25 @@ registerAppTool(
|
|
|
1186
891
|
const progress = makeProgressReporter(extra);
|
|
1187
892
|
logger.info("findall: Starting", { description: params.description, timeout: params.timeout });
|
|
1188
893
|
|
|
1189
|
-
const sessionCheck = requireActiveSession();
|
|
1190
|
-
if (!sessionCheck.valid) {
|
|
1191
|
-
logger.warn("findall: No active session");
|
|
1192
|
-
return sessionCheck.error;
|
|
1193
|
-
}
|
|
1194
|
-
|
|
1195
894
|
try {
|
|
1196
895
|
logger.debug("findall: Calling SDK findAll");
|
|
1197
896
|
const stopHeartbeat = progress.heartbeat(`Looking for all "${params.description}"...`);
|
|
1198
|
-
let
|
|
897
|
+
let result: ActionResult;
|
|
1199
898
|
try {
|
|
1200
|
-
|
|
1201
|
-
extra.signal,
|
|
1202
|
-
"findall",
|
|
1203
|
-
sdk.findAll(params.description, params.timeout ? { timeout: params.timeout } : undefined)
|
|
1204
|
-
);
|
|
899
|
+
result = await raceAbort(extra.signal, "findall", core.findAll(params.description, params.timeout));
|
|
1205
900
|
} finally {
|
|
1206
901
|
stopHeartbeat();
|
|
1207
902
|
}
|
|
1208
|
-
const count = elements.length;
|
|
1209
903
|
|
|
1210
|
-
// Store element refs for later use
|
|
1211
|
-
const refs: string[] = [];
|
|
1212
|
-
const elementInfos: Array<{ ref: string; x: number; y: number; centerX: number; centerY: number; confidence: number }> = [];
|
|
1213
|
-
|
|
1214
|
-
for (let i = 0; i < elements.length; i++) {
|
|
1215
|
-
const element = elements[i];
|
|
1216
|
-
const coords = element.getCoordinates();
|
|
1217
|
-
const elementRef = `el-${Date.now()}-${i}`;
|
|
1218
|
-
|
|
1219
|
-
if (coords) {
|
|
1220
|
-
elementRefs.set(elementRef, {
|
|
1221
|
-
element: element,
|
|
1222
|
-
description: `${params.description} [${i}]`,
|
|
1223
|
-
coords: {
|
|
1224
|
-
x: coords.x,
|
|
1225
|
-
y: coords.y,
|
|
1226
|
-
centerX: coords.centerX,
|
|
1227
|
-
centerY: coords.centerY,
|
|
1228
|
-
},
|
|
1229
|
-
});
|
|
1230
|
-
refs.push(elementRef);
|
|
1231
|
-
elementInfos.push({
|
|
1232
|
-
ref: elementRef,
|
|
1233
|
-
x: coords.x,
|
|
1234
|
-
y: coords.y,
|
|
1235
|
-
centerX: coords.centerX,
|
|
1236
|
-
centerY: coords.centerY,
|
|
1237
|
-
confidence: element.confidence,
|
|
1238
|
-
});
|
|
1239
|
-
}
|
|
1240
|
-
}
|
|
1241
|
-
|
|
1242
|
-
logger.info("findall: Elements found", {
|
|
1243
|
-
description: params.description,
|
|
1244
|
-
count,
|
|
1245
|
-
refs
|
|
1246
|
-
});
|
|
1247
|
-
|
|
1248
|
-
// Get the first element's response for the image (shows all highlights)
|
|
1249
|
-
const rawResponse = elements[0]?._response || {};
|
|
1250
904
|
const duration = Date.now() - startTime;
|
|
1251
|
-
|
|
1252
|
-
// Store cropped image for resource serving (instead of inline data URL)
|
|
1253
|
-
let croppedImageResourceUri: string | undefined;
|
|
1254
|
-
let screenshotResourceUri: string | undefined;
|
|
1255
|
-
const croppedImage = rawResponse.croppedImage;
|
|
1256
|
-
if (croppedImage) {
|
|
1257
|
-
const imageData = croppedImage.startsWith('data:')
|
|
1258
|
-
? croppedImage.replace(/^data:image\/\w+;base64,/, '')
|
|
1259
|
-
: croppedImage;
|
|
1260
|
-
croppedImageResourceUri = storeImage(imageData, "cropped");
|
|
1261
|
-
// Remove croppedImage from response to avoid context bloat
|
|
1262
|
-
delete rawResponse.croppedImage;
|
|
1263
|
-
} else if (count === 0) {
|
|
1264
|
-
// No elements found and no cropped image - capture a fresh screenshot
|
|
1265
|
-
// so the user can see what's currently visible on screen
|
|
1266
|
-
try {
|
|
1267
|
-
const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
1268
|
-
if (screenshotBase64) {
|
|
1269
|
-
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
1270
|
-
logger.debug("findall: Captured screenshot for not-found state");
|
|
1271
|
-
}
|
|
1272
|
-
} catch (e) {
|
|
1273
|
-
logger.warn("findall: Failed to capture screenshot for not-found state", { error: String(e) });
|
|
1274
|
-
}
|
|
1275
|
-
}
|
|
1276
|
-
|
|
1277
|
-
// Remove extractedText and pixelDiffImage from response to reduce context bloat
|
|
1278
|
-
delete rawResponse.extractedText;
|
|
1279
|
-
delete rawResponse.pixelDiffImage;
|
|
1280
|
-
|
|
1281
|
-
// Generate code for this findall action
|
|
1282
|
-
const generatedCode = count > 0 ? generateActionCode("findall", { description: params.description }) : undefined;
|
|
905
|
+
logger.info("findall: Completed", { description: params.description, count: result.data.count, duration });
|
|
1283
906
|
|
|
1284
|
-
|
|
1285
|
-
const refsList = refs.map((ref, i) => ` [${i}] ${ref}`).join('\n');
|
|
1286
|
-
|
|
1287
|
-
return createToolResult(
|
|
1288
|
-
count > 0,
|
|
1289
|
-
count > 0
|
|
1290
|
-
? `Found ${count} elements matching "${params.description}":\n${refsList}`
|
|
1291
|
-
: `No elements found matching: "${params.description}"`,
|
|
1292
|
-
{
|
|
1293
|
-
...rawResponse,
|
|
1294
|
-
count,
|
|
1295
|
-
refs,
|
|
1296
|
-
elements: elementInfos,
|
|
1297
|
-
croppedImageResourceUri,
|
|
1298
|
-
screenshotResourceUri,
|
|
1299
|
-
duration,
|
|
1300
|
-
},
|
|
1301
|
-
generatedCode
|
|
1302
|
-
);
|
|
907
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1303
908
|
} catch (error) {
|
|
909
|
+
if (error instanceof NoActiveSessionError) {
|
|
910
|
+
logger.warn("findall: No active session");
|
|
911
|
+
return noSessionResult(error);
|
|
912
|
+
}
|
|
1304
913
|
const cancelled = cancelledResultOrNull(error, "findall");
|
|
1305
914
|
if (cancelled) return cancelled;
|
|
1306
915
|
logger.error("findall: Failed", { error: String(error), description: params.description });
|
|
@@ -1327,70 +936,20 @@ registerAppTool(
|
|
|
1327
936
|
const startTime = Date.now();
|
|
1328
937
|
logger.info("click: Starting", { elementRef: params.elementRef, action: params.action });
|
|
1329
938
|
|
|
1330
|
-
const sessionCheck = requireActiveSession();
|
|
1331
|
-
if (!sessionCheck.valid) {
|
|
1332
|
-
logger.warn("click: No active session");
|
|
1333
|
-
return sessionCheck.error;
|
|
1334
|
-
}
|
|
1335
|
-
|
|
1336
|
-
// Look up the element reference
|
|
1337
|
-
const ref = elementRefs.get(params.elementRef);
|
|
1338
|
-
if (!ref) {
|
|
1339
|
-
logger.warn("click: Element reference not found", { elementRef: params.elementRef });
|
|
1340
|
-
return createToolResult(false, `Element reference "${params.elementRef}" not found. Use 'find' first to locate the element.`, { error: "Element reference not found" });
|
|
1341
|
-
}
|
|
1342
|
-
|
|
1343
|
-
const { element, description, coords } = ref;
|
|
1344
|
-
|
|
1345
939
|
try {
|
|
1346
|
-
logger.debug("click: Executing click on element", {
|
|
1347
|
-
|
|
1348
|
-
// Use the Element's click method instead of raw coordinates
|
|
1349
|
-
if (params.action === "click") {
|
|
1350
|
-
await element.click();
|
|
1351
|
-
} else if (params.action === "double-click") {
|
|
1352
|
-
await element.doubleClick();
|
|
1353
|
-
} else if (params.action === "right-click") {
|
|
1354
|
-
await element.rightClick();
|
|
1355
|
-
}
|
|
940
|
+
logger.debug("click: Executing click on element", { elementRef: params.elementRef, action: params.action });
|
|
941
|
+
const result = await core.click(params.elementRef, params.action);
|
|
1356
942
|
|
|
1357
|
-
// Capture screenshot after click to show result
|
|
1358
|
-
logger.debug("click: Capturing screenshot after click");
|
|
1359
|
-
const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
1360
|
-
|
|
1361
|
-
let screenshotResourceUri: string | undefined;
|
|
1362
|
-
if (screenshotBase64) {
|
|
1363
|
-
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
1364
|
-
lastScreenshotBase64 = screenshotBase64;
|
|
1365
|
-
}
|
|
1366
|
-
|
|
1367
|
-
const rawResponse = element._response || {};
|
|
1368
|
-
// Remove large data from response to reduce context bloat
|
|
1369
|
-
delete rawResponse.croppedImage;
|
|
1370
|
-
delete rawResponse.extractedText;
|
|
1371
|
-
delete rawResponse.pixelDiffImage;
|
|
1372
|
-
|
|
1373
943
|
const duration = Date.now() - startTime;
|
|
1374
|
-
logger.info("click: Completed", {
|
|
1375
|
-
|
|
1376
|
-
// Generate code for this click action
|
|
1377
|
-
const generatedCode = generateActionCode("click", { action: params.action });
|
|
944
|
+
logger.info("click: Completed", { elementRef: params.elementRef, duration });
|
|
1378
945
|
|
|
1379
|
-
return
|
|
1380
|
-
true,
|
|
1381
|
-
`Clicked on "${description}"`,
|
|
1382
|
-
{
|
|
1383
|
-
...rawResponse,
|
|
1384
|
-
action: "click",
|
|
1385
|
-
clickAction: params.action,
|
|
1386
|
-
clickPosition: coords,
|
|
1387
|
-
screenshotResourceUri,
|
|
1388
|
-
duration
|
|
1389
|
-
},
|
|
1390
|
-
generatedCode
|
|
1391
|
-
);
|
|
946
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1392
947
|
} catch (error) {
|
|
1393
|
-
|
|
948
|
+
if (error instanceof NoActiveSessionError) {
|
|
949
|
+
logger.warn("click: No active session");
|
|
950
|
+
return noSessionResult(error);
|
|
951
|
+
}
|
|
952
|
+
logger.error("click: Failed", { error: String(error), elementRef: params.elementRef });
|
|
1394
953
|
captureException(error as Error, { tags: { tool: "click" }, extra: { elementRef: params.elementRef, action: params.action } });
|
|
1395
954
|
throw error;
|
|
1396
955
|
}
|
|
@@ -1413,60 +972,20 @@ registerAppTool(
|
|
|
1413
972
|
const startTime = Date.now();
|
|
1414
973
|
logger.info("hover: Starting", { elementRef: params.elementRef });
|
|
1415
974
|
|
|
1416
|
-
const sessionCheck = requireActiveSession();
|
|
1417
|
-
if (!sessionCheck.valid) {
|
|
1418
|
-
logger.warn("hover: No active session");
|
|
1419
|
-
return sessionCheck.error;
|
|
1420
|
-
}
|
|
1421
|
-
|
|
1422
|
-
// Look up the element reference
|
|
1423
|
-
const ref = elementRefs.get(params.elementRef);
|
|
1424
|
-
if (!ref) {
|
|
1425
|
-
logger.warn("hover: Element reference not found", { elementRef: params.elementRef });
|
|
1426
|
-
return createToolResult(false, `Element reference "${params.elementRef}" not found. Use 'find' first to locate the element.`, { error: "Element reference not found" });
|
|
1427
|
-
}
|
|
1428
|
-
|
|
1429
|
-
const { element, description, coords } = ref;
|
|
1430
|
-
|
|
1431
975
|
try {
|
|
1432
|
-
logger.debug("hover: Executing hover on element", {
|
|
1433
|
-
await
|
|
976
|
+
logger.debug("hover: Executing hover on element", { elementRef: params.elementRef });
|
|
977
|
+
const result = await core.hover(params.elementRef);
|
|
1434
978
|
|
|
1435
|
-
// Capture screenshot after hover to show result
|
|
1436
|
-
logger.debug("hover: Capturing screenshot after hover");
|
|
1437
|
-
const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
1438
|
-
|
|
1439
|
-
let screenshotResourceUri: string | undefined;
|
|
1440
|
-
if (screenshotBase64) {
|
|
1441
|
-
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
1442
|
-
lastScreenshotBase64 = screenshotBase64;
|
|
1443
|
-
}
|
|
1444
|
-
|
|
1445
|
-
const rawResponse = element._response || {};
|
|
1446
|
-
// Remove large data from response to reduce context bloat
|
|
1447
|
-
delete rawResponse.croppedImage;
|
|
1448
|
-
delete rawResponse.extractedText;
|
|
1449
|
-
delete rawResponse.pixelDiffImage;
|
|
1450
|
-
|
|
1451
979
|
const duration = Date.now() - startTime;
|
|
1452
|
-
logger.info("hover: Completed", {
|
|
980
|
+
logger.info("hover: Completed", { elementRef: params.elementRef, duration });
|
|
1453
981
|
|
|
1454
|
-
|
|
1455
|
-
const generatedCode = generateActionCode("hover", {});
|
|
1456
|
-
|
|
1457
|
-
return createToolResult(
|
|
1458
|
-
true,
|
|
1459
|
-
`Hovered over "${description}"`,
|
|
1460
|
-
{
|
|
1461
|
-
...rawResponse,
|
|
1462
|
-
action: "hover",
|
|
1463
|
-
screenshotResourceUri,
|
|
1464
|
-
duration
|
|
1465
|
-
},
|
|
1466
|
-
generatedCode
|
|
1467
|
-
);
|
|
982
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1468
983
|
} catch (error) {
|
|
1469
|
-
|
|
984
|
+
if (error instanceof NoActiveSessionError) {
|
|
985
|
+
logger.warn("hover: No active session");
|
|
986
|
+
return noSessionResult(error);
|
|
987
|
+
}
|
|
988
|
+
logger.error("hover: Failed", { error: String(error), elementRef: params.elementRef });
|
|
1470
989
|
captureException(error as Error, { tags: { tool: "hover" }, extra: { elementRef: params.elementRef } });
|
|
1471
990
|
throw error;
|
|
1472
991
|
}
|
|
@@ -1486,29 +1005,19 @@ server.registerTool(
|
|
|
1486
1005
|
const startTime = Date.now();
|
|
1487
1006
|
logger.info("wait: Starting", { timeout: params.timeout });
|
|
1488
1007
|
|
|
1489
|
-
const sessionCheck = requireActiveSession();
|
|
1490
|
-
if (!sessionCheck.valid) {
|
|
1491
|
-
logger.warn("wait: No active session");
|
|
1492
|
-
return sessionCheck.error;
|
|
1493
|
-
}
|
|
1494
|
-
|
|
1495
1008
|
try {
|
|
1496
1009
|
logger.debug("wait: Waiting", { timeout: params.timeout });
|
|
1497
|
-
await
|
|
1010
|
+
const result = await core.wait(params.timeout);
|
|
1498
1011
|
|
|
1499
1012
|
const duration = Date.now() - startTime;
|
|
1500
1013
|
logger.info("wait: Completed", { timeout: params.timeout, duration });
|
|
1501
1014
|
|
|
1502
|
-
|
|
1503
|
-
const generatedCode = generateActionCode("wait", { timeout: params.timeout });
|
|
1504
|
-
|
|
1505
|
-
return createToolResult(
|
|
1506
|
-
true,
|
|
1507
|
-
`Waited for ${params.timeout}ms`,
|
|
1508
|
-
{ action: "wait", timeout: params.timeout, duration },
|
|
1509
|
-
generatedCode
|
|
1510
|
-
);
|
|
1015
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1511
1016
|
} catch (error) {
|
|
1017
|
+
if (error instanceof NoActiveSessionError) {
|
|
1018
|
+
logger.warn("wait: No active session");
|
|
1019
|
+
return noSessionResult(error);
|
|
1020
|
+
}
|
|
1512
1021
|
logger.error("wait: Failed", { error: String(error) });
|
|
1513
1022
|
captureException(error as Error, { tags: { tool: "wait" }, extra: { timeout: params.timeout } });
|
|
1514
1023
|
throw error;
|
|
@@ -1529,29 +1038,19 @@ server.registerTool(
|
|
|
1529
1038
|
const startTime = Date.now();
|
|
1530
1039
|
logger.info("focus_application: Starting", { name: params.name });
|
|
1531
1040
|
|
|
1532
|
-
const sessionCheck = requireActiveSession();
|
|
1533
|
-
if (!sessionCheck.valid) {
|
|
1534
|
-
logger.warn("focus_application: No active session");
|
|
1535
|
-
return sessionCheck.error;
|
|
1536
|
-
}
|
|
1537
|
-
|
|
1538
1041
|
try {
|
|
1539
1042
|
logger.debug("focus_application: Focusing", { name: params.name });
|
|
1540
|
-
await
|
|
1043
|
+
const result = await core.focusApplication(params.name);
|
|
1541
1044
|
|
|
1542
1045
|
const duration = Date.now() - startTime;
|
|
1543
1046
|
logger.info("focus_application: Completed", { name: params.name, duration });
|
|
1544
1047
|
|
|
1545
|
-
|
|
1546
|
-
const generatedCode = generateActionCode("focus_application", { name: params.name });
|
|
1547
|
-
|
|
1548
|
-
return createToolResult(
|
|
1549
|
-
true,
|
|
1550
|
-
`Focused application: "${params.name}"`,
|
|
1551
|
-
{ action: "focus", name: params.name, duration },
|
|
1552
|
-
generatedCode
|
|
1553
|
-
);
|
|
1048
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1554
1049
|
} catch (error) {
|
|
1050
|
+
if (error instanceof NoActiveSessionError) {
|
|
1051
|
+
logger.warn("focus_application: No active session");
|
|
1052
|
+
return noSessionResult(error);
|
|
1053
|
+
}
|
|
1555
1054
|
logger.error("focus_application: Failed", { error: String(error), name: params.name });
|
|
1556
1055
|
captureException(error as Error, { tags: { tool: "focus_application" }, extra: { name: params.name } });
|
|
1557
1056
|
throw error;
|
|
@@ -1577,146 +1076,25 @@ registerAppTool(
|
|
|
1577
1076
|
const progress = makeProgressReporter(extra);
|
|
1578
1077
|
logger.info("find_and_click: Starting", { description: params.description, action: params.action });
|
|
1579
1078
|
|
|
1580
|
-
const sessionCheck = requireActiveSession();
|
|
1581
|
-
if (!sessionCheck.valid) {
|
|
1582
|
-
logger.warn("find_and_click: No active session");
|
|
1583
|
-
return sessionCheck.error;
|
|
1584
|
-
}
|
|
1585
|
-
|
|
1586
1079
|
try {
|
|
1587
1080
|
logger.debug("find_and_click: Finding element");
|
|
1588
1081
|
const stopHeartbeat = progress.heartbeat(`Looking for "${params.description}"...`);
|
|
1589
|
-
let
|
|
1082
|
+
let result: ActionResult;
|
|
1590
1083
|
try {
|
|
1591
|
-
|
|
1084
|
+
result = await raceAbort(extra.signal, "find_and_click", core.findAndClick(params.description, params.action));
|
|
1592
1085
|
} finally {
|
|
1593
1086
|
stopHeartbeat();
|
|
1594
1087
|
}
|
|
1595
|
-
const found = element.found();
|
|
1596
|
-
|
|
1597
|
-
if (!found) {
|
|
1598
|
-
logger.warn("find_and_click: Element not found", { description: params.description });
|
|
1599
|
-
|
|
1600
|
-
// Capture screenshot to show current state even when element not found
|
|
1601
|
-
const rawResponse = element._response || {};
|
|
1602
|
-
const duration = Date.now() - startTime;
|
|
1603
|
-
|
|
1604
|
-
// Store cropped image (screenshot) for resource serving
|
|
1605
|
-
let croppedImageResourceUri: string | undefined;
|
|
1606
|
-
let screenshotResourceUri: string | undefined;
|
|
1607
|
-
const croppedImage = rawResponse.croppedImage;
|
|
1608
|
-
if (croppedImage) {
|
|
1609
|
-
const imageData = croppedImage.startsWith('data:')
|
|
1610
|
-
? croppedImage.replace(/^data:image\/\w+;base64,/, '')
|
|
1611
|
-
: croppedImage;
|
|
1612
|
-
croppedImageResourceUri = storeImage(imageData, "screenshot");
|
|
1613
|
-
delete rawResponse.croppedImage;
|
|
1614
|
-
} else {
|
|
1615
|
-
// No cropped image - capture a fresh screenshot so the user can see
|
|
1616
|
-
// what's currently visible on screen when element was not found
|
|
1617
|
-
try {
|
|
1618
|
-
const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
1619
|
-
if (screenshotBase64) {
|
|
1620
|
-
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
1621
|
-
logger.debug("find_and_click: Captured screenshot for not-found state");
|
|
1622
|
-
}
|
|
1623
|
-
} catch (e) {
|
|
1624
|
-
logger.warn("find_and_click: Failed to capture screenshot for not-found state", { error: String(e) });
|
|
1625
|
-
}
|
|
1626
|
-
}
|
|
1627
|
-
|
|
1628
|
-
// Remove extractedText and pixelDiffImage from response to reduce context bloat
|
|
1629
|
-
delete rawResponse.extractedText;
|
|
1630
|
-
delete rawResponse.pixelDiffImage;
|
|
1631
|
-
|
|
1632
|
-
return createToolResult(
|
|
1633
|
-
false,
|
|
1634
|
-
`Element not found: "${params.description}"`,
|
|
1635
|
-
{
|
|
1636
|
-
...rawResponse,
|
|
1637
|
-
action: "find_and_click",
|
|
1638
|
-
error: "Element not found",
|
|
1639
|
-
croppedImageResourceUri,
|
|
1640
|
-
screenshotResourceUri,
|
|
1641
|
-
duration
|
|
1642
|
-
}
|
|
1643
|
-
);
|
|
1644
|
-
}
|
|
1645
|
-
|
|
1646
|
-
const coords = element.getCoordinates();
|
|
1647
|
-
|
|
1648
|
-
// Store element ref for later use (in case user wants to interact again)
|
|
1649
|
-
const elementRef = `el-${Date.now()}`;
|
|
1650
|
-
if (coords) {
|
|
1651
|
-
elementRefs.set(elementRef, {
|
|
1652
|
-
element: element,
|
|
1653
|
-
description: params.description,
|
|
1654
|
-
coords: {
|
|
1655
|
-
x: coords.x,
|
|
1656
|
-
y: coords.y,
|
|
1657
|
-
centerX: coords.centerX,
|
|
1658
|
-
centerY: coords.centerY,
|
|
1659
|
-
},
|
|
1660
|
-
});
|
|
1661
|
-
}
|
|
1662
|
-
|
|
1663
|
-
logger.debug("find_and_click: Element found, clicking", { action: params.action, elementRef });
|
|
1664
|
-
if (params.action === "click") {
|
|
1665
|
-
await element.click();
|
|
1666
|
-
} else if (params.action === "double-click") {
|
|
1667
|
-
await element.doubleClick();
|
|
1668
|
-
} else if (params.action === "right-click") {
|
|
1669
|
-
await element.rightClick();
|
|
1670
|
-
}
|
|
1671
1088
|
|
|
1672
|
-
// Return raw SDK response directly
|
|
1673
|
-
const rawResponse = element._response || {};
|
|
1674
1089
|
const duration = Date.now() - startTime;
|
|
1675
|
-
|
|
1676
|
-
// Store cropped image for resource serving (instead of inline data URL)
|
|
1677
|
-
let croppedImageResourceUri: string | undefined;
|
|
1678
|
-
const croppedImage = rawResponse.croppedImage;
|
|
1679
|
-
if (croppedImage) {
|
|
1680
|
-
const imageData = croppedImage.startsWith('data:')
|
|
1681
|
-
? croppedImage.replace(/^data:image\/\w+;base64,/, '')
|
|
1682
|
-
: croppedImage;
|
|
1683
|
-
croppedImageResourceUri = storeImage(imageData, "cropped");
|
|
1684
|
-
// Remove croppedImage from response to avoid context bloat
|
|
1685
|
-
delete rawResponse.croppedImage;
|
|
1686
|
-
}
|
|
1687
|
-
|
|
1688
|
-
// Remove extractedText and pixelDiffImage from response to reduce context bloat
|
|
1689
|
-
delete rawResponse.extractedText;
|
|
1690
|
-
delete rawResponse.pixelDiffImage;
|
|
1691
|
-
|
|
1692
|
-
// Generate code for this find_and_click action
|
|
1693
|
-
const generatedCode = generateActionCode("find_and_click", { description: params.description, action: params.action });
|
|
1694
|
-
|
|
1695
|
-
// Build element info for display (match find action format)
|
|
1696
|
-
const elementInfo = coords ? {
|
|
1697
|
-
description: params.description,
|
|
1698
|
-
centerX: coords.centerX,
|
|
1699
|
-
centerY: coords.centerY,
|
|
1700
|
-
confidence: element.confidence,
|
|
1701
|
-
ref: elementRef,
|
|
1702
|
-
} : undefined;
|
|
1090
|
+
logger.info("find_and_click: Completed", { description: params.description, found: result.ok, duration });
|
|
1703
1091
|
|
|
1704
|
-
return
|
|
1705
|
-
true,
|
|
1706
|
-
`Found and clicked: "${params.description}" at (${rawResponse.coordinates?.x}, ${rawResponse.coordinates?.y})\nRef: ${elementRef}`,
|
|
1707
|
-
{
|
|
1708
|
-
...rawResponse,
|
|
1709
|
-
action: "find_and_click",
|
|
1710
|
-
element: elementInfo,
|
|
1711
|
-
ref: elementRef,
|
|
1712
|
-
clickAction: params.action,
|
|
1713
|
-
clickPosition: coords ? { x: coords.centerX, y: coords.centerY } : undefined,
|
|
1714
|
-
croppedImageResourceUri,
|
|
1715
|
-
duration,
|
|
1716
|
-
},
|
|
1717
|
-
generatedCode
|
|
1718
|
-
);
|
|
1092
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1719
1093
|
} catch (error) {
|
|
1094
|
+
if (error instanceof NoActiveSessionError) {
|
|
1095
|
+
logger.warn("find_and_click: No active session");
|
|
1096
|
+
return noSessionResult(error);
|
|
1097
|
+
}
|
|
1720
1098
|
const cancelled = cancelledResultOrNull(error, "find_and_click");
|
|
1721
1099
|
if (cancelled) return cancelled;
|
|
1722
1100
|
logger.error("find_and_click: Failed", { error: String(error), description: params.description });
|
|
@@ -1741,29 +1119,19 @@ server.registerTool(
|
|
|
1741
1119
|
const startTime = Date.now();
|
|
1742
1120
|
logger.info("type: Starting", { textLength: params.text.length, secret: params.secret });
|
|
1743
1121
|
|
|
1744
|
-
const sessionCheck = requireActiveSession();
|
|
1745
|
-
if (!sessionCheck.valid) {
|
|
1746
|
-
logger.warn("type: No active session");
|
|
1747
|
-
return sessionCheck.error;
|
|
1748
|
-
}
|
|
1749
|
-
|
|
1750
1122
|
try {
|
|
1751
1123
|
logger.debug("type: Typing text");
|
|
1752
|
-
await
|
|
1124
|
+
const result = await core.type(params.text, params.secret, params.delay);
|
|
1753
1125
|
|
|
1754
1126
|
const duration = Date.now() - startTime;
|
|
1755
1127
|
logger.info("type: Completed", { duration });
|
|
1756
1128
|
|
|
1757
|
-
|
|
1758
|
-
const generatedCode = generateActionCode("type", { text: params.text, secret: params.secret });
|
|
1759
|
-
|
|
1760
|
-
return createToolResult(
|
|
1761
|
-
true,
|
|
1762
|
-
`Typed: ${params.secret ? "[secret text]" : `"${params.text}"`}`,
|
|
1763
|
-
{ action: "type", text: params.secret ? "[SECRET]" : params.text, duration },
|
|
1764
|
-
generatedCode
|
|
1765
|
-
);
|
|
1129
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1766
1130
|
} catch (error) {
|
|
1131
|
+
if (error instanceof NoActiveSessionError) {
|
|
1132
|
+
logger.warn("type: No active session");
|
|
1133
|
+
return noSessionResult(error);
|
|
1134
|
+
}
|
|
1767
1135
|
logger.error("type: Failed", { error: String(error) });
|
|
1768
1136
|
captureException(error as Error, { tags: { tool: "type" }, extra: { textLength: params.text.length, secret: params.secret } });
|
|
1769
1137
|
throw error;
|
|
@@ -1784,29 +1152,19 @@ server.registerTool(
|
|
|
1784
1152
|
const startTime = Date.now();
|
|
1785
1153
|
logger.info("press_keys: Starting", { keys: params.keys });
|
|
1786
1154
|
|
|
1787
|
-
const sessionCheck = requireActiveSession();
|
|
1788
|
-
if (!sessionCheck.valid) {
|
|
1789
|
-
logger.warn("press_keys: No active session");
|
|
1790
|
-
return sessionCheck.error;
|
|
1791
|
-
}
|
|
1792
|
-
|
|
1793
1155
|
try {
|
|
1794
1156
|
logger.debug("press_keys: Pressing keys");
|
|
1795
|
-
await
|
|
1157
|
+
const result = await core.pressKeys(params.keys);
|
|
1796
1158
|
|
|
1797
1159
|
const duration = Date.now() - startTime;
|
|
1798
1160
|
logger.info("press_keys: Completed", { keys: params.keys, duration });
|
|
1799
1161
|
|
|
1800
|
-
|
|
1801
|
-
const generatedCode = generateActionCode("press_keys", { keys: params.keys });
|
|
1802
|
-
|
|
1803
|
-
return createToolResult(
|
|
1804
|
-
true,
|
|
1805
|
-
`Pressed keys: ${params.keys.join(" + ")}`,
|
|
1806
|
-
{ action: "press_keys", keys: params.keys, duration },
|
|
1807
|
-
generatedCode
|
|
1808
|
-
);
|
|
1162
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1809
1163
|
} catch (error) {
|
|
1164
|
+
if (error instanceof NoActiveSessionError) {
|
|
1165
|
+
logger.warn("press_keys: No active session");
|
|
1166
|
+
return noSessionResult(error);
|
|
1167
|
+
}
|
|
1810
1168
|
logger.error("press_keys: Failed", { error: String(error), keys: params.keys });
|
|
1811
1169
|
captureException(error as Error, { tags: { tool: "press_keys" }, extra: { keys: params.keys } });
|
|
1812
1170
|
throw error;
|
|
@@ -1828,29 +1186,19 @@ server.registerTool(
|
|
|
1828
1186
|
const startTime = Date.now();
|
|
1829
1187
|
logger.info("scroll: Starting", { direction: params.direction, amount: params.amount });
|
|
1830
1188
|
|
|
1831
|
-
const sessionCheck = requireActiveSession();
|
|
1832
|
-
if (!sessionCheck.valid) {
|
|
1833
|
-
logger.warn("scroll: No active session");
|
|
1834
|
-
return sessionCheck.error;
|
|
1835
|
-
}
|
|
1836
|
-
|
|
1837
1189
|
try {
|
|
1838
1190
|
logger.debug("scroll: Scrolling");
|
|
1839
|
-
await
|
|
1191
|
+
const result = await core.scroll(params.direction, params.amount);
|
|
1840
1192
|
|
|
1841
1193
|
const duration = Date.now() - startTime;
|
|
1842
1194
|
logger.info("scroll: Completed", { direction: params.direction, duration });
|
|
1843
1195
|
|
|
1844
|
-
|
|
1845
|
-
const generatedCode = generateActionCode("scroll", { direction: params.direction, amount: params.amount });
|
|
1846
|
-
|
|
1847
|
-
return createToolResult(
|
|
1848
|
-
true,
|
|
1849
|
-
`Scrolled ${params.direction}${params.amount ? ` by ${params.amount}px` : ""}`,
|
|
1850
|
-
{ action: "scroll", scrollDirection: params.direction, direction: params.direction, amount: params.amount, duration },
|
|
1851
|
-
generatedCode
|
|
1852
|
-
);
|
|
1196
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1853
1197
|
} catch (error) {
|
|
1198
|
+
if (error instanceof NoActiveSessionError) {
|
|
1199
|
+
logger.warn("scroll: No active session");
|
|
1200
|
+
return noSessionResult(error);
|
|
1201
|
+
}
|
|
1854
1202
|
logger.error("scroll: Failed", { error: String(error), direction: params.direction });
|
|
1855
1203
|
captureException(error as Error, { tags: { tool: "scroll" }, extra: { direction: params.direction, amount: params.amount } });
|
|
1856
1204
|
throw error;
|
|
@@ -1877,29 +1225,19 @@ Unlike 'check' which is for your understanding during development, 'assert' crea
|
|
|
1877
1225
|
const startTime = Date.now();
|
|
1878
1226
|
logger.info("assert: Starting", { assertion: params.assertion });
|
|
1879
1227
|
|
|
1880
|
-
const sessionCheck = requireActiveSession();
|
|
1881
|
-
if (!sessionCheck.valid) {
|
|
1882
|
-
logger.warn("assert: No active session");
|
|
1883
|
-
return sessionCheck.error;
|
|
1884
|
-
}
|
|
1885
|
-
|
|
1886
1228
|
try {
|
|
1887
1229
|
logger.debug("assert: Running assertion");
|
|
1888
|
-
const result = await
|
|
1230
|
+
const result = await core.assert(params.assertion);
|
|
1889
1231
|
|
|
1890
1232
|
const duration = Date.now() - startTime;
|
|
1891
|
-
logger.info("assert: Completed", { assertion: params.assertion, passed: result, duration });
|
|
1892
|
-
|
|
1893
|
-
// Generate code for this assert action
|
|
1894
|
-
const generatedCode = generateActionCode("assert", { assertion: params.assertion });
|
|
1233
|
+
logger.info("assert: Completed", { assertion: params.assertion, passed: result.ok, duration });
|
|
1895
1234
|
|
|
1896
|
-
return
|
|
1897
|
-
result,
|
|
1898
|
-
result ? `✓ Assertion passed: "${params.assertion}"` : `✗ Assertion failed: "${params.assertion}"`,
|
|
1899
|
-
{ action: "assert", assertion: params.assertion, passed: result, success: result, duration },
|
|
1900
|
-
generatedCode
|
|
1901
|
-
);
|
|
1235
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
1902
1236
|
} catch (error) {
|
|
1237
|
+
if (error instanceof NoActiveSessionError) {
|
|
1238
|
+
logger.warn("assert: No active session");
|
|
1239
|
+
return noSessionResult(error);
|
|
1240
|
+
}
|
|
1903
1241
|
logger.error("assert: Failed", { error: String(error), assertion: params.assertion });
|
|
1904
1242
|
captureException(error as Error, { tags: { tool: "assert" }, extra: { assertion: params.assertion } });
|
|
1905
1243
|
throw error;
|
|
@@ -1943,122 +1281,61 @@ You can optionally provide a reference image URI to compare against a previous s
|
|
|
1943
1281
|
const progress = makeProgressReporter(extra);
|
|
1944
1282
|
logger.info("check: Starting", { task: params.task, hasReferenceImageUri: !!params.referenceImageUri });
|
|
1945
1283
|
|
|
1946
|
-
const sessionCheck = requireActiveSession();
|
|
1947
|
-
if (!sessionCheck.valid) {
|
|
1948
|
-
logger.warn("check: No active session");
|
|
1949
|
-
return sessionCheck.error;
|
|
1950
|
-
}
|
|
1951
|
-
|
|
1952
1284
|
try {
|
|
1953
|
-
//
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
// Use provided reference image URI, last screenshot as "before" state, or current if no previous screenshot
|
|
1959
|
-
let beforeScreenshot: string;
|
|
1285
|
+
// Resolve the optional reference image from the image store (a server/UI
|
|
1286
|
+
// concern). The core handles "last screenshot" / current-screenshot
|
|
1287
|
+
// fallback internally when no reference image is passed.
|
|
1288
|
+
let referenceImage: string | undefined;
|
|
1960
1289
|
if (params.referenceImageUri) {
|
|
1961
1290
|
// Extract image ID from URI (e.g., "screenshot://testdriver/screenshot/screenshot-1" -> "screenshot-1")
|
|
1962
|
-
const uriParts = params.referenceImageUri.split(
|
|
1291
|
+
const uriParts = params.referenceImageUri.split("/");
|
|
1963
1292
|
const imageId = uriParts[uriParts.length - 1];
|
|
1964
|
-
|
|
1965
|
-
logger.info("check: Looking up reference image", {
|
|
1966
|
-
referenceImageUri: params.referenceImageUri,
|
|
1293
|
+
|
|
1294
|
+
logger.info("check: Looking up reference image", {
|
|
1295
|
+
referenceImageUri: params.referenceImageUri,
|
|
1967
1296
|
extractedImageId: imageId,
|
|
1968
1297
|
imageStoreSize: imageStore.size,
|
|
1969
|
-
availableKeys: Array.from(imageStore.keys())
|
|
1298
|
+
availableKeys: Array.from(imageStore.keys()),
|
|
1970
1299
|
});
|
|
1971
|
-
|
|
1300
|
+
|
|
1972
1301
|
const storedImage = getStoredImage(imageId);
|
|
1973
|
-
|
|
1302
|
+
|
|
1974
1303
|
if (storedImage) {
|
|
1975
|
-
logger.info("check: Found reference image", {
|
|
1976
|
-
imageId,
|
|
1304
|
+
logger.info("check: Found reference image", {
|
|
1305
|
+
imageId,
|
|
1977
1306
|
dataLength: storedImage.data?.length,
|
|
1978
1307
|
type: storedImage.type,
|
|
1979
|
-
hasData: !!storedImage.data
|
|
1308
|
+
hasData: !!storedImage.data,
|
|
1980
1309
|
});
|
|
1981
|
-
|
|
1310
|
+
referenceImage = storedImage.data;
|
|
1982
1311
|
} else {
|
|
1983
|
-
logger.warn("check: Reference image NOT found in store, falling back to last screenshot", {
|
|
1984
|
-
referenceImageUri: params.referenceImageUri,
|
|
1312
|
+
logger.warn("check: Reference image NOT found in store, falling back to last screenshot", {
|
|
1313
|
+
referenceImageUri: params.referenceImageUri,
|
|
1985
1314
|
imageId,
|
|
1986
1315
|
imageStoreSize: imageStore.size,
|
|
1987
|
-
availableKeys: Array.from(imageStore.keys())
|
|
1316
|
+
availableKeys: Array.from(imageStore.keys()),
|
|
1988
1317
|
});
|
|
1989
|
-
beforeScreenshot = lastScreenshotBase64 || currentScreenshot;
|
|
1990
1318
|
}
|
|
1991
|
-
} else {
|
|
1992
|
-
beforeScreenshot = lastScreenshotBase64 || currentScreenshot;
|
|
1993
1319
|
}
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
lastScreenshotBase64 = currentScreenshot;
|
|
1997
|
-
|
|
1998
|
-
// Get system state
|
|
1999
|
-
const mousePosition = await sdk.agent.system.getMousePosition();
|
|
2000
|
-
const activeWindow = await sdk.agent.system.activeWin();
|
|
2001
|
-
|
|
2002
|
-
// Call the check endpoint
|
|
2003
|
-
logger.info("check: Calling check API endpoint", {
|
|
2004
|
-
hasLastScreenshot: beforeScreenshot !== currentScreenshot,
|
|
2005
|
-
usingReferenceImageUri: !!params.referenceImageUri,
|
|
2006
|
-
beforeScreenshotLength: beforeScreenshot?.length || 0,
|
|
2007
|
-
currentScreenshotLength: currentScreenshot?.length || 0,
|
|
2008
|
-
beforeScreenshotPreview: beforeScreenshot?.substring(0, 50),
|
|
2009
|
-
currentScreenshotPreview: currentScreenshot?.substring(0, 50)
|
|
2010
|
-
});
|
|
1320
|
+
|
|
1321
|
+
progress.report("Capturing screenshot...");
|
|
2011
1322
|
const stopHeartbeat = progress.heartbeat(`Checking: "${params.task}"...`);
|
|
2012
|
-
let
|
|
1323
|
+
let result: ActionResult;
|
|
2013
1324
|
try {
|
|
2014
|
-
|
|
2015
|
-
tasks: [params.task],
|
|
2016
|
-
images: [beforeScreenshot, currentScreenshot],
|
|
2017
|
-
mousePosition,
|
|
2018
|
-
activeWindow,
|
|
2019
|
-
}));
|
|
1325
|
+
result = await raceAbort(extra.signal, "check", core.check(params.task, referenceImage));
|
|
2020
1326
|
} finally {
|
|
2021
1327
|
stopHeartbeat();
|
|
2022
1328
|
}
|
|
2023
1329
|
|
|
2024
|
-
const aiResponse = response.data;
|
|
2025
|
-
|
|
2026
|
-
// Store screenshot for resource serving
|
|
2027
|
-
let screenshotResourceUri: string | undefined;
|
|
2028
|
-
if (currentScreenshot) {
|
|
2029
|
-
screenshotResourceUri = storeImage(currentScreenshot, "screenshot");
|
|
2030
|
-
}
|
|
2031
|
-
|
|
2032
|
-
// Determine if the check passed based on the AI response
|
|
2033
|
-
// The AI typically returns markdown with its analysis
|
|
2034
|
-
// We consider it "complete" if the response doesn't contain code blocks (indicating more work needed)
|
|
2035
|
-
const hasCodeBlocks = aiResponse && (
|
|
2036
|
-
aiResponse.includes("```yml") ||
|
|
2037
|
-
aiResponse.includes("```yaml") ||
|
|
2038
|
-
aiResponse.includes("- command:")
|
|
2039
|
-
);
|
|
2040
|
-
const isComplete = !hasCodeBlocks;
|
|
2041
|
-
|
|
2042
1330
|
const duration = Date.now() - startTime;
|
|
2043
|
-
logger.info("check: Completed", { task: params.task, complete:
|
|
1331
|
+
logger.info("check: Completed", { task: params.task, complete: result.ok, duration });
|
|
2044
1332
|
|
|
2045
|
-
|
|
2046
|
-
return createToolResult(
|
|
2047
|
-
isComplete,
|
|
2048
|
-
isComplete
|
|
2049
|
-
? `✓ Task appears complete: "${params.task}"\n\nAI Analysis:\n${aiResponse}`
|
|
2050
|
-
: `⚠ Task may not be complete: "${params.task}"\n\nAI Analysis:\n${aiResponse}`,
|
|
2051
|
-
{
|
|
2052
|
-
action: "check",
|
|
2053
|
-
task: params.task,
|
|
2054
|
-
complete: isComplete,
|
|
2055
|
-
success: isComplete,
|
|
2056
|
-
aiResponse,
|
|
2057
|
-
screenshotResourceUri,
|
|
2058
|
-
duration
|
|
2059
|
-
}
|
|
2060
|
-
);
|
|
1333
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
2061
1334
|
} catch (error) {
|
|
1335
|
+
if (error instanceof NoActiveSessionError) {
|
|
1336
|
+
logger.warn("check: No active session");
|
|
1337
|
+
return noSessionResult(error);
|
|
1338
|
+
}
|
|
2062
1339
|
const cancelled = cancelledResultOrNull(error, "check");
|
|
2063
1340
|
if (cancelled) return cancelled;
|
|
2064
1341
|
logger.error("check: Failed", { error: String(error), task: params.task });
|
|
@@ -2083,29 +1360,19 @@ server.registerTool(
|
|
|
2083
1360
|
const startTime = Date.now();
|
|
2084
1361
|
logger.info("exec: Starting", { language: params.language, codeLength: params.code.length, timeout: params.timeout });
|
|
2085
1362
|
|
|
2086
|
-
const sessionCheck = requireActiveSession();
|
|
2087
|
-
if (!sessionCheck.valid) {
|
|
2088
|
-
logger.warn("exec: No active session");
|
|
2089
|
-
return sessionCheck.error;
|
|
2090
|
-
}
|
|
2091
|
-
|
|
2092
1363
|
try {
|
|
2093
1364
|
logger.debug("exec: Executing code", { language: params.language });
|
|
2094
|
-
const
|
|
1365
|
+
const result = await core.exec(params.language, params.code, params.timeout);
|
|
2095
1366
|
|
|
2096
1367
|
const duration = Date.now() - startTime;
|
|
2097
|
-
logger.info("exec: Completed", { language: params.language, outputLength: output?.length || 0, duration });
|
|
1368
|
+
logger.info("exec: Completed", { language: params.language, outputLength: (result.data.output as string)?.length || 0, duration });
|
|
2098
1369
|
|
|
2099
|
-
|
|
2100
|
-
const generatedCode = generateActionCode("exec", { language: params.language, code: params.code, timeout: params.timeout });
|
|
2101
|
-
|
|
2102
|
-
return createToolResult(
|
|
2103
|
-
true,
|
|
2104
|
-
`Executed ${params.language} code:\n${output || "(no output)"}`,
|
|
2105
|
-
{ action: "exec", language: params.language, output, duration },
|
|
2106
|
-
generatedCode
|
|
2107
|
-
);
|
|
1370
|
+
return resultToMcp({ ...result, data: { ...result.data, duration } });
|
|
2108
1371
|
} catch (error) {
|
|
1372
|
+
if (error instanceof NoActiveSessionError) {
|
|
1373
|
+
logger.warn("exec: No active session");
|
|
1374
|
+
return noSessionResult(error);
|
|
1375
|
+
}
|
|
2109
1376
|
logger.error("exec: Failed", { error: String(error), language: params.language });
|
|
2110
1377
|
captureException(error as Error, { tags: { tool: "exec" }, extra: { language: params.language, codeLength: params.code.length } });
|
|
2111
1378
|
throw error;
|
|
@@ -2485,37 +1752,27 @@ Only use 'screenshot' when you explicitly want to show something to the human us
|
|
|
2485
1752
|
const startTime = Date.now();
|
|
2486
1753
|
logger.info("screenshot: Starting");
|
|
2487
1754
|
|
|
2488
|
-
const sessionCheck = requireActiveSession();
|
|
2489
|
-
if (!sessionCheck.valid) {
|
|
2490
|
-
logger.warn("screenshot: No active session");
|
|
2491
|
-
return sessionCheck.error;
|
|
2492
|
-
}
|
|
2493
|
-
|
|
2494
1755
|
try {
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
1756
|
+
const result = await core.screenshot();
|
|
1757
|
+
|
|
1758
|
+
// Store the captured image as a resource URI (kept OUT of AI context — the
|
|
1759
|
+
// MCP app fetches it via resources/read). Preserve the original user-facing
|
|
1760
|
+
// text rather than the core's neutral text.
|
|
1761
|
+
const data: Record<string, unknown> = { action: "screenshot" };
|
|
1762
|
+
for (const img of result.images ?? []) {
|
|
1763
|
+
data.screenshotResourceUri = storeImage(img.base64, "screenshot");
|
|
2502
1764
|
}
|
|
2503
|
-
|
|
1765
|
+
|
|
2504
1766
|
const duration = Date.now() - startTime;
|
|
2505
|
-
|
|
1767
|
+
data.duration = duration;
|
|
1768
|
+
logger.info("screenshot: Completed", { duration, hasImage: (result.images?.length ?? 0) > 0 });
|
|
2506
1769
|
|
|
2507
|
-
|
|
2508
|
-
// This keeps the base64 image data OUT of AI context
|
|
2509
|
-
return createToolResult(
|
|
2510
|
-
true,
|
|
2511
|
-
"Screenshot captured and displayed to user",
|
|
2512
|
-
{
|
|
2513
|
-
action: "screenshot",
|
|
2514
|
-
screenshotResourceUri,
|
|
2515
|
-
duration
|
|
2516
|
-
}
|
|
2517
|
-
);
|
|
1770
|
+
return createToolResult(true, "Screenshot captured and displayed to user", data);
|
|
2518
1771
|
} catch (error) {
|
|
1772
|
+
if (error instanceof NoActiveSessionError) {
|
|
1773
|
+
logger.warn("screenshot: No active session");
|
|
1774
|
+
return noSessionResult(error);
|
|
1775
|
+
}
|
|
2519
1776
|
logger.error("screenshot: Failed", { error: String(error) });
|
|
2520
1777
|
return createToolResult(false, `Screenshot failed: ${error}`, { error: String(error) });
|
|
2521
1778
|
}
|
|
@@ -2612,27 +1869,111 @@ Learn more at https://docs.testdriver.ai/v7/getting-started/
|
|
|
2612
1869
|
);
|
|
2613
1870
|
|
|
2614
1871
|
|
|
1872
|
+
// =============================================================================
|
|
1873
|
+
// HTTP transport (Streamable HTTP, no auth)
|
|
1874
|
+
// =============================================================================
|
|
1875
|
+
|
|
1876
|
+
/**
|
|
1877
|
+
* Serve the MCP server over Streamable HTTP with no authentication.
|
|
1878
|
+
*
|
|
1879
|
+
* This is what the eve agent (and any other Streamable-HTTP MCP client) connects
|
|
1880
|
+
* to. The server holds single-session global state (the active `sdk`, element
|
|
1881
|
+
* refs, image store), so we mirror that with a single long-lived MCP transport:
|
|
1882
|
+
* the first `initialize` request mints a session id and every subsequent request
|
|
1883
|
+
* reuses the same `server` + transport. Concurrent sessions are intentionally not
|
|
1884
|
+
* supported — this server provisions one sandbox at a time.
|
|
1885
|
+
*
|
|
1886
|
+
* No auth is applied here by design (the connection is expected to be local-only
|
|
1887
|
+
* or otherwise protected outside the MCP layer). Do not expose this on a public
|
|
1888
|
+
* network without putting a real authenticating proxy in front of it.
|
|
1889
|
+
*/
|
|
1890
|
+
async function startHttpServer() {
|
|
1891
|
+
const host = process.env.TD_MCP_HOST || "127.0.0.1";
|
|
1892
|
+
const port = Number(process.env.TD_MCP_PORT || process.env.PORT || 8788);
|
|
1893
|
+
const mcpPath = process.env.TD_MCP_PATH || "/mcp";
|
|
1894
|
+
|
|
1895
|
+
// One transport for the lifetime of the process (single-session design).
|
|
1896
|
+
const transport = new StreamableHTTPServerTransport({
|
|
1897
|
+
sessionIdGenerator: () => randomUUID(),
|
|
1898
|
+
});
|
|
1899
|
+
await server.connect(transport);
|
|
1900
|
+
|
|
1901
|
+
const httpServer = http.createServer((req, res) => {
|
|
1902
|
+
const url = new URL(req.url || "/", `http://${req.headers.host}`);
|
|
1903
|
+
|
|
1904
|
+
// Lightweight health check for readiness probes / `eve dev`.
|
|
1905
|
+
if (req.method === "GET" && url.pathname === "/health") {
|
|
1906
|
+
res.writeHead(200, { "content-type": "application/json" });
|
|
1907
|
+
res.end(JSON.stringify({ status: "ok", server: "testdriver", version }));
|
|
1908
|
+
return;
|
|
1909
|
+
}
|
|
1910
|
+
|
|
1911
|
+
if (url.pathname !== mcpPath) {
|
|
1912
|
+
res.writeHead(404, { "content-type": "application/json" });
|
|
1913
|
+
res.end(JSON.stringify({ error: "Not found", hint: `MCP endpoint is ${mcpPath}` }));
|
|
1914
|
+
return;
|
|
1915
|
+
}
|
|
1916
|
+
|
|
1917
|
+
// The Streamable HTTP transport handles POST (requests), GET (SSE stream),
|
|
1918
|
+
// and DELETE (session teardown) on the same path. Body parsing is left to
|
|
1919
|
+
// the transport so it can read the raw stream.
|
|
1920
|
+
transport.handleRequest(req, res).catch((error) => {
|
|
1921
|
+
logger.error("http: handleRequest failed", { error: String(error) });
|
|
1922
|
+
captureException(error as Error, { tags: { phase: "http" } });
|
|
1923
|
+
if (!res.headersSent) {
|
|
1924
|
+
res.writeHead(500, { "content-type": "application/json" });
|
|
1925
|
+
res.end(JSON.stringify({ error: "Internal server error" }));
|
|
1926
|
+
}
|
|
1927
|
+
});
|
|
1928
|
+
});
|
|
1929
|
+
|
|
1930
|
+
await new Promise<void>((resolve) => httpServer.listen(port, host, resolve));
|
|
1931
|
+
logger.info("TestDriver MCP Server running on Streamable HTTP", {
|
|
1932
|
+
url: `http://${host}:${port}${mcpPath}`,
|
|
1933
|
+
});
|
|
1934
|
+
|
|
1935
|
+
const shutdown = async () => {
|
|
1936
|
+
logger.info("Shutting down MCP Server");
|
|
1937
|
+
httpServer.close();
|
|
1938
|
+
await transport.close().catch(() => {});
|
|
1939
|
+
await flushSentry();
|
|
1940
|
+
process.exit(0);
|
|
1941
|
+
};
|
|
1942
|
+
process.on("SIGINT", shutdown);
|
|
1943
|
+
process.on("SIGTERM", shutdown);
|
|
1944
|
+
}
|
|
1945
|
+
|
|
2615
1946
|
// Start the server
|
|
2616
1947
|
async function main() {
|
|
2617
|
-
|
|
1948
|
+
// Transport selection: TD_MCP_TRANSPORT=http serves Streamable HTTP (used by
|
|
1949
|
+
// the eve agent); anything else (default) uses stdio for local CLI clients.
|
|
1950
|
+
const transportMode = (process.env.TD_MCP_TRANSPORT || "stdio").toLowerCase();
|
|
1951
|
+
|
|
1952
|
+
logger.info("Starting TestDriver MCP Server", {
|
|
2618
1953
|
version,
|
|
1954
|
+
transport: transportMode,
|
|
2619
1955
|
logLevel: process.env.TD_LOG_LEVEL || "INFO",
|
|
2620
1956
|
distDir: DIST_DIR,
|
|
2621
1957
|
sentryEnabled: isSentryEnabled(),
|
|
2622
1958
|
});
|
|
2623
|
-
|
|
1959
|
+
|
|
1960
|
+
if (transportMode === "http") {
|
|
1961
|
+
await startHttpServer();
|
|
1962
|
+
return;
|
|
1963
|
+
}
|
|
1964
|
+
|
|
2624
1965
|
const transport = new StdioServerTransport();
|
|
2625
1966
|
await server.connect(transport);
|
|
2626
|
-
|
|
1967
|
+
|
|
2627
1968
|
logger.info("TestDriver MCP Server running on stdio");
|
|
2628
|
-
|
|
1969
|
+
|
|
2629
1970
|
// Handle graceful shutdown
|
|
2630
1971
|
const shutdown = async () => {
|
|
2631
1972
|
logger.info("Shutting down MCP Server");
|
|
2632
1973
|
await flushSentry();
|
|
2633
1974
|
process.exit(0);
|
|
2634
1975
|
};
|
|
2635
|
-
|
|
1976
|
+
|
|
2636
1977
|
process.on("SIGINT", shutdown);
|
|
2637
1978
|
process.on("SIGTERM", shutdown);
|
|
2638
1979
|
}
|