@chrrxs/robloxstudio-mcp 2.12.0 → 2.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,18 @@
1
1
  import { HttpService, Players, ReplicatedStorage, RunService, ServerStorage } from "@rbxts/services";
2
2
  import RuntimeLogBuffer from "./RuntimeLogBuffer";
3
3
  import MemoryHandlers from "./handlers/MemoryHandlers";
4
+ import CaptureHandlers from "./handlers/CaptureHandlers";
5
+ import InputHandlers from "./handlers/InputHandlers";
4
6
  import LuauExec from "./LuauExec";
5
7
 
8
+ interface StudioTestServiceMultiplayer extends StudioTestService {
9
+ CanLeaveTest(): boolean;
10
+ LeaveTest(): void;
11
+ EditModeActive: boolean;
12
+ }
13
+
14
+ const StudioTestService = game.GetService("StudioTestService") as StudioTestServiceMultiplayer;
15
+
6
16
  // Mirror of Communication.computeInstanceId() — duplicated here because the
7
17
  // client broker runs in the play-server DM where it can't easily import from
8
18
  // the edit-side module, and the place identifier must match what the edit-DM
@@ -53,6 +63,7 @@ function resolvePlaceName(): string {
53
63
 
54
64
  const MCP_URL = "http://localhost:58741";
55
65
  const BROKER_NAME = "__MCPClientBroker";
66
+ const BROKER_OWNER_ATTRIBUTE = "__MCPBrokerOwner";
56
67
 
57
68
  interface ProxyEntry {
58
69
  pluginSessionId: string;
@@ -76,6 +87,15 @@ const CLIENT_BROKER_ALLOWED_ENDPOINTS = new Set<string>([
76
87
  "/api/execute-luau",
77
88
  "/api/get-runtime-logs",
78
89
  "/api/get-memory-breakdown",
90
+ "/api/multiplayer-test-state",
91
+ "/api/multiplayer-test-leave-client",
92
+ // Screenshot capture must run in the client peer (CaptureService captures
93
+ // the play viewport there); the edit DM reads the temp id back separately.
94
+ "/api/capture-begin",
95
+ // Virtual input (CreateVirtualInput) drives the running client's input
96
+ // pipeline, so it must execute in the client peer's VM.
97
+ "/api/simulate-mouse-input",
98
+ "/api/simulate-keyboard-input",
79
99
  ]);
80
100
 
81
101
  interface ReadyResponseBody {
@@ -155,6 +175,52 @@ function handleGetRuntimeLogs(data: Record<string, unknown> | undefined): unknow
155
175
  return RuntimeLogBuffer.query({ since, tail, filter }, "client");
156
176
  }
157
177
 
178
+ function handleMultiplayerTestState(): unknown {
179
+ const [argsOk, args] = pcall(() => StudioTestService.GetTestArgs());
180
+ const [canLeaveOk, canLeave] = pcall(() => StudioTestService.CanLeaveTest());
181
+ const players = Players.GetPlayers().map((player) => ({
182
+ name: player.Name,
183
+ userId: player.UserId,
184
+ displayName: player.DisplayName,
185
+ }));
186
+ players.sort((a, b) => a.name < b.name);
187
+ return {
188
+ success: true,
189
+ peer: "client",
190
+ isRunning: RunService.IsRunning(),
191
+ isRunMode: RunService.IsRunMode(),
192
+ editModeActive: StudioTestService.EditModeActive,
193
+ testArgsOk: argsOk,
194
+ testArgs: argsOk ? args : undefined,
195
+ testArgsError: argsOk ? undefined : tostring(args),
196
+ players,
197
+ playerCount: players.size(),
198
+ localPlayer: Players.LocalPlayer ? Players.LocalPlayer.Name : undefined,
199
+ canLeaveOk,
200
+ canLeave: canLeaveOk ? canLeave : false,
201
+ canLeaveError: canLeaveOk ? undefined : tostring(canLeave),
202
+ };
203
+ }
204
+
205
+ function handleMultiplayerTestLeaveClient(): unknown {
206
+ const [canLeaveOk, canLeave] = pcall(() => StudioTestService.CanLeaveTest());
207
+ if (!canLeaveOk) {
208
+ return { error: tostring(canLeave), canLeaveOk: false };
209
+ }
210
+ if (!canLeave) {
211
+ return { error: "This client cannot leave the current test session.", canLeaveOk: true, canLeave: false };
212
+ }
213
+ const localPlayer = Players.LocalPlayer ? Players.LocalPlayer.Name : undefined;
214
+ task.defer(() => {
215
+ pcall(() => StudioTestService.LeaveTest());
216
+ });
217
+ return {
218
+ success: true,
219
+ message: "Client leave requested.",
220
+ localPlayer,
221
+ };
222
+ }
223
+
158
224
  function setupClientBroker() {
159
225
  const rf = ReplicatedStorage.WaitForChild(BROKER_NAME, 10);
160
226
  if (!rf || !rf.IsA("RemoteFunction")) {
@@ -174,6 +240,21 @@ function setupClientBroker() {
174
240
  if (payload && payload.endpoint === "/api/get-memory-breakdown") {
175
241
  return MemoryHandlers.getMemoryBreakdown(payload.data ?? {});
176
242
  }
243
+ if (payload && payload.endpoint === "/api/multiplayer-test-state") {
244
+ return handleMultiplayerTestState();
245
+ }
246
+ if (payload && payload.endpoint === "/api/multiplayer-test-leave-client") {
247
+ return handleMultiplayerTestLeaveClient();
248
+ }
249
+ if (payload && payload.endpoint === "/api/capture-begin") {
250
+ return CaptureHandlers.captureBegin();
251
+ }
252
+ if (payload && payload.endpoint === "/api/simulate-mouse-input") {
253
+ return InputHandlers.simulateMouseInput(payload.data ?? {});
254
+ }
255
+ if (payload && payload.endpoint === "/api/simulate-keyboard-input") {
256
+ return InputHandlers.simulateKeyboardInput(payload.data ?? {});
257
+ }
177
258
  if (payload && payload.endpoint === "/api/execute-luau") {
178
259
  return handleExecuteLuau(payload.data);
179
260
  }
@@ -183,6 +264,7 @@ function setupClientBroker() {
183
264
  }
184
265
 
185
266
  const proxyByPlayer = new Map<Player, ProxyEntry>();
267
+ let serverBrokerStarted = false;
186
268
 
187
269
  function pollProxy(proxyId: string, player: Player, rf: RemoteFunction) {
188
270
  while (player.Parent !== undefined && proxyByPlayer.has(player)) {
@@ -215,10 +297,12 @@ function pollProxy(proxyId: string, player: Player, rf: RemoteFunction) {
215
297
  response = { success: false, error: `InvokeClient failed: ${tostring(invokeRes)}` };
216
298
  }
217
299
  } else {
300
+ const allowed: string[] = [];
301
+ for (const ep of CLIENT_BROKER_ALLOWED_ENDPOINTS) allowed.push(ep);
218
302
  response = {
219
303
  error:
220
304
  `Client-proxy does not forward ${tostring(request.endpoint)}. ` +
221
- `Allowed: /api/execute-luau, /api/get-runtime-logs.`,
305
+ `Allowed: ${allowed.join(", ")}.`,
222
306
  };
223
307
  }
224
308
  postJson("/response", { requestId: body.requestId, response });
@@ -257,12 +341,18 @@ function registerProxy(player: Player, rf: RemoteFunction) {
257
341
  // which doesn't depend on MCP server state or peer registration at all.)
258
342
 
259
343
  function setupServerBroker() {
344
+ if (serverBrokerStarted) return;
260
345
  let rf = ReplicatedStorage.FindFirstChild(BROKER_NAME) as RemoteFunction | undefined;
261
346
  if (!rf) {
262
347
  rf = new Instance("RemoteFunction");
263
348
  rf.Name = BROKER_NAME;
264
349
  rf.Parent = ReplicatedStorage;
265
350
  }
351
+ if (rf.GetAttribute(BROKER_OWNER_ATTRIBUTE) !== undefined) {
352
+ return;
353
+ }
354
+ rf.SetAttribute(BROKER_OWNER_ATTRIBUTE, HttpService.GenerateGUID(false));
355
+ serverBrokerStarted = true;
266
356
  const broker = rf;
267
357
  Players.PlayerAdded.Connect((p) => registerProxy(p, broker));
268
358
  for (const p of Players.GetPlayers()) {
@@ -2,6 +2,7 @@ import { HttpService, RunService, ServerStorage } from "@rbxts/services";
2
2
  import State from "./State";
3
3
  import Utils from "./Utils";
4
4
  import UI from "./UI";
5
+ import { ensureBridgesInstalled } from "./EvalBridges";
5
6
  import QueryHandlers from "./handlers/QueryHandlers";
6
7
  import PropertyHandlers from "./handlers/PropertyHandlers";
7
8
  import InstanceHandlers from "./handlers/InstanceHandlers";
@@ -132,6 +133,11 @@ const routeMap: Record<string, Handler> = {
132
133
  "/api/start-playtest": TestHandlers.startPlaytest,
133
134
  "/api/stop-playtest": TestHandlers.stopPlaytest,
134
135
  "/api/get-playtest-output": TestHandlers.getPlaytestOutput,
136
+ "/api/multiplayer-test-start": TestHandlers.multiplayerTestStart,
137
+ "/api/multiplayer-test-state": TestHandlers.multiplayerTestState,
138
+ "/api/multiplayer-test-add-players": TestHandlers.multiplayerTestAddPlayers,
139
+ "/api/multiplayer-test-leave-client": TestHandlers.multiplayerTestLeaveClient,
140
+ "/api/multiplayer-test-end": TestHandlers.multiplayerTestEnd,
135
141
  "/api/character-navigation": TestHandlers.characterNavigation,
136
142
 
137
143
  "/api/export-build": BuildHandlers.exportBuild,
@@ -143,6 +149,8 @@ const routeMap: Record<string, Handler> = {
143
149
  "/api/preview-asset": AssetHandlers.previewAsset,
144
150
 
145
151
  "/api/capture-screenshot": CaptureHandlers.captureScreenshot,
152
+ "/api/capture-begin": CaptureHandlers.captureBegin,
153
+ "/api/capture-read": CaptureHandlers.captureRead,
146
154
  "/api/simulate-mouse-input": InputHandlers.simulateMouseInput,
147
155
  "/api/simulate-keyboard-input": InputHandlers.simulateKeyboardInput,
148
156
 
@@ -458,6 +466,20 @@ function activatePlugin(connIndex?: number) {
458
466
  // later reports knownInstance=false (process restart, etc).
459
467
  sendReady(conn);
460
468
 
469
+ // Keep the eval bridges present in the edit DM so that ANY playtest —
470
+ // including one the dev starts manually via the Studio Play button —
471
+ // clones them into the play DMs and eval_*_runtime works with no setup
472
+ // roundtrip. Only the edit DM installs; play DMs already have the cloned
473
+ // copies. Idempotent, so reconnects don't re-dirty the place.
474
+ if (!RunService.IsRunning()) {
475
+ task.spawn(() => {
476
+ const result = ensureBridgesInstalled();
477
+ if (!result.installed) {
478
+ warn(`[MCPPlugin] Eval bridge install failed: ${result.error}`);
479
+ }
480
+ });
481
+ }
482
+
461
483
  // Watch for game.Name updates so a stale "Place1" captured at first
462
484
  // /ready gets refreshed once Studio settles on the real DM name.
463
485
  ensureNameChangeWatcher(conn);
@@ -21,11 +21,16 @@
21
21
  // ServerScriptService.LoadStringEnabled, so eval_server_runtime works even
22
22
  // when LoadStringEnabled=false (the default in fresh places).
23
23
  //
24
- // Lifecycle: TestHandlers.startPlaytest inserts both scripts into the EDIT
25
- // DM right before ExecutePlayModeAsync. ExecutePlayModeAsync clones the
26
- // DataModel into the play DMs, so the scripts come along and run there.
27
- // TestHandlers cleans them up from the edit DM when ExecutePlayModeAsync
28
- // returns (test ended for any reason: stop_playtest, manual close, EndTest).
24
+ // Lifecycle: the bridges live PERMANENTLY in the edit DM. Communication
25
+ // installs them (ensureBridgesInstalled) when the plugin connects in edit,
26
+ // and TestHandlers.startPlaytest force-refreshes them right before
27
+ // ExecutePlayModeAsync. ExecutePlayModeAsync clones the DataModel into the
28
+ // play DMs, so the scripts come along and run there. We keep them in the edit
29
+ // DM after a playtest ends (rather than cleaning up) so that a playtest the
30
+ // dev starts MANUALLY via the Studio Play button — not the MCP start_playtest
31
+ // tool — also gets the bridges cloned in. This is intentionally a little
32
+ // intrusive (two helper scripts visible in Explorer) in exchange for a
33
+ // zero-roundtrip eval_*_runtime experience for devs working 1:1 with an agent.
29
34
  //
30
35
  // Archivable handling: ExecutePlayModeAsync's deep-clone SKIPS instances
31
36
  // with Archivable=false (verified empirically in v2.9.0 testing - bridges
@@ -61,9 +66,9 @@ export const BRIDGE_NAMES = {
61
66
  // Embedded Luau. The double `${...}` references our exported names so a
62
67
  // rename here propagates to both the script source and the tool wrappers.
63
68
  const SERVER_BRIDGE_SOURCE = `
64
- -- Auto-installed by @chrrxs/robloxstudio-mcp at start_playtest, removed at
65
- -- stop_playtest. Provides shared-require-cache eval on the server peer for
66
- -- the eval_server_runtime MCP tool.
69
+ -- Installed by @chrrxs/robloxstudio-mcp to power the eval_server_runtime MCP
70
+ -- tool (shared-require-cache eval on the server during playtests). Inert
71
+ -- outside Studio (no-ops in live games); safe to leave in place.
67
72
 
68
73
  local ServerScriptService = game:GetService("ServerScriptService")
69
74
  local RunService = game:GetService("RunService")
@@ -88,9 +93,9 @@ end
88
93
  `;
89
94
 
90
95
  const CLIENT_BRIDGE_SOURCE = `
91
- -- Auto-installed by @chrrxs/robloxstudio-mcp at start_playtest, removed at
92
- -- stop_playtest. Provides shared-require-cache eval on the client peer for
93
- -- the eval_client_runtime MCP tool.
96
+ -- Installed by @chrrxs/robloxstudio-mcp to power the eval_client_runtime MCP
97
+ -- tool (shared-require-cache eval on the client during playtests). Inert
98
+ -- outside Studio (no-ops in live games); safe to leave in place.
94
99
 
95
100
  local ReplicatedStorage = game:GetService("ReplicatedStorage")
96
101
  local RunService = game:GetService("RunService")
@@ -114,6 +119,28 @@ bf.OnInvoke = function(payload)
114
119
  end
115
120
  `;
116
121
 
122
+ // Stamp written onto each installed bridge Script so we can tell whether the
123
+ // bridge currently in the DM was produced by THIS plugin build. It's a djb2
124
+ // hash of the actual bridge source plus the plugin version, so ANY change to
125
+ // the source (or a version bump) yields a new stamp — which makes
126
+ // ensureBridgesInstalled() force a refresh on the next plugin load instead of
127
+ // keeping a stale bridge that happens to still be present (e.g. one saved into
128
+ // the .rbxl from an older build).
129
+ const STAMP_ATTR = "__MCPBridgeStamp";
130
+
131
+ function computeBridgeStamp(): string {
132
+ const combined = `${SERVER_BRIDGE_SOURCE}|${CLIENT_BRIDGE_SOURCE}`;
133
+ let h = 5381;
134
+ for (let i = 1; i <= combined.size(); i++) {
135
+ h = (h * 33 + string.byte(combined, i)[0]) % 2147483647;
136
+ }
137
+ // "__VERSION__" is replaced with the package version at package time
138
+ // (scripts/build-plugin.mjs injectVersion), so a release bump also restamps.
139
+ return `${tostring(h)}-__VERSION__`;
140
+ }
141
+
142
+ const BRIDGE_STAMP = computeBridgeStamp();
143
+
117
144
  function setSource(scriptInst: Script | LocalScript, source: string): void {
118
145
  // ScriptEditorService is the cleaner API and integrates with Studio's
119
146
  // edit history; fall back to direct Source mutation (allowed in plugin
@@ -144,6 +171,26 @@ export function cleanupBridges(): void {
144
171
  }
145
172
  }
146
173
 
174
+ // Idempotent variant: install only if the bridge scripts aren't already
175
+ // present in the edit DM. Used to keep the bridges always available (so a
176
+ // playtest the dev starts manually — not via the MCP start_playtest tool —
177
+ // still clones them into the play DMs). Cheap no-op when already installed,
178
+ // which avoids re-dirtying the place on every plugin reconnect.
179
+ export function ensureBridgesInstalled(): { installed: boolean; error?: string } {
180
+ const { server, client } = findBridges();
181
+ if (server && client) {
182
+ // Both present — but only skip the reinstall if they were produced by
183
+ // THIS build. A mismatched/absent stamp means a stale bridge (older
184
+ // plugin, or one persisted in the saved place), so force a refresh.
185
+ const sStamp = server.GetAttribute(STAMP_ATTR);
186
+ const cStamp = client.GetAttribute(STAMP_ATTR);
187
+ if (sStamp === BRIDGE_STAMP && cStamp === BRIDGE_STAMP) {
188
+ return { installed: true };
189
+ }
190
+ }
191
+ return installBridges();
192
+ }
193
+
147
194
  export function installBridges(): { installed: boolean; error?: string } {
148
195
  // Defensive: clear any stale bridges from a prior unclean exit before
149
196
  // inserting fresh. The injected script also self-cleans its
@@ -158,6 +205,7 @@ export function installBridges(): { installed: boolean; error?: string } {
158
205
  // script. cleanupBridges() removes it from the edit DM when the
159
206
  // playtest ends.
160
207
  setSource(serverScript, SERVER_BRIDGE_SOURCE);
208
+ serverScript.SetAttribute(STAMP_ATTR, BRIDGE_STAMP);
161
209
  serverScript.Parent = ServerScriptService;
162
210
 
163
211
  const sps = getStarterPlayerScripts();
@@ -167,6 +215,7 @@ export function installBridges(): { installed: boolean; error?: string } {
167
215
  const clientScript = new Instance("LocalScript");
168
216
  clientScript.Name = CLIENT_SCRIPT_NAME;
169
217
  setSource(clientScript, CLIENT_BRIDGE_SOURCE);
218
+ clientScript.SetAttribute(STAMP_ATTR, BRIDGE_STAMP);
170
219
  clientScript.Parent = sps;
171
220
  });
172
221
 
@@ -0,0 +1,60 @@
1
+ // Detects whether the Studio window is actually rendering, so virtual input
2
+ // and screenshot tools can surface a clear reason instead of silently failing.
3
+ //
4
+ // When a Studio window is MINIMIZED, the engine suspends the render loop AND
5
+ // input processing, but keeps running scripts (Heartbeat keeps firing). That's
6
+ // why simulate_*_input would return success while having zero effect, and
7
+ // CaptureService:CaptureScreenshot would time out. Validated live: during a 3s
8
+ // minimize, RenderStepped's max inter-frame gap was 5.08s while Heartbeat's was
9
+ // 0.10s. So RenderStepped freshness is the reliable "is this window rendering?"
10
+ // signal; Heartbeat is not.
11
+
12
+ import { RunService } from "@rbxts/services";
13
+
14
+ let lastFrame = 0;
15
+ let connected = false;
16
+
17
+ // Above this many seconds since the last rendered frame, we treat the window
18
+ // as not rendering. RenderStepped normally fires every ~16ms; a multi-second
19
+ // gap only happens when minimized/suspended, so 1s cleanly avoids false
20
+ // positives from ordinary frame hitches while still catching the real case.
21
+ const STALE_THRESHOLD = 1.0;
22
+
23
+ export function start(): void {
24
+ if (connected) return;
25
+ // RenderStepped can only be connected from a client/edit render loop; it
26
+ // throws in the play-server DM. pcall so a server-DM call is a safe no-op
27
+ // (connected stays false → notRenderingReason() returns undefined there).
28
+ const [ok] = pcall(() => {
29
+ RunService.RenderStepped.Connect(() => {
30
+ lastFrame = tick();
31
+ });
32
+ });
33
+ if (ok) {
34
+ connected = true;
35
+ lastFrame = tick();
36
+ }
37
+ }
38
+
39
+ export function secondsSinceFrame(): number {
40
+ if (!connected) return 0;
41
+ return tick() - lastFrame;
42
+ }
43
+
44
+ // Returns a human-readable reason if the window appears minimized / not
45
+ // rendering (so input + screenshots won't work), else undefined. Fail-open:
46
+ // when the monitor isn't active in this DM (server peer, or connect failed) it
47
+ // returns undefined so we never block on a false signal.
48
+ export function notRenderingReason(): string | undefined {
49
+ if (!connected) return undefined;
50
+ const gap = secondsSinceFrame();
51
+ if (gap > STALE_THRESHOLD) {
52
+ return string.format(
53
+ "Studio window appears minimized or not rendering (no frame in %.1fs). " +
54
+ "Virtual input and screenshots only work while the window is visible — " +
55
+ "restore/un-minimize the Studio window and retry.",
56
+ gap,
57
+ );
58
+ }
59
+ return undefined;
60
+ }
@@ -1,3 +1,5 @@
1
+ import * as RenderMonitor from "../RenderMonitor";
2
+
1
3
  const CaptureService = game.GetService("CaptureService");
2
4
  const AssetService = game.GetService("AssetService");
3
5
 
@@ -71,7 +73,17 @@ function readPixelsTiled(img: EditableImage, w: number, h: number): buffer {
71
73
  return fullBuf;
72
74
  }
73
75
 
74
- function captureScreenshotData(): unknown {
76
+ // Triggers CaptureService:CaptureScreenshot and waits for the temporary
77
+ // content id. Works in any DM, including the play CLIENT (where reading the
78
+ // pixels back is blocked, but capturing is not). The returned rbxtemp:// id is
79
+ // a process-scoped handle: it can be dereferenced from a DIFFERENT, more
80
+ // privileged DM (the edit DM) — see captureRead.
81
+ function doCaptureScreenshot(): { contentId: string } | { error: string } {
82
+ // Fast-fail with a clear reason if the window isn't rendering — otherwise
83
+ // CaptureScreenshot's callback never fires and we'd block for the full 10s.
84
+ const notRendering = RenderMonitor.notRenderingReason();
85
+ if (notRendering !== undefined) return { error: notRendering };
86
+
75
87
  let contentId: string | undefined;
76
88
 
77
89
  CaptureService.CaptureScreenshot((id: string) => {
@@ -82,14 +94,23 @@ function captureScreenshotData(): unknown {
82
94
  while (contentId === undefined) {
83
95
  if (tick() - startTime > 10) {
84
96
  return {
85
- error: "Screenshot capture timed out. Ensure the Studio viewport is visible and you are in Edit mode (not Play mode). Known Roblox bug: capture may fail if viewport renders a solid color.",
97
+ error: "Screenshot capture timed out (CaptureScreenshot callback never fired). The Studio window is likely minimized or occluded restore it so the viewport renders. (Known Roblox bug: capture can also fail if the viewport renders a solid color.)",
86
98
  };
87
99
  }
88
100
  task.wait(0.1);
89
101
  }
90
102
 
103
+ return { contentId };
104
+ }
105
+
106
+ // Promotes a CaptureScreenshot content id into an EditableImage and reads its
107
+ // RGBA pixels. MUST run in the edit/plugin context: the running game VM lacks
108
+ // the privilege to create an EditableImage from a temporary texture id (errors
109
+ // "cannot currently create editable image from temporary texture id"), while
110
+ // the edit DM can — even for an id captured in the play client DM.
111
+ function readContentToBase64(contentId: string): unknown {
91
112
  const [editableOk, editableResult] = pcall(() => {
92
- return AssetService.CreateEditableImageAsync(Content.fromUri(contentId!));
113
+ return AssetService.CreateEditableImageAsync(Content.fromUri(contentId));
93
114
  });
94
115
 
95
116
  if (!editableOk) {
@@ -118,11 +139,32 @@ function captureScreenshotData(): unknown {
118
139
  return { success: true, width: w, height: h, data: base64Data };
119
140
  }
120
141
 
142
+ // Edit-mode single shot: capture and read back in the same (edit) context.
143
+ function captureScreenshotData(): unknown {
144
+ const cap = doCaptureScreenshot();
145
+ if ("error" in cap) return cap;
146
+ return readContentToBase64(cap.contentId);
147
+ }
148
+
121
149
  function captureScreenshot(): unknown {
122
150
  return captureScreenshotData();
123
151
  }
124
152
 
153
+ // Play-mode step 1 (run on the CLIENT): capture only, return the temp id.
154
+ function captureBegin(): unknown {
155
+ return doCaptureScreenshot();
156
+ }
157
+
158
+ // Play-mode step 2 (run on EDIT): read pixels from a temp id captured elsewhere.
159
+ function captureRead(requestData: Record<string, unknown>): unknown {
160
+ const contentId = requestData.contentId as string | undefined;
161
+ if (!contentId) return { error: "contentId is required" };
162
+ return readContentToBase64(contentId);
163
+ }
164
+
125
165
  export = {
126
166
  captureScreenshotData,
127
167
  captureScreenshot,
168
+ captureBegin,
169
+ captureRead,
128
170
  };
@@ -1,55 +1,101 @@
1
- interface VIMethods {
2
- SendMouseButtonEvent(x: number, y: number, button: number, isDown: boolean): void;
3
- SendMouseMoveEvent(x: number, y: number): void;
4
- SendMouseWheelEvent(x: number, y: number, isForward: boolean): void;
5
- SendKeyEvent(isPressed: boolean, keyCode: Enum.KeyCode, isRepeatedKey: boolean): void;
1
+ // Virtual input via UserInputService:CreateVirtualInput().
2
+ //
3
+ // We deliberately do NOT use VirtualInputManager:Send*Event — those methods
4
+ // are gated behind RobloxScriptSecurity ("lacking capability RobloxScript")
5
+ // in every context a plugin can reach (edit DM, play server/client DMs), so
6
+ // they silently never worked. CreateVirtualInput() is callable without that
7
+ // capability and drives the REAL input pipeline: SendKey feeds
8
+ // UserInputService.InputBegan/Ended and the control modules (so WASD walks the
9
+ // character at full WalkSpeed with controls intact, no Humanoid hijack),
10
+ // SendMouseButton feeds UIS and activates GUI buttons (and hit-tests against
11
+ // CoreGui), and SendTextInput types into the focused TextBox.
12
+ //
13
+ // Method set on the VirtualInput object (verified live):
14
+ // SendKey(isDown: boolean, keyCode: Enum.KeyCode)
15
+ // SendMouseButton(position: Vector2, inputType: Enum.UserInputType, isDown: boolean)
16
+ // SendTextInput(text: string)
17
+ // There is NO SendMouseMove / SendMouseWheel / SendKeyEvent — so "move" and
18
+ // "scroll" mouse actions are not supported.
19
+ //
20
+ // Coordinate space: SendMouseButton coordinates are viewport pixels matching
21
+ // what capture_screenshot returns (window space, origin at the top-left of the
22
+ // rendered viewport). Pass screenshot pixel coordinates straight through. Note
23
+ // that UserInputService reports input positions in GUI space, which is offset
24
+ // from this by GuiService:GetGuiInset() (~58px on the Y axis) — irrelevant for
25
+ // callers who pick coordinates off a screenshot, which is why we do not
26
+ // translate here.
27
+
28
+ import * as RenderMonitor from "../RenderMonitor";
29
+
30
+ const UserInputService = game.GetService("UserInputService");
31
+
32
+ interface VirtualInput {
33
+ SendKey(isDown: boolean, keyCode: Enum.KeyCode): void;
34
+ SendMouseButton(position: Vector2, inputType: Enum.UserInputType, isDown: boolean): void;
35
+ SendTextInput(text: string): void;
6
36
  }
7
37
 
8
- function getVIM(): VIMethods | undefined {
9
- const [ok, result] = pcall(() => {
10
- return (game as unknown as { GetService(name: string): Instance }).GetService("VirtualInputManager");
38
+ // One VirtualInput per plugin VM, reused across calls so that a key held down
39
+ // in one call (action="press") and released in a later call (action="release")
40
+ // share the same input source.
41
+ let cachedVI: VirtualInput | undefined;
42
+
43
+ function getVI(): VirtualInput | undefined {
44
+ if (cachedVI) return cachedVI;
45
+ const [ok, vi] = pcall(() => {
46
+ return (UserInputService as unknown as { CreateVirtualInput(): unknown }).CreateVirtualInput();
11
47
  });
12
- if (ok && result) return result as unknown as VIMethods;
48
+ if (ok && vi !== undefined) {
49
+ cachedVI = vi as VirtualInput;
50
+ return cachedVI;
51
+ }
13
52
  return undefined;
14
53
  }
15
54
 
16
- const BUTTON_MAP: Record<string, number> = { Left: 0, Right: 1, Middle: 2 };
55
+ const MOUSE_TYPE_MAP: Record<string, Enum.UserInputType> = {
56
+ Left: Enum.UserInputType.MouseButton1,
57
+ Right: Enum.UserInputType.MouseButton2,
58
+ Middle: Enum.UserInputType.MouseButton3,
59
+ };
17
60
 
18
61
  function simulateMouseInput(requestData: Record<string, unknown>) {
19
62
  const action = requestData.action as string;
20
63
  const x = requestData.x as number | undefined;
21
64
  const y = requestData.y as number | undefined;
22
65
  const button = (requestData.button as string) ?? "Left";
23
- const scrollDirection = requestData.scrollDirection as string | undefined;
24
66
 
25
67
  if (!action) return { error: "action is required" };
68
+ if (x === undefined || y === undefined) {
69
+ return { error: "x and y are required" };
70
+ }
26
71
 
27
- const vim = getVIM();
28
- if (!vim) return { error: "VirtualInputManager is not available in this context" };
72
+ // Input is silently dropped by the engine when the window isn't rendering
73
+ // (e.g. minimized). Surface that instead of returning a false success.
74
+ const notRendering = RenderMonitor.notRenderingReason();
75
+ if (notRendering !== undefined) return { error: notRendering };
76
+
77
+ const vi = getVI();
78
+ if (!vi) {
79
+ return { error: "UserInputService:CreateVirtualInput() is not available in this context" };
80
+ }
29
81
 
30
- const buttonNum = BUTTON_MAP[button] ?? 0;
82
+ const inputType = MOUSE_TYPE_MAP[button] ?? Enum.UserInputType.MouseButton1;
83
+ const pos = new Vector2(x, y);
31
84
 
32
85
  const [success, err] = pcall(() => {
33
86
  if (action === "click") {
34
- if (x === undefined || y === undefined) error("x and y are required for click");
35
- vim.SendMouseButtonEvent(x, y, buttonNum, true);
87
+ vi.SendMouseButton(pos, inputType, true);
36
88
  task.wait(0.05);
37
- vim.SendMouseButtonEvent(x, y, buttonNum, false);
89
+ vi.SendMouseButton(pos, inputType, false);
38
90
  } else if (action === "mouseDown") {
39
- if (x === undefined || y === undefined) error("x and y are required for mouseDown");
40
- vim.SendMouseButtonEvent(x, y, buttonNum, true);
91
+ vi.SendMouseButton(pos, inputType, true);
41
92
  } else if (action === "mouseUp") {
42
- if (x === undefined || y === undefined) error("x and y are required for mouseUp");
43
- vim.SendMouseButtonEvent(x, y, buttonNum, false);
44
- } else if (action === "move") {
45
- if (x === undefined || y === undefined) error("x and y are required for move");
46
- vim.SendMouseMoveEvent(x, y);
47
- } else if (action === "scroll") {
48
- if (x === undefined || y === undefined) error("x and y are required for scroll");
49
- if (!scrollDirection) error("scrollDirection is required for scroll");
50
- vim.SendMouseWheelEvent(x, y, scrollDirection === "up");
93
+ vi.SendMouseButton(pos, inputType, false);
51
94
  } else {
52
- error(`Unknown action: ${action}`);
95
+ error(
96
+ `Unsupported action "${action}". CreateVirtualInput supports click, mouseDown, mouseUp ` +
97
+ `(no move/scroll — those methods don't exist on VirtualInput).`,
98
+ );
53
99
  }
54
100
  });
55
101
 
@@ -60,31 +106,46 @@ function simulateMouseInput(requestData: Record<string, unknown>) {
60
106
  }
61
107
 
62
108
  function simulateKeyboardInput(requestData: Record<string, unknown>) {
109
+ const notRendering = RenderMonitor.notRenderingReason();
110
+ if (notRendering !== undefined) return { error: notRendering };
111
+
112
+ const vi = getVI();
113
+ if (!vi) {
114
+ return { error: "UserInputService:CreateVirtualInput() is not available in this context" };
115
+ }
116
+
117
+ // Text mode: type a string into the focused TextBox.
118
+ const text = requestData.text as string | undefined;
119
+ if (text !== undefined) {
120
+ const [ok, err] = pcall(() => vi.SendTextInput(text));
121
+ if (ok) return { success: true, text };
122
+ return { error: `Failed to send text input: ${err}` };
123
+ }
124
+
63
125
  const keyCodeName = requestData.keyCode as string;
126
+ if (!keyCodeName) return { error: "keyCode (or text) is required" };
127
+
64
128
  const action = (requestData.action as string) ?? "tap";
65
129
  const duration = (requestData.duration as number) ?? 0.1;
66
130
 
67
- if (!keyCodeName) return { error: "keyCode is required" };
68
-
69
- const vim = getVIM();
70
- if (!vim) return { error: "VirtualInputManager is not available in this context" };
71
-
72
131
  const [enumOk, keyCode] = pcall(() => {
73
132
  return (Enum.KeyCode as unknown as Record<string, Enum.KeyCode>)[keyCodeName];
74
133
  });
75
134
  if (!enumOk || !keyCode) {
76
- return { error: `Unknown keyCode: ${keyCodeName}. Use Enum.KeyCode names like "W", "Space", "E", "LeftShift", etc.` };
135
+ return {
136
+ error: `Unknown keyCode: ${keyCodeName}. Use Enum.KeyCode names like "W", "Space", "E", "LeftShift", etc.`,
137
+ };
77
138
  }
78
139
 
79
140
  const [success, err] = pcall(() => {
80
141
  if (action === "press") {
81
- vim.SendKeyEvent(true, keyCode, false);
142
+ vi.SendKey(true, keyCode);
82
143
  } else if (action === "release") {
83
- vim.SendKeyEvent(false, keyCode, false);
144
+ vi.SendKey(false, keyCode);
84
145
  } else if (action === "tap") {
85
- vim.SendKeyEvent(true, keyCode, false);
146
+ vi.SendKey(true, keyCode);
86
147
  task.wait(duration);
87
- vim.SendKeyEvent(false, keyCode, false);
148
+ vi.SendKey(false, keyCode);
88
149
  } else {
89
150
  error(`Unknown action: ${action}`);
90
151
  }