@chrrxs/robloxstudio-mcp 2.12.0 → 2.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1508 -64
- package/package.json +2 -2
- package/studio-plugin/INSTALLATION.md +1 -0
- package/studio-plugin/MCPInspectorPlugin.rbxmx +1312 -325
- package/studio-plugin/MCPPlugin.rbxmx +705 -97
- package/studio-plugin/src/modules/ClientBroker.ts +91 -1
- package/studio-plugin/src/modules/Communication.ts +22 -0
- package/studio-plugin/src/modules/EvalBridges.ts +60 -11
- package/studio-plugin/src/modules/RenderMonitor.ts +60 -0
- package/studio-plugin/src/modules/handlers/CaptureHandlers.ts +45 -3
- package/studio-plugin/src/modules/handlers/InputHandlers.ts +100 -39
- package/studio-plugin/src/modules/handlers/TestHandlers.ts +257 -18
- package/studio-plugin/src/server/index.server.ts +6 -0
|
@@ -1,8 +1,18 @@
|
|
|
1
1
|
import { HttpService, Players, ReplicatedStorage, RunService, ServerStorage } from "@rbxts/services";
|
|
2
2
|
import RuntimeLogBuffer from "./RuntimeLogBuffer";
|
|
3
3
|
import MemoryHandlers from "./handlers/MemoryHandlers";
|
|
4
|
+
import CaptureHandlers from "./handlers/CaptureHandlers";
|
|
5
|
+
import InputHandlers from "./handlers/InputHandlers";
|
|
4
6
|
import LuauExec from "./LuauExec";
|
|
5
7
|
|
|
8
|
+
interface StudioTestServiceMultiplayer extends StudioTestService {
|
|
9
|
+
CanLeaveTest(): boolean;
|
|
10
|
+
LeaveTest(): void;
|
|
11
|
+
EditModeActive: boolean;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
const StudioTestService = game.GetService("StudioTestService") as StudioTestServiceMultiplayer;
|
|
15
|
+
|
|
6
16
|
// Mirror of Communication.computeInstanceId() — duplicated here because the
|
|
7
17
|
// client broker runs in the play-server DM where it can't easily import from
|
|
8
18
|
// the edit-side module, and the place identifier must match what the edit-DM
|
|
@@ -53,6 +63,7 @@ function resolvePlaceName(): string {
|
|
|
53
63
|
|
|
54
64
|
const MCP_URL = "http://localhost:58741";
|
|
55
65
|
const BROKER_NAME = "__MCPClientBroker";
|
|
66
|
+
const BROKER_OWNER_ATTRIBUTE = "__MCPBrokerOwner";
|
|
56
67
|
|
|
57
68
|
interface ProxyEntry {
|
|
58
69
|
pluginSessionId: string;
|
|
@@ -76,6 +87,15 @@ const CLIENT_BROKER_ALLOWED_ENDPOINTS = new Set<string>([
|
|
|
76
87
|
"/api/execute-luau",
|
|
77
88
|
"/api/get-runtime-logs",
|
|
78
89
|
"/api/get-memory-breakdown",
|
|
90
|
+
"/api/multiplayer-test-state",
|
|
91
|
+
"/api/multiplayer-test-leave-client",
|
|
92
|
+
// Screenshot capture must run in the client peer (CaptureService captures
|
|
93
|
+
// the play viewport there); the edit DM reads the temp id back separately.
|
|
94
|
+
"/api/capture-begin",
|
|
95
|
+
// Virtual input (CreateVirtualInput) drives the running client's input
|
|
96
|
+
// pipeline, so it must execute in the client peer's VM.
|
|
97
|
+
"/api/simulate-mouse-input",
|
|
98
|
+
"/api/simulate-keyboard-input",
|
|
79
99
|
]);
|
|
80
100
|
|
|
81
101
|
interface ReadyResponseBody {
|
|
@@ -155,6 +175,52 @@ function handleGetRuntimeLogs(data: Record<string, unknown> | undefined): unknow
|
|
|
155
175
|
return RuntimeLogBuffer.query({ since, tail, filter }, "client");
|
|
156
176
|
}
|
|
157
177
|
|
|
178
|
+
function handleMultiplayerTestState(): unknown {
|
|
179
|
+
const [argsOk, args] = pcall(() => StudioTestService.GetTestArgs());
|
|
180
|
+
const [canLeaveOk, canLeave] = pcall(() => StudioTestService.CanLeaveTest());
|
|
181
|
+
const players = Players.GetPlayers().map((player) => ({
|
|
182
|
+
name: player.Name,
|
|
183
|
+
userId: player.UserId,
|
|
184
|
+
displayName: player.DisplayName,
|
|
185
|
+
}));
|
|
186
|
+
players.sort((a, b) => a.name < b.name);
|
|
187
|
+
return {
|
|
188
|
+
success: true,
|
|
189
|
+
peer: "client",
|
|
190
|
+
isRunning: RunService.IsRunning(),
|
|
191
|
+
isRunMode: RunService.IsRunMode(),
|
|
192
|
+
editModeActive: StudioTestService.EditModeActive,
|
|
193
|
+
testArgsOk: argsOk,
|
|
194
|
+
testArgs: argsOk ? args : undefined,
|
|
195
|
+
testArgsError: argsOk ? undefined : tostring(args),
|
|
196
|
+
players,
|
|
197
|
+
playerCount: players.size(),
|
|
198
|
+
localPlayer: Players.LocalPlayer ? Players.LocalPlayer.Name : undefined,
|
|
199
|
+
canLeaveOk,
|
|
200
|
+
canLeave: canLeaveOk ? canLeave : false,
|
|
201
|
+
canLeaveError: canLeaveOk ? undefined : tostring(canLeave),
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function handleMultiplayerTestLeaveClient(): unknown {
|
|
206
|
+
const [canLeaveOk, canLeave] = pcall(() => StudioTestService.CanLeaveTest());
|
|
207
|
+
if (!canLeaveOk) {
|
|
208
|
+
return { error: tostring(canLeave), canLeaveOk: false };
|
|
209
|
+
}
|
|
210
|
+
if (!canLeave) {
|
|
211
|
+
return { error: "This client cannot leave the current test session.", canLeaveOk: true, canLeave: false };
|
|
212
|
+
}
|
|
213
|
+
const localPlayer = Players.LocalPlayer ? Players.LocalPlayer.Name : undefined;
|
|
214
|
+
task.defer(() => {
|
|
215
|
+
pcall(() => StudioTestService.LeaveTest());
|
|
216
|
+
});
|
|
217
|
+
return {
|
|
218
|
+
success: true,
|
|
219
|
+
message: "Client leave requested.",
|
|
220
|
+
localPlayer,
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
|
|
158
224
|
function setupClientBroker() {
|
|
159
225
|
const rf = ReplicatedStorage.WaitForChild(BROKER_NAME, 10);
|
|
160
226
|
if (!rf || !rf.IsA("RemoteFunction")) {
|
|
@@ -174,6 +240,21 @@ function setupClientBroker() {
|
|
|
174
240
|
if (payload && payload.endpoint === "/api/get-memory-breakdown") {
|
|
175
241
|
return MemoryHandlers.getMemoryBreakdown(payload.data ?? {});
|
|
176
242
|
}
|
|
243
|
+
if (payload && payload.endpoint === "/api/multiplayer-test-state") {
|
|
244
|
+
return handleMultiplayerTestState();
|
|
245
|
+
}
|
|
246
|
+
if (payload && payload.endpoint === "/api/multiplayer-test-leave-client") {
|
|
247
|
+
return handleMultiplayerTestLeaveClient();
|
|
248
|
+
}
|
|
249
|
+
if (payload && payload.endpoint === "/api/capture-begin") {
|
|
250
|
+
return CaptureHandlers.captureBegin();
|
|
251
|
+
}
|
|
252
|
+
if (payload && payload.endpoint === "/api/simulate-mouse-input") {
|
|
253
|
+
return InputHandlers.simulateMouseInput(payload.data ?? {});
|
|
254
|
+
}
|
|
255
|
+
if (payload && payload.endpoint === "/api/simulate-keyboard-input") {
|
|
256
|
+
return InputHandlers.simulateKeyboardInput(payload.data ?? {});
|
|
257
|
+
}
|
|
177
258
|
if (payload && payload.endpoint === "/api/execute-luau") {
|
|
178
259
|
return handleExecuteLuau(payload.data);
|
|
179
260
|
}
|
|
@@ -183,6 +264,7 @@ function setupClientBroker() {
|
|
|
183
264
|
}
|
|
184
265
|
|
|
185
266
|
const proxyByPlayer = new Map<Player, ProxyEntry>();
|
|
267
|
+
let serverBrokerStarted = false;
|
|
186
268
|
|
|
187
269
|
function pollProxy(proxyId: string, player: Player, rf: RemoteFunction) {
|
|
188
270
|
while (player.Parent !== undefined && proxyByPlayer.has(player)) {
|
|
@@ -215,10 +297,12 @@ function pollProxy(proxyId: string, player: Player, rf: RemoteFunction) {
|
|
|
215
297
|
response = { success: false, error: `InvokeClient failed: ${tostring(invokeRes)}` };
|
|
216
298
|
}
|
|
217
299
|
} else {
|
|
300
|
+
const allowed: string[] = [];
|
|
301
|
+
for (const ep of CLIENT_BROKER_ALLOWED_ENDPOINTS) allowed.push(ep);
|
|
218
302
|
response = {
|
|
219
303
|
error:
|
|
220
304
|
`Client-proxy does not forward ${tostring(request.endpoint)}. ` +
|
|
221
|
-
`Allowed:
|
|
305
|
+
`Allowed: ${allowed.join(", ")}.`,
|
|
222
306
|
};
|
|
223
307
|
}
|
|
224
308
|
postJson("/response", { requestId: body.requestId, response });
|
|
@@ -257,12 +341,18 @@ function registerProxy(player: Player, rf: RemoteFunction) {
|
|
|
257
341
|
// which doesn't depend on MCP server state or peer registration at all.)
|
|
258
342
|
|
|
259
343
|
function setupServerBroker() {
|
|
344
|
+
if (serverBrokerStarted) return;
|
|
260
345
|
let rf = ReplicatedStorage.FindFirstChild(BROKER_NAME) as RemoteFunction | undefined;
|
|
261
346
|
if (!rf) {
|
|
262
347
|
rf = new Instance("RemoteFunction");
|
|
263
348
|
rf.Name = BROKER_NAME;
|
|
264
349
|
rf.Parent = ReplicatedStorage;
|
|
265
350
|
}
|
|
351
|
+
if (rf.GetAttribute(BROKER_OWNER_ATTRIBUTE) !== undefined) {
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
rf.SetAttribute(BROKER_OWNER_ATTRIBUTE, HttpService.GenerateGUID(false));
|
|
355
|
+
serverBrokerStarted = true;
|
|
266
356
|
const broker = rf;
|
|
267
357
|
Players.PlayerAdded.Connect((p) => registerProxy(p, broker));
|
|
268
358
|
for (const p of Players.GetPlayers()) {
|
|
@@ -2,6 +2,7 @@ import { HttpService, RunService, ServerStorage } from "@rbxts/services";
|
|
|
2
2
|
import State from "./State";
|
|
3
3
|
import Utils from "./Utils";
|
|
4
4
|
import UI from "./UI";
|
|
5
|
+
import { ensureBridgesInstalled } from "./EvalBridges";
|
|
5
6
|
import QueryHandlers from "./handlers/QueryHandlers";
|
|
6
7
|
import PropertyHandlers from "./handlers/PropertyHandlers";
|
|
7
8
|
import InstanceHandlers from "./handlers/InstanceHandlers";
|
|
@@ -132,6 +133,11 @@ const routeMap: Record<string, Handler> = {
|
|
|
132
133
|
"/api/start-playtest": TestHandlers.startPlaytest,
|
|
133
134
|
"/api/stop-playtest": TestHandlers.stopPlaytest,
|
|
134
135
|
"/api/get-playtest-output": TestHandlers.getPlaytestOutput,
|
|
136
|
+
"/api/multiplayer-test-start": TestHandlers.multiplayerTestStart,
|
|
137
|
+
"/api/multiplayer-test-state": TestHandlers.multiplayerTestState,
|
|
138
|
+
"/api/multiplayer-test-add-players": TestHandlers.multiplayerTestAddPlayers,
|
|
139
|
+
"/api/multiplayer-test-leave-client": TestHandlers.multiplayerTestLeaveClient,
|
|
140
|
+
"/api/multiplayer-test-end": TestHandlers.multiplayerTestEnd,
|
|
135
141
|
"/api/character-navigation": TestHandlers.characterNavigation,
|
|
136
142
|
|
|
137
143
|
"/api/export-build": BuildHandlers.exportBuild,
|
|
@@ -143,6 +149,8 @@ const routeMap: Record<string, Handler> = {
|
|
|
143
149
|
"/api/preview-asset": AssetHandlers.previewAsset,
|
|
144
150
|
|
|
145
151
|
"/api/capture-screenshot": CaptureHandlers.captureScreenshot,
|
|
152
|
+
"/api/capture-begin": CaptureHandlers.captureBegin,
|
|
153
|
+
"/api/capture-read": CaptureHandlers.captureRead,
|
|
146
154
|
"/api/simulate-mouse-input": InputHandlers.simulateMouseInput,
|
|
147
155
|
"/api/simulate-keyboard-input": InputHandlers.simulateKeyboardInput,
|
|
148
156
|
|
|
@@ -458,6 +466,20 @@ function activatePlugin(connIndex?: number) {
|
|
|
458
466
|
// later reports knownInstance=false (process restart, etc).
|
|
459
467
|
sendReady(conn);
|
|
460
468
|
|
|
469
|
+
// Keep the eval bridges present in the edit DM so that ANY playtest —
|
|
470
|
+
// including one the dev starts manually via the Studio Play button —
|
|
471
|
+
// clones them into the play DMs and eval_*_runtime works with no setup
|
|
472
|
+
// roundtrip. Only the edit DM installs; play DMs already have the cloned
|
|
473
|
+
// copies. Idempotent, so reconnects don't re-dirty the place.
|
|
474
|
+
if (!RunService.IsRunning()) {
|
|
475
|
+
task.spawn(() => {
|
|
476
|
+
const result = ensureBridgesInstalled();
|
|
477
|
+
if (!result.installed) {
|
|
478
|
+
warn(`[MCPPlugin] Eval bridge install failed: ${result.error}`);
|
|
479
|
+
}
|
|
480
|
+
});
|
|
481
|
+
}
|
|
482
|
+
|
|
461
483
|
// Watch for game.Name updates so a stale "Place1" captured at first
|
|
462
484
|
// /ready gets refreshed once Studio settles on the real DM name.
|
|
463
485
|
ensureNameChangeWatcher(conn);
|
|
@@ -21,11 +21,16 @@
|
|
|
21
21
|
// ServerScriptService.LoadStringEnabled, so eval_server_runtime works even
|
|
22
22
|
// when LoadStringEnabled=false (the default in fresh places).
|
|
23
23
|
//
|
|
24
|
-
// Lifecycle:
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
//
|
|
28
|
-
//
|
|
24
|
+
// Lifecycle: the bridges live PERMANENTLY in the edit DM. Communication
|
|
25
|
+
// installs them (ensureBridgesInstalled) when the plugin connects in edit,
|
|
26
|
+
// and TestHandlers.startPlaytest force-refreshes them right before
|
|
27
|
+
// ExecutePlayModeAsync. ExecutePlayModeAsync clones the DataModel into the
|
|
28
|
+
// play DMs, so the scripts come along and run there. We keep them in the edit
|
|
29
|
+
// DM after a playtest ends (rather than cleaning up) so that a playtest the
|
|
30
|
+
// dev starts MANUALLY via the Studio Play button — not the MCP start_playtest
|
|
31
|
+
// tool — also gets the bridges cloned in. This is intentionally a little
|
|
32
|
+
// intrusive (two helper scripts visible in Explorer) in exchange for a
|
|
33
|
+
// zero-roundtrip eval_*_runtime experience for devs working 1:1 with an agent.
|
|
29
34
|
//
|
|
30
35
|
// Archivable handling: ExecutePlayModeAsync's deep-clone SKIPS instances
|
|
31
36
|
// with Archivable=false (verified empirically in v2.9.0 testing - bridges
|
|
@@ -61,9 +66,9 @@ export const BRIDGE_NAMES = {
|
|
|
61
66
|
// Embedded Luau. The double `${...}` references our exported names so a
|
|
62
67
|
// rename here propagates to both the script source and the tool wrappers.
|
|
63
68
|
const SERVER_BRIDGE_SOURCE = `
|
|
64
|
-
--
|
|
65
|
-
--
|
|
66
|
-
--
|
|
69
|
+
-- Installed by @chrrxs/robloxstudio-mcp to power the eval_server_runtime MCP
|
|
70
|
+
-- tool (shared-require-cache eval on the server during playtests). Inert
|
|
71
|
+
-- outside Studio (no-ops in live games); safe to leave in place.
|
|
67
72
|
|
|
68
73
|
local ServerScriptService = game:GetService("ServerScriptService")
|
|
69
74
|
local RunService = game:GetService("RunService")
|
|
@@ -88,9 +93,9 @@ end
|
|
|
88
93
|
`;
|
|
89
94
|
|
|
90
95
|
const CLIENT_BRIDGE_SOURCE = `
|
|
91
|
-
--
|
|
92
|
-
--
|
|
93
|
-
--
|
|
96
|
+
-- Installed by @chrrxs/robloxstudio-mcp to power the eval_client_runtime MCP
|
|
97
|
+
-- tool (shared-require-cache eval on the client during playtests). Inert
|
|
98
|
+
-- outside Studio (no-ops in live games); safe to leave in place.
|
|
94
99
|
|
|
95
100
|
local ReplicatedStorage = game:GetService("ReplicatedStorage")
|
|
96
101
|
local RunService = game:GetService("RunService")
|
|
@@ -114,6 +119,28 @@ bf.OnInvoke = function(payload)
|
|
|
114
119
|
end
|
|
115
120
|
`;
|
|
116
121
|
|
|
122
|
+
// Stamp written onto each installed bridge Script so we can tell whether the
|
|
123
|
+
// bridge currently in the DM was produced by THIS plugin build. It's a djb2
|
|
124
|
+
// hash of the actual bridge source plus the plugin version, so ANY change to
|
|
125
|
+
// the source (or a version bump) yields a new stamp — which makes
|
|
126
|
+
// ensureBridgesInstalled() force a refresh on the next plugin load instead of
|
|
127
|
+
// keeping a stale bridge that happens to still be present (e.g. one saved into
|
|
128
|
+
// the .rbxl from an older build).
|
|
129
|
+
const STAMP_ATTR = "__MCPBridgeStamp";
|
|
130
|
+
|
|
131
|
+
function computeBridgeStamp(): string {
|
|
132
|
+
const combined = `${SERVER_BRIDGE_SOURCE}|${CLIENT_BRIDGE_SOURCE}`;
|
|
133
|
+
let h = 5381;
|
|
134
|
+
for (let i = 1; i <= combined.size(); i++) {
|
|
135
|
+
h = (h * 33 + string.byte(combined, i)[0]) % 2147483647;
|
|
136
|
+
}
|
|
137
|
+
// "__VERSION__" is replaced with the package version at package time
|
|
138
|
+
// (scripts/build-plugin.mjs injectVersion), so a release bump also restamps.
|
|
139
|
+
return `${tostring(h)}-__VERSION__`;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const BRIDGE_STAMP = computeBridgeStamp();
|
|
143
|
+
|
|
117
144
|
function setSource(scriptInst: Script | LocalScript, source: string): void {
|
|
118
145
|
// ScriptEditorService is the cleaner API and integrates with Studio's
|
|
119
146
|
// edit history; fall back to direct Source mutation (allowed in plugin
|
|
@@ -144,6 +171,26 @@ export function cleanupBridges(): void {
|
|
|
144
171
|
}
|
|
145
172
|
}
|
|
146
173
|
|
|
174
|
+
// Idempotent variant: install only if the bridge scripts aren't already
|
|
175
|
+
// present in the edit DM. Used to keep the bridges always available (so a
|
|
176
|
+
// playtest the dev starts manually — not via the MCP start_playtest tool —
|
|
177
|
+
// still clones them into the play DMs). Cheap no-op when already installed,
|
|
178
|
+
// which avoids re-dirtying the place on every plugin reconnect.
|
|
179
|
+
export function ensureBridgesInstalled(): { installed: boolean; error?: string } {
|
|
180
|
+
const { server, client } = findBridges();
|
|
181
|
+
if (server && client) {
|
|
182
|
+
// Both present — but only skip the reinstall if they were produced by
|
|
183
|
+
// THIS build. A mismatched/absent stamp means a stale bridge (older
|
|
184
|
+
// plugin, or one persisted in the saved place), so force a refresh.
|
|
185
|
+
const sStamp = server.GetAttribute(STAMP_ATTR);
|
|
186
|
+
const cStamp = client.GetAttribute(STAMP_ATTR);
|
|
187
|
+
if (sStamp === BRIDGE_STAMP && cStamp === BRIDGE_STAMP) {
|
|
188
|
+
return { installed: true };
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
return installBridges();
|
|
192
|
+
}
|
|
193
|
+
|
|
147
194
|
export function installBridges(): { installed: boolean; error?: string } {
|
|
148
195
|
// Defensive: clear any stale bridges from a prior unclean exit before
|
|
149
196
|
// inserting fresh. The injected script also self-cleans its
|
|
@@ -158,6 +205,7 @@ export function installBridges(): { installed: boolean; error?: string } {
|
|
|
158
205
|
// script. cleanupBridges() removes it from the edit DM when the
|
|
159
206
|
// playtest ends.
|
|
160
207
|
setSource(serverScript, SERVER_BRIDGE_SOURCE);
|
|
208
|
+
serverScript.SetAttribute(STAMP_ATTR, BRIDGE_STAMP);
|
|
161
209
|
serverScript.Parent = ServerScriptService;
|
|
162
210
|
|
|
163
211
|
const sps = getStarterPlayerScripts();
|
|
@@ -167,6 +215,7 @@ export function installBridges(): { installed: boolean; error?: string } {
|
|
|
167
215
|
const clientScript = new Instance("LocalScript");
|
|
168
216
|
clientScript.Name = CLIENT_SCRIPT_NAME;
|
|
169
217
|
setSource(clientScript, CLIENT_BRIDGE_SOURCE);
|
|
218
|
+
clientScript.SetAttribute(STAMP_ATTR, BRIDGE_STAMP);
|
|
170
219
|
clientScript.Parent = sps;
|
|
171
220
|
});
|
|
172
221
|
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
// Detects whether the Studio window is actually rendering, so virtual input
|
|
2
|
+
// and screenshot tools can surface a clear reason instead of silently failing.
|
|
3
|
+
//
|
|
4
|
+
// When a Studio window is MINIMIZED, the engine suspends the render loop AND
|
|
5
|
+
// input processing, but keeps running scripts (Heartbeat keeps firing). That's
|
|
6
|
+
// why simulate_*_input would return success while having zero effect, and
|
|
7
|
+
// CaptureService:CaptureScreenshot would time out. Validated live: during a 3s
|
|
8
|
+
// minimize, RenderStepped's max inter-frame gap was 5.08s while Heartbeat's was
|
|
9
|
+
// 0.10s. So RenderStepped freshness is the reliable "is this window rendering?"
|
|
10
|
+
// signal; Heartbeat is not.
|
|
11
|
+
|
|
12
|
+
import { RunService } from "@rbxts/services";
|
|
13
|
+
|
|
14
|
+
let lastFrame = 0;
|
|
15
|
+
let connected = false;
|
|
16
|
+
|
|
17
|
+
// Above this many seconds since the last rendered frame, we treat the window
|
|
18
|
+
// as not rendering. RenderStepped normally fires every ~16ms; a multi-second
|
|
19
|
+
// gap only happens when minimized/suspended, so 1s cleanly avoids false
|
|
20
|
+
// positives from ordinary frame hitches while still catching the real case.
|
|
21
|
+
const STALE_THRESHOLD = 1.0;
|
|
22
|
+
|
|
23
|
+
export function start(): void {
|
|
24
|
+
if (connected) return;
|
|
25
|
+
// RenderStepped can only be connected from a client/edit render loop; it
|
|
26
|
+
// throws in the play-server DM. pcall so a server-DM call is a safe no-op
|
|
27
|
+
// (connected stays false → notRenderingReason() returns undefined there).
|
|
28
|
+
const [ok] = pcall(() => {
|
|
29
|
+
RunService.RenderStepped.Connect(() => {
|
|
30
|
+
lastFrame = tick();
|
|
31
|
+
});
|
|
32
|
+
});
|
|
33
|
+
if (ok) {
|
|
34
|
+
connected = true;
|
|
35
|
+
lastFrame = tick();
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function secondsSinceFrame(): number {
|
|
40
|
+
if (!connected) return 0;
|
|
41
|
+
return tick() - lastFrame;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// Returns a human-readable reason if the window appears minimized / not
|
|
45
|
+
// rendering (so input + screenshots won't work), else undefined. Fail-open:
|
|
46
|
+
// when the monitor isn't active in this DM (server peer, or connect failed) it
|
|
47
|
+
// returns undefined so we never block on a false signal.
|
|
48
|
+
export function notRenderingReason(): string | undefined {
|
|
49
|
+
if (!connected) return undefined;
|
|
50
|
+
const gap = secondsSinceFrame();
|
|
51
|
+
if (gap > STALE_THRESHOLD) {
|
|
52
|
+
return string.format(
|
|
53
|
+
"Studio window appears minimized or not rendering (no frame in %.1fs). " +
|
|
54
|
+
"Virtual input and screenshots only work while the window is visible — " +
|
|
55
|
+
"restore/un-minimize the Studio window and retry.",
|
|
56
|
+
gap,
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
return undefined;
|
|
60
|
+
}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import * as RenderMonitor from "../RenderMonitor";
|
|
2
|
+
|
|
1
3
|
const CaptureService = game.GetService("CaptureService");
|
|
2
4
|
const AssetService = game.GetService("AssetService");
|
|
3
5
|
|
|
@@ -71,7 +73,17 @@ function readPixelsTiled(img: EditableImage, w: number, h: number): buffer {
|
|
|
71
73
|
return fullBuf;
|
|
72
74
|
}
|
|
73
75
|
|
|
74
|
-
|
|
76
|
+
// Triggers CaptureService:CaptureScreenshot and waits for the temporary
|
|
77
|
+
// content id. Works in any DM, including the play CLIENT (where reading the
|
|
78
|
+
// pixels back is blocked, but capturing is not). The returned rbxtemp:// id is
|
|
79
|
+
// a process-scoped handle: it can be dereferenced from a DIFFERENT, more
|
|
80
|
+
// privileged DM (the edit DM) — see captureRead.
|
|
81
|
+
function doCaptureScreenshot(): { contentId: string } | { error: string } {
|
|
82
|
+
// Fast-fail with a clear reason if the window isn't rendering — otherwise
|
|
83
|
+
// CaptureScreenshot's callback never fires and we'd block for the full 10s.
|
|
84
|
+
const notRendering = RenderMonitor.notRenderingReason();
|
|
85
|
+
if (notRendering !== undefined) return { error: notRendering };
|
|
86
|
+
|
|
75
87
|
let contentId: string | undefined;
|
|
76
88
|
|
|
77
89
|
CaptureService.CaptureScreenshot((id: string) => {
|
|
@@ -82,14 +94,23 @@ function captureScreenshotData(): unknown {
|
|
|
82
94
|
while (contentId === undefined) {
|
|
83
95
|
if (tick() - startTime > 10) {
|
|
84
96
|
return {
|
|
85
|
-
error: "Screenshot capture timed out.
|
|
97
|
+
error: "Screenshot capture timed out (CaptureScreenshot callback never fired). The Studio window is likely minimized or occluded — restore it so the viewport renders. (Known Roblox bug: capture can also fail if the viewport renders a solid color.)",
|
|
86
98
|
};
|
|
87
99
|
}
|
|
88
100
|
task.wait(0.1);
|
|
89
101
|
}
|
|
90
102
|
|
|
103
|
+
return { contentId };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// Promotes a CaptureScreenshot content id into an EditableImage and reads its
|
|
107
|
+
// RGBA pixels. MUST run in the edit/plugin context: the running game VM lacks
|
|
108
|
+
// the privilege to create an EditableImage from a temporary texture id (errors
|
|
109
|
+
// "cannot currently create editable image from temporary texture id"), while
|
|
110
|
+
// the edit DM can — even for an id captured in the play client DM.
|
|
111
|
+
function readContentToBase64(contentId: string): unknown {
|
|
91
112
|
const [editableOk, editableResult] = pcall(() => {
|
|
92
|
-
return AssetService.CreateEditableImageAsync(Content.fromUri(contentId
|
|
113
|
+
return AssetService.CreateEditableImageAsync(Content.fromUri(contentId));
|
|
93
114
|
});
|
|
94
115
|
|
|
95
116
|
if (!editableOk) {
|
|
@@ -118,11 +139,32 @@ function captureScreenshotData(): unknown {
|
|
|
118
139
|
return { success: true, width: w, height: h, data: base64Data };
|
|
119
140
|
}
|
|
120
141
|
|
|
142
|
+
// Edit-mode single shot: capture and read back in the same (edit) context.
|
|
143
|
+
function captureScreenshotData(): unknown {
|
|
144
|
+
const cap = doCaptureScreenshot();
|
|
145
|
+
if ("error" in cap) return cap;
|
|
146
|
+
return readContentToBase64(cap.contentId);
|
|
147
|
+
}
|
|
148
|
+
|
|
121
149
|
function captureScreenshot(): unknown {
|
|
122
150
|
return captureScreenshotData();
|
|
123
151
|
}
|
|
124
152
|
|
|
153
|
+
// Play-mode step 1 (run on the CLIENT): capture only, return the temp id.
|
|
154
|
+
function captureBegin(): unknown {
|
|
155
|
+
return doCaptureScreenshot();
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// Play-mode step 2 (run on EDIT): read pixels from a temp id captured elsewhere.
|
|
159
|
+
function captureRead(requestData: Record<string, unknown>): unknown {
|
|
160
|
+
const contentId = requestData.contentId as string | undefined;
|
|
161
|
+
if (!contentId) return { error: "contentId is required" };
|
|
162
|
+
return readContentToBase64(contentId);
|
|
163
|
+
}
|
|
164
|
+
|
|
125
165
|
export = {
|
|
126
166
|
captureScreenshotData,
|
|
127
167
|
captureScreenshot,
|
|
168
|
+
captureBegin,
|
|
169
|
+
captureRead,
|
|
128
170
|
};
|
|
@@ -1,55 +1,101 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
// Virtual input via UserInputService:CreateVirtualInput().
|
|
2
|
+
//
|
|
3
|
+
// We deliberately do NOT use VirtualInputManager:Send*Event — those methods
|
|
4
|
+
// are gated behind RobloxScriptSecurity ("lacking capability RobloxScript")
|
|
5
|
+
// in every context a plugin can reach (edit DM, play server/client DMs), so
|
|
6
|
+
// they silently never worked. CreateVirtualInput() is callable without that
|
|
7
|
+
// capability and drives the REAL input pipeline: SendKey feeds
|
|
8
|
+
// UserInputService.InputBegan/Ended and the control modules (so WASD walks the
|
|
9
|
+
// character at full WalkSpeed with controls intact, no Humanoid hijack),
|
|
10
|
+
// SendMouseButton feeds UIS and activates GUI buttons (and hit-tests against
|
|
11
|
+
// CoreGui), and SendTextInput types into the focused TextBox.
|
|
12
|
+
//
|
|
13
|
+
// Method set on the VirtualInput object (verified live):
|
|
14
|
+
// SendKey(isDown: boolean, keyCode: Enum.KeyCode)
|
|
15
|
+
// SendMouseButton(position: Vector2, inputType: Enum.UserInputType, isDown: boolean)
|
|
16
|
+
// SendTextInput(text: string)
|
|
17
|
+
// There is NO SendMouseMove / SendMouseWheel / SendKeyEvent — so "move" and
|
|
18
|
+
// "scroll" mouse actions are not supported.
|
|
19
|
+
//
|
|
20
|
+
// Coordinate space: SendMouseButton coordinates are viewport pixels matching
|
|
21
|
+
// what capture_screenshot returns (window space, origin at the top-left of the
|
|
22
|
+
// rendered viewport). Pass screenshot pixel coordinates straight through. Note
|
|
23
|
+
// that UserInputService reports input positions in GUI space, which is offset
|
|
24
|
+
// from this by GuiService:GetGuiInset() (~58px on the Y axis) — irrelevant for
|
|
25
|
+
// callers who pick coordinates off a screenshot, which is why we do not
|
|
26
|
+
// translate here.
|
|
27
|
+
|
|
28
|
+
import * as RenderMonitor from "../RenderMonitor";
|
|
29
|
+
|
|
30
|
+
const UserInputService = game.GetService("UserInputService");
|
|
31
|
+
|
|
32
|
+
interface VirtualInput {
|
|
33
|
+
SendKey(isDown: boolean, keyCode: Enum.KeyCode): void;
|
|
34
|
+
SendMouseButton(position: Vector2, inputType: Enum.UserInputType, isDown: boolean): void;
|
|
35
|
+
SendTextInput(text: string): void;
|
|
6
36
|
}
|
|
7
37
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
38
|
+
// One VirtualInput per plugin VM, reused across calls so that a key held down
|
|
39
|
+
// in one call (action="press") and released in a later call (action="release")
|
|
40
|
+
// share the same input source.
|
|
41
|
+
let cachedVI: VirtualInput | undefined;
|
|
42
|
+
|
|
43
|
+
function getVI(): VirtualInput | undefined {
|
|
44
|
+
if (cachedVI) return cachedVI;
|
|
45
|
+
const [ok, vi] = pcall(() => {
|
|
46
|
+
return (UserInputService as unknown as { CreateVirtualInput(): unknown }).CreateVirtualInput();
|
|
11
47
|
});
|
|
12
|
-
if (ok &&
|
|
48
|
+
if (ok && vi !== undefined) {
|
|
49
|
+
cachedVI = vi as VirtualInput;
|
|
50
|
+
return cachedVI;
|
|
51
|
+
}
|
|
13
52
|
return undefined;
|
|
14
53
|
}
|
|
15
54
|
|
|
16
|
-
const
|
|
55
|
+
const MOUSE_TYPE_MAP: Record<string, Enum.UserInputType> = {
|
|
56
|
+
Left: Enum.UserInputType.MouseButton1,
|
|
57
|
+
Right: Enum.UserInputType.MouseButton2,
|
|
58
|
+
Middle: Enum.UserInputType.MouseButton3,
|
|
59
|
+
};
|
|
17
60
|
|
|
18
61
|
function simulateMouseInput(requestData: Record<string, unknown>) {
|
|
19
62
|
const action = requestData.action as string;
|
|
20
63
|
const x = requestData.x as number | undefined;
|
|
21
64
|
const y = requestData.y as number | undefined;
|
|
22
65
|
const button = (requestData.button as string) ?? "Left";
|
|
23
|
-
const scrollDirection = requestData.scrollDirection as string | undefined;
|
|
24
66
|
|
|
25
67
|
if (!action) return { error: "action is required" };
|
|
68
|
+
if (x === undefined || y === undefined) {
|
|
69
|
+
return { error: "x and y are required" };
|
|
70
|
+
}
|
|
26
71
|
|
|
27
|
-
|
|
28
|
-
|
|
72
|
+
// Input is silently dropped by the engine when the window isn't rendering
|
|
73
|
+
// (e.g. minimized). Surface that instead of returning a false success.
|
|
74
|
+
const notRendering = RenderMonitor.notRenderingReason();
|
|
75
|
+
if (notRendering !== undefined) return { error: notRendering };
|
|
76
|
+
|
|
77
|
+
const vi = getVI();
|
|
78
|
+
if (!vi) {
|
|
79
|
+
return { error: "UserInputService:CreateVirtualInput() is not available in this context" };
|
|
80
|
+
}
|
|
29
81
|
|
|
30
|
-
const
|
|
82
|
+
const inputType = MOUSE_TYPE_MAP[button] ?? Enum.UserInputType.MouseButton1;
|
|
83
|
+
const pos = new Vector2(x, y);
|
|
31
84
|
|
|
32
85
|
const [success, err] = pcall(() => {
|
|
33
86
|
if (action === "click") {
|
|
34
|
-
|
|
35
|
-
vim.SendMouseButtonEvent(x, y, buttonNum, true);
|
|
87
|
+
vi.SendMouseButton(pos, inputType, true);
|
|
36
88
|
task.wait(0.05);
|
|
37
|
-
|
|
89
|
+
vi.SendMouseButton(pos, inputType, false);
|
|
38
90
|
} else if (action === "mouseDown") {
|
|
39
|
-
|
|
40
|
-
vim.SendMouseButtonEvent(x, y, buttonNum, true);
|
|
91
|
+
vi.SendMouseButton(pos, inputType, true);
|
|
41
92
|
} else if (action === "mouseUp") {
|
|
42
|
-
|
|
43
|
-
vim.SendMouseButtonEvent(x, y, buttonNum, false);
|
|
44
|
-
} else if (action === "move") {
|
|
45
|
-
if (x === undefined || y === undefined) error("x and y are required for move");
|
|
46
|
-
vim.SendMouseMoveEvent(x, y);
|
|
47
|
-
} else if (action === "scroll") {
|
|
48
|
-
if (x === undefined || y === undefined) error("x and y are required for scroll");
|
|
49
|
-
if (!scrollDirection) error("scrollDirection is required for scroll");
|
|
50
|
-
vim.SendMouseWheelEvent(x, y, scrollDirection === "up");
|
|
93
|
+
vi.SendMouseButton(pos, inputType, false);
|
|
51
94
|
} else {
|
|
52
|
-
error(
|
|
95
|
+
error(
|
|
96
|
+
`Unsupported action "${action}". CreateVirtualInput supports click, mouseDown, mouseUp ` +
|
|
97
|
+
`(no move/scroll — those methods don't exist on VirtualInput).`,
|
|
98
|
+
);
|
|
53
99
|
}
|
|
54
100
|
});
|
|
55
101
|
|
|
@@ -60,31 +106,46 @@ function simulateMouseInput(requestData: Record<string, unknown>) {
|
|
|
60
106
|
}
|
|
61
107
|
|
|
62
108
|
function simulateKeyboardInput(requestData: Record<string, unknown>) {
|
|
109
|
+
const notRendering = RenderMonitor.notRenderingReason();
|
|
110
|
+
if (notRendering !== undefined) return { error: notRendering };
|
|
111
|
+
|
|
112
|
+
const vi = getVI();
|
|
113
|
+
if (!vi) {
|
|
114
|
+
return { error: "UserInputService:CreateVirtualInput() is not available in this context" };
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// Text mode: type a string into the focused TextBox.
|
|
118
|
+
const text = requestData.text as string | undefined;
|
|
119
|
+
if (text !== undefined) {
|
|
120
|
+
const [ok, err] = pcall(() => vi.SendTextInput(text));
|
|
121
|
+
if (ok) return { success: true, text };
|
|
122
|
+
return { error: `Failed to send text input: ${err}` };
|
|
123
|
+
}
|
|
124
|
+
|
|
63
125
|
const keyCodeName = requestData.keyCode as string;
|
|
126
|
+
if (!keyCodeName) return { error: "keyCode (or text) is required" };
|
|
127
|
+
|
|
64
128
|
const action = (requestData.action as string) ?? "tap";
|
|
65
129
|
const duration = (requestData.duration as number) ?? 0.1;
|
|
66
130
|
|
|
67
|
-
if (!keyCodeName) return { error: "keyCode is required" };
|
|
68
|
-
|
|
69
|
-
const vim = getVIM();
|
|
70
|
-
if (!vim) return { error: "VirtualInputManager is not available in this context" };
|
|
71
|
-
|
|
72
131
|
const [enumOk, keyCode] = pcall(() => {
|
|
73
132
|
return (Enum.KeyCode as unknown as Record<string, Enum.KeyCode>)[keyCodeName];
|
|
74
133
|
});
|
|
75
134
|
if (!enumOk || !keyCode) {
|
|
76
|
-
return {
|
|
135
|
+
return {
|
|
136
|
+
error: `Unknown keyCode: ${keyCodeName}. Use Enum.KeyCode names like "W", "Space", "E", "LeftShift", etc.`,
|
|
137
|
+
};
|
|
77
138
|
}
|
|
78
139
|
|
|
79
140
|
const [success, err] = pcall(() => {
|
|
80
141
|
if (action === "press") {
|
|
81
|
-
|
|
142
|
+
vi.SendKey(true, keyCode);
|
|
82
143
|
} else if (action === "release") {
|
|
83
|
-
|
|
144
|
+
vi.SendKey(false, keyCode);
|
|
84
145
|
} else if (action === "tap") {
|
|
85
|
-
|
|
146
|
+
vi.SendKey(true, keyCode);
|
|
86
147
|
task.wait(duration);
|
|
87
|
-
|
|
148
|
+
vi.SendKey(false, keyCode);
|
|
88
149
|
} else {
|
|
89
150
|
error(`Unknown action: ${action}`);
|
|
90
151
|
}
|